def configure(self):
self.verbose("Configuring running instances...\n")
session = botocore.session.get_session()
access_key = session.get_credentials().access_key
secret_key = session.get_credentials().secret_key
retry = 0
for instance in self.get_instances():
client = paramiko.SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
while retry < self.opts.retry:
try:
client.connect(instance.public_ip_address, port=22, username='******',
key_filename='/home/leo_zou/.ssh/aws_211.pem')
break
except (BadHostKeyException, AuthenticationException,
NoValidConnectionsError, SSHException, socket.error) as e:
self.verbose(" failed to connect %s, retry...\n" % instance.public_ip_address, 0)
time.sleep(10)
retry += 1
self.ssh[instance.instance_id] = client
channel = client.invoke_shell()
stdin = channel.makefile('wb')
stdout = channel.makefile('rb')
self.verbose(" %s: installing packages...\n" % instance.instance_id, 1)
stdin.write('''
sudo yum-config-manager --enable epel
sudo yum install -y iperf3 fio python27-test
sudo `which pip` install -U pip
sudo `which pip` install -U boto boto3 awscli
mkdir -p ~/.aws
exit
''')
self.verbose(stdout.read(), 2)
sftp = client.open_sftp()
aws_config = sftp.file("/home/ec2-user/.aws/config", "w+", -1)
aws_config.write('''
[default]
region = us-west-2
aws_access_key_id = %s
aws_secret_access_key = %s
s3 =
max_concurrent_requests = 20
max_queue_size = 10000
multipart_threshold = 64MB
multipart_chunksize = 16MB
addressing_style = path
''' % (access_key, secret_key))
# TODO: use_accelerate_endpoint
aws_config.flush()
aws_config.close()
def get_user_data_configuration():
"""Retrieve and update the application configuration with information from the user-data
Returns:
`None`
"""
from cloud_inquisitor import get_local_aws_session, app_config
kms_region = app_config.kms_region
session = get_local_aws_session()
if session.get_credentials().method == 'iam-role':
kms = session.client('kms', region_name=kms_region)
else:
sts = session.client('sts')
audit_role = sts.assume_role(RoleArn=app_config.aws_api.instance_role_arn, RoleSessionName='cloud_inquisitor')
kms = boto3.session.Session(
audit_role['Credentials']['AccessKeyId'],
audit_role['Credentials']['SecretAccessKey'],
audit_role['Credentials']['SessionToken'],
).client('kms', region_name=kms_region)
user_data_url = app_config.user_data_url
res = requests.get(user_data_url)
if res.status_code == 200:
data = kms.decrypt(CiphertextBlob=b64decode(res.content))
kms_config = json.loads(zlib.decompress(data['Plaintext']).decode('utf-8'))
app_config.database_uri = kms_config['db_uri']
else:
raise RuntimeError('Failed loading user-data, cannot continue: {}: {}'.format(res.status_code, res.content))
def connection(self):
connection = getattr(self._connections, 'connection', None)
if connection is None:
session = boto3.session.Session()
print('key: {}, secret: {}'.format(self.access_key,
self.secret_key))
if self.use_instance_metadata:
creds = session.get_credentials()
self.access_key = creds.access_key
self.secret_key = creds.secret_key
self.security_token = creds.token
print('using metadata: {}, {}, {}'.format(
self.access_key, self.secret_key, self.security_token))
self._connections.connection = session.resource(
's3',
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
aws_session_token=self.security_token,
region_name=self.region_name,
use_ssl=self.use_ssl,
endpoint_url=self.endpoint_url,
config=self.config,
verify=self.verify,
)
return self._connections.connection
def _parse_aws(self):
"""
Parse the AWS args and attempt to obtain credentials using
:class:`boto3.session.Session`, which follows the AWS documentation at
http://amzn.to/2fRCGCt
"""
self.logger.debug('self.aws = {}'.format(self.aws))
self.logger.debug('self.client_args = {}'.format(self.client_args))
if self.use_aws:
if not 'aws_region' in self.aws or self.aws['aws_region'] is None:
raise MissingArgument('Missing "aws_region".')
from boto3 import session
from botocore.exceptions import NoCredentialsError
from requests_aws4auth import AWS4Auth
try:
session = session.Session()
credentials = session.get_credentials()
self.aws['aws_key'] = credentials.access_key
self.aws['aws_secret_key'] = credentials.secret_key
self.aws['aws_token'] = credentials.token
# If an attribute doesn't exist, we were not able to retrieve credentials as expected so we can't continue
except AttributeError:
self.logger.debug('Unable to locate AWS credentials')
raise NoCredentialsError
# Override these self.client_args
self.client_args['use_ssl'] = True
self.client_args['verify_certs'] = True
self.client_args[
'connection_class'] = elasticsearch.RequestsHttpConnection
self.client_args['http_auth'] = (AWS4Auth(
self.aws['aws_key'],
self.aws['aws_secret_key'],
self.aws['aws_region'],
'es',
session_token=self.aws['aws_token']))
def get_aws_session(account):
"""Function to return a boto3 Session based on the account passed in the first argument.
Args:
account (:obj:`Account`): Account to create the session object for
Returns:
:obj:`boto3:boto3.session.Session`
"""
from cloud_inquisitor.config import dbconfig
from cloud_inquisitor.plugins.types.accounts import AWSAccount
if not isinstance(account, AWSAccount):
raise InquisitorError('Non AWSAccount passed to get_aws_session, got {}'.format(account.__class__.__name__))
# If no keys are on supplied for the account, use sts.assume_role instead
session = get_local_aws_session()
if session.get_credentials().method == 'iam-role':
sts = session.client('sts')
else:
# If we are not running on an EC2 instance, assume the instance role
# first, then assume the remote role
temp_sts = session.client('sts')
audit_sts_role = temp_sts.assume_role(
RoleArn=app_config.aws_api.instance_role_arn,
RoleSessionName='inquisitor'
)
sts = boto3.session.Session(
audit_sts_role['Credentials']['AccessKeyId'],
audit_sts_role['Credentials']['SecretAccessKey'],
audit_sts_role['Credentials']['SessionToken']
).client('sts')
role = sts.assume_role(
RoleArn='arn:aws:iam::{}:role/{}'.format(
account.account_number,
dbconfig.get('role_name', default='cinq_role')
),
RoleSessionName='inquisitor'
)
sess = boto3.session.Session(
role['Credentials']['AccessKeyId'],
role['Credentials']['SecretAccessKey'],
role['Credentials']['SessionToken']
)
return sess
def get_client(self, settings_module):
profile = setting(self.profile_names, settings_module=settings_module)
access_key = setting(self.access_key_names, settings_module=settings_module)
secret_key = setting(self.secret_key_names, settings_module=settings_module)
region_name = setting(self.region_name_names, settings_module=settings_module)
credentials = {}
if access_key and secret_key:
credentials['aws_access_key_id'] = access_key
credentials['aws_secret_access_key'] = secret_key
elif profile:
credentials['profile_name'] = profile
credentials['region_name'] = region_name
session = boto3.session.Session(**credentials)
if session.get_credentials() is None:
raise CredentialsNotExists()
return session.client('secretsmanager')
wc = sys.argv[4]
s3_client = boto3.client('s3')
dynamodb = boto3.resource('dynamodb', region_name='us-east-1')
scheduler_table = dynamodb.Table('v486121-GPS-PathFinder-Scheduler-Table-' + env)
validation_table = dynamodb.Table('v486121-GPS-PathFinder-Validation-Rules-' + env)
response = scheduler_table.scan()
schedule_data = eval(json.dumps(response))
response = validation_table.scan()
validate_data = eval(json.dumps(response))
#########################Create elasticsearch session instance###################################
session = session.Session()
credentials = session.get_credentials()
region = 'us-east-1'
service = 'es'
auth = AWSV4Sign(credentials, region, service)
try:
es = Elasticsearch(host=host,
port=443,
connection_class=RequestsHttpConnection,
http_auth=auth,
use_ssl=True,
verify_ssl=True,
timeout=30,
max_retries=10,
retry_on_timeout=True
)
except Exception, err:
DOCTYPES = {
'data_obj': os.environ.get('DATA_OBJ_DOCTYPE', 'meta'),
'data_bdl': os.environ.get('DATA_BDL_DOCTYPE', 'databundle'),
}
try:
es_host = os.environ['ES_HOST']
except KeyError:
raise RuntimeError("You must specify the domain name of your ElasticSearch"
" instance with the ES_HOST environment variable.")
es_region = os.environ.get('ES_REGION', DEFAULT_REGION)
access_token = os.environ.get('ACCESS_KEY', DEFAULT_ACCESS_TOKEN)
session = boto3.session.Session()
credentials = session.get_credentials().get_frozen_credentials()
awsauth = aws_requests_auth.aws_auth.AWSRequestsAuth(
aws_access_key=credentials.access_key,
aws_secret_access_key=credentials.secret_key,
aws_token=credentials.token,
aws_host=es_host,
aws_region=session.region_name,
aws_service='es'
)
es = elasticsearch.Elasticsearch(
hosts=[{'host': es_host, 'port': 443}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=elasticsearch.RequestsHttpConnection
)
