def configure(self): self.verbose("Configuring running instances...\n") session = botocore.session.get_session() access_key = session.get_credentials().access_key secret_key = session.get_credentials().secret_key retry = 0 for instance in self.get_instances(): client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) while retry < self.opts.retry: try: client.connect(instance.public_ip_address, port=22, username='******', key_filename='/home/leo_zou/.ssh/aws_211.pem') break except (BadHostKeyException, AuthenticationException, NoValidConnectionsError, SSHException, socket.error) as e: self.verbose(" failed to connect %s, retry...\n" % instance.public_ip_address, 0) time.sleep(10) retry += 1 self.ssh[instance.instance_id] = client channel = client.invoke_shell() stdin = channel.makefile('wb') stdout = channel.makefile('rb') self.verbose(" %s: installing packages...\n" % instance.instance_id, 1) stdin.write(''' sudo yum-config-manager --enable epel sudo yum install -y iperf3 fio python27-test sudo `which pip` install -U pip sudo `which pip` install -U boto boto3 awscli mkdir -p ~/.aws exit ''') self.verbose(stdout.read(), 2) sftp = client.open_sftp() aws_config = sftp.file("/home/ec2-user/.aws/config", "w+", -1) aws_config.write(''' [default] region = us-west-2 aws_access_key_id = %s aws_secret_access_key = %s s3 = max_concurrent_requests = 20 max_queue_size = 10000 multipart_threshold = 64MB multipart_chunksize = 16MB addressing_style = path ''' % (access_key, secret_key)) # TODO: use_accelerate_endpoint aws_config.flush() aws_config.close()
def get_user_data_configuration(): """Retrieve and update the application configuration with information from the user-data Returns: `None` """ from cloud_inquisitor import get_local_aws_session, app_config kms_region = app_config.kms_region session = get_local_aws_session() if session.get_credentials().method == 'iam-role': kms = session.client('kms', region_name=kms_region) else: sts = session.client('sts') audit_role = sts.assume_role(RoleArn=app_config.aws_api.instance_role_arn, RoleSessionName='cloud_inquisitor') kms = boto3.session.Session( audit_role['Credentials']['AccessKeyId'], audit_role['Credentials']['SecretAccessKey'], audit_role['Credentials']['SessionToken'], ).client('kms', region_name=kms_region) user_data_url = app_config.user_data_url res = requests.get(user_data_url) if res.status_code == 200: data = kms.decrypt(CiphertextBlob=b64decode(res.content)) kms_config = json.loads(zlib.decompress(data['Plaintext']).decode('utf-8')) app_config.database_uri = kms_config['db_uri'] else: raise RuntimeError('Failed loading user-data, cannot continue: {}: {}'.format(res.status_code, res.content))
def connection(self): connection = getattr(self._connections, 'connection', None) if connection is None: session = boto3.session.Session() print('key: {}, secret: {}'.format(self.access_key, self.secret_key)) if self.use_instance_metadata: creds = session.get_credentials() self.access_key = creds.access_key self.secret_key = creds.secret_key self.security_token = creds.token print('using metadata: {}, {}, {}'.format( self.access_key, self.secret_key, self.security_token)) self._connections.connection = session.resource( 's3', aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key, aws_session_token=self.security_token, region_name=self.region_name, use_ssl=self.use_ssl, endpoint_url=self.endpoint_url, config=self.config, verify=self.verify, ) return self._connections.connection
def _parse_aws(self): """ Parse the AWS args and attempt to obtain credentials using :class:`boto3.session.Session`, which follows the AWS documentation at http://amzn.to/2fRCGCt """ self.logger.debug('self.aws = {}'.format(self.aws)) self.logger.debug('self.client_args = {}'.format(self.client_args)) if self.use_aws: if not 'aws_region' in self.aws or self.aws['aws_region'] is None: raise MissingArgument('Missing "aws_region".') from boto3 import session from botocore.exceptions import NoCredentialsError from requests_aws4auth import AWS4Auth try: session = session.Session() credentials = session.get_credentials() self.aws['aws_key'] = credentials.access_key self.aws['aws_secret_key'] = credentials.secret_key self.aws['aws_token'] = credentials.token # If an attribute doesn't exist, we were not able to retrieve credentials as expected so we can't continue except AttributeError: self.logger.debug('Unable to locate AWS credentials') raise NoCredentialsError # Override these self.client_args self.client_args['use_ssl'] = True self.client_args['verify_certs'] = True self.client_args[ 'connection_class'] = elasticsearch.RequestsHttpConnection self.client_args['http_auth'] = (AWS4Auth( self.aws['aws_key'], self.aws['aws_secret_key'], self.aws['aws_region'], 'es', session_token=self.aws['aws_token']))
def get_aws_session(account): """Function to return a boto3 Session based on the account passed in the first argument. Args: account (:obj:`Account`): Account to create the session object for Returns: :obj:`boto3:boto3.session.Session` """ from cloud_inquisitor.config import dbconfig from cloud_inquisitor.plugins.types.accounts import AWSAccount if not isinstance(account, AWSAccount): raise InquisitorError('Non AWSAccount passed to get_aws_session, got {}'.format(account.__class__.__name__)) # If no keys are on supplied for the account, use sts.assume_role instead session = get_local_aws_session() if session.get_credentials().method == 'iam-role': sts = session.client('sts') else: # If we are not running on an EC2 instance, assume the instance role # first, then assume the remote role temp_sts = session.client('sts') audit_sts_role = temp_sts.assume_role( RoleArn=app_config.aws_api.instance_role_arn, RoleSessionName='inquisitor' ) sts = boto3.session.Session( audit_sts_role['Credentials']['AccessKeyId'], audit_sts_role['Credentials']['SecretAccessKey'], audit_sts_role['Credentials']['SessionToken'] ).client('sts') role = sts.assume_role( RoleArn='arn:aws:iam::{}:role/{}'.format( account.account_number, dbconfig.get('role_name', default='cinq_role') ), RoleSessionName='inquisitor' ) sess = boto3.session.Session( role['Credentials']['AccessKeyId'], role['Credentials']['SecretAccessKey'], role['Credentials']['SessionToken'] ) return sess
def get_client(self, settings_module): profile = setting(self.profile_names, settings_module=settings_module) access_key = setting(self.access_key_names, settings_module=settings_module) secret_key = setting(self.secret_key_names, settings_module=settings_module) region_name = setting(self.region_name_names, settings_module=settings_module) credentials = {} if access_key and secret_key: credentials['aws_access_key_id'] = access_key credentials['aws_secret_access_key'] = secret_key elif profile: credentials['profile_name'] = profile credentials['region_name'] = region_name session = boto3.session.Session(**credentials) if session.get_credentials() is None: raise CredentialsNotExists() return session.client('secretsmanager')
wc = sys.argv[4] s3_client = boto3.client('s3') dynamodb = boto3.resource('dynamodb', region_name='us-east-1') scheduler_table = dynamodb.Table('v486121-GPS-PathFinder-Scheduler-Table-' + env) validation_table = dynamodb.Table('v486121-GPS-PathFinder-Validation-Rules-' + env) response = scheduler_table.scan() schedule_data = eval(json.dumps(response)) response = validation_table.scan() validate_data = eval(json.dumps(response)) #########################Create elasticsearch session instance################################### session = session.Session() credentials = session.get_credentials() region = 'us-east-1' service = 'es' auth = AWSV4Sign(credentials, region, service) try: es = Elasticsearch(host=host, port=443, connection_class=RequestsHttpConnection, http_auth=auth, use_ssl=True, verify_ssl=True, timeout=30, max_retries=10, retry_on_timeout=True ) except Exception, err:
DOCTYPES = { 'data_obj': os.environ.get('DATA_OBJ_DOCTYPE', 'meta'), 'data_bdl': os.environ.get('DATA_BDL_DOCTYPE', 'databundle'), } try: es_host = os.environ['ES_HOST'] except KeyError: raise RuntimeError("You must specify the domain name of your ElasticSearch" " instance with the ES_HOST environment variable.") es_region = os.environ.get('ES_REGION', DEFAULT_REGION) access_token = os.environ.get('ACCESS_KEY', DEFAULT_ACCESS_TOKEN) session = boto3.session.Session() credentials = session.get_credentials().get_frozen_credentials() awsauth = aws_requests_auth.aws_auth.AWSRequestsAuth( aws_access_key=credentials.access_key, aws_secret_access_key=credentials.secret_key, aws_token=credentials.token, aws_host=es_host, aws_region=session.region_name, aws_service='es' ) es = elasticsearch.Elasticsearch( hosts=[{'host': es_host, 'port': 443}], http_auth=awsauth, use_ssl=True, verify_certs=True, connection_class=elasticsearch.RequestsHttpConnection )