def create_bucket(region, session, bucket_prefix): bucket = create_bucket_name(bucket_prefix) if region != 'us-east-1': session.resource('s3').create_bucket( Bucket=bucket, CreateBucketConfiguration={'LocationConstraint': region}) else: session.resource('s3').create_bucket(Bucket=bucket) return bucket
def getResource(self, resourceType): session = self.getSession() resource = session.resource(resourceType) if hasattr(session, "aws_session_token"): return session.resource( resourceType, aws_access_key_id=session.aws_access_key, aws_secret_access_key=session.aws_secret_key, aws_session_token=session.aws_session_token) else: return session.resource( resourceType, aws_access_key_id=session.aws_access_key, aws_secret_access_key=session.aws_secret_key)
def run(self): session_config = Config( connect_timeout=self.config.C_TIME, read_timeout=self.config.R_TIME, retries={'max_attempts': self.config.MAX_RETRIES } #We're gonna declare 0 attempts for now ) session = boto3.session.Session() #boto3.set_stream_logger('') self.s3 = session.resource( 's3', aws_access_key_id=self.config.key_id, aws_secret_access_key=self.config.access_key, region_name=self.device.region, config=session_config) #if isConnected(): try: device_folder = self.device.location + '-' + self.device.name file_timestamp = self.file_path.split('/')[-1] key = os.path.join(device_folder, self.config.DATA_FOLDER, file_timestamp) self.s3.Bucket(self.config.BUCKET_NAME).upload_file( self.file_path, key) #print("[LOGS] Uploaded image {0} to {1}".format(self.file_path, key)) return 200 except ClientError as e: print('[ALERT] Boto upload FAILED: ', e) return 400
def test_can_create_all_resources(resource_args): """Verify we can create all existing resources.""" session, service_name = resource_args resource = session.resource(service_name) # Verifying we have a "meta" attr is just an arbitrary # sanity check. assert hasattr(resource, 'meta')
def test_can_inject_method_onto_resource(self): session = boto3.Session(botocore_session=self.botocore_session) self.botocore_session.register('creating-resource-class.s3', self.add_new_method(name='my_method')) resource = session.resource('s3') self.assertTrue(hasattr(resource, 'my_method')) self.assertEqual(resource.my_method('anything'), 'anything')
def run_annotation(job_id, user, filename, bucket_name, s3key): # Get the s3 bucket s3 = session.resource('s3') bucket = s3.Bucket(bucket_name) # Check if the user exists d = 'AnnotationResult/user/' + user + '/' if os.path.exists(d) == False: os.mkdir(d) # Create a directory for each submitted job_id request directory = 'AnnotationResult/user/' + user + '/' + job_id + '/' os.mkdir(directory) # Download the input file from s3 input bucket bucket.download_file(s3key, 'AnnotationResult/user/' + user + '/' + job_id + '/' + filename) # Update the job status only if the current status is pending DBresponse = ann_table.get_item(Key={'job_id': job_id}) item = DBresponse['Item'] if item['job_status'] == 'PENDING': ann_table.update_item(Key={'job_id': job_id}, UpdateExpression='SET job_status = :v', ExpressionAttributeValues= {':v': 'RUNNING'}) # Launch annotation job as a background process subprocess.Popen(["python", "run.py", 'AnnotationResult/user/' + user + '/' + job_id + '/' + filename]) response.status = 200 f = open('AnnotationResult/user/' + user + '/' + job_id + '/status.txt', 'w+') f.write("code:\t" + response.status + "\tdata:\t" + str(job_id) + "\tfilename:\t" + filename + '\n') f.close()
def __init__(self, keyname=None, use_akms=None, host=None, region='us-east-1', *args, **kwargs): import botocore.session # DEPRECATED - `host` if host: logger.warning('the host param is deprecated') use_akms = (use_akms if use_akms is not None else 'USE_AKMS' in os.environ) args = (keyname, use_akms) with self.lock: if args not in self.sessions: session = Session(botocore_session=botocore.session.Session( session_vars={ 'akms_keyname': keyname, 'use_akms': use_akms, })) client = session.client('s3') s3 = session.resource('s3') self.sessions[args] = (session, client, s3) (self.session, self.client, self.s3) = self.sessions[args]
def save(self, commit=True): short_url_data = self.data.get('short_url') url = expandedurl.objects.create() url.short_url = short_url_data site = requests.get(short_url_data) url.http_status_code = site.status_code url.destination_url = site.url session = Session(aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, region_name='us-west-2') client = session.resource('s3') driver = webdriver.PhantomJS( service_log_path=os.path.devnull) # or add to your PATH driver.set_window_size(1024, 768) # optional driver.get(str(url.destination_url)) url.page_title = driver.title filepath = '/tmp/' + str(url.id) + '.png' s3path = 'captures/' + str(url.id) + '.png' driver.save_screenshot(filepath) data = open(filepath, 'rb') # save a screenshot to disk client.Object(AWS_STORAGE_BUCKET_NAME, s3path).put( Body=data ) #upload_file(filepath, AWS_STORAGE_BUCKET_NAME, AWS_ACCESS_KEY_ID) url.screen_capture = 'https://s3.amazonaws.com/%s' % AWS_STORAGE_BUCKET_NAME + '/' + s3path driver.service.process.kill() os.remove(filepath) return url
def get_resource_type( self, resource_type: Optional[str] = None, region_name: Optional[str] = None, config: Optional[Config] = None, ) -> boto3.resource: """Get the underlying boto3 resource using boto3 session""" session, endpoint_url = self._get_credentials(region_name=region_name) if resource_type: warnings.warn( "resource_type is deprecated. Set resource_type from class attribute.", DeprecationWarning, stacklevel=2, ) else: resource_type = self.resource_type # No AWS Operators use the config argument to this method. # Keep backward compatibility with other users who might use it if config is None: config = self.config return session.resource(resource_type, endpoint_url=endpoint_url, config=config, verify=self.verify)
def test_can_inject_method_onto_resource(self): session = boto3.Session(botocore_session=self.botocore_session) self.botocore_session.register('creating-resource-class.s3', self.add_new_method(name='my_method')) resource = session.resource('s3') assert hasattr(resource, 'my_method') assert resource.my_method('anything') == 'anything'
def resource(self, *args, **kwargs): """ Outside of a context established by `.assumed_role_credentials()` or `.direct_access_credentials()` this is the same as boto3.resource. Within such a context, it returns boto3 clients that use different, temporary credentials. """ session = self._per_thread.session or boto3 return session.resource(*args, **kwargs)
def suspend_bucket_versioning(session, bucket_name): """ Suspending the bucket versioning :param session: :param bucket_name: :return: """ s3 = session.resource('s3') bucket_versioning = s3.BucketVersioning(bucket_name) response = bucket_versioning.suspend() logger.info(response) return
def get_resource_type( self, resource_type: str, region_name: Optional[str] = None, config: Optional[Config] = None, ) -> boto3.resource: """Get the underlying boto3 resource using boto3 session""" session, endpoint_url = self._get_credentials(region_name) # No AWS Operators use the config argument to this method. # Keep backward compatibility with other users who might use it if config is None: config = self.config return session.resource(resource_type, endpoint_url=endpoint_url, config=config, verify=self.verify)
def resource(session=None, region_name=None, api_version=None, use_ssl=False, verify=None, endpoint_url=None, aws_access_key_id=None, aws_secret_access_key=None, aws_session_token=None, config=None, endpoints=None): ''' Create a high-level Boto Resource matching DynamoDB. Instead of ddb = boto3.resource('dynamodb') use dax = AmazonDaxClient.resource() ''' # Local import so it doesn't fail if boto3 is not available from .Resource import is_boto3_session, DaxSession if session is None or isinstance(session, botocore.session.Session): session = DaxSession(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token, region_name=region_name, botocore_session=session, endpoints=endpoints) else: if is_boto3_session(session): session = DaxSession.from_boto3(session, endpoints) else: raise ValueError('session must be a botocore or boto3 session') # Create the resource res = session.resource('dynamodb', region_name=region_name, api_version=api_version, use_ssl=use_ssl, verify=verify, endpoint_url=endpoint_url, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token, config=config) return res
def main(aws_profile, output_file): """ This script audits invalid records in the signature table and generates a report """ start = datetime.now() try: if os.environ.get("STAGE") is None: logging.warning( "Please set the 'STAGE' environment variable - typically one of: {dev, staging, prod}" ) return stage = os.environ.get("STAGE", "dev") signatures_table_name = "cla-{}-signatures".format(stage) companies_table_name = "cla-{}-companies".format(stage) users_table_name = "cla-{}-users".format(stage) session = boto3.Session(profile_name=aws_profile) dynamodb = session.resource("dynamodb") signatures_table = dynamodb.Table(signatures_table_name) companies_table = dynamodb.Table(companies_table_name) users_table = dynamodb.Table(users_table_name) signatures = signatures_table.scan()["Items"] logging.info("{} number of signatures".format(len(signatures))) # set tables used in the audit process audit_signature = AuditSignature(dynamodb, batch=signatures) audit_signature.set_signatures_table(signatures_table) audit_signature.set_companies_table(companies_table) audit_signature.set_users_table(users_table) invalid_fields = audit_signature.process_batch() columns = ["signature_id", "error_type", "column", "data"] with open(output_file, "w", newline="") as csv_file: writer = csv.DictWriter(csv_file, fieldnames=columns, delimiter=" ") writer.writeheader() writer.writerows({ "signature_id": audit["signature_id"], "error_type": audit["error_type"], "column": audit["column"], "data": audit["data"], } for audit in invalid_fields) logging.info("Auditing Duration : {}".format(datetime.now() - start)) except (Exception, ClientError) as err: logging.error(err, exc_info=True)
def run_annotation(): # Extract job parameters from the request body (NOT the URL query string!) body = request.body.read() res = json.loads(body) bucket_name = res['s3_inputs_bucket'] job_id = res['job_id'] filename = res['input_file_name'] s3key = res['s3_key_input_file'] user = res['username'] # Get the input file S3 object and copy it to a local file session = Session(aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_KEY) s3 = session.resource('s3') bucket = s3.Bucket(bucket_name) # Check if the user exists d = 'AnnotationResult/user/' + user + '/' if os.path.exists(d) == False: os.mkdir(d) directory = 'AnnotationResult/user/' + user + '/' + job_id + '/' os.mkdir(directory) # Download the input file from s3 input bucket bucket.download_file( s3key, 'AnnotationResult/user/' + user + '/' + job_id + '/' + filename) # Launch annotation job as a background process subprocess.Popen([ "python", "run.py", 'AnnotationResult/user/' + user + '/' + job_id + '/' + filename ]) response.status = 200 f = open('AnnotationResult/user/' + user + '/' + job_id + '/status.txt', 'w+') f.write("code:\t" + response.status + "\tdata:\t" + str(job_id) + "\tfilename:\t" + filename + '\n') f.close() # Return response to notify user of successful submission data = {'id': job_id, 'input_file': filename} return json.dumps({'code': response.status, 'data': data})
def capture(self, url): session = Session(aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, region_name='us-west-2') s3 = session.resource('s3') driver = webdriver.PhantomJS( service_log_path=os.path.devnull) # or add to your PATH driver.set_window_size(1024, 768) # optional driver.get(str(url.destination_url)) url.page_title = driver.title filepath = '/tmp/' + str(url.id) + '.png' s3path = 'captures/' + str(url.id) + '.png' driver.save_screenshot(filepath) data = open(filepath, 'rb') # save a screenshot to disk s3.Object(AWS_STORAGE_BUCKET_NAME, s3path).put( Body=data ) #upload_file(filepath, AWS_STORAGE_BUCKET_NAME, AWS_ACCESS_KEY_ID) url.screen_capture = 'https://s3.amazonaws.com/%s' % AWS_STORAGE_BUCKET_NAME + '/' + s3path driver.service.process.kill() os.remove(filepath) return url
def delete_s3_bucket(session, s3conn, bucket_name): """ Clean up all S3 buckets :param session: :param s3conn: :param bucket_name: :return: """ client = session.client('s3') resource = session.resource('s3') list_of_s3_buckets = client.list_buckets()['Buckets'] for bucket in list_of_s3_buckets: if bucket['Name'] == bucket_name: try: if re.search('portal', bucket_name): logger.info("Portal bucket %s will not be deleted only delete the files." % bucket_name) s3_delete_files_with_prefix(client, bucket_name, 'admin') s3_delete_files_with_prefix(client, bucket_name, 'auth') s3_delete_files_with_prefix(client, bucket_name, 'logs') else: client.get_bucket_location(Bucket=bucket_name) logger.info("Bucket %s OK" % bucket_name) s3_bucket_logging_disable(s3conn, bucket_name) remove_bucket_replication(client, bucket_name) disable_bucket_versioning(client, bucket_name) logger.info("Delete s3 bucket lifecycle configuration.") client.delete_bucket_lifecycle(Bucket=bucket_name) delete_objects_in_bucket(resource, bucket_name) full_bucket = s3conn.get_bucket(bucket_name) for key in full_bucket.list(): key.delete() logger.info("Delete bucket %s" % bucket_name) s3conn.delete_bucket(bucket_name) except (S3ResponseError, ClientError) as e: message = e.response['Error']['Message'] logger.info("Bucket %s - %s" % (bucket_name, message)) pass break return
def init(profile, conf, bucket=None, path=None): """Init Initialises the module so that it can be used Args: profile (str): The name of the profile to use to connect conf (dict): The configuration parameters, see boto3.resource for more info bucket (str): Optional bucket (needed if not passed in init) path (str): Optional path to prepend to all keys Returns: None """ # Pull in the module variable global _c, _r, _s3 # Init the conf _s3 = { "profile": profile, "conf": BotoConfig(**conf), "bucket": bucket, "path": path } # Create a new session using the profile session = boto3.Session(profile_name=profile) # Get an S3 resource _r = session.resource('s3', config=_s3['conf']) # Get a client _c = session.client('s3', config=boto3.session.Config( s3={'addressing_style': 'path'}, signature_version='s3v4'))
def main(aws_profile, output_file): """ This script audits invalid records in the projects table - specifically projects template pdfs """ try: if os.environ.get("STAGE") is None: logging.warning( "Please set the 'STAGE' environment variable - typically one of: {dev, staging, prod}" ) return stage = os.environ.get("STAGE", "dev") projects_table_name = "cla-{}-projects".format(stage) session = boto3.Session(profile_name=aws_profile) dynamodb = session.resource("dynamodb") projects_table = dynamodb.Table(projects_table_name) projects = projects_table.scan()["Items"] # set the projects table used in the audit process audit_project = ProjectAudit(dynamodb, batch=projects) invalid_fields = audit_project.process_batch() columns = ["project_id", "error_type", "column", "data"] with open(output_file, "w", newline="") as csv_file: writer = csv.DictWriter(csv_file, fieldnames=columns, delimiter=" ") writer.writeheader() writer.writerows({ "project_id": audit["project_id"], "error_type": audit["error_type"], "column": audit["column"], "data": audit["data"], } for audit in invalid_fields) except (Exception, ClientError) as err: logging.error(err, exc_info=True)
def archive_data(job_id, user_role, result_file): # Pass in user role, job_id, completion_time, username glacier_client = boto3.client('glacier') s3 = session.resource('s3') client = boto3.client('s3') bucket = s3.Bucket('gas-results') # Get the corresponding job id res = ann_table.query(KeyConditionExpression=Key('job_id').eq(job_id)) items = res['Items'] # When the 30 mins reach, we check again if the user is premium in case he upgrade to premium within 30 time frame # We only archive result file if the user is a free_user role if items[0]['user_role'] == 'free_user': for obj in bucket.objects.filter(Prefix=result_file): # Archive result file to Glacier archive_res = glacier_client.upload_archive(vaultName='ucmpcs',body=obj.get()['Body'].read()) print 'Archive id: ' + archive_res['archiveId'] archive_id = archive_res['archiveId'] # Update the job details within DynamoDB updateData = ann_table.update_item( Key={ 'job_id': job_id }, UpdateExpression="set results_file_archive_id = :a", ExpressionAttributeValues={ ':a': archive_id }, ReturnValues="UPDATED_NEW" ) # Remove the result file from s3 bucket client.delete_object(Bucket='gas-results', Key=result_file)
def _test_create_resource(session, service_name): resource = session.resource(service_name) # Verifying we have a "meta" attr is just an arbitrary # sanity check. assert_true(hasattr(resource, 'meta'))
def create_keypair(region, session, key_name, save_path): new_keypair = session.resource('ec2').create_key_pair(KeyName=key_name) with open(save_path, 'w') as file: file.write(new_keypair.key_material) print(new_keypair.key_fingerprint)
def connect(account_name, connection_type, **args): """ Examples of use: ec2 = sts_connect.connect(environment, 'ec2', region=region, validate_certs=False) ec2 = sts_connect.connect(environment, 'ec2', validate_certs=False, debug=1000) ec2 = sts_connect.connect(environment, 'ec2') where environment is ( test, prod, dev ) s3 = sts_connect.connect(environment, 's3') ses = sts_connect.connect(environment, 'ses') :param account: Account to connect with (i.e. test, prod, dev) :raises Exception: RDS Region not valid AWS Tech not supported. :returns: STS Connection Object for given tech :note: To use this method a SecurityMonkey role must be created in the target account with full read only privileges. """ if 'assumed_role' in args: role = args['assumed_role'] else: account = Account.query.filter(Account.name == account_name).first() sts = boto3.client('sts') role_name = 'SecurityMonkey' if account.getCustom( "role_name") and account.getCustom("role_name") != '': role_name = account.getCustom("role_name") role = sts.assume_role(RoleArn='arn:aws:iam::' + account.identifier + ':role/' + role_name, RoleSessionName='secmonkey') from security_monkey import app if connection_type == 'botocore': botocore_session = botocore.session.get_session() botocore_session.set_credentials( role['Credentials']['AccessKeyId'], role['Credentials']['SecretAccessKey'], token=role['Credentials']['SessionToken']) return botocore_session region = 'us-east-1' if 'region' in args: region = args.pop('region') if hasattr(region, 'name'): region = region.name if 'boto3' in connection_type: # Should be called in this format: boto3.iam.client _, tech, api = connection_type.split('.') session = boto3.Session( aws_access_key_id=role['Credentials']['AccessKeyId'], aws_secret_access_key=role['Credentials']['SecretAccessKey'], aws_session_token=role['Credentials']['SessionToken'], region_name=region) if api == 'resource': return session.resource(tech) return session.client(tech) module = __import__("boto.{}".format(connection_type)) for subm in connection_type.split('.'): module = getattr(module, subm) return module.connect_to_region( region, aws_access_key_id=role['Credentials']['AccessKeyId'], aws_secret_access_key=role['Credentials']['SecretAccessKey'], security_token=role['Credentials']['SessionToken'])
# Get job_id, s3key and file name input_file_name = sys.argv[1] files = input_file_name.split('/') first = input_file_name.find('/') second = input_file_name.rindex('/') s3key = input_file_name[first + 1:second] job_id = files[3] # Calculate how much time taken to run the annotation job request with Timer() as t: driver.run(input_file_name, 'vcf') print "Total runtime: %s seconds" % t.secs # Get the input file upload path s3 = session.resource('s3') bucket = s3.Bucket('gas-results') index = input_file_name.rindex('/') prefix = input_file_name[:index] rindex = input_file_name.find('.') filename = input_file_name[index + 1:rindex] findex = prefix.find('/') path = prefix[findex + 1:index] # Upload the result file bucket.upload_file(prefix + '/' + filename + '.annot.vcf', 'zhuoyuzhu/' + path + '/' + filename + '.annot.vcf') # Upload the log file bucket.upload_file( prefix + '/' + filename + '.vcf.count.log',
def connect(account_name, connection_type, **args): """ Examples of use: ec2 = sts_connect.connect(environment, 'ec2', region=region, validate_certs=False) ec2 = sts_connect.connect(environment, 'ec2', validate_certs=False, debug=1000) ec2 = sts_connect.connect(environment, 'ec2') where environment is ( test, prod, dev ) s3 = sts_connect.connect(environment, 's3') ses = sts_connect.connect(environment, 'ses') :param account: Account to connect with (i.e. test, prod, dev) :raises Exception: RDS Region not valid AWS Tech not supported. :returns: STS Connection Object for given tech :note: To use this method a SecurityMonkey role must be created in the target account with full read only privileges. """ account = Account.query.filter(Account.name == account_name).first() sts = boto.connect_sts() role_name = 'SecurityMonkey' if account.role_name and account.role_name != '': role_name = account.role_name role = sts.assume_role('arn:aws:iam::' + account.number + ':role/' + role_name, 'secmonkey') if connection_type == 'botocore': botocore_session = botocore.session.get_session() botocore_session.set_credentials( role.credentials.access_key, role.credentials.secret_key, token=role.credentials.session_token ) return botocore_session if connection_type == 'iam_boto3': session = boto3.Session( aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, aws_session_token=role.credentials.session_token ) return session.resource('iam') region = 'us-east-1' if args.has_key('region'): region = args.pop('region') if hasattr(region, 'name'): region = region.name # ElasticSearch Service: if connection_type == 'es': session = boto3.Session( aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, aws_session_token=role.credentials.session_token, region_name=region ) return session.client('es') module = __import__("boto.{}".format(connection_type)) for subm in connection_type.split('.'): module = getattr(module, subm) return module.connect_to_region( region, aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token )
def connect(account_name, connection_type, **args): """ Examples of use: ec2 = sts_connect.connect(environment, 'ec2', region=region, validate_certs=False) ec2 = sts_connect.connect(environment, 'ec2', validate_certs=False, debug=1000) ec2 = sts_connect.connect(environment, 'ec2') where environment is ( test, prod, dev ) s3 = sts_connect.connect(environment, 's3') ses = sts_connect.connect(environment, 'ses') :param account: Account to connect with (i.e. test, prod, dev) :raises Exception: RDS Region not valid AWS Tech not supported. :returns: STS Connection Object for given tech :note: To use this method a SecurityMonkey role must be created in the target account with full read only privileges. """ if 'assumed_role' in args: role = args['assumed_role'] else: account = Account.query.filter(Account.name == account_name).first() sts = boto3.client('sts') role_name = 'SecurityMonkey' if account.role_name and account.role_name != '': role_name = account.role_name role = sts.assume_role(RoleArn='arn:aws:iam::' + account.number + ':role/' + role_name, RoleSessionName='secmonkey') from security_monkey import app if connection_type == 'botocore': botocore_session = botocore.session.get_session() botocore_session.set_credentials( role['Credentials']['AccessKeyId'], role['Credentials']['SecretAccessKey'], token=role['Credentials']['SessionToken'] ) return botocore_session region = 'us-east-1' if 'region' in args: region = args.pop('region') if hasattr(region, 'name'): region = region.name if 'boto3' in connection_type: # Should be called in this format: boto3.iam.client _, tech, api = connection_type.split('.') session = boto3.Session( aws_access_key_id=role['Credentials']['AccessKeyId'], aws_secret_access_key=role['Credentials']['SecretAccessKey'], aws_session_token=role['Credentials']['SessionToken'], region_name=region ) if api == 'resource': return session.resource(tech) return session.client(tech) module = __import__("boto.{}".format(connection_type)) for subm in connection_type.split('.'): module = getattr(module, subm) return module.connect_to_region( region, aws_access_key_id=role['Credentials']['AccessKeyId'], aws_secret_access_key=role['Credentials']['SecretAccessKey'], security_token=role['Credentials']['SessionToken'] )
def writeObjectsToDatabase(self, lookupTableObjectList, objectTableObjectList, lookupTableName, objectTableName): client = None lookupTable = None objectTable = None try: session = boto3.Session(profile_name=self.profile) client = session.resource('dynamodb', region_name=str(self.region)) except Exception as e: return { "status": "success", "payload": { "error": "There was a problem obtaining the connection to DynamoDB.", "traceback": ''.join(traceback.format_exc()) } } try: lookupTable = client.Table(str(lookupTableName)) objectTable = client.Table(str(objectTableName)) except Exception as e: return { "status": "success", "payload": { "error": "There was a problem connecting to the DynamoDB tables.", "traceback": ''.join(traceback.format_exc()) } } try: with lookupTable.batch_writer() as batch: for item in lookupTableObjectList: batch.put_item(Item=item) except Exception as e: return { "status": "success", "payload": { "error": "There was a problem adding the provided data to the Lookup DynamoDB table.", "traceback": ''.join(traceback.format_exc()) } } try: with objectTable.batch_writer() as batch: for item in objectTableObjectList: batch.put_item(Item=item) except Exception as e: return { "status": "success", "payload": { "error": "There was a problem adding the provided data to the Object DynamoDB table.", "traceback": ''.join(traceback.format_exc()) } } return { "status": "success", "payload": "Successfully pre-populated the Lookup and Object database tables with the provided information." }
def connect(account_name, connection_type, **args): """ Examples of use: ec2 = sts_connect.connect(environment, 'ec2', region=region, validate_certs=False) ec2 = sts_connect.connect(environment, 'ec2', validate_certs=False, debug=1000) ec2 = sts_connect.connect(environment, 'ec2') where environment is ( test, prod, dev ) s3 = sts_connect.connect(environment, 's3') ses = sts_connect.connect(environment, 'ses') :param account: Account to connect with (i.e. test, prod, dev) :raises Exception: RDS Region not valid AWS Tech not supported. :returns: STS Connection Object for given tech :note: To use this method a SecurityMonkey role must be created in the target account with full read only privileges. """ account = Account.query.filter(Account.name == account_name).first() sts = boto.connect_sts() role_name = 'SecurityMonkey' if account.role_name and account.role_name != '': role_name = account.role_name role = sts.assume_role('arn:aws:iam::' + account.number + ':role/' + role_name, 'secmonkey') if connection_type == 'botocore': botocore_session = botocore.session.get_session() botocore_session.set_credentials( role.credentials.access_key, role.credentials.secret_key, token=role.credentials.session_token ) return botocore_session if connection_type == 'ec2': return boto.connect_ec2( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'elb': if 'region' in args: region = args['region'] del args['region'] else: region = 'us-east-1' return boto.ec2.elb.connect_to_region( region, aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 's3': if 'region' in args: region = args['region'] # drop region key-val pair from args or you'll get an exception del args['region'] return boto.s3.connect_to_region( region, aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token, **args) return boto.connect_s3( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'ses': if 'region' in args: region = args['region'] del args['region'] return boto.ses.connect_to_region( region, aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token, **args) return boto.connect_ses( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'iam_boto3': session = boto3.Session( aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, aws_session_token=role.credentials.session_token ) return session.resource('iam') if connection_type == 'iam': if 'region' in args: region = args['region'] # drop region key-val pair from args or you'll get an exception del args['region'] return boto.iam.connect_to_region( region, aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token, **args) return boto.connect_iam( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'route53': return boto.connect_route53( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'sns': if 'region' in args: region = args['region'] del args['region'] return boto.sns.connect_to_region( region.name, aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token, **args) return boto.connect_sns( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'sqs': if 'region' in args: region = args['region'] del args['region'] return boto.sqs.connect_to_region( region.name, aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token, **args) return boto.connect_sqs( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'vpc': return boto.connect_vpc( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'rds': if 'region' in args: reg = args['region'] rds_region = None for boto_region in boto.rds.regions(): if reg.name == boto_region.name: rds_region = boto_region if rds_region is None: raise Exception('The supplied region {0} is not in boto.rds.regions. {1}'.format(reg, boto.rds.regions())) return boto.connect_rds( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'redshift': if 'region' in args: region = args['region'] del args['region'] return boto.redshift.connect_to_region( region.name, aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token, **args) return boto.connect_redshift( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'vpc': if 'region' in args: region = args['region'] del args['region'] return boto.vpc.connect_to_region( region.name, aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token, **args) return boto.connect_vpc( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) err_msg = 'The connection_type supplied (%s) is not implemented.' % connection_type raise Exception(err_msg)
import boto3 import botocore.session session = boto3.session.Session(profile_name='power-user') dynamodb = session.resource('dynamodb') table = dynamodb.Table('prod-visual-schedules-data-table') print(table.creation_date_time) with table.batch_writer() as batch: batch.put_item( Item={ 'pk': "KID_SCHEDULE|2020|Kiera", 'sk': "PERIOD|1", 'start': "08:45", 'end': "09:35", 'TERMS|Nine Weeks 1': { "Class": "Pre AP English 2", "Category": "English" }, 'TERMS|Nine Weeks 2': { "Class": "Pre AP English 2", "Category": "English" }, 'TERMS|Nine Weeks 3': { "Class": "Pre AP English 2", "Category": "English" }, 'TERMS|Nine Weeks 4': { "Class": "Pre AP English 2", "Category": "English" },
AWS_KEY_ID = 'AKIAIZMP2MTECH75C74A' AWS_SECRET_KEY = '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' AWS_BUCKET_NAME = 'memrix1' ALLOWED_EXTENSIONS = set(['jpg', 'jpeg', 'gif', 'png']) GMAPS_API_KEY = "AIzaSyDEgKQ8xiwb_MHBJXT6wL_t2p_mH1REWZ4" from boto3.session import Session import botocore import botocore.session session = Session(aws_access_key_id=AWS_KEY_ID, aws_secret_access_key=AWS_SECRET_KEY, region_name='us-west-2') s3 = session.resource('s3') s3Client = session.client('s3') bucket = s3.Bucket('mybucket') exists = True try: s3.meta.client.head_bucket(Bucket='mybucket') except botocore.exceptions.ClientError as e: # If a client error is thrown, then check that it was a 404 error. # If it was a 404 error, then the bucket does not exist. error_code = int(e.response['Error']['Code']) if error_code == 404: exists = False # for getting image urls # session = botocore.session.get_session()
def connect(account_name, connection_type, **args): """ Examples of use: ec2 = sts_connect.connect(environment, 'ec2', region=region, validate_certs=False) ec2 = sts_connect.connect(environment, 'ec2', validate_certs=False, debug=1000) ec2 = sts_connect.connect(environment, 'ec2') where environment is ( test, prod, dev ) s3 = sts_connect.connect(environment, 's3') ses = sts_connect.connect(environment, 'ses') :param account: Account to connect with (i.e. test, prod, dev) :raises Exception: RDS Region not valid AWS Tech not supported. :returns: STS Connection Object for given tech :note: To use this method a SecurityMonkey role must be created in the target account with full read only privledges. """ account = Account.query.filter(Account.name == account_name).first() sts = boto.connect_sts() role = sts.assume_role('arn:aws:iam::' + account.number + ':role/SecurityMonkey', 'secmonkey') if connection_type == 'botocore': botocore_session = botocore.session.get_session() botocore_session.set_credentials( role.credentials.access_key, role.credentials.secret_key, token=role.credentials.session_token ) return botocore_session if connection_type == 'ec2': return boto.connect_ec2( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'elb': if 'region' in args: region = args['region'] del args['region'] else: region = 'us-east-1' return boto.ec2.elb.connect_to_region( region, aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 's3': if 'region' in args: region = args['region'] # drop region key-val pair from args or you'll get an exception del args['region'] return boto.s3.connect_to_region( region, aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token, **args) return boto.connect_s3( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'ses': if 'region' in args: region = args['region'] del args['region'] return boto.ses.connect_to_region( region, aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token, **args) return boto.connect_ses( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'iam_boto3': session = boto3.Session( aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, aws_session_token=role.credentials.session_token ) return session.resource('iam') if connection_type == 'iam': if 'region' in args: region = args['region'] # drop region key-val pair from args or you'll get an exception del args['region'] return boto.iam.connect_to_region( region, aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token, **args) return boto.connect_iam( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'route53': return boto.connect_route53( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'sns': if 'region' in args: region = args['region'] del args['region'] return boto.sns.connect_to_region( region.name, aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token, **args) return boto.connect_sns( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'sqs': if 'region' in args: region = args['region'] del args['region'] return boto.sqs.connect_to_region( region.name, aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token, **args) return boto.connect_sqs( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'vpc': return boto.connect_vpc( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'rds': if 'region' in args: reg = args['region'] rds_region = None for boto_region in boto.rds.regions(): if reg.name == boto_region.name: rds_region = boto_region if rds_region is None: raise Exception('The supplied region {0} is not in boto.rds.regions. {1}'.format(reg, boto.rds.regions())) return boto.connect_rds( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'redshift': if 'region' in args: region = args['region'] del args['region'] return boto.redshift.connect_to_region( region.name, aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token, **args) return boto.connect_redshift( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) if connection_type == 'vpc': if 'region' in args: region = args['region'] del args['region'] return boto.vpc.connect_to_region( region.name, aws_access_key_id=role.credentials.access_key, aws_secret_access_key=role.credentials.secret_key, security_token=role.credentials.session_token, **args) return boto.connect_vpc( role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token, **args) err_msg = 'The connection_type supplied (%s) is not implemented.' % connection_type raise Exception(err_msg)