def ship_style(args, conf): """Ship style to bucket.""" # get bucket name bucket_name = conf.get( 'DATASET_BUCKET') if args.bucket is None else args.bucket # get fab function func = get_func('sdscli.adapters.{}.fabfile'.format(args.type), args.subparser2) # execute execute(func, bucket_name, args.encrypt, roles=['mozart']) # turn on website hosting and set index and error docs bw_args = { 'WebsiteConfiguration': { 'ErrorDocument': { 'Key': 'index.html' }, 'IndexDocument': { 'Suffix': 'index.html' } } } configure_bucket_website(bucket_name, **bw_args)
def get_adapter_func(sds_type, mod_name, func_name): """Import adapter function.""" adapter_mod = 'sdscli.adapters.%s.%s' % (sds_type, mod_name) try: return get_func(adapter_mod, func_name) except ImportError: logger.error('Failed to import adapter module "%s" for SDS type "%s".' % (mod_name, sds_type)) logger.error('Not implemented yet. Mahalo for trying. ;)') sys.exit(1) except AttributeError: logger.error('Failed to get function "%s" from adapter module "%s".' % (func_name, adapter_mod)) logger.error('Not implemented yet. Mahalo for trying. ;)') sys.exit(1)
def storage(args): """Cloud storage management functions.""" # print args logger.debug("In storage(): {}".format(args)) # get user's SDS conf settings conf = SettingsConf() # get func try: func = get_func( 'sdscli.cloud.{}.{}'.format(args.cloud, args.subparser), args.subparser2) except (ImportError, AttributeError): logger.error('Not implemented yet. Mahalo for trying. ;)') return 1 # run return func(args, conf)
def create_staging_area(args, conf): """Provision staging area in bucket.""" # create boto3 clients s3_res = boto3.resource('s3') sns_client = boto3.client('sns') # get bucket name bucket_name = conf.get( 'DATASET_BUCKET') if args.bucket is None else args.bucket bucket = get_bucket(bucket_name, c=s3_res) logger.debug("bucket: {}".format(bucket)) # create SNS topic logger.debug("prefix: {}".format(args.prefix)) logger.debug("suffix: {}".format(args.suffix)) shasum = hashlib.sha224("{}-{}".format(args.prefix, args.suffix)).hexdigest() topic_name = "{}-{}-{}".format(conf.get('VENUE'), bucket_name, shasum[:4]) logger.debug("topic_name: {}".format(topic_name)) topic_arn = create_topic(Name=topic_name, c=sns_client)['TopicArn'] logger.debug("topic_arn: {}".format(topic_arn)) # add policy to allow S3 to publish to topic pol = { "Version": "2008-10-17", "Id": "__default_policy_ID", "Statement": [{ "Sid": "__default_statement_ID", "Effect": "Allow", "Principal": { "AWS": "*" }, "Action": [ "SNS:GetTopicAttributes", "SNS:SetTopicAttributes", "SNS:AddPermission", "SNS:RemovePermission", "SNS:DeleteTopic", "SNS:Subscribe", "SNS:ListSubscriptionsByTopic", "SNS:Publish", "SNS:Receive" ], "Resource": topic_arn, "Condition": { "StringEquals": { "aws:SourceArn": "arn:aws:s3:::{}".format(bucket_name) } } }] } sns_client.set_topic_attributes(TopicArn=topic_arn, AttributeName="Policy", AttributeValue=json.dumps(pol)) sns_client.set_topic_attributes(TopicArn=topic_arn, AttributeName="DisplayName", AttributeValue=topic_name) # create notification event on bucket staging area notification_name = "data-staged-{}".format(topic_name) logger.debug("notification_name: {}".format(notification_name)) bn_args = { 'NotificationConfiguration': { 'TopicConfigurations': [{ 'Id': notification_name, 'TopicArn': topic_arn, 'Events': [ 's3:ObjectCreated:*', ], 'Filter': { 'Key': { 'FilterRules': [ { 'Name': 'prefix', 'Value': args.prefix, }, { 'Name': 'suffix', 'Value': args.suffix, }, ] } } }] } } configure_bucket_notification(bucket_name, c=s3_res, **bn_args) # create lambda zip file and upload to code bucket zip_file = "/tmp/data-staged.zip" func = get_func('sdscli.adapters.{}.fabfile'.format(args.type), 'create_zip') if args.debug: execute(func, "mozart/ops/hysds-cloud-functions/aws/data-staged", zip_file, roles=['mozart']) else: with hide('everything'): execute(func, "mozart/ops/hysds-cloud-functions/aws/data-staged", zip_file, roles=['mozart']) #code_bucket = "s3://{}/{}".format(conf.get('S3_ENDPOINT'), conf.get('CODE_BUCKET')) #zip_url = os.path.join(code_bucket, 'data-staged.zip') #put(zip_file, zip_url) # prompt for security groups cur_sgs = {i['GroupId']: i for i in get_sgs()} logger.debug("cur_sgs: {}".format(pformat(cur_sgs))) desc = "\nSelect security groups lambda will use (space between each selected): " sgs, vpc_id = prompt_secgroup(cur_sgs, desc) logger.debug("security groups: {}".format(sgs)) logger.debug("VPC ID: {}".format(vpc_id)) # get current AZs cur_azs = {i['ZoneName']: i for i in get_azs()} logger.debug("cur_azs: {}".format(pformat(cur_azs))) # get subnet IDs and corresponding AZs for VPC subnets = [] azs = set() for sn in get_subnets_by_vpc(vpc_id): sn_id = sn.subnet_id sn_az = sn.availability_zone if cur_azs[sn_az]['State'] == 'available': subnets.append(sn_id) azs.add(sn_az) azs = list(azs) logger.debug("subnets: {}".format(pformat(subnets))) logger.debug("azs: {}".format(pformat(azs))) # prompt for role roles = get_roles() logger.debug("Found {} roles.".format(len(roles))) cur_roles = OrderedDict([ (i['Arn'], i) for i in sorted(roles, key=itemgetter('CreateDate')) ]) role = prompt_role(cur_roles) logger.debug("Selected role: {}".format(role)) # prompt for job type, release, and queue job_type = prompt(get_prompt_tokens=lambda x: [( Token, "Enter job type to submit on data staged event: ")], style=prompt_style).strip() logger.debug("job type: {}".format(job_type)) job_release = prompt(get_prompt_tokens=lambda x: [( Token, "Enter release version for {}: ".format(job_type))], style=prompt_style).strip() logger.debug("job release: {}".format(job_release)) job_queue = prompt(get_prompt_tokens=lambda x: [( Token, "Enter queue name to submit {}-{} jobs to: ".format( job_type, job_release))], style=prompt_style).strip() logger.debug("job queue: {}".format(job_queue)) # create lambda function_name = "{}-{}-submit_data_staged_ingest".format( conf.get('VENUE'), args.prefix.replace('/', '')) lambda_client = boto3.client('lambda') cf_args = { "FunctionName": function_name, "Runtime": "python2.7", "Role": role, "Handler": "lambda_function.lambda_handler", "Code": { "ZipFile": open(zip_file, 'rb').read(), #"S3Bucket": conf.get('CODE_BUCKET'), #"S3Key": os.path.basename(zip_url) }, "Description": "Lambda function to submit ingest job for data staged to S3 staging area.", "VpcConfig": { "SubnetIds": subnets, "SecurityGroupIds": sgs, }, "Environment": { "Variables": { "DATASET_S3_ENDPOINT": conf.get('DATASET_S3_ENDPOINT'), "JOB_TYPE": job_type, "JOB_RELEASE": job_release, "JOB_QUEUE": job_queue, "MOZART_URL": "https://{}/mozart".format(conf.get('MOZART_PVT_IP')) } } } lambda_resp = lambda_client.create_function(**cf_args) logger.debug("lambda_resp: {}".format(lambda_resp)) # add permission for sns to invoke lambda function lambda_client.add_permission(Action="lambda:InvokeFunction", FunctionName=function_name, Principal="sns.amazonaws.com", StatementId="ID-1", SourceArn=topic_arn) # subscribe lambda endpoint to sns sns_resp = sns_client.subscribe(TopicArn=topic_arn, Protocol="lambda", Endpoint=lambda_resp['FunctionArn']) logger.debug("sns_resp: {}".format(sns_resp))