class S3Client: """ S3 helper class. """ def __init__(self, endpoint_args, session): self.s3 = Service('s3', endpoint_args['region_name'], session=session) @staticmethod def _bundle_size(bundle): bundle.seek(0, 2) size = bundle.tell() bundle.seek(0) return size def upload_to_s3(self, parsed_args, bundle): size_remaining = self._bundle_size(bundle) if size_remaining < MULTIPART_LIMIT: return self.s3.PutObject(bucket=parsed_args.bucket, key=parsed_args.key, body=bundle) else: return self._multipart_upload_to_s3(parsed_args, bundle, size_remaining) def _multipart_upload_to_s3(self, parsed_args, bundle, size_remaining): create_response = self.s3.CreateMultipartUpload( bucket=parsed_args.bucket, key=parsed_args.key) upload_id = create_response['UploadId'] try: part_num = 1 multipart_list = [] bundle.seek(0) while size_remaining > 0: data = bundle.read(MULTIPART_LIMIT) upload_response = self.s3.UploadPart(bucket=parsed_args.bucket, key=parsed_args.key, uploadId=upload_id, partNumber=part_num, body=six.BytesIO(data)) multipart_list.append({ 'PartNumber': part_num, 'ETag': upload_response['ETag'] }) part_num += 1 size_remaining -= len(data) return self.s3.CompleteMultipartUpload( bucket=parsed_args.bucket, key=parsed_args.key, uploadId=upload_id, multipartUpload={'Parts': multipart_list}) except Exception as e: self.s3.AbortMultipartUpload(bucket=parsed_args.bucket, key=parsed_args.key, uploadId=upload_id) raise e
def setup_services(self, args, parsed_globals): endpoint_args = { 'region_name': None, 'endpoint_url': None, 'verify': None } if 'region' in parsed_globals: endpoint_args['region_name'] = parsed_globals.region if 'verify_ssl' in parsed_globals: endpoint_args['verify'] = parsed_globals.verify_ssl # Initialize services LOG.debug('Initializing S3, SNS and CloudTrail...') self.iam = Service('iam', endpoint_args=endpoint_args, session=self._session) self.s3 = Service('s3', endpoint_args=endpoint_args, session=self._session) self.sns = Service('sns', endpoint_args=endpoint_args, session=self._session) # If the endpoint is specified, it is designated for the cloudtrail # service. Not all of the other services will use it. if 'endpoint_url' in parsed_globals: endpoint_args['endpoint_url'] = parsed_globals.endpoint_url self.cloudtrail = Service('cloudtrail', endpoint_args=endpoint_args, session=self._session)
class CodeDeployClient: """ CodeDeploy helper class for defined commands. """ def __init__(self, endpoint_args, session): self.codedeploy = Service('codedeploy', endpoint_args=endpoint_args, session=session) @staticmethod def _get_revision(parsed_args): revision = { 'revisionType': 'S3', 's3Location': { 'bucket': parsed_args.bucket, 'key': parsed_args.key, 'bundleType': 'zip', 'eTag': parsed_args.eTag } } if 'version' in parsed_args: revision['s3Location']['version'] = parsed_args.version return revision def register_revision(self, parsed_args): self.codedeploy.RegisterApplicationRevision( applicationName=parsed_args.application_name, revision=self._get_revision(parsed_args), description=parsed_args.description)
def _run_main(self, args, parsed_globals): endpoint_args = {'region_name': None, 'endpoint_url': None} if 'region' in parsed_globals: endpoint_args['region_name'] = parsed_globals.region if 'endpoint_url' in parsed_globals: endpoint_args['endpoint_url'] = parsed_globals.endpoint_url # Initialize services LOG.debug('Initializing S3, SNS and CloudTrail...') self.iam = Service('iam', session=self._session) self.s3 = Service('s3', endpoint_args['region_name'], session=self._session) self.sns = Service('sns', endpoint_args['region_name'], session=self._session) self.cloudtrail = Service('cloudtrail', endpoint_args=endpoint_args, session=self._session) # Run the command and report success self._call(args, parsed_globals) return 1
class CloudTrailSubscribe(BasicCommand): """ Subscribe/update a user account to CloudTrail, creating the required S3 bucket, the optional SNS topic, and starting the CloudTrail monitoring and logging. """ NAME = 'create-subscription' DESCRIPTION = ('Creates and configures the AWS resources necessary to use' ' CloudTrail, creates a trail using those resources, and ' 'turns on logging.') SYNOPSIS = ('aws cloudtrail create-subscription' ' (--s3-use-bucket|--s3-new-bucket) bucket-name' ' [--sns-new-topic topic-name]\n') ARG_TABLE = [{ 'name': 'name', 'required': True, 'help_text': 'Cloudtrail name' }, { 'name': 's3-new-bucket', 'help_text': 'Create a new S3 bucket with this name' }, { 'name': 's3-use-bucket', 'help_text': 'Use an existing S3 bucket with this name' }, { 'name': 's3-prefix', 'help_text': 'S3 object prefix' }, { 'name': 'sns-new-topic', 'help_text': 'Create a new SNS topic with this name' }, { 'name': 'include-global-service-events', 'help_text': 'Whether to include global service events' }, { 'name': 's3-custom-policy', 'help_text': 'Optional URL to a custom S3 policy template' }, { 'name': 'sns-custom-policy', 'help_text': 'Optional URL to a custom SNS policy template' }] UPDATE = False def _run_main(self, args, parsed_globals): self.setup_services(args, parsed_globals) # Run the command and report success self._call(args, parsed_globals) return 1 def setup_services(self, args, parsed_globals): endpoint_args = { 'region_name': None, 'endpoint_url': None, 'verify': None } if 'region' in parsed_globals: endpoint_args['region_name'] = parsed_globals.region if 'verify_ssl' in parsed_globals: endpoint_args['verify'] = parsed_globals.verify_ssl # Initialize services LOG.debug('Initializing S3, SNS and CloudTrail...') self.iam = Service('iam', endpoint_args=endpoint_args, session=self._session) self.s3 = Service('s3', endpoint_args=endpoint_args, session=self._session) self.sns = Service('sns', endpoint_args=endpoint_args, session=self._session) # If the endpoint is specified, it is designated for the cloudtrail # service. Not all of the other services will use it. if 'endpoint_url' in parsed_globals: endpoint_args['endpoint_url'] = parsed_globals.endpoint_url self.cloudtrail = Service('cloudtrail', endpoint_args=endpoint_args, session=self._session) def _call(self, options, parsed_globals): """ Run the command. Calls various services based on input options and outputs the final CloudTrail configuration. """ gse = options.include_global_service_events if gse: if gse.lower() == 'true': gse = True elif gse.lower() == 'false': gse = False else: raise ValueError('You must pass either true or false to' ' --include-global-service-events.') bucket = options.s3_use_bucket if options.s3_new_bucket: bucket = options.s3_new_bucket if self.UPDATE and options.s3_prefix is None: # Prefix was not passed and this is updating the S3 bucket, # so let's find the existing prefix and use that if possible res = self.cloudtrail.DescribeTrails( trail_name_list=[options.name]) trail_info = res['trailList'][0] if 'S3KeyPrefix' in trail_info: LOG.debug('Setting S3 prefix to {0}'.format( trail_info['S3KeyPrefix'])) options.s3_prefix = trail_info['S3KeyPrefix'] self.setup_new_bucket(bucket, options.s3_prefix, options.s3_custom_policy) elif not bucket and not self.UPDATE: # No bucket was passed for creation. raise ValueError('You must pass either --s3-use-bucket or' ' --s3-new-bucket to create.') if options.sns_new_topic: try: topic_result = self.setup_new_topic(options.sns_new_topic, options.sns_custom_policy) except Exception: # Roll back any S3 bucket creation if options.s3_new_bucket: self.s3.DeleteBucket(bucket=options.s3_new_bucket) raise try: cloudtrail_config = self.upsert_cloudtrail_config( options.name, bucket, options.s3_prefix, options.sns_new_topic, gse) except Exception: # Roll back any S3 bucket / SNS topic creations if options.s3_new_bucket: self.s3.DeleteBucket(bucket=options.s3_new_bucket) if options.sns_new_topic: self.sns.DeleteTopic(topic_arn=topic_result['TopicArn']) raise sys.stdout.write('CloudTrail configuration:\n{config}\n'.format( config=json.dumps(cloudtrail_config, indent=2))) if not self.UPDATE: # If the configure call command above completes then this should # have a really high chance of also completing self.start_cloudtrail(options.name) sys.stdout.write( 'Logs will be delivered to {bucket}:{prefix}\n'.format( bucket=bucket, prefix=options.s3_prefix or '')) def setup_new_bucket(self, bucket, prefix, policy_url=None): """ Creates a new S3 bucket with an appropriate policy to let CloudTrail write to the prefix path. """ sys.stdout.write( 'Setting up new S3 bucket {bucket}...\n'.format(bucket=bucket)) # Who am I? response = self.iam.GetUser() account_id = response['User']['Arn'].split(':')[4] # Clean up the prefix - it requires a trailing slash if set if prefix and not prefix.endswith('/'): prefix += '/' # Fetch policy data from S3 or a custom URL if policy_url: policy = requests.get(policy_url).text else: data = self.s3.GetObject(bucket='awscloudtrail', key=S3_POLICY_TEMPLATE) policy = data['Body'].read().decode('utf-8') policy = policy.replace('<BucketName>', bucket)\ .replace('<CustomerAccountID>', account_id) if '<Prefix>/' in policy: policy = policy.replace('<Prefix>/', prefix or '') else: policy = policy.replace('<Prefix>', prefix or '') LOG.debug('Bucket policy:\n{0}'.format(policy)) # Make sure bucket doesn't already exist # Warn but do not fail if ListBucket permissions # are missing from the IAM role try: buckets = self.s3.ListBuckets()['Buckets'] except Exception: buckets = [] LOG.warn('Unable to list buckets, continuing...') if [b for b in buckets if b['Name'] == bucket]: raise Exception( 'Bucket {bucket} already exists.'.format(bucket=bucket)) # If we are not using the us-east-1 region, then we must set # a location constraint on the new bucket. region_name = self.s3.endpoint.region_name params = {'bucket': bucket} if region_name != 'us-east-1': bucket_config = {'LocationConstraint': region_name} params['create_bucket_configuration'] = bucket_config data = self.s3.CreateBucket(**params) try: self.s3.PutBucketPolicy(bucket=bucket, policy=policy) except Exception: # Roll back bucket creation self.s3.DeleteBucket(bucket=bucket) raise return data def setup_new_topic(self, topic, policy_url=None): """ Creates a new SNS topic with an appropriate policy to let CloudTrail post messages to the topic. """ sys.stdout.write( 'Setting up new SNS topic {topic}...\n'.format(topic=topic)) # Who am I? response = self.iam.GetUser() account_id = response['User']['Arn'].split(':')[4] # Make sure topic doesn't already exist # Warn but do not fail if ListTopics permissions # are missing from the IAM role? try: topics = self.sns.ListTopics()['Topics'] except Exception: topics = [] LOG.warn('Unable to list topics, continuing...') if [t for t in topics if t['TopicArn'].split(':')[-1] == topic]: raise Exception( 'Topic {topic} already exists.'.format(topic=topic)) region = self.sns.endpoint.region_name # Get the SNS topic policy information to allow CloudTrail # write-access. if policy_url: policy = requests.get(policy_url).text else: data = self.s3.GetObject(bucket='awscloudtrail', key=SNS_POLICY_TEMPLATE) policy = data['Body'].read().decode('utf-8') policy = policy.replace('<Region>', region)\ .replace('<SNSTopicOwnerAccountId>', account_id)\ .replace('<SNSTopicName>', topic) topic_result = self.sns.CreateTopic(name=topic) try: # Merge any existing topic policy with our new policy statements topic_attr = self.sns.GetTopicAttributes( topic_arn=topic_result['TopicArn']) policy = self.merge_sns_policy(topic_attr['Attributes']['Policy'], policy) LOG.debug('Topic policy:\n{0}'.format(policy)) # Set the topic policy self.sns.SetTopicAttributes(topic_arn=topic_result['TopicArn'], attribute_name='Policy', attribute_value=policy) except Exception: # Roll back topic creation self.sns.DeleteTopic(topic_arn=topic_result['TopicArn']) raise return topic_result def merge_sns_policy(self, left, right): """ Merge two SNS topic policy documents. The id information from ``left`` is used in the final document, and the statements from ``right`` are merged into ``left``. http://docs.aws.amazon.com/sns/latest/dg/BasicStructure.html :type left: string :param left: First policy JSON document :type right: string :param right: Second policy JSON document :rtype: string :return: Merged policy JSON """ left_parsed = json.loads(left) right_parsed = json.loads(right) left_parsed['Statement'] += right_parsed['Statement'] return json.dumps(left_parsed) def upsert_cloudtrail_config(self, name, bucket, prefix, topic, gse): """ Either create or update the CloudTrail configuration depending on whether this command is a create or update command. """ sys.stdout.write('Creating/updating CloudTrail configuration...\n') config = {'name': name} if bucket is not None: config['s3_bucket_name'] = bucket if prefix is not None: config['s3_key_prefix'] = prefix if topic is not None: config['sns_topic_name'] = topic if gse is not None: config['include_global_service_events'] = gse if not self.UPDATE: self.cloudtrail.CreateTrail(**config) else: self.cloudtrail.UpdateTrail(**config) return self.cloudtrail.DescribeTrails() def start_cloudtrail(self, name): """ Start the CloudTrail service, which begins logging. """ sys.stdout.write('Starting CloudTrail service...\n') return self.cloudtrail.StartLogging(name=name)
def __init__(self, endpoint_args, session): self.codedeploy = Service('codedeploy', endpoint_args=endpoint_args, session=session)
def __init__(self, endpoint_args, session): self.s3 = Service('s3', endpoint_args['region_name'], session=session)