def parse_notification_configuration(notification_config: Dict, pool=None) -> List[S3Notification]: # notification_config returned by: # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.get_bucket_notification_configuration notifications = list() arn_selectors = { "QueueConfigurations": "QueueArn", "TopicConfigurations": "TopicArn", "LambdaFunctionConfigurations": "LambdaFunctionArn", } for config_type, configs in notification_config.items(): if config_type not in arn_selectors: continue for config in configs: try: arn = config[arn_selectors[config_type]] target = EventSource.get(arn, pool=pool) notification = S3Notification(target.id) notification.target = target notifications.append(notification) except Exception as e: LOG.warning("error parsing s3 notification: %s", e) return notifications
def extract_endpoints(code_map, pool=None): if pool is None: pool = {} result = [] identifiers = [] for key, code in code_map.items(): # Elasticsearch references pattern = r'[\'"](.*\.es\.amazonaws\.com)[\'"]' for es in re.findall(pattern, code): if es not in identifiers: identifiers.append(es) es = EventSource.get(es, pool=pool, type=ElasticSearch) if es: result.append(es) # Elasticsearch references pattern = r"\.put_record_batch\([^,]+,\s*([^,\s]+)\s*," for firehose in re.findall(pattern, code): if firehose not in identifiers: identifiers.append(firehose) firehose = EventSource.get(firehose, pool=pool, type=FirehoseStream) if firehose: result.append(firehose) # DynamoDB references # TODO fix pattern to be generic pattern = r"\.(insert|get)_document\s*\([^,]+,\s*([^,\s]+)\s*," for (op, dynamo) in re.findall(pattern, code): dynamo = resolve_string_or_variable(dynamo, code_map) if dynamo not in identifiers: identifiers.append(dynamo) dynamo = EventSource.get(dynamo, pool=pool, type=DynamoDB) if dynamo: result.append(dynamo) # S3 references pattern = r"\.upload_file\([^,]+,\s*([^,\s]+)\s*," for s3 in re.findall(pattern, code): s3 = resolve_string_or_variable(s3, code_map) if s3 not in identifiers: identifiers.append(s3) s3 = EventSource.get(s3, pool=pool, type=S3Bucket) if s3: result.append(s3) return result
def extract_endpoints(code_map, pool={}): result = [] identifiers = [] for key, code in iteritems(code_map): # Elasticsearch references pattern = r'[\'"](.*\.es\.amazonaws\.com)[\'"]' for es in re.findall(pattern, code): if es not in identifiers: identifiers.append(es) es = EventSource.get(es, pool=pool, type=ElasticSearch) if es: result.append(es) # Elasticsearch references pattern = r'\.put_record_batch\([^,]+,\s*([^,\s]+)\s*,' for firehose in re.findall(pattern, code): if firehose not in identifiers: identifiers.append(firehose) firehose = EventSource.get(firehose, pool=pool, type=FirehoseStream) if firehose: result.append(firehose) # DynamoDB references # TODO fix pattern to be generic pattern = r'\.(insert|get)_document\s*\([^,]+,\s*([^,\s]+)\s*,' for (op, dynamo) in re.findall(pattern, code): dynamo = resolve_string_or_variable(dynamo, code_map) if dynamo not in identifiers: identifiers.append(dynamo) dynamo = EventSource.get(dynamo, pool=pool, type=DynamoDB) if dynamo: result.append(dynamo) # S3 references pattern = r'\.upload_file\([^,]+,\s*([^,\s]+)\s*,' for s3 in re.findall(pattern, code): s3 = resolve_string_or_variable(s3, code_map) if s3 not in identifiers: identifiers.append(s3) s3 = EventSource.get(s3, pool=pool, type=S3Bucket) if s3: result.append(s3) return result
def handle(func): func_name = func['FunctionName'] if re.match(filter, func_name): arn = func['FunctionArn'] f = LambdaFunction(arn) pool[arn] = f result.append(f) if details: sources = get_lambda_event_sources(f.name(), env=env) for src in sources: arn = src['EventSourceArn'] f.event_sources.append(EventSource.get(arn, pool=pool)) try: code_map = get_lambda_code(func_name, env=env) f.targets = extract_endpoints(code_map, pool) except Exception: LOG.warning("Unable to get code for lambda '%s'" % func_name)
def get_firehose_streams(filter=".*", pool={}, env=None, region=None): result = [] try: firehose_client = _connect("firehose", env=env, region=region) out = firehose_client.list_delivery_streams() for stream_name in out["DeliveryStreamNames"]: if re.match(filter, stream_name): details = firehose_client.describe_delivery_stream(DeliveryStreamName=stream_name) details = details["DeliveryStreamDescription"] arn = details["DeliveryStreamARN"] s = FirehoseStream(arn) for dest in details["Destinations"]: dest_s3 = dest["S3DestinationDescription"]["BucketARN"] bucket = EventSource.get(dest_s3, pool=pool) s.destinations.append(bucket) result.append(s) except Exception: pass return result
def handle(bucket): bucket_name = bucket["Name"] if re.match(filter, bucket_name): arn = "arn:aws:s3:::%s" % bucket_name bucket = S3Bucket(arn) result.append(bucket) pool[arn] = bucket if details: try: out = s3_client.get_bucket_notification(Bucket=bucket_name) if out: if "CloudFunctionConfiguration" in out: func = out["CloudFunctionConfiguration"]["CloudFunction"] func = EventSource.get(func, pool=pool) n = S3Notification(func.id) n.target = func bucket.notifications.append(n) except Exception as e: print("WARNING: Unable to get details for bucket: %s" % e)
def get_firehose_streams(filter='.*', pool={}, env=None): result = [] try: out = cmd_firehose('list-delivery-streams', env) out = json.loads(out) for stream_name in out['DeliveryStreamNames']: if re.match(filter, stream_name): details = cmd_firehose( 'describe-delivery-stream --delivery-stream-name %s' % stream_name, env) details = json.loads(details)['DeliveryStreamDescription'] arn = details['DeliveryStreamARN'] s = FirehoseStream(arn) for dest in details['Destinations']: dest_s3 = dest['S3DestinationDescription']['BucketARN'] bucket = EventSource.get(dest_s3, pool=pool) s.destinations.append(bucket) result.append(s) except socket.error: pass return result
def handle(bucket): bucket_name = bucket['Name'] if re.match(filter, bucket_name): arn = 'arn:aws:s3:::%s' % bucket_name bucket = S3Bucket(arn) result.append(bucket) pool[arn] = bucket if details: try: out = cmd_s3api('get-bucket-notification-configuration --bucket %s' % bucket_name, env=env) if out: out = json.loads(out) if 'CloudFunctionConfiguration' in out: func = out['CloudFunctionConfiguration']['CloudFunction'] func = EventSource.get(func, pool=pool) n = S3Notification(func.id) n.target = func bucket.notifications.append(n) except Exception as e: print('WARNING: Unable to get details for bucket: %s' % e)
def get_firehose_streams(filter='.*', pool={}, env=None): result = [] try: firehose_client = aws_stack.connect_to_service('firehose') out = firehose_client.list_delivery_streams() for stream_name in out['DeliveryStreamNames']: if re.match(filter, stream_name): details = firehose_client.describe_delivery_stream( DeliveryStreamName=stream_name) details = details['DeliveryStreamDescription'] arn = details['DeliveryStreamARN'] s = FirehoseStream(arn) for dest in details['Destinations']: dest_s3 = dest['S3DestinationDescription']['BucketARN'] bucket = EventSource.get(dest_s3, pool=pool) s.destinations.append(bucket) result.append(s) except Exception: pass return result
def handle(bucket): bucket_name = bucket['Name'] if re.match(filter, bucket_name): arn = 'arn:aws:s3:::%s' % bucket_name bucket = S3Bucket(arn) result.append(bucket) pool[arn] = bucket if details: try: s3_client = aws_stack.connect_to_service('s3') out = s3_client.get_bucket_notification(Bucket=bucket_name) if out: if 'CloudFunctionConfiguration' in out: func = out['CloudFunctionConfiguration'][ 'CloudFunction'] func = EventSource.get(func, pool=pool) n = S3Notification(func.id) n.target = func bucket.notifications.append(n) except Exception as e: print('WARNING: Unable to get details for bucket: %s' % e)