def test_aws_get_sqs_queues_list_queues_is_empty(monkeypatch): sqs_client = MagicMock() sqs_client.list_queues.return_value = {} boto = get_boto_client(monkeypatch, sqs_client) assert aws.get_sqs_queues(REGION, ACCOUNT) == [] boto.assert_called_with('sqs', region_name=REGION) sqs_client.list_queues.assert_called()
def test_aws_get_sqs_queues_list_queues_is_empty(monkeypatch): sqs_client = MagicMock() sqs_client.list_queues.return_value = {} boto = get_boto_client(monkeypatch, sqs_client) assert aws.get_sqs_queues(REGION, ACCOUNT) == [] boto.assert_called_with('sqs', region_name=REGION) sqs_client.list_queues.assert_called()
def test_aws_get_sqs_queues_fails_to_list_queues(monkeypatch): sqs_client = MagicMock() sqs_client.list_queues.side_effect = RuntimeError('Oops') boto = get_boto_client(monkeypatch, sqs_client) assert aws.get_sqs_queues(REGION, ACCOUNT) == [] boto.assert_called_with('sqs', region_name=REGION) sqs_client.list_queues.assert_called()
def test_aws_get_sqs_queues_fails_to_list_queues(monkeypatch): sqs_client = MagicMock() sqs_client.list_queues.side_effect = RuntimeError('Oops') boto = get_boto_client(monkeypatch, sqs_client) assert aws.get_sqs_queues(REGION, ACCOUNT) == [] boto.assert_called_with('sqs', region_name=REGION) sqs_client.list_queues.assert_called()
def test_aws_get_sqs_queues_access_denied(monkeypatch): sqs_client = MagicMock() sqs_client.list_queues.side_effect = ClientError(operation_name='foo', error_response={'Error': {'Code': 'AccessDenied'}}) boto = get_boto_client(monkeypatch, sqs_client) assert aws.get_sqs_queues(REGION, ACCOUNT) == [] boto.assert_called_with('sqs', region_name=REGION) sqs_client.list_queues.assert_called()
def test_aws_get_sqs_queues_access_denied(monkeypatch): sqs_client = MagicMock() sqs_client.list_queues.side_effect = ClientError(operation_name='foo', error_response={'Error': {'Code': 'AccessDenied'}}) boto = get_boto_client(monkeypatch, sqs_client) assert aws.get_sqs_queues(REGION, ACCOUNT) == [] boto.assert_called_with('sqs', region_name=REGION) sqs_client.list_queues.assert_called()
def test_aws_get_sqs_queues(monkeypatch): urls, attributes, dead_letter_sources, result = get_sqs_queues() sqs_client = MagicMock() sqs_client.list_queues.return_value = urls sqs_client.get_queue_attributes.side_effect = attributes sqs_client.list_dead_letter_source_queues.side_effect = dead_letter_sources boto = get_boto_client(monkeypatch, sqs_client) res = aws.get_sqs_queues(REGION, ACCOUNT) assert res == result boto.assert_called_with('sqs', region_name=REGION) sqs_client.list_queues.assert_called() attribute_calls = [call(QueueUrl=url, AttributeNames=['All']) for url in urls['QueueUrls']] sqs_client.get_queue_attributes.assert_has_calls(attribute_calls) dl_sources_calls = [call(QueueUrl=url) for url in urls['QueueUrls']] sqs_client.list_dead_letter_source_queues.assert_has_calls(dl_sources_calls)
def test_aws_get_sqs_queues(monkeypatch): urls, attributes, dead_letter_sources, result = get_sqs_queues() sqs_client = MagicMock() sqs_client.list_queues.return_value = urls sqs_client.get_queue_attributes.side_effect = attributes sqs_client.list_dead_letter_source_queues.side_effect = dead_letter_sources boto = get_boto_client(monkeypatch, sqs_client) res = aws.get_sqs_queues(REGION, ACCOUNT) assert res == result boto.assert_called_with('sqs', region_name=REGION) sqs_client.list_queues.assert_called() attribute_calls = [call(QueueUrl=url, AttributeNames=['All']) for url in urls['QueueUrls']] sqs_client.get_queue_attributes.assert_has_calls(attribute_calls) dl_sources_calls = [call(QueueUrl=url) for url in urls['QueueUrls']] sqs_client.list_dead_letter_source_queues.assert_has_calls(dl_sources_calls)
def test_aws_get_sqs_queues_fails_on_weird_arn(monkeypatch): urls, attributes, dead_letter_sources, result = get_sqs_queues() attributes[0]['Attributes']['QueueArn'] = 'arn:aws:i-am-not-a-valid-arn' dead_letter_sources.pop(0) result.pop(0) sqs_client = MagicMock() sqs_client.list_queues.return_value = urls sqs_client.get_queue_attributes.side_effect = attributes sqs_client.list_dead_letter_source_queues.side_effect = dead_letter_sources boto = get_boto_client(monkeypatch, sqs_client) res = aws.get_sqs_queues(REGION, ACCOUNT) assert res == result boto.assert_called_with('sqs', region_name=REGION) sqs_client.list_queues.assert_called() calls = [call(QueueUrl=url, AttributeNames=['All']) for url in urls['QueueUrls']] sqs_client.get_queue_attributes.assert_has_calls(calls)
def test_aws_get_sqs_queues_fails_on_weird_arn(monkeypatch): urls, attributes, dead_letter_sources, result = get_sqs_queues() attributes[0]['Attributes']['QueueArn'] = 'arn:aws:i-am-not-a-valid-arn' dead_letter_sources.pop(0) result.pop(0) sqs_client = MagicMock() sqs_client.list_queues.return_value = urls sqs_client.get_queue_attributes.side_effect = attributes sqs_client.list_dead_letter_source_queues.side_effect = dead_letter_sources boto = get_boto_client(monkeypatch, sqs_client) res = aws.get_sqs_queues(REGION, ACCOUNT) assert res == result boto.assert_called_with('sqs', region_name=REGION) sqs_client.list_queues.assert_called() calls = [call(QueueUrl=url, AttributeNames=['All']) for url in urls['QueueUrls']] sqs_client.get_queue_attributes.assert_has_calls(calls)
def main(): argp = argparse.ArgumentParser(description='ZMON AWS Agent') argp.add_argument('-e', '--entity-service', dest='entityservice') argp.add_argument('-r', '--region', dest='region', default=None) argp.add_argument('-j', '--json', dest='json', action='store_true') argp.add_argument('--no-oauth2', dest='disable_oauth2', action='store_true', default=False) argp.add_argument('--postgresql-user', dest='postgresql_user', default=os.environ.get('AGENT_POSTGRESQL_USER')) argp.add_argument('--postgresql-pass', dest='postgresql_pass', default=os.environ.get('AGENT_POSTGRESQL_PASS')) args = argp.parse_args() if not args.disable_oauth2: tokens.configure() tokens.manage('uid', ['uid']) tokens.start() logging.basicConfig(level=logging.INFO) # 1. Determine region if not args.region: logger.info('Trying to figure out region..') try: response = requests.get( 'http://169.254.169.254/latest/meta-data/placement/availability-zone', timeout=2) except: logger.exception( 'Region was not specified as a parameter and can not be fetched from instance meta-data!' ) raise region = response.text[:-1] else: region = args.region logger.info('Using region: {}'.format(region)) logger.info('Entity service URL: %s', args.entityservice) logger.info('Reading DNS data for hosted zones') aws.populate_dns_data() aws_account_id = aws.get_account_id(region) infrastructure_account = 'aws:{}'.format( aws_account_id) if aws_account_id else None if not infrastructure_account: logger.error( 'AWS agent: Cannot determine infrastructure account ID. Terminating!' ) return # 2. ZMON entities token = None if args.disable_oauth2 else tokens.get('uid') zmon_client = Zmon(args.entityservice, token=token, user_agent=get_user_agent()) query = { 'infrastructure_account': infrastructure_account, 'region': region, 'created_by': 'agent' } entities = zmon_client.get_entities(query) # 3. Get running apps apps = aws.get_running_apps(region, entities) elbs = [] scaling_groups = [] rds = [] elasticaches = [] dynamodbs = [] sqs = [] new_entities = [] to_be_removed = [] if len(apps) > 0: elbs = aws.get_running_elbs(region, infrastructure_account) scaling_groups = aws.get_auto_scaling_groups(region, infrastructure_account) rds = aws.get_rds_instances(region, infrastructure_account, entities) elasticaches = aws.get_elasticache_nodes(region, infrastructure_account) dynamodbs = aws.get_dynamodb_tables(region, infrastructure_account) certificates = aws.get_certificates(region, infrastructure_account) aws_limits = aws.get_limits(region, infrastructure_account, apps, elbs) sqs = aws.get_sqs_queues(region, infrastructure_account, entities) account_alias = aws.get_account_alias(region) ia_entity = { 'type': 'local', 'infrastructure_account': infrastructure_account, 'account_alias': account_alias, 'region': region, 'id': 'aws-ac[{}:{}]'.format(infrastructure_account, region), 'created_by': 'agent', } application_entities = aws.get_apps_from_entities(apps, infrastructure_account, region) if args.postgresql_user and args.postgresql_pass: postgresql_clusters = zmon_client.get_entities({ 'infrastructure_account': infrastructure_account, 'region': region, 'type': 'postgresql_cluster' }) postgresql_databases = postgresql.get_databases_from_clusters( postgresql_clusters, infrastructure_account, region, args.postgresql_user, args.postgresql_pass) else: # Pretend the list of DBs is empty, but also make sure we don't remove # any pre-existing database entities because we don't know about them. postgresql_databases = [] entities = [ e for e in entities if e.get('type') != 'postgresql_database' ] current_entities = (elbs + scaling_groups + apps + application_entities + rds + postgresql_databases + elasticaches + dynamodbs + certificates + sqs) current_entities.append(aws_limits) current_entities.append(ia_entity) # 4. Removing misssing entities existing_ids = get_existing_ids(entities) current_entities_ids = {e['id'] for e in current_entities} to_be_removed, delete_error_count = remove_missing_entities( existing_ids, current_entities_ids, zmon_client, json=args.json) logger.info( 'Found {} removed entities from {} entities ({} failed)'.format( len(new_entities), len(current_entities), delete_error_count)) # 5. Get new/updated entities new_entities, add_error_count = add_new_entities(current_entities, entities, zmon_client, json=args.json) logger.info('Found {} new entities from {} entities ({} failed)'.format( len(new_entities), len(current_entities), add_error_count)) # 6. Always add Local entity if not args.json: ia_entity['errors'] = { 'delete_count': delete_error_count, 'add_count': add_error_count } try: zmon_client.add_entity(ia_entity) except: logger.exception( 'Failed to add Local entity: {}'.format(ia_entity)) types = { e['type']: len([t for t in new_entities if t['type'] == e['type']]) for e in new_entities } for t, v in types.items(): logger.info('Found {} new entities of type: {}'.format(v, t)) # Check if it is a dry run! if args.json: d = { 'applications': application_entities, 'apps': apps, 'dynamodb': dynamodbs, 'elbs': elbs, 'elc': elasticaches, 'rds': rds, 'certificates': certificates, 'aws_limits': aws_limits, 'sqs_queues': sqs, 'new_entities': new_entities, 'to_be_removed': to_be_removed, } print(json.dumps(d, indent=4))
def main(): argp = argparse.ArgumentParser(description='ZMON AWS Agent') argp.add_argument('-e', '--entity-service', dest='entityservice') argp.add_argument('-r', '--region', dest='region', default=None) argp.add_argument('-j', '--json', dest='json', action='store_true') argp.add_argument('-t', '--tracer', dest='tracer', default=os.environ.get('OPENTRACING_TRACER', 'noop')) argp.add_argument('--no-oauth2', dest='disable_oauth2', action='store_true', default=False) argp.add_argument('--postgresql-user', dest='postgresql_user', default=os.environ.get('AGENT_POSTGRESQL_USER')) argp.add_argument('--postgresql-pass', dest='postgresql_pass', default=os.environ.get('AGENT_POSTGRESQL_PASS')) args = argp.parse_args() if not args.disable_oauth2: tokens.configure() tokens.manage('uid', ['uid']) tokens.start() init_opentracing_tracer(args.tracer) root_span = opentracing.tracer.start_span( operation_name='aws_entity_discovery') with root_span: logging.basicConfig(level=logging.INFO) # 0. Fetch extra data for entities entity_extras = {} for ex in os.getenv('EXTRA_ENTITY_FIELDS', '').split(','): if '=' not in ex: continue k, v = ex.split('=', 1) if k and v: entity_extras[k] = v # 1. Determine region if not args.region: logger.info('Trying to figure out region..') try: response = requests.get( 'http://169.254.169.254/latest/meta-data/placement/availability-zone', timeout=2) except Exception: root_span.set_tag('error', True) root_span.log_kv({'exception': traceback.format_exc()}) logger.exception( 'Region was not specified as a parameter and' + 'can not be fetched from instance meta-data!') raise region = response.text[:-1] else: region = args.region root_span.set_tag('region', region) logger.info('Using region: {}'.format(region)) logger.info('Entity service URL: %s', args.entityservice) logger.info('Reading DNS data for hosted zones') aws.populate_dns_data() aws_account_id = aws.get_account_id(region) infrastructure_account = 'aws:{}'.format( aws_account_id) if aws_account_id else None if not infrastructure_account: logger.error( 'AWS agent: Cannot determine infrastructure account ID. Terminating!' ) return root_span.set_tag('account', infrastructure_account) # 2. ZMON entities if not args.disable_oauth2: token = os.getenv('ZMON_TOKEN', None) or tokens.get('uid') zmon_client = Zmon(args.entityservice, token=token, user_agent=get_user_agent()) query = { 'infrastructure_account': infrastructure_account, 'region': region, 'created_by': 'agent' } entities = zmon_client.get_entities(query) # 3. Get running apps apps = aws.get_running_apps(region, entities) elbs = [] scaling_groups = [] rds = [] elasticaches = [] dynamodbs = [] sqs = [] new_entities = [] to_be_removed = [] if len(apps) > 0: elbs = aws.get_running_elbs(region, infrastructure_account) scaling_groups = aws.get_auto_scaling_groups( region, infrastructure_account) rds = aws.get_rds_instances(region, infrastructure_account, entities) elasticaches = aws.get_elasticache_nodes(region, infrastructure_account) dynamodbs = aws.get_dynamodb_tables(region, infrastructure_account) certificates = aws.get_certificates(region, infrastructure_account) aws_limits = aws.get_limits(region, infrastructure_account, apps, elbs, entities) sqs = aws.get_sqs_queues(region, infrastructure_account, entities) postgresql_clusters = postgresql.get_postgresql_clusters( region, infrastructure_account, scaling_groups, apps) account_alias = aws.get_account_alias(region) ia_entity = { 'type': 'local', 'infrastructure_account': infrastructure_account, 'account_alias': account_alias, 'region': region, 'id': 'aws-ac[{}:{}]'.format(infrastructure_account, region), 'created_by': 'agent', } account_alias_prefix = os.getenv('ACCOUNT_ALIAS_PREFIX', None) owner = account_alias if account_alias_prefix: owner = owner.replace(account_alias_prefix, '', 1) root_span.set_tag('team', owner) application_entities = aws.get_apps_from_entities( apps, infrastructure_account, region) if args.postgresql_user and args.postgresql_pass: postgresql_databases = postgresql.get_databases_from_clusters( postgresql_clusters, infrastructure_account, region, args.postgresql_user, args.postgresql_pass) else: # Pretend the list of DBs is empty, but also make sure we don't remove # any pre-existing database entities because we don't know about them. postgresql_databases = [] entities = [ e for e in entities if e.get('type') != 'postgresql_database' ] current_entities = (elbs + scaling_groups + apps + application_entities + rds + postgresql_databases + postgresql_clusters + elasticaches + dynamodbs + certificates + sqs) current_entities.append(aws_limits) current_entities.append(ia_entity) for entity in current_entities: entity.update(entity_extras) # 4. Removing misssing entities existing_ids = get_existing_ids(entities) current_entities_ids = {e['id'] for e in current_entities} to_be_removed, delete_error_count = remove_missing_entities( existing_ids, current_entities_ids, zmon_client, json=args.json) root_span.log_kv({'total_entitites': str(len(current_entities))}) root_span.log_kv({'removed_entities': str(len(to_be_removed))}) logger.info( 'Found {} removed entities from {} entities ({} failed)'.format( len(to_be_removed), len(current_entities), delete_error_count)) # 5. Get new/updated entities new_entities, add_error_count = add_new_entities(current_entities, entities, zmon_client, json=args.json) root_span.log_kv({'new_entities': str(len(new_entities))}) logger.info( 'Found {} new entities from {} entities ({} failed)'.format( len(new_entities), len(current_entities), add_error_count)) # 6. Always add Local entity if not args.json: ia_entity['errors'] = { 'delete_count': delete_error_count, 'add_count': add_error_count } update_local_entity(zmon_client, ia_entity) types = { e['type']: len([t for t in new_entities if t['type'] == e['type']]) for e in new_entities } for t, v in types.items(): logger.info('Found {} new entities of type: {}'.format(v, t)) # Check if it is a dry run! if args.json: d = { 'applications': application_entities, 'apps': apps, 'dynamodb': dynamodbs, 'elbs': elbs, 'elc': elasticaches, 'rds': rds, 'certificates': certificates, 'aws_limits': aws_limits, 'sqs_queues': sqs, 'new_entities': new_entities, 'to_be_removed': to_be_removed, 'posgresql_clusters': postgresql_clusters } print(json.dumps(d, indent=4))
def main(): argp = argparse.ArgumentParser(description='ZMON AWS Agent') argp.add_argument('-e', '--entity-service', dest='entityservice') argp.add_argument('-r', '--region', dest='region', default=None) argp.add_argument('-j', '--json', dest='json', action='store_true') argp.add_argument('-t', '--tracer', dest='tracer', default=os.environ.get('OPENTRACING_TRACER', 'noop')) argp.add_argument('--no-oauth2', dest='disable_oauth2', action='store_true', default=False) argp.add_argument('--postgresql-user', dest='postgresql_user', default=os.environ.get('AGENT_POSTGRESQL_USER')) argp.add_argument('--postgresql-pass', dest='postgresql_pass', default=os.environ.get('AGENT_POSTGRESQL_PASS')) args = argp.parse_args() if not args.disable_oauth2: tokens.configure() tokens.manage('uid', ['uid']) tokens.start() init_opentracing_tracer(args.tracer) root_span = opentracing.tracer.start_span(operation_name='aws_entity_discovery') with root_span: logging.basicConfig(level=logging.INFO) # 0. Fetch extra data for entities entity_extras = {} for ex in os.getenv('EXTRA_ENTITY_FIELDS', '').split(','): if '=' not in ex: continue k, v = ex.split('=', 1) if k and v: entity_extras[k] = v # 1. Determine region if not args.region: logger.info('Trying to figure out region..') try: response = requests.get('http://169.254.169.254/latest/meta-data/placement/availability-zone', timeout=2) except Exception: root_span.set_tag('error', True) root_span.log_kv({'exception': traceback.format_exc()}) logger.exception('Region was not specified as a parameter and' + 'can not be fetched from instance meta-data!') raise region = response.text[:-1] else: region = args.region root_span.set_tag('region', region) logger.info('Using region: {}'.format(region)) logger.info('Entity service URL: %s', args.entityservice) logger.info('Reading DNS data for hosted zones') aws.populate_dns_data() aws_account_id = aws.get_account_id(region) infrastructure_account = 'aws:{}'.format(aws_account_id) if aws_account_id else None if not infrastructure_account: logger.error('AWS agent: Cannot determine infrastructure account ID. Terminating!') return root_span.set_tag('account', infrastructure_account) # 2. ZMON entities if not args.disable_oauth2: token = os.getenv('ZMON_TOKEN', None) or tokens.get('uid') zmon_client = Zmon(args.entityservice, token=token, user_agent=get_user_agent()) query = {'infrastructure_account': infrastructure_account, 'region': region, 'created_by': 'agent'} entities = zmon_client.get_entities(query) # 3. Get running apps apps = aws.get_running_apps(region, entities) elbs = [] scaling_groups = [] elastigroups = [] certificates = [] rds = [] elasticaches = [] dynamodbs = [] sqs = [] postgresql_clusters = [] aws_limits = [] new_entities = [] to_be_removed = [] if len(apps) > 0: elbs = aws.get_running_elbs(region, infrastructure_account) scaling_groups = aws.get_auto_scaling_groups(region, infrastructure_account) elastigroups = elastigroup.get_elastigroup_entities(region, infrastructure_account) rds = aws.get_rds_instances(region, infrastructure_account, entities) elasticaches = aws.get_elasticache_nodes(region, infrastructure_account) dynamodbs = aws.get_dynamodb_tables(region, infrastructure_account) certificates = aws.get_certificates(region, infrastructure_account) aws_limits = aws.get_limits(region, infrastructure_account, apps, elbs, entities) sqs = aws.get_sqs_queues(region, infrastructure_account, entities) postgresql_clusters = postgresql.get_postgresql_clusters(region, infrastructure_account, scaling_groups, apps) account_alias = aws.get_account_alias(region) ia_entity = { 'type': 'local', 'infrastructure_account': infrastructure_account, 'account_alias': account_alias, 'region': region, 'id': 'aws-ac[{}:{}]'.format(infrastructure_account, region), 'created_by': 'agent', } account_alias_prefix = os.getenv('ACCOUNT_ALIAS_PREFIX', None) owner = account_alias if account_alias_prefix: owner = owner.replace(account_alias_prefix, '', 1) root_span.set_tag('team', owner) application_entities = aws.get_apps_from_entities(apps, infrastructure_account, region) if args.postgresql_user and args.postgresql_pass: postgresql_databases = postgresql.get_databases_from_clusters(postgresql_clusters, infrastructure_account, region, args.postgresql_user, args.postgresql_pass) else: # Pretend the list of DBs is empty, but also make sure we don't remove # any pre-existing database entities because we don't know about them. postgresql_databases = [] entities = [e for e in entities if e.get('type') != 'postgresql_database'] current_entities = ( elbs + scaling_groups + elastigroups + apps + application_entities + rds + postgresql_databases + postgresql_clusters + elasticaches + dynamodbs + certificates + sqs) current_entities.append(aws_limits) current_entities.append(ia_entity) for entity in current_entities: entity.update(entity_extras) # 4. Removing misssing entities existing_ids = get_existing_ids(entities) current_entities_ids = {e['id'] for e in current_entities} to_be_removed, delete_error_count = remove_missing_entities( existing_ids, current_entities_ids, zmon_client, json=args.json) root_span.log_kv({'total_entitites': str(len(current_entities))}) root_span.log_kv({'removed_entities': str(len(to_be_removed))}) logger.info('Found {} removed entities from {} entities ({} failed)'.format( len(to_be_removed), len(current_entities), delete_error_count)) # 5. Get new/updated entities new_entities, add_error_count = add_new_entities(current_entities, entities, zmon_client, json=args.json) root_span.log_kv({'new_entities': str(len(new_entities))}) logger.info('Found {} new entities from {} entities ({} failed)'.format( len(new_entities), len(current_entities), add_error_count)) # 6. Always add Local entity if not args.json: ia_entity['errors'] = {'delete_count': delete_error_count, 'add_count': add_error_count} update_local_entity(zmon_client, ia_entity) types = {e['type']: len([t for t in new_entities if t['type'] == e['type']]) for e in new_entities} for t, v in types.items(): logger.info('Found {} new entities of type: {}'.format(v, t)) # Check if it is a dry run! if args.json: d = { 'applications': application_entities, 'apps': apps, 'elastigroups': elastigroups, 'dynamodb': dynamodbs, 'elbs': elbs, 'elc': elasticaches, 'rds': rds, 'certificates': certificates, 'aws_limits': aws_limits, 'sqs_queues': sqs, 'new_entities': new_entities, 'to_be_removed': to_be_removed, 'posgresql_clusters': postgresql_clusters } print(json.dumps(d, indent=4))