def test_init_basic():
recorder = Recorder()
init_opentracing_tracer(OPENTRACING_BASIC, recorder=recorder)
assert isinstance(opentracing.tracer, BasicTracer)
assert opentracing.tracer.recorder == recorder
def test_init_jaeger(monkeypatch):
config = MagicMock()
config.return_value.initialize_tracer.return_value = 'jaeger'
monkeypatch.setattr('jaeger_client.Config', config)
init_opentracing_tracer(OPENTRACING_JAEGER, service_name='test_jaeger')
config.assert_called_once_with(config={}, service_name='test_jaeger')
assert opentracing.tracer == 'jaeger'
def test_init_jaeger_with_config(monkeypatch):
config = MagicMock()
config.return_value.initialize_tracer.return_value = 'jaeger'
monkeypatch.setattr('jaeger_client.Config', config)
monkeypatch.setenv('OPENTRACING_JAEGER_SERVICE_NAME', 'component')
init_opentracing_tracer(OPENTRACING_JAEGER, config={'logging': True})
config.assert_called_once_with(config={'logging': True},
service_name='component')
assert opentracing.tracer == 'jaeger'
def test_init_lightstep(monkeypatch):
tracer = MagicMock()
monkeypatch.setattr('lightstep.Tracer', tracer)
init_opentracing_tracer(OPENTRACING_LIGHTSTEP,
component_name='test_lightstep',
verbosity=2)
tracer.assert_called_once_with(component_name='test_lightstep',
access_token=None,
collector_host='collector.lightstep.com',
collector_port=443,
verbosity=2)
def test_init_instana(monkeypatch):
init = MagicMock()
opts = MagicMock()
monkeypatch.setattr('instana.tracer.init', init)
monkeypatch.setattr('instana.options', opts)
init_opentracing_tracer(OPENTRACING_INSTANA,
log_level=logging.DEBUG,
service=SERVICE_NAME)
opts.Options.assert_called_once_with(service=SERVICE_NAME,
log_level=logging.DEBUG)
init.assert_called_once()
def test_init_lightstep_env_vars(monkeypatch):
tracer = MagicMock()
monkeypatch.setattr('lightstep.Tracer', tracer)
monkeypatch.setenv('OPENTRACING_LIGHTSTEP_COMPONENT_NAME', 'component')
monkeypatch.setenv('OPENTRACING_LIGHTSTEP_ACCESS_TOKEN', '1234')
monkeypatch.setenv('OPENTRACING_LIGHTSTEP_COLLECTOR_HOST',
'tracer.example.org')
monkeypatch.setenv('OPENTRACING_LIGHTSTEP_COLLECTOR_PORT', '8443')
monkeypatch.setenv('OPENTRACING_LIGHTSTEP_VERBOSITY', '1')
init_opentracing_tracer(OPENTRACING_LIGHTSTEP)
tracer.assert_called_once_with(component_name='component',
access_token='1234',
collector_host='tracer.example.org',
collector_port=8443,
verbosity=1)
def start_worker(**kwargs):
"""
A simple wrapper for workflow.start_worker(role) , needed to solve the logger import problem with multiprocessing
:param role: one of the constants workflow.ROLE_...
:return:
"""
_set_logging(settings.LOGGING)
logger = logging.getLogger(__name__)
logger.info('ZMON Worker running with {} OpenTracing tracer!'.format(
kwargs.get('tracer', 'noop')))
init_opentracing_tracer(kwargs.pop('tracer', None),
tags=kwargs.pop('tracer_tags', {}))
import workflow
workflow.start_worker_for_queue(**kwargs)
def create_app(*args, **kwargs):
connexion_app = connexion.App(__name__)
connexion_app.app.json_encoder = DecimalEncoder
connexion_app.app.config.from_object('app.config')
app = connexion_app.app
init_opentracing_tracer(OPENTRACING_TRACER)
register_extensions(app)
register_middleware(app)
register_routes(connexion_app)
register_errors(app)
if kwargs.get('connexion_app'):
return connexion_app
return app
def main():
argp = argparse.ArgumentParser(description='ZMON Kubernetes Agent')
argp.add_argument(
'-i',
'--infrastructure-account',
dest='infrastructure_account',
default=None,
help=
'Infrastructure account which identifies this agent. Can be set via '
'ZMON_AGENT_INFRASTRUCTURE_ACCOUNT env variable.')
argp.add_argument(
'-r',
'--region',
dest='region',
help='Cluster region. Can be set via ZMON_AGENT_REGION env variable.')
argp.add_argument(
'-d',
'--discover',
dest='discover',
help=
('Comma separated list of builtin discovery agents to be used. Current supported discovery '
'agents are {}. Can be set via ZMON_AGENT_BUILTIN_DISCOVERY env variable.'
).format(BUILTIN_DISCOVERY))
argp.add_argument(
'-e',
'--entity-service',
dest='entity_service',
help=
'ZMON backend URL. Can be set via ZMON_AGENT_ENTITY_SERVICE_URL env variable.'
)
argp.add_argument(
'--interval',
dest='interval',
help=
'Interval for agent sync. If not set then agent will run once. Can be set via '
'ZMON_AGENT_INTERVAL env variable.')
# OPENTRACING SUPPORT
argp.add_argument(
'--opentracing',
dest='opentracing',
default=os.environ.get('ZMON_AGENT_OPENTRACING_TRACER'),
help=
'OpenTracing tracer name as supported by opentracing-utils. Please Ignore for NOOP tracer.'
)
argp.add_argument('-j',
'--json',
dest='json',
action='store_true',
help='Print JSON output only.',
default=False)
argp.add_argument('--skip-ssl',
dest='skip_ssl',
action='store_true',
default=False)
argp.add_argument('-v',
'--verbose',
dest='verbose',
action='store_true',
default=False,
help='Verbose output.')
args = argp.parse_args()
logger.info('Initializing opentracing tracer: {}'.format(
args.opentracing if args.opentracing else 'noop'))
init_opentracing_tracer(args.opentracing)
# Give some time for tracer initialization
if args.opentracing:
time.sleep(2)
init_span = opentracing.tracer.start_span(operation_name='zmon-agent-init')
with init_span:
# Hard requirements
infrastructure_account = (
args.infrastructure_account if args.infrastructure_account else
os.environ.get('ZMON_AGENT_INFRASTRUCTURE_ACCOUNT'))
if not infrastructure_account:
init_span.set_tag('error', True)
raise RuntimeError(
'Cannot determine infrastructure account. Please use --infrastructure-account option or '
'set env variable ZMON_AGENT_INFRASTRUCTURE_ACCOUNT.')
region = os.environ.get('ZMON_AGENT_REGION', args.region)
entity_service = os.environ.get('ZMON_AGENT_ENTITY_SERVICE_URL',
args.entity_service)
interval = os.environ.get('ZMON_AGENT_INTERVAL', args.interval)
init_span.set_tag('account', infrastructure_account)
init_span.set_tag('region', region)
if interval:
interval = int(interval)
# OAUTH2 tokens
tokens.configure()
tokens.manage('uid', ['uid'])
verbose = args.verbose if args.verbose else os.environ.get(
'ZMON_AGENT_DEBUG', False)
if verbose:
logger.setLevel(logging.DEBUG)
verify = True
if args.skip_ssl:
logger.warning('ZMON agent will skip SSL verification!')
verify = False
if not region:
# Assuming running on AWS
logger.info('Trying to figure out region ...')
try:
response = requests.get(
'http://169.254.169.254/latest/meta-data/placement/availability-zone',
timeout=2)
response.raise_for_status()
region = response.text[:-1]
except Exception:
init_span.set_tag('error', True)
logger.error(
'AWS region was not specified and can not be fetched from instance meta-data!'
)
raise
logger.info('Starting sync operations!')
sync(infrastructure_account, region, entity_service, verify, args.json,
interval)
if args.opentracing:
time.sleep(5)
logger.exception('Failed to assume role {}'.format(arn))
return connexion.problem(500, 'AWS Error', error_message)
credentials = role['Credentials']
logger.info(
'Handing out credentials for {account_id}/{role_name} to {uid}'.format(
account_id=account_id, role_name=role_name, uid=uid))
return {
'access_key_id': credentials['AccessKeyId'],
'secret_access_key': credentials['SecretAccessKey'],
'session_token': credentials['SessionToken'],
'expiration': credentials['Expiration']
}
app = connexion.App(__name__)
app.add_api('swagger.yaml')
if __name__ == '__main__':
tracer = os.getenv('OPENTRACING_TRACER')
logger.info('Starting server with OpenTracing tracer: {}'.format(tracer))
init_opentracing_tracer(tracer)
# Trace all incoming requests
trace_flask(app.app)
# run our standalone gevent server
app.run(port=8080, server='gevent')
def test_init_instana(monkeypatch):
init_opentracing_tracer(OPENTRACING_INSTANA)
def test_init_noop():
t = init_opentracing_tracer(None)
assert isinstance(opentracing.tracer, opentracing.Tracer)
assert t == opentracing.tracer
def main():
argp = argparse.ArgumentParser(description='ZMON AWS Agent')
argp.add_argument('-e', '--entity-service', dest='entityservice')
argp.add_argument('-r', '--region', dest='region', default=None)
argp.add_argument('-j', '--json', dest='json', action='store_true')
argp.add_argument('-t',
'--tracer',
dest='tracer',
default=os.environ.get('OPENTRACING_TRACER', 'noop'))
argp.add_argument('--no-oauth2',
dest='disable_oauth2',
action='store_true',
default=False)
argp.add_argument('--postgresql-user',
dest='postgresql_user',
default=os.environ.get('AGENT_POSTGRESQL_USER'))
argp.add_argument('--postgresql-pass',
dest='postgresql_pass',
default=os.environ.get('AGENT_POSTGRESQL_PASS'))
args = argp.parse_args()
if not args.disable_oauth2:
tokens.configure()
tokens.manage('uid', ['uid'])
tokens.start()
init_opentracing_tracer(args.tracer)
root_span = opentracing.tracer.start_span(
operation_name='aws_entity_discovery')
with root_span:
logging.basicConfig(level=logging.INFO)
# 0. Fetch extra data for entities
entity_extras = {}
for ex in os.getenv('EXTRA_ENTITY_FIELDS', '').split(','):
if '=' not in ex:
continue
k, v = ex.split('=', 1)
if k and v:
entity_extras[k] = v
# 1. Determine region
if not args.region:
logger.info('Trying to figure out region..')
try:
response = requests.get(
'http://169.254.169.254/latest/meta-data/placement/availability-zone',
timeout=2)
except Exception:
root_span.set_tag('error', True)
root_span.log_kv({'exception': traceback.format_exc()})
logger.exception(
'Region was not specified as a parameter and' +
'can not be fetched from instance meta-data!')
raise
region = response.text[:-1]
else:
region = args.region
root_span.set_tag('region', region)
logger.info('Using region: {}'.format(region))
logger.info('Entity service URL: %s', args.entityservice)
logger.info('Reading DNS data for hosted zones')
aws.populate_dns_data()
aws_account_id = aws.get_account_id(region)
infrastructure_account = 'aws:{}'.format(
aws_account_id) if aws_account_id else None
if not infrastructure_account:
logger.error(
'AWS agent: Cannot determine infrastructure account ID. Terminating!'
)
return
root_span.set_tag('account', infrastructure_account)
# 2. ZMON entities
if not args.disable_oauth2:
token = os.getenv('ZMON_TOKEN', None) or tokens.get('uid')
zmon_client = Zmon(args.entityservice,
token=token,
user_agent=get_user_agent())
query = {
'infrastructure_account': infrastructure_account,
'region': region,
'created_by': 'agent'
}
entities = zmon_client.get_entities(query)
# 3. Get running apps
apps = aws.get_running_apps(region, entities)
elbs = []
scaling_groups = []
rds = []
elasticaches = []
dynamodbs = []
sqs = []
new_entities = []
to_be_removed = []
if len(apps) > 0:
elbs = aws.get_running_elbs(region, infrastructure_account)
scaling_groups = aws.get_auto_scaling_groups(
region, infrastructure_account)
rds = aws.get_rds_instances(region, infrastructure_account,
entities)
elasticaches = aws.get_elasticache_nodes(region,
infrastructure_account)
dynamodbs = aws.get_dynamodb_tables(region, infrastructure_account)
certificates = aws.get_certificates(region, infrastructure_account)
aws_limits = aws.get_limits(region, infrastructure_account, apps,
elbs, entities)
sqs = aws.get_sqs_queues(region, infrastructure_account, entities)
postgresql_clusters = postgresql.get_postgresql_clusters(
region, infrastructure_account, scaling_groups, apps)
account_alias = aws.get_account_alias(region)
ia_entity = {
'type': 'local',
'infrastructure_account': infrastructure_account,
'account_alias': account_alias,
'region': region,
'id': 'aws-ac[{}:{}]'.format(infrastructure_account, region),
'created_by': 'agent',
}
account_alias_prefix = os.getenv('ACCOUNT_ALIAS_PREFIX', None)
owner = account_alias
if account_alias_prefix:
owner = owner.replace(account_alias_prefix, '', 1)
root_span.set_tag('team', owner)
application_entities = aws.get_apps_from_entities(
apps, infrastructure_account, region)
if args.postgresql_user and args.postgresql_pass:
postgresql_databases = postgresql.get_databases_from_clusters(
postgresql_clusters, infrastructure_account, region,
args.postgresql_user, args.postgresql_pass)
else:
# Pretend the list of DBs is empty, but also make sure we don't remove
# any pre-existing database entities because we don't know about them.
postgresql_databases = []
entities = [
e for e in entities if e.get('type') != 'postgresql_database'
]
current_entities = (elbs + scaling_groups + apps +
application_entities + rds + postgresql_databases +
postgresql_clusters + elasticaches + dynamodbs +
certificates + sqs)
current_entities.append(aws_limits)
current_entities.append(ia_entity)
for entity in current_entities:
entity.update(entity_extras)
# 4. Removing misssing entities
existing_ids = get_existing_ids(entities)
current_entities_ids = {e['id'] for e in current_entities}
to_be_removed, delete_error_count = remove_missing_entities(
existing_ids, current_entities_ids, zmon_client, json=args.json)
root_span.log_kv({'total_entitites': str(len(current_entities))})
root_span.log_kv({'removed_entities': str(len(to_be_removed))})
logger.info(
'Found {} removed entities from {} entities ({} failed)'.format(
len(to_be_removed), len(current_entities), delete_error_count))
# 5. Get new/updated entities
new_entities, add_error_count = add_new_entities(current_entities,
entities,
zmon_client,
json=args.json)
root_span.log_kv({'new_entities': str(len(new_entities))})
logger.info(
'Found {} new entities from {} entities ({} failed)'.format(
len(new_entities), len(current_entities), add_error_count))
# 6. Always add Local entity
if not args.json:
ia_entity['errors'] = {
'delete_count': delete_error_count,
'add_count': add_error_count
}
update_local_entity(zmon_client, ia_entity)
types = {
e['type']: len([t for t in new_entities if t['type'] == e['type']])
for e in new_entities
}
for t, v in types.items():
logger.info('Found {} new entities of type: {}'.format(v, t))
# Check if it is a dry run!
if args.json:
d = {
'applications': application_entities,
'apps': apps,
'dynamodb': dynamodbs,
'elbs': elbs,
'elc': elasticaches,
'rds': rds,
'certificates': certificates,
'aws_limits': aws_limits,
'sqs_queues': sqs,
'new_entities': new_entities,
'to_be_removed': to_be_removed,
'posgresql_clusters': postgresql_clusters
}
print(json.dumps(d, indent=4))
def main():
argp = argparse.ArgumentParser(description='ZMON AWS Agent')
argp.add_argument('-e', '--entity-service', dest='entityservice')
argp.add_argument('-r', '--region', dest='region', default=None)
argp.add_argument('-j', '--json', dest='json', action='store_true')
argp.add_argument('-t', '--tracer', dest='tracer', default=os.environ.get('OPENTRACING_TRACER', 'noop'))
argp.add_argument('--no-oauth2', dest='disable_oauth2', action='store_true', default=False)
argp.add_argument('--postgresql-user', dest='postgresql_user', default=os.environ.get('AGENT_POSTGRESQL_USER'))
argp.add_argument('--postgresql-pass', dest='postgresql_pass', default=os.environ.get('AGENT_POSTGRESQL_PASS'))
args = argp.parse_args()
if not args.disable_oauth2:
tokens.configure()
tokens.manage('uid', ['uid'])
tokens.start()
init_opentracing_tracer(args.tracer)
root_span = opentracing.tracer.start_span(operation_name='aws_entity_discovery')
with root_span:
logging.basicConfig(level=logging.INFO)
# 0. Fetch extra data for entities
entity_extras = {}
for ex in os.getenv('EXTRA_ENTITY_FIELDS', '').split(','):
if '=' not in ex:
continue
k, v = ex.split('=', 1)
if k and v:
entity_extras[k] = v
# 1. Determine region
if not args.region:
logger.info('Trying to figure out region..')
try:
response = requests.get('http://169.254.169.254/latest/meta-data/placement/availability-zone',
timeout=2)
except Exception:
root_span.set_tag('error', True)
root_span.log_kv({'exception': traceback.format_exc()})
logger.exception('Region was not specified as a parameter and' +
'can not be fetched from instance meta-data!')
raise
region = response.text[:-1]
else:
region = args.region
root_span.set_tag('region', region)
logger.info('Using region: {}'.format(region))
logger.info('Entity service URL: %s', args.entityservice)
logger.info('Reading DNS data for hosted zones')
aws.populate_dns_data()
aws_account_id = aws.get_account_id(region)
infrastructure_account = 'aws:{}'.format(aws_account_id) if aws_account_id else None
if not infrastructure_account:
logger.error('AWS agent: Cannot determine infrastructure account ID. Terminating!')
return
root_span.set_tag('account', infrastructure_account)
# 2. ZMON entities
if not args.disable_oauth2:
token = os.getenv('ZMON_TOKEN', None) or tokens.get('uid')
zmon_client = Zmon(args.entityservice, token=token, user_agent=get_user_agent())
query = {'infrastructure_account': infrastructure_account, 'region': region, 'created_by': 'agent'}
entities = zmon_client.get_entities(query)
# 3. Get running apps
apps = aws.get_running_apps(region, entities)
elbs = []
scaling_groups = []
elastigroups = []
certificates = []
rds = []
elasticaches = []
dynamodbs = []
sqs = []
postgresql_clusters = []
aws_limits = []
new_entities = []
to_be_removed = []
if len(apps) > 0:
elbs = aws.get_running_elbs(region, infrastructure_account)
scaling_groups = aws.get_auto_scaling_groups(region, infrastructure_account)
elastigroups = elastigroup.get_elastigroup_entities(region, infrastructure_account)
rds = aws.get_rds_instances(region, infrastructure_account, entities)
elasticaches = aws.get_elasticache_nodes(region, infrastructure_account)
dynamodbs = aws.get_dynamodb_tables(region, infrastructure_account)
certificates = aws.get_certificates(region, infrastructure_account)
aws_limits = aws.get_limits(region, infrastructure_account, apps, elbs, entities)
sqs = aws.get_sqs_queues(region, infrastructure_account, entities)
postgresql_clusters = postgresql.get_postgresql_clusters(region, infrastructure_account,
scaling_groups, apps)
account_alias = aws.get_account_alias(region)
ia_entity = {
'type': 'local',
'infrastructure_account': infrastructure_account,
'account_alias': account_alias,
'region': region,
'id': 'aws-ac[{}:{}]'.format(infrastructure_account, region),
'created_by': 'agent',
}
account_alias_prefix = os.getenv('ACCOUNT_ALIAS_PREFIX', None)
owner = account_alias
if account_alias_prefix:
owner = owner.replace(account_alias_prefix, '', 1)
root_span.set_tag('team', owner)
application_entities = aws.get_apps_from_entities(apps, infrastructure_account, region)
if args.postgresql_user and args.postgresql_pass:
postgresql_databases = postgresql.get_databases_from_clusters(postgresql_clusters,
infrastructure_account,
region,
args.postgresql_user,
args.postgresql_pass)
else:
# Pretend the list of DBs is empty, but also make sure we don't remove
# any pre-existing database entities because we don't know about them.
postgresql_databases = []
entities = [e for e in entities if e.get('type') != 'postgresql_database']
current_entities = (
elbs + scaling_groups + elastigroups + apps + application_entities +
rds + postgresql_databases + postgresql_clusters + elasticaches + dynamodbs +
certificates + sqs)
current_entities.append(aws_limits)
current_entities.append(ia_entity)
for entity in current_entities:
entity.update(entity_extras)
# 4. Removing misssing entities
existing_ids = get_existing_ids(entities)
current_entities_ids = {e['id'] for e in current_entities}
to_be_removed, delete_error_count = remove_missing_entities(
existing_ids, current_entities_ids, zmon_client, json=args.json)
root_span.log_kv({'total_entitites': str(len(current_entities))})
root_span.log_kv({'removed_entities': str(len(to_be_removed))})
logger.info('Found {} removed entities from {} entities ({} failed)'.format(
len(to_be_removed), len(current_entities), delete_error_count))
# 5. Get new/updated entities
new_entities, add_error_count = add_new_entities(current_entities, entities, zmon_client, json=args.json)
root_span.log_kv({'new_entities': str(len(new_entities))})
logger.info('Found {} new entities from {} entities ({} failed)'.format(
len(new_entities), len(current_entities), add_error_count))
# 6. Always add Local entity
if not args.json:
ia_entity['errors'] = {'delete_count': delete_error_count, 'add_count': add_error_count}
update_local_entity(zmon_client, ia_entity)
types = {e['type']: len([t for t in new_entities if t['type'] == e['type']]) for e in new_entities}
for t, v in types.items():
logger.info('Found {} new entities of type: {}'.format(v, t))
# Check if it is a dry run!
if args.json:
d = {
'applications': application_entities,
'apps': apps,
'elastigroups': elastigroups,
'dynamodb': dynamodbs,
'elbs': elbs,
'elc': elasticaches,
'rds': rds,
'certificates': certificates,
'aws_limits': aws_limits,
'sqs_queues': sqs,
'new_entities': new_entities,
'to_be_removed': to_be_removed,
'posgresql_clusters': postgresql_clusters
}
print(json.dumps(d, indent=4))
