def test_init_basic(): recorder = Recorder() init_opentracing_tracer(OPENTRACING_BASIC, recorder=recorder) assert isinstance(opentracing.tracer, BasicTracer) assert opentracing.tracer.recorder == recorder
def test_init_jaeger(monkeypatch): config = MagicMock() config.return_value.initialize_tracer.return_value = 'jaeger' monkeypatch.setattr('jaeger_client.Config', config) init_opentracing_tracer(OPENTRACING_JAEGER, service_name='test_jaeger') config.assert_called_once_with(config={}, service_name='test_jaeger') assert opentracing.tracer == 'jaeger'
def test_init_jaeger_with_config(monkeypatch): config = MagicMock() config.return_value.initialize_tracer.return_value = 'jaeger' monkeypatch.setattr('jaeger_client.Config', config) monkeypatch.setenv('OPENTRACING_JAEGER_SERVICE_NAME', 'component') init_opentracing_tracer(OPENTRACING_JAEGER, config={'logging': True}) config.assert_called_once_with(config={'logging': True}, service_name='component') assert opentracing.tracer == 'jaeger'
def test_init_lightstep(monkeypatch): tracer = MagicMock() monkeypatch.setattr('lightstep.Tracer', tracer) init_opentracing_tracer(OPENTRACING_LIGHTSTEP, component_name='test_lightstep', verbosity=2) tracer.assert_called_once_with(component_name='test_lightstep', access_token=None, collector_host='collector.lightstep.com', collector_port=443, verbosity=2)
def test_init_instana(monkeypatch): init = MagicMock() opts = MagicMock() monkeypatch.setattr('instana.tracer.init', init) monkeypatch.setattr('instana.options', opts) init_opentracing_tracer(OPENTRACING_INSTANA, log_level=logging.DEBUG, service=SERVICE_NAME) opts.Options.assert_called_once_with(service=SERVICE_NAME, log_level=logging.DEBUG) init.assert_called_once()
def test_init_lightstep_env_vars(monkeypatch): tracer = MagicMock() monkeypatch.setattr('lightstep.Tracer', tracer) monkeypatch.setenv('OPENTRACING_LIGHTSTEP_COMPONENT_NAME', 'component') monkeypatch.setenv('OPENTRACING_LIGHTSTEP_ACCESS_TOKEN', '1234') monkeypatch.setenv('OPENTRACING_LIGHTSTEP_COLLECTOR_HOST', 'tracer.example.org') monkeypatch.setenv('OPENTRACING_LIGHTSTEP_COLLECTOR_PORT', '8443') monkeypatch.setenv('OPENTRACING_LIGHTSTEP_VERBOSITY', '1') init_opentracing_tracer(OPENTRACING_LIGHTSTEP) tracer.assert_called_once_with(component_name='component', access_token='1234', collector_host='tracer.example.org', collector_port=8443, verbosity=1)
def start_worker(**kwargs): """ A simple wrapper for workflow.start_worker(role) , needed to solve the logger import problem with multiprocessing :param role: one of the constants workflow.ROLE_... :return: """ _set_logging(settings.LOGGING) logger = logging.getLogger(__name__) logger.info('ZMON Worker running with {} OpenTracing tracer!'.format( kwargs.get('tracer', 'noop'))) init_opentracing_tracer(kwargs.pop('tracer', None), tags=kwargs.pop('tracer_tags', {})) import workflow workflow.start_worker_for_queue(**kwargs)
def create_app(*args, **kwargs): connexion_app = connexion.App(__name__) connexion_app.app.json_encoder = DecimalEncoder connexion_app.app.config.from_object('app.config') app = connexion_app.app init_opentracing_tracer(OPENTRACING_TRACER) register_extensions(app) register_middleware(app) register_routes(connexion_app) register_errors(app) if kwargs.get('connexion_app'): return connexion_app return app
def main(): argp = argparse.ArgumentParser(description='ZMON Kubernetes Agent') argp.add_argument( '-i', '--infrastructure-account', dest='infrastructure_account', default=None, help= 'Infrastructure account which identifies this agent. Can be set via ' 'ZMON_AGENT_INFRASTRUCTURE_ACCOUNT env variable.') argp.add_argument( '-r', '--region', dest='region', help='Cluster region. Can be set via ZMON_AGENT_REGION env variable.') argp.add_argument( '-d', '--discover', dest='discover', help= ('Comma separated list of builtin discovery agents to be used. Current supported discovery ' 'agents are {}. Can be set via ZMON_AGENT_BUILTIN_DISCOVERY env variable.' ).format(BUILTIN_DISCOVERY)) argp.add_argument( '-e', '--entity-service', dest='entity_service', help= 'ZMON backend URL. Can be set via ZMON_AGENT_ENTITY_SERVICE_URL env variable.' ) argp.add_argument( '--interval', dest='interval', help= 'Interval for agent sync. If not set then agent will run once. Can be set via ' 'ZMON_AGENT_INTERVAL env variable.') # OPENTRACING SUPPORT argp.add_argument( '--opentracing', dest='opentracing', default=os.environ.get('ZMON_AGENT_OPENTRACING_TRACER'), help= 'OpenTracing tracer name as supported by opentracing-utils. Please Ignore for NOOP tracer.' ) argp.add_argument('-j', '--json', dest='json', action='store_true', help='Print JSON output only.', default=False) argp.add_argument('--skip-ssl', dest='skip_ssl', action='store_true', default=False) argp.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=False, help='Verbose output.') args = argp.parse_args() logger.info('Initializing opentracing tracer: {}'.format( args.opentracing if args.opentracing else 'noop')) init_opentracing_tracer(args.opentracing) # Give some time for tracer initialization if args.opentracing: time.sleep(2) init_span = opentracing.tracer.start_span(operation_name='zmon-agent-init') with init_span: # Hard requirements infrastructure_account = ( args.infrastructure_account if args.infrastructure_account else os.environ.get('ZMON_AGENT_INFRASTRUCTURE_ACCOUNT')) if not infrastructure_account: init_span.set_tag('error', True) raise RuntimeError( 'Cannot determine infrastructure account. Please use --infrastructure-account option or ' 'set env variable ZMON_AGENT_INFRASTRUCTURE_ACCOUNT.') region = os.environ.get('ZMON_AGENT_REGION', args.region) entity_service = os.environ.get('ZMON_AGENT_ENTITY_SERVICE_URL', args.entity_service) interval = os.environ.get('ZMON_AGENT_INTERVAL', args.interval) init_span.set_tag('account', infrastructure_account) init_span.set_tag('region', region) if interval: interval = int(interval) # OAUTH2 tokens tokens.configure() tokens.manage('uid', ['uid']) verbose = args.verbose if args.verbose else os.environ.get( 'ZMON_AGENT_DEBUG', False) if verbose: logger.setLevel(logging.DEBUG) verify = True if args.skip_ssl: logger.warning('ZMON agent will skip SSL verification!') verify = False if not region: # Assuming running on AWS logger.info('Trying to figure out region ...') try: response = requests.get( 'http://169.254.169.254/latest/meta-data/placement/availability-zone', timeout=2) response.raise_for_status() region = response.text[:-1] except Exception: init_span.set_tag('error', True) logger.error( 'AWS region was not specified and can not be fetched from instance meta-data!' ) raise logger.info('Starting sync operations!') sync(infrastructure_account, region, entity_service, verify, args.json, interval) if args.opentracing: time.sleep(5)
logger.exception('Failed to assume role {}'.format(arn)) return connexion.problem(500, 'AWS Error', error_message) credentials = role['Credentials'] logger.info( 'Handing out credentials for {account_id}/{role_name} to {uid}'.format( account_id=account_id, role_name=role_name, uid=uid)) return { 'access_key_id': credentials['AccessKeyId'], 'secret_access_key': credentials['SecretAccessKey'], 'session_token': credentials['SessionToken'], 'expiration': credentials['Expiration'] } app = connexion.App(__name__) app.add_api('swagger.yaml') if __name__ == '__main__': tracer = os.getenv('OPENTRACING_TRACER') logger.info('Starting server with OpenTracing tracer: {}'.format(tracer)) init_opentracing_tracer(tracer) # Trace all incoming requests trace_flask(app.app) # run our standalone gevent server app.run(port=8080, server='gevent')
def test_init_instana(monkeypatch): init_opentracing_tracer(OPENTRACING_INSTANA)
def test_init_noop(): t = init_opentracing_tracer(None) assert isinstance(opentracing.tracer, opentracing.Tracer) assert t == opentracing.tracer
def main(): argp = argparse.ArgumentParser(description='ZMON AWS Agent') argp.add_argument('-e', '--entity-service', dest='entityservice') argp.add_argument('-r', '--region', dest='region', default=None) argp.add_argument('-j', '--json', dest='json', action='store_true') argp.add_argument('-t', '--tracer', dest='tracer', default=os.environ.get('OPENTRACING_TRACER', 'noop')) argp.add_argument('--no-oauth2', dest='disable_oauth2', action='store_true', default=False) argp.add_argument('--postgresql-user', dest='postgresql_user', default=os.environ.get('AGENT_POSTGRESQL_USER')) argp.add_argument('--postgresql-pass', dest='postgresql_pass', default=os.environ.get('AGENT_POSTGRESQL_PASS')) args = argp.parse_args() if not args.disable_oauth2: tokens.configure() tokens.manage('uid', ['uid']) tokens.start() init_opentracing_tracer(args.tracer) root_span = opentracing.tracer.start_span( operation_name='aws_entity_discovery') with root_span: logging.basicConfig(level=logging.INFO) # 0. Fetch extra data for entities entity_extras = {} for ex in os.getenv('EXTRA_ENTITY_FIELDS', '').split(','): if '=' not in ex: continue k, v = ex.split('=', 1) if k and v: entity_extras[k] = v # 1. Determine region if not args.region: logger.info('Trying to figure out region..') try: response = requests.get( 'http://169.254.169.254/latest/meta-data/placement/availability-zone', timeout=2) except Exception: root_span.set_tag('error', True) root_span.log_kv({'exception': traceback.format_exc()}) logger.exception( 'Region was not specified as a parameter and' + 'can not be fetched from instance meta-data!') raise region = response.text[:-1] else: region = args.region root_span.set_tag('region', region) logger.info('Using region: {}'.format(region)) logger.info('Entity service URL: %s', args.entityservice) logger.info('Reading DNS data for hosted zones') aws.populate_dns_data() aws_account_id = aws.get_account_id(region) infrastructure_account = 'aws:{}'.format( aws_account_id) if aws_account_id else None if not infrastructure_account: logger.error( 'AWS agent: Cannot determine infrastructure account ID. Terminating!' ) return root_span.set_tag('account', infrastructure_account) # 2. ZMON entities if not args.disable_oauth2: token = os.getenv('ZMON_TOKEN', None) or tokens.get('uid') zmon_client = Zmon(args.entityservice, token=token, user_agent=get_user_agent()) query = { 'infrastructure_account': infrastructure_account, 'region': region, 'created_by': 'agent' } entities = zmon_client.get_entities(query) # 3. Get running apps apps = aws.get_running_apps(region, entities) elbs = [] scaling_groups = [] rds = [] elasticaches = [] dynamodbs = [] sqs = [] new_entities = [] to_be_removed = [] if len(apps) > 0: elbs = aws.get_running_elbs(region, infrastructure_account) scaling_groups = aws.get_auto_scaling_groups( region, infrastructure_account) rds = aws.get_rds_instances(region, infrastructure_account, entities) elasticaches = aws.get_elasticache_nodes(region, infrastructure_account) dynamodbs = aws.get_dynamodb_tables(region, infrastructure_account) certificates = aws.get_certificates(region, infrastructure_account) aws_limits = aws.get_limits(region, infrastructure_account, apps, elbs, entities) sqs = aws.get_sqs_queues(region, infrastructure_account, entities) postgresql_clusters = postgresql.get_postgresql_clusters( region, infrastructure_account, scaling_groups, apps) account_alias = aws.get_account_alias(region) ia_entity = { 'type': 'local', 'infrastructure_account': infrastructure_account, 'account_alias': account_alias, 'region': region, 'id': 'aws-ac[{}:{}]'.format(infrastructure_account, region), 'created_by': 'agent', } account_alias_prefix = os.getenv('ACCOUNT_ALIAS_PREFIX', None) owner = account_alias if account_alias_prefix: owner = owner.replace(account_alias_prefix, '', 1) root_span.set_tag('team', owner) application_entities = aws.get_apps_from_entities( apps, infrastructure_account, region) if args.postgresql_user and args.postgresql_pass: postgresql_databases = postgresql.get_databases_from_clusters( postgresql_clusters, infrastructure_account, region, args.postgresql_user, args.postgresql_pass) else: # Pretend the list of DBs is empty, but also make sure we don't remove # any pre-existing database entities because we don't know about them. postgresql_databases = [] entities = [ e for e in entities if e.get('type') != 'postgresql_database' ] current_entities = (elbs + scaling_groups + apps + application_entities + rds + postgresql_databases + postgresql_clusters + elasticaches + dynamodbs + certificates + sqs) current_entities.append(aws_limits) current_entities.append(ia_entity) for entity in current_entities: entity.update(entity_extras) # 4. Removing misssing entities existing_ids = get_existing_ids(entities) current_entities_ids = {e['id'] for e in current_entities} to_be_removed, delete_error_count = remove_missing_entities( existing_ids, current_entities_ids, zmon_client, json=args.json) root_span.log_kv({'total_entitites': str(len(current_entities))}) root_span.log_kv({'removed_entities': str(len(to_be_removed))}) logger.info( 'Found {} removed entities from {} entities ({} failed)'.format( len(to_be_removed), len(current_entities), delete_error_count)) # 5. Get new/updated entities new_entities, add_error_count = add_new_entities(current_entities, entities, zmon_client, json=args.json) root_span.log_kv({'new_entities': str(len(new_entities))}) logger.info( 'Found {} new entities from {} entities ({} failed)'.format( len(new_entities), len(current_entities), add_error_count)) # 6. Always add Local entity if not args.json: ia_entity['errors'] = { 'delete_count': delete_error_count, 'add_count': add_error_count } update_local_entity(zmon_client, ia_entity) types = { e['type']: len([t for t in new_entities if t['type'] == e['type']]) for e in new_entities } for t, v in types.items(): logger.info('Found {} new entities of type: {}'.format(v, t)) # Check if it is a dry run! if args.json: d = { 'applications': application_entities, 'apps': apps, 'dynamodb': dynamodbs, 'elbs': elbs, 'elc': elasticaches, 'rds': rds, 'certificates': certificates, 'aws_limits': aws_limits, 'sqs_queues': sqs, 'new_entities': new_entities, 'to_be_removed': to_be_removed, 'posgresql_clusters': postgresql_clusters } print(json.dumps(d, indent=4))
def main(): argp = argparse.ArgumentParser(description='ZMON AWS Agent') argp.add_argument('-e', '--entity-service', dest='entityservice') argp.add_argument('-r', '--region', dest='region', default=None) argp.add_argument('-j', '--json', dest='json', action='store_true') argp.add_argument('-t', '--tracer', dest='tracer', default=os.environ.get('OPENTRACING_TRACER', 'noop')) argp.add_argument('--no-oauth2', dest='disable_oauth2', action='store_true', default=False) argp.add_argument('--postgresql-user', dest='postgresql_user', default=os.environ.get('AGENT_POSTGRESQL_USER')) argp.add_argument('--postgresql-pass', dest='postgresql_pass', default=os.environ.get('AGENT_POSTGRESQL_PASS')) args = argp.parse_args() if not args.disable_oauth2: tokens.configure() tokens.manage('uid', ['uid']) tokens.start() init_opentracing_tracer(args.tracer) root_span = opentracing.tracer.start_span(operation_name='aws_entity_discovery') with root_span: logging.basicConfig(level=logging.INFO) # 0. Fetch extra data for entities entity_extras = {} for ex in os.getenv('EXTRA_ENTITY_FIELDS', '').split(','): if '=' not in ex: continue k, v = ex.split('=', 1) if k and v: entity_extras[k] = v # 1. Determine region if not args.region: logger.info('Trying to figure out region..') try: response = requests.get('http://169.254.169.254/latest/meta-data/placement/availability-zone', timeout=2) except Exception: root_span.set_tag('error', True) root_span.log_kv({'exception': traceback.format_exc()}) logger.exception('Region was not specified as a parameter and' + 'can not be fetched from instance meta-data!') raise region = response.text[:-1] else: region = args.region root_span.set_tag('region', region) logger.info('Using region: {}'.format(region)) logger.info('Entity service URL: %s', args.entityservice) logger.info('Reading DNS data for hosted zones') aws.populate_dns_data() aws_account_id = aws.get_account_id(region) infrastructure_account = 'aws:{}'.format(aws_account_id) if aws_account_id else None if not infrastructure_account: logger.error('AWS agent: Cannot determine infrastructure account ID. Terminating!') return root_span.set_tag('account', infrastructure_account) # 2. ZMON entities if not args.disable_oauth2: token = os.getenv('ZMON_TOKEN', None) or tokens.get('uid') zmon_client = Zmon(args.entityservice, token=token, user_agent=get_user_agent()) query = {'infrastructure_account': infrastructure_account, 'region': region, 'created_by': 'agent'} entities = zmon_client.get_entities(query) # 3. Get running apps apps = aws.get_running_apps(region, entities) elbs = [] scaling_groups = [] elastigroups = [] certificates = [] rds = [] elasticaches = [] dynamodbs = [] sqs = [] postgresql_clusters = [] aws_limits = [] new_entities = [] to_be_removed = [] if len(apps) > 0: elbs = aws.get_running_elbs(region, infrastructure_account) scaling_groups = aws.get_auto_scaling_groups(region, infrastructure_account) elastigroups = elastigroup.get_elastigroup_entities(region, infrastructure_account) rds = aws.get_rds_instances(region, infrastructure_account, entities) elasticaches = aws.get_elasticache_nodes(region, infrastructure_account) dynamodbs = aws.get_dynamodb_tables(region, infrastructure_account) certificates = aws.get_certificates(region, infrastructure_account) aws_limits = aws.get_limits(region, infrastructure_account, apps, elbs, entities) sqs = aws.get_sqs_queues(region, infrastructure_account, entities) postgresql_clusters = postgresql.get_postgresql_clusters(region, infrastructure_account, scaling_groups, apps) account_alias = aws.get_account_alias(region) ia_entity = { 'type': 'local', 'infrastructure_account': infrastructure_account, 'account_alias': account_alias, 'region': region, 'id': 'aws-ac[{}:{}]'.format(infrastructure_account, region), 'created_by': 'agent', } account_alias_prefix = os.getenv('ACCOUNT_ALIAS_PREFIX', None) owner = account_alias if account_alias_prefix: owner = owner.replace(account_alias_prefix, '', 1) root_span.set_tag('team', owner) application_entities = aws.get_apps_from_entities(apps, infrastructure_account, region) if args.postgresql_user and args.postgresql_pass: postgresql_databases = postgresql.get_databases_from_clusters(postgresql_clusters, infrastructure_account, region, args.postgresql_user, args.postgresql_pass) else: # Pretend the list of DBs is empty, but also make sure we don't remove # any pre-existing database entities because we don't know about them. postgresql_databases = [] entities = [e for e in entities if e.get('type') != 'postgresql_database'] current_entities = ( elbs + scaling_groups + elastigroups + apps + application_entities + rds + postgresql_databases + postgresql_clusters + elasticaches + dynamodbs + certificates + sqs) current_entities.append(aws_limits) current_entities.append(ia_entity) for entity in current_entities: entity.update(entity_extras) # 4. Removing misssing entities existing_ids = get_existing_ids(entities) current_entities_ids = {e['id'] for e in current_entities} to_be_removed, delete_error_count = remove_missing_entities( existing_ids, current_entities_ids, zmon_client, json=args.json) root_span.log_kv({'total_entitites': str(len(current_entities))}) root_span.log_kv({'removed_entities': str(len(to_be_removed))}) logger.info('Found {} removed entities from {} entities ({} failed)'.format( len(to_be_removed), len(current_entities), delete_error_count)) # 5. Get new/updated entities new_entities, add_error_count = add_new_entities(current_entities, entities, zmon_client, json=args.json) root_span.log_kv({'new_entities': str(len(new_entities))}) logger.info('Found {} new entities from {} entities ({} failed)'.format( len(new_entities), len(current_entities), add_error_count)) # 6. Always add Local entity if not args.json: ia_entity['errors'] = {'delete_count': delete_error_count, 'add_count': add_error_count} update_local_entity(zmon_client, ia_entity) types = {e['type']: len([t for t in new_entities if t['type'] == e['type']]) for e in new_entities} for t, v in types.items(): logger.info('Found {} new entities of type: {}'.format(v, t)) # Check if it is a dry run! if args.json: d = { 'applications': application_entities, 'apps': apps, 'elastigroups': elastigroups, 'dynamodb': dynamodbs, 'elbs': elbs, 'elc': elasticaches, 'rds': rds, 'certificates': certificates, 'aws_limits': aws_limits, 'sqs_queues': sqs, 'new_entities': new_entities, 'to_be_removed': to_be_removed, 'posgresql_clusters': postgresql_clusters } print(json.dumps(d, indent=4))