def test_logging_retry(caplog, integrated_ff): # get es_client info from the health page es_url = ff_utils.get_health_page( key=integrated_ff['ff_key'])['elasticsearch'] log_utils.set_logging(env='fourfront-mastertest', es_server=es_url, in_prod=True) log = structlog.getLogger(__name__) log.warning('test_retry', _test_log_utils=True) assert len(caplog.records) == 1 assert caplog.records[0].__dict__['msg']['event'] == 'test_retry' log_uuid = caplog.records[0].__dict__['msg']['log_uuid'] # retrying will take 5 sec, so log shoudldn't be in ES yet time.sleep(1) es_client = es_utils.create_es_client(es_url, use_aws_auth=True) es_res = ff_utils.get_es_metadata([log_uuid], es_client=es_client, key=integrated_ff['ff_key']) assert len(es_res) == 0 # wait to allow logs to retry time.sleep(7) es_client = es_utils.create_es_client(es_url, use_aws_auth=True) es_res = ff_utils.get_es_metadata([log_uuid], es_client=es_client, key=integrated_ff['ff_key']) assert len(es_res) == 1 assert es_res[0]['log_uuid'] == log_uuid assert es_res[0]['event'] == 'test_retry'
def main(global_config, **local_config): """ This function returns a Pyramid WSGI application. """ settings = global_config settings.update(local_config) # TODO: move to dcicutils set_logging(settings.get('elasticsearch.server'), settings.get('production')) # TODO - these need to be set for dummy app # settings['snovault.jsonld.namespaces'] = json_asset('snovault:schemas/namespaces.json') # settings['snovault.jsonld.terms_namespace'] = 'https://www.encodeproject.org/terms/' settings['snovault.jsonld.terms_prefix'] = 'snovault' config = Configurator(settings=settings) from snovault.elasticsearch import APP_FACTORY config.registry[APP_FACTORY] = main # used by mp_indexer config.include(app_version) config.include('pyramid_multiauth') # must be before calling set_authorization_policy from pyramid_localroles import LocalRolesAuthorizationPolicy # Override default authz policy set by pyramid_multiauth config.set_authorization_policy(LocalRolesAuthorizationPolicy()) config.include(session) config.include(configure_dbsession) config.include('snovault') config.commit() # commit so search can override listing # Render an HTML page to browsers and a JSON document for API clients config.include('snowflakes.renderers') # these two should be application specific config.include('.authentication') config.include('snowflakes.root') if 'elasticsearch.server' in config.registry.settings: config.include('snovault.elasticsearch') # needed for /search/? config.include('snowflakes.search') config.include(static_resources) config.include(changelogs) # TODO This is optional AWS only - possibly move to a plug-in aws_ip_ranges = json_from_path(settings.get('aws_ip_ranges_path'), {'prefixes': []}) config.registry['aws_ipset'] = netaddr.IPSet( record['ip_prefix'] for record in aws_ip_ranges['prefixes'] if record['service'] == 'AMAZON') if asbool(settings.get('testing', False)): config.include('.tests.testing_views') # Load upgrades last so that all views (including testing views) are # registered. # TODO we would need a generic upgrade audit PACKAGE (__init__) # config.include('.audit) # config.include('.upgrade') app = config.make_wsgi_app() return app
def main(): parser = argparse.ArgumentParser( description="Create Elasticsearch mapping on deployment", epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument('config_uri', help="path to configfile") parser.add_argument('--app-name', help="Pyramid app name in configfile") args = parser.parse_args() app = get_app(args.config_uri, args.app_name) # Loading app will have configured from config file. Reconfigure here: set_logging(in_prod=app.registry.settings.get('production'), level=logging.DEBUG) # set_logging(app.registry.settings.get('elasticsearch.server'), app.registry.settings.get('production'), level=logging.DEBUG) # check if staging try: data_env = whodaman() env = app.registry.settings.get('env.name') if 'webprod' in env: if data_env != env: log.info("looks like we are on staging, run create mapping without check first") run_create_mapping(app, check_first=False) return # handle mastertest ... by blowing away all data first if 'mastertest' in env: log.info("looks like we are on mastertest, run create mapping without check first") run_create_mapping(app, check_first=False, purge_queue=True) return log.info("looks like we are NOT on staging or mastertest so run create mapping with check first") except Exception: import traceback log.warning("error checking whodaman: %s " % traceback.format_exc()) log.warning("couldn't get wodaman, so assuming NOT staging") log.info("... using default create mapping case") run_create_mapping(app, check_first=True, purge_queue=True)
def test_set_logging_level(caplog, integrated_ff): """ Provides log_dir, log_name and level args to set_logging """ es_url = ff_utils.get_health_page( key=integrated_ff['ff_key'])['elasticsearch'] log_utils.set_logging(es_server=es_url, level=logging.ERROR, log_name='Errors', log_dir='.') log = structlog.getLogger('Errors') log.error('oh no an error!', foo='faux') assert len(caplog.records) == 1
def test_set_logging_prod_but_no_es(caplog): # omitting es_server while using in_prod=True will cause dictionary # log messages but no additional Elasticsearch logging handlers log_utils.set_logging(es_server=None, in_prod=True) log = structlog.getLogger(__name__) log.error('bleh', foo='bean') assert len(caplog.records) == 1 log_record = caplog.records[0] assert isinstance(log_record.__dict__['msg'], dict) assert 'log_uuid' in log_record.__dict__['msg'] assert len(log_record._logger.handlers) == 0
def test_set_logging_not_prod(caplog): # setting in_prod to False will invalidate es_server setting log_utils.set_logging(es_server='not_a_real_server', in_prod=False) log = structlog.getLogger(__name__) log.error('bleh', foo='baz') assert len(caplog.records) == 1 log_record = caplog.records[0] # make sure there are no handlers and the message is non-dictionary assert len(log_record._logger.handlers) == 0 assert 'baz' in log_record.__dict__['msg'] assert 'error' in log_record.__dict__['msg'] assert 'log_uuid' in log_record.__dict__['msg'] assert not isinstance(log_record.__dict__['msg'], dict)
def test_set_logging_in_prod(caplog, integrated_ff): # get es_client info from the health page health = ff_utils.get_health_page(key=integrated_ff['ff_key']) es_url = health['elasticsearch'] log_utils.set_logging(env='fourfront-mastertest', es_server=es_url, in_prod=True) log = structlog.getLogger(__name__) log.warning('meh', foo='bar') assert len(caplog.records) == 1 log_record = caplog.records[0] # make sure the ES handler is present assert len(log_record._logger.handlers) == 1 assert 'log_uuid' in caplog.records[0].__dict__['msg'] assert log_record.__dict__['msg']['event'] == 'meh' assert log_record.__dict__['msg']['foo'] == 'bar' assert log_record.__dict__['msg']['level'] == 'warning' log_uuid = log_record.__dict__['msg']['log_uuid'] # make sure the log was written successfully to mastertest ES time.sleep(1) es_client = es_utils.create_es_client(es_url, use_aws_auth=True) es_res = ff_utils.get_es_metadata([log_uuid], es_client=es_client, key=integrated_ff['ff_key']) assert len(es_res) == 1 assert es_res[0]['event'] == 'meh' assert es_res[0]['foo'] == 'bar' assert es_res[0]['log_uuid'] == log_uuid assert es_res[0]['level'] == 'warning' # setting _skip_es = True will cause the log not to be shipped to ES log.warning('test_skip', _skip_es=True) assert len(caplog.records) == 2 # two logs now log_record2 = caplog.records[1] # make sure the ES handler is present assert len(log_record2._logger.handlers) == 1 assert 'log_uuid' in log_record2.__dict__['msg'] assert log_record2.__dict__['msg']['event'] == 'test_skip' log_uuid = log_record2.__dict__['msg']['log_uuid'] time.sleep(1) es_client = es_utils.create_es_client(es_url, use_aws_auth=True) es_res = ff_utils.get_es_metadata([log_uuid], es_client=es_client, key=integrated_ff['ff_key']) assert len(es_res) == 0 # log is not in ES, as anticipated
def main(): parser = argparse.ArgumentParser( # noqa - PyCharm wrongly thinks the formatter_class is invalid description="Create Elasticsearch mapping on deployment", epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument('config_uri', help="path to configfile") parser.add_argument('--app-name', help="Pyramid app name in configfile") parser.add_argument('--wipe-es', help="Specify to wipe ES", action='store_true', default=False) parser.add_argument('--clear-queue', help="Specify to clear the SQS queue", action='store_true', default=False) args = parser.parse_args() app = get_app(args.config_uri, args.app_name) # Loading app will have configured from config file. Reconfigure here: set_logging(in_prod=app.registry.settings.get('production'), log_name=__name__, level=logging.DEBUG) # set_logging(app.registry.settings.get('elasticsearch.server'), app.registry.settings.get('production'), # level=logging.DEBUG) _run_create_mapping(app, args) exit(0)
def main(global_config, **local_config): """ This function returns a Pyramid WSGI application. """ settings = global_config settings.update(local_config) # BEGIN PART THAT'S NOT IN FOURFRONT # adjust log levels for some annoying loggers lnames = ['boto', 'urllib', 'elasticsearch', 'dcicutils'] for name in logging.Logger.manager.loggerDict: if any(logname in name for logname in lnames): logging.getLogger(name).setLevel(logging.WARNING) # END PART THAT'S NOT IN FOURFRONT set_logging(in_prod=settings.get('production')) # set_logging(settings.get('elasticsearch.server'), settings.get('production')) # source environment variables on elastic beanstalk source_beanstalk_env_vars() # settings['snovault.jsonld.namespaces'] = json_asset('encoded:schemas/namespaces.json') # settings['snovault.jsonld.terms_namespace'] = 'https://www.encodeproject.org/terms/' settings['snovault.jsonld.terms_prefix'] = 'encode' # set auth0 keys settings['auth0.secret'] = os.environ.get("Auth0Secret") settings['auth0.client'] = os.environ.get("Auth0Client") # set google reCAPTCHA keys settings['g.recaptcha.key'] = os.environ.get('reCaptchaKey') settings['g.recaptcha.secret'] = os.environ.get('reCaptchaSecret') # enable invalidation scope settings[INVALIDATION_SCOPE_ENABLED] = True # set mirrored Elasticsearch location (for staging and production servers) # does not exist for CGAP currently mirror = get_mirror_env_from_context(settings) if mirror is not None: settings['mirror.env.name'] = mirror settings['mirror_health'] = get_health_page(ff_env=mirror) config = Configurator(settings=settings) config.registry[APP_FACTORY] = main # used by mp_indexer config.include(app_version) config.include( 'pyramid_multiauth') # must be before calling set_authorization_policy # Override default authz policy set by pyramid_multiauth config.set_authorization_policy(LocalRolesAuthorizationPolicy()) config.include(session) # must include, as tm.attempts was removed from pyramid_tm config.include('pyramid_retry') # for CGAP, always enable type=nested mapping # NOTE: this MUST occur prior to including Snovault, otherwise it will not work config.add_settings({'mappings.use_nested': True}) config.include(configure_dbsession) config.include('snovault') config.commit() # commit so search can override listing # Render an HTML page to browsers and a JSON document for API clients # config.include(add_schemas_to_html_responses) config.include('.renderers') config.include('.authentication') config.include('.server_defaults') config.include('.root') config.include('.types') # Fourfront does this. Do we need that here? -kmp 8-Apr-2020 # config.include('.batch_download') config.include('.loadxl') config.include('.visualization') config.include('.ingestion_listener') config.include('.custom_embed') if 'elasticsearch.server' in config.registry.settings: config.include('snovault.elasticsearch') config.include('.search.search') config.include( '.search.compound_search') # could make enabling configurable # this contains fall back url, so make sure it comes just before static_resoruces config.include('.types.page') config.include(static_resources) config.include(changelogs) aws_ip_ranges = json_from_path(settings.get('aws_ip_ranges_path'), {'prefixes': []}) config.registry['aws_ipset'] = netaddr.IPSet( record['ip_prefix'] for record in aws_ip_ranges['prefixes'] if record['service'] == 'AMAZON') if asbool(settings.get('testing', False)): config.include('.tests.testing_views') # Load upgrades last so that all views (including testing views) are # registered. config.include('.upgrade') # initialize sentry reporting init_sentry(settings.get('sentry_dsn', None)) app = config.make_wsgi_app() workbook_filename = settings.get('load_workbook', '') load_test_only = asbool(settings.get('load_test_only', False)) docsdir = settings.get('load_docsdir', None) if docsdir is not None: docsdir = [path.strip() for path in docsdir.strip().split('\n')] if workbook_filename: load_workbook(app, workbook_filename, docsdir) return app