def test_logging_retry(caplog, integrated_ff):
# get es_client info from the health page
es_url = ff_utils.get_health_page(
key=integrated_ff['ff_key'])['elasticsearch']
log_utils.set_logging(env='fourfront-mastertest',
es_server=es_url,
in_prod=True)
log = structlog.getLogger(__name__)
log.warning('test_retry', _test_log_utils=True)
assert len(caplog.records) == 1
assert caplog.records[0].__dict__['msg']['event'] == 'test_retry'
log_uuid = caplog.records[0].__dict__['msg']['log_uuid']
# retrying will take 5 sec, so log shoudldn't be in ES yet
time.sleep(1)
es_client = es_utils.create_es_client(es_url, use_aws_auth=True)
es_res = ff_utils.get_es_metadata([log_uuid],
es_client=es_client,
key=integrated_ff['ff_key'])
assert len(es_res) == 0
# wait to allow logs to retry
time.sleep(7)
es_client = es_utils.create_es_client(es_url, use_aws_auth=True)
es_res = ff_utils.get_es_metadata([log_uuid],
es_client=es_client,
key=integrated_ff['ff_key'])
assert len(es_res) == 1
assert es_res[0]['log_uuid'] == log_uuid
assert es_res[0]['event'] == 'test_retry'
def main(global_config, **local_config):
""" This function returns a Pyramid WSGI application.
"""
settings = global_config
settings.update(local_config)
# TODO: move to dcicutils
set_logging(settings.get('elasticsearch.server'), settings.get('production'))
# TODO - these need to be set for dummy app
# settings['snovault.jsonld.namespaces'] = json_asset('snovault:schemas/namespaces.json')
# settings['snovault.jsonld.terms_namespace'] = 'https://www.encodeproject.org/terms/'
settings['snovault.jsonld.terms_prefix'] = 'snovault'
config = Configurator(settings=settings)
from snovault.elasticsearch import APP_FACTORY
config.registry[APP_FACTORY] = main # used by mp_indexer
config.include(app_version)
config.include('pyramid_multiauth') # must be before calling set_authorization_policy
from pyramid_localroles import LocalRolesAuthorizationPolicy
# Override default authz policy set by pyramid_multiauth
config.set_authorization_policy(LocalRolesAuthorizationPolicy())
config.include(session)
config.include(configure_dbsession)
config.include('snovault')
config.commit() # commit so search can override listing
# Render an HTML page to browsers and a JSON document for API clients
config.include('snowflakes.renderers')
# these two should be application specific
config.include('.authentication')
config.include('snowflakes.root')
if 'elasticsearch.server' in config.registry.settings:
config.include('snovault.elasticsearch')
# needed for /search/?
config.include('snowflakes.search')
config.include(static_resources)
config.include(changelogs)
# TODO This is optional AWS only - possibly move to a plug-in
aws_ip_ranges = json_from_path(settings.get('aws_ip_ranges_path'), {'prefixes': []})
config.registry['aws_ipset'] = netaddr.IPSet(
record['ip_prefix'] for record in aws_ip_ranges['prefixes'] if record['service'] == 'AMAZON')
if asbool(settings.get('testing', False)):
config.include('.tests.testing_views')
# Load upgrades last so that all views (including testing views) are
# registered.
# TODO we would need a generic upgrade audit PACKAGE (__init__)
# config.include('.audit)
# config.include('.upgrade')
app = config.make_wsgi_app()
return app
def main():
parser = argparse.ArgumentParser(
description="Create Elasticsearch mapping on deployment", epilog=EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument('config_uri', help="path to configfile")
parser.add_argument('--app-name', help="Pyramid app name in configfile")
args = parser.parse_args()
app = get_app(args.config_uri, args.app_name)
# Loading app will have configured from config file. Reconfigure here:
set_logging(in_prod=app.registry.settings.get('production'), level=logging.DEBUG)
# set_logging(app.registry.settings.get('elasticsearch.server'), app.registry.settings.get('production'), level=logging.DEBUG)
# check if staging
try:
data_env = whodaman()
env = app.registry.settings.get('env.name')
if 'webprod' in env:
if data_env != env:
log.info("looks like we are on staging, run create mapping without check first")
run_create_mapping(app, check_first=False)
return
# handle mastertest ... by blowing away all data first
if 'mastertest' in env:
log.info("looks like we are on mastertest, run create mapping without check first")
run_create_mapping(app, check_first=False, purge_queue=True)
return
log.info("looks like we are NOT on staging or mastertest so run create mapping with check first")
except Exception:
import traceback
log.warning("error checking whodaman: %s " % traceback.format_exc())
log.warning("couldn't get wodaman, so assuming NOT staging")
log.info("... using default create mapping case")
run_create_mapping(app, check_first=True, purge_queue=True)
def test_set_logging_level(caplog, integrated_ff):
""" Provides log_dir, log_name and level args to set_logging """
es_url = ff_utils.get_health_page(
key=integrated_ff['ff_key'])['elasticsearch']
log_utils.set_logging(es_server=es_url,
level=logging.ERROR,
log_name='Errors',
log_dir='.')
log = structlog.getLogger('Errors')
log.error('oh no an error!', foo='faux')
assert len(caplog.records) == 1
def test_set_logging_prod_but_no_es(caplog):
# omitting es_server while using in_prod=True will cause dictionary
# log messages but no additional Elasticsearch logging handlers
log_utils.set_logging(es_server=None, in_prod=True)
log = structlog.getLogger(__name__)
log.error('bleh', foo='bean')
assert len(caplog.records) == 1
log_record = caplog.records[0]
assert isinstance(log_record.__dict__['msg'], dict)
assert 'log_uuid' in log_record.__dict__['msg']
assert len(log_record._logger.handlers) == 0
def test_set_logging_not_prod(caplog):
# setting in_prod to False will invalidate es_server setting
log_utils.set_logging(es_server='not_a_real_server', in_prod=False)
log = structlog.getLogger(__name__)
log.error('bleh', foo='baz')
assert len(caplog.records) == 1
log_record = caplog.records[0]
# make sure there are no handlers and the message is non-dictionary
assert len(log_record._logger.handlers) == 0
assert 'baz' in log_record.__dict__['msg']
assert 'error' in log_record.__dict__['msg']
assert 'log_uuid' in log_record.__dict__['msg']
assert not isinstance(log_record.__dict__['msg'], dict)
def test_set_logging_in_prod(caplog, integrated_ff):
# get es_client info from the health page
health = ff_utils.get_health_page(key=integrated_ff['ff_key'])
es_url = health['elasticsearch']
log_utils.set_logging(env='fourfront-mastertest',
es_server=es_url,
in_prod=True)
log = structlog.getLogger(__name__)
log.warning('meh', foo='bar')
assert len(caplog.records) == 1
log_record = caplog.records[0]
# make sure the ES handler is present
assert len(log_record._logger.handlers) == 1
assert 'log_uuid' in caplog.records[0].__dict__['msg']
assert log_record.__dict__['msg']['event'] == 'meh'
assert log_record.__dict__['msg']['foo'] == 'bar'
assert log_record.__dict__['msg']['level'] == 'warning'
log_uuid = log_record.__dict__['msg']['log_uuid']
# make sure the log was written successfully to mastertest ES
time.sleep(1)
es_client = es_utils.create_es_client(es_url, use_aws_auth=True)
es_res = ff_utils.get_es_metadata([log_uuid],
es_client=es_client,
key=integrated_ff['ff_key'])
assert len(es_res) == 1
assert es_res[0]['event'] == 'meh'
assert es_res[0]['foo'] == 'bar'
assert es_res[0]['log_uuid'] == log_uuid
assert es_res[0]['level'] == 'warning'
# setting _skip_es = True will cause the log not to be shipped to ES
log.warning('test_skip', _skip_es=True)
assert len(caplog.records) == 2 # two logs now
log_record2 = caplog.records[1]
# make sure the ES handler is present
assert len(log_record2._logger.handlers) == 1
assert 'log_uuid' in log_record2.__dict__['msg']
assert log_record2.__dict__['msg']['event'] == 'test_skip'
log_uuid = log_record2.__dict__['msg']['log_uuid']
time.sleep(1)
es_client = es_utils.create_es_client(es_url, use_aws_auth=True)
es_res = ff_utils.get_es_metadata([log_uuid],
es_client=es_client,
key=integrated_ff['ff_key'])
assert len(es_res) == 0 # log is not in ES, as anticipated
def main():
parser = argparse.ArgumentParser( # noqa - PyCharm wrongly thinks the formatter_class is invalid
description="Create Elasticsearch mapping on deployment", epilog=EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument('config_uri', help="path to configfile")
parser.add_argument('--app-name', help="Pyramid app name in configfile")
parser.add_argument('--wipe-es', help="Specify to wipe ES", action='store_true', default=False)
parser.add_argument('--clear-queue', help="Specify to clear the SQS queue", action='store_true', default=False)
args = parser.parse_args()
app = get_app(args.config_uri, args.app_name)
# Loading app will have configured from config file. Reconfigure here:
set_logging(in_prod=app.registry.settings.get('production'), log_name=__name__, level=logging.DEBUG)
# set_logging(app.registry.settings.get('elasticsearch.server'), app.registry.settings.get('production'),
# level=logging.DEBUG)
_run_create_mapping(app, args)
exit(0)
def main(global_config, **local_config):
"""
This function returns a Pyramid WSGI application.
"""
settings = global_config
settings.update(local_config)
# BEGIN PART THAT'S NOT IN FOURFRONT
# adjust log levels for some annoying loggers
lnames = ['boto', 'urllib', 'elasticsearch', 'dcicutils']
for name in logging.Logger.manager.loggerDict:
if any(logname in name for logname in lnames):
logging.getLogger(name).setLevel(logging.WARNING)
# END PART THAT'S NOT IN FOURFRONT
set_logging(in_prod=settings.get('production'))
# set_logging(settings.get('elasticsearch.server'), settings.get('production'))
# source environment variables on elastic beanstalk
source_beanstalk_env_vars()
# settings['snovault.jsonld.namespaces'] = json_asset('encoded:schemas/namespaces.json')
# settings['snovault.jsonld.terms_namespace'] = 'https://www.encodeproject.org/terms/'
settings['snovault.jsonld.terms_prefix'] = 'encode'
# set auth0 keys
settings['auth0.secret'] = os.environ.get("Auth0Secret")
settings['auth0.client'] = os.environ.get("Auth0Client")
# set google reCAPTCHA keys
settings['g.recaptcha.key'] = os.environ.get('reCaptchaKey')
settings['g.recaptcha.secret'] = os.environ.get('reCaptchaSecret')
# enable invalidation scope
settings[INVALIDATION_SCOPE_ENABLED] = True
# set mirrored Elasticsearch location (for staging and production servers)
# does not exist for CGAP currently
mirror = get_mirror_env_from_context(settings)
if mirror is not None:
settings['mirror.env.name'] = mirror
settings['mirror_health'] = get_health_page(ff_env=mirror)
config = Configurator(settings=settings)
config.registry[APP_FACTORY] = main # used by mp_indexer
config.include(app_version)
config.include(
'pyramid_multiauth') # must be before calling set_authorization_policy
# Override default authz policy set by pyramid_multiauth
config.set_authorization_policy(LocalRolesAuthorizationPolicy())
config.include(session)
# must include, as tm.attempts was removed from pyramid_tm
config.include('pyramid_retry')
# for CGAP, always enable type=nested mapping
# NOTE: this MUST occur prior to including Snovault, otherwise it will not work
config.add_settings({'mappings.use_nested': True})
config.include(configure_dbsession)
config.include('snovault')
config.commit() # commit so search can override listing
# Render an HTML page to browsers and a JSON document for API clients
# config.include(add_schemas_to_html_responses)
config.include('.renderers')
config.include('.authentication')
config.include('.server_defaults')
config.include('.root')
config.include('.types')
# Fourfront does this. Do we need that here? -kmp 8-Apr-2020
# config.include('.batch_download')
config.include('.loadxl')
config.include('.visualization')
config.include('.ingestion_listener')
config.include('.custom_embed')
if 'elasticsearch.server' in config.registry.settings:
config.include('snovault.elasticsearch')
config.include('.search.search')
config.include(
'.search.compound_search') # could make enabling configurable
# this contains fall back url, so make sure it comes just before static_resoruces
config.include('.types.page')
config.include(static_resources)
config.include(changelogs)
aws_ip_ranges = json_from_path(settings.get('aws_ip_ranges_path'),
{'prefixes': []})
config.registry['aws_ipset'] = netaddr.IPSet(
record['ip_prefix'] for record in aws_ip_ranges['prefixes']
if record['service'] == 'AMAZON')
if asbool(settings.get('testing', False)):
config.include('.tests.testing_views')
# Load upgrades last so that all views (including testing views) are
# registered.
config.include('.upgrade')
# initialize sentry reporting
init_sentry(settings.get('sentry_dsn', None))
app = config.make_wsgi_app()
workbook_filename = settings.get('load_workbook', '')
load_test_only = asbool(settings.get('load_test_only', False))
docsdir = settings.get('load_docsdir', None)
if docsdir is not None:
docsdir = [path.strip() for path in docsdir.strip().split('\n')]
if workbook_filename:
load_workbook(app, workbook_filename, docsdir)
return app