def _test_max_age(testdir):
confpath = os.path.join(testdir, 'config.json')
def config_reader():
with open(confpath) as f:
data = f.read()
return json.loads(data), hashlib.md5(data).hexdigest()
def schema_reader():
with open(os.path.join(testdir, 'config.schema.json')) as f:
data = f.read()
return json.loads(data), hashlib.md5(data).hexdigest()
settings = SearchSettings(config_reader, schema_reader, max_age=0.1)
assert settings._default_interval == 31536000
time.sleep(.1)
with open(confpath) as f:
data = json.loads(f.read())
data['defaultInterval'] = -1
with open(confpath, 'w') as fw:
fw.write(json.dumps(data))
# reading again should reload the file
try:
settings.get('prod', '39', 'channel', 'loc', 'terr',
'dist', 'distver')
except KeyError:
pass
assert settings._default_interval == -1
def main(args=sys.argv[1:]):
parser = argparse.ArgumentParser(description='Simple HTTP Load runner.')
parser.add_argument('-d', '--data-dir', help='Data directory',
type=str, default=DEFAULT_DATADIR)
parser.add_argument('-c', '--config-file', help='Config File',
type=str, default='config.json')
parser.add_argument('-s', '--schema-file', help='Schema File',
type=str, default='config.schema.json')
args = parser.parse_args(args=args)
print('Validating file...')
configpath = os.path.join(args.data_dir, args.config_file)
schemapath = os.path.join(args.data_dir, args.schema_file)
def read_config():
with open(configpath) as f:
data = f.read()
return json.loads(data), hashlib.md5(data).hexdigest()
def read_schema():
with open(schemapath) as f:
data = f.read()
return json.loads(data), hashlib.md5(data).hexdigest()
try:
SearchSettings(read_config, read_schema)
except ValueError as e:
print('Not a valid JSON file')
print(str(e))
return 1
print('OK')
return 0
def initialize_app(config):
# logging configuration
logging.config.fileConfig(config, disable_existing_loggers=False)
logger.info("Read configuration from %r" % config)
app._config_file = config
app._config = Config(config)
app.add_hook('before_request', before_request)
app.add_hook('after_request', after_request)
# statsd configuration
app._statsd = _Statsd(app._config['statsd'])
# sentry configuration
if app._config['sentry']['enabled']:
app._sentry = Sentry(app._config['sentry']['dsn'])
else:
app._sentry = None
# backend configuration
configfile = app._config['absearch']['config']
schemafile = app._config['absearch']['schema']
if app._config['absearch']['backend'] == 'aws':
logger.info("Read config and schema from AWS")
config_reader = partial(get_s3_file, configfile, app._config,
app._statsd)
schema_reader = partial(get_s3_file, schemafile, app._config,
app._statsd)
else:
# directory
datadir = app._config['directory']['path']
logger.info("Read config and schema from %r on disk" % datadir)
def config_reader():
with open(os.path.join(datadir, configfile)) as f:
data = f.read()
return json.loads(data), hashlib.md5(data).hexdigest()
def schema_reader():
with open(os.path.join(datadir, schemafile)) as f:
data = f.read()
return json.loads(data), hashlib.md5(data).hexdigest()
# counter configuration
counter = app._config['absearch']['counter']
if counter == 'redis':
counter_options = dict(app._config['redis'])
else:
counter_options = {}
counter_options['statsd'] = app._statsd
max_age = app._config['absearch']['max_age']
app.settings = SearchSettings(config_reader, schema_reader, counter,
counter_options, max_age)
def test_loaded_broken():
def config_reader():
raise ReadError()
try:
SearchSettings(config_reader, schema_reader=None)
except Exception:
pass
else:
raise AssertionError()
def test_no_schema_validator():
datadir = os.path.join(os.path.dirname(__file__), '..', '..', 'data')
testdir = tempfile.mkdtemp()
confpath = os.path.join(testdir, 'config.json')
shutil.copyfile(os.path.join(datadir, 'config.json'),
os.path.join(testdir, 'config.json'))
def config_reader():
with open(confpath) as f:
data = f.read()
return json.loads(data), hashlib.md5(data).hexdigest()
try:
settings = SearchSettings(config_reader, schema_reader=None)
finally:
shutil.rmtree(testdir)
default = settings.get('firefox', '45', 'release', 'fr', 'fr',
'default', 'default')
assert 'interval' in default
def test_immutable_locales():
datadir = os.path.join(os.path.dirname(__file__), '..', '..', 'data')
testdir = tempfile.mkdtemp()
confpath = os.path.join(testdir, 'config.json')
shutil.copyfile(os.path.join(datadir, 'config.json'),
os.path.join(testdir, 'config.json'))
def config_reader():
with open(confpath) as f:
data = f.read()
return json.loads(data), hashlib.md5(data).hexdigest()
try:
settings = SearchSettings(config_reader, schema_reader=None)
finally:
shutil.rmtree(testdir)
# the data sent by _get_cohort or _pick_cohort should be a copy
res, __ = settings._get_cohort('fr-fr', 'fr', 'default')
res['bah'] = 1
res, __ = settings._get_cohort('fr-fr', 'fr', 'default')
assert 'bah' not in res
res = settings._pick_cohort('fr-fr', 'fr', 'firefox', 42, 'release')
setting = res['settings']
setting['bah'] = 1
res = settings._pick_cohort('fr-fr', 'fr', 'firefox', 42, 'release')
setting = res['settings']
assert 'bah' not in setting
def initialize_app(config):
app._config_file = config
app._config = Config(config)
# logging configuration
logging.config.fileConfig(config)
# statsd configuration
app._statsd = _Statsd(app._config['statsd'])
# sentry configuration
if app._config['sentry']['enabled']:
app._sentry = Sentry(app._config['sentry']['dsn'])
else:
app._sentry = None
# backend configuration
configfile = app._config['absearch']['config']
schemafile = app._config['absearch']['schema']
if app._config['absearch']['backend'] == 'aws':
config_reader = partial(get_s3_file, configfile, app._config,
app._statsd)
schema_reader = partial(get_s3_file, schemafile, app._config,
app._statsd)
else:
# directory
datadir = app._config['directory']['path']
def config_reader():
with open(os.path.join(datadir, configfile)) as f:
data = f.read()
return json.loads(data), hashlib.md5(data).hexdigest()
def schema_reader():
with open(os.path.join(datadir, schemafile)) as f:
data = f.read()
return json.loads(data), hashlib.md5(data).hexdigest()
# counter configuration
counter = app._config['absearch']['counter']
if counter == 'redis':
counter_options = dict(app._config['redis'])
else:
counter_options = {}
counter_options['statsd'] = app._statsd
max_age = app._config['absearch']['max_age']
app.settings = SearchSettings(config_reader, schema_reader, counter,
counter_options, max_age)
def initialize_app(config):
# logging configuration
logging.config.fileConfig(config, disable_existing_loggers=False)
logger.info("Read configuration from %r" % config)
app._config_file = config
app._config = Config(config)
app.add_hook('before_request', before_request)
app.add_hook('after_request', after_request)
# sentry configuration
if app._config['sentry']['enabled']:
app._sentry = Sentry(app._config['sentry']['dsn'])
else:
app._sentry = None
# backend configuration
configfile = app._config['absearch']['config']
schemafile = app._config['absearch']['schema']
# directory
datadir = app._config['directory']['path']
logger.info("Read config and schema from %r on disk" % datadir)
def config_reader():
with open(os.path.join(datadir, configfile)) as f:
data = f.read()
return (
json.loads(data),
hashlib.md5(data.encode("utf8")).hexdigest(),
)
def schema_reader():
with open(os.path.join(datadir, schemafile)) as f:
data = f.read()
return (
json.loads(data),
hashlib.md5(data.encode("utf8")).hexdigest(),
)
# counter configuration
counter = app._config['absearch']['counter']
counter_options = {}
max_age = app._config['absearch']['max_age']
app.settings = SearchSettings(config_reader, schema_reader, counter,
counter_options, max_age)