def discover_config(config, prefix):
'''Look out in the world to find kvlayer configuration.
This will generally try to populate the `storage_type` and
`storage_addresses` parameters, if a reasonable default for those
can be found.
'''
# If the caller specified a top-level storage_addresses, stop
# (Even if they didn't specify storage_type)
if 'storage_addresses' in config:
return
# Or, if they specified both storage_type and a subconfig with
# storage_addresses, stop
storage_type = config.get('storage_type', None)
if (storage_type is not None and
storage_type in config and
'storage_addresses' in config[storage_type]):
return
# Otherwise we can try to do discovery (either specifically on
# storage_type, or on the alphabetically-first thing that has
# anything)
# Do we have a storage_addresses for some storage_type already?
for name in sorted(STORAGE_CLIENTS.iterkeys()):
if storage_type is not None and storage_type != name:
continue
if name not in config:
continue
if 'storage_addresses' not in config[name]:
continue
# We found something!
if storage_type is None:
config['storage_type'] = name
return
# Is there some storage_type that can find itself?
for name in sorted(STORAGE_CLIENTS.iterkeys()):
if storage_type is not None and storage_type != name:
continue
impl = STORAGE_CLIENTS[name]
if not hasattr(impl, 'discover_config'):
continue
sub_config = config.get(name, {})
impl.discover_config(sub_config, prefix + '.' + name)
if 'storage_addresses' not in sub_config:
continue
# Again, we found something!
config[name] = sub_config
if storage_type is None:
config['storage_type'] = name
return
def run_all_perftests(redis_address=None, clientlist=None):
'''Run all of the performance tests, on every backend.
This is intended to be a fully-automated answer for
:program:`kvlayer_test`.
'''
rc = 0
for name in STORAGE_CLIENTS.iterkeys():
if clientlist is not None:
if name not in clientlist:
logger.info('skipping backend %r', name)
continue
elif name in ['cassandra', 'accumulo', 'postgres', 'cborproxy']:
logger.info('skipping backend %r', name)
continue
config = os.path.join(os.path.dirname(__file__),
'config_{0}.yaml'.format(name))
if not os.path.exists(config):
continue
params = {'app_name': 'kvlayer_performance',
'namespace': 'ns' + uuid.uuid1().hex}
if name == 'filestorage':
params['kvlayer_filename'] = os.tmpnam()
if name == 'redis' and redis_address is not None:
params['storage_addresses'] = [redis_address]
with yakonfig.defaulted_config(
[kvlayer], filename=config, params=params):
ret = run_perftests()
if rc == 0:
rc = ret
if name == 'filestorage':
os.unlink(params['kvlayer_filename'])
return rc
name = entry_point.name
constructor = entry_point.load()
_extension_test_configs[name] = constructor()
except:
logger.error('failed loading kvlayer test_config %r',
entry_point and entry_point.name, exc_info=True)
# run at global scope to ensure it's before we might possibly need
# STORAGE_CLIENTS in any context
load_entry_point_kvlayer_impls()
load_entry_point_kvlayer_test_configs()
@pytest.fixture(scope='module',
params=STORAGE_CLIENTS.keys())
def backend(request):
backend = request.param
if backend == 'cassandra':
pytest.skip('cassandra doesn\'t support non-UUID keys')
if backend in _extension_test_configs:
pass # okay
elif not request.fspath.dirpath('config_{0}.yaml'.format(backend)).exists():
pytest.skip('no configuration file for backend {0}'.format(backend))
#if backend not in ('local', 'redis', 'riak', 'accumulo', 'cborproxy'):
#if backend != 'local':
# pytest.skip('TODO DELETE TEMPROARY SKIP ' + backend)
return backend
@pytest.yield_fixture(scope='function')