Ejemplo n.º 1
0
def discover_config(config, prefix):
    '''Look out in the world to find kvlayer configuration.

    This will generally try to populate the `storage_type` and
    `storage_addresses` parameters, if a reasonable default for those
    can be found.

    '''

    # If the caller specified a top-level storage_addresses, stop
    # (Even if they didn't specify storage_type)
    if 'storage_addresses' in config:
        return

    # Or, if they specified both storage_type and a subconfig with
    # storage_addresses, stop
    storage_type = config.get('storage_type', None)
    if (storage_type is not None and
        storage_type in config and
        'storage_addresses' in config[storage_type]):
        return

    # Otherwise we can try to do discovery (either specifically on
    # storage_type, or on the alphabetically-first thing that has
    # anything)

    # Do we have a storage_addresses for some storage_type already?
    for name in sorted(STORAGE_CLIENTS.iterkeys()):
        if storage_type is not None and storage_type != name:
            continue
        if name not in config:
            continue
        if 'storage_addresses' not in config[name]:
            continue
        # We found something!
        if storage_type is None:
            config['storage_type'] = name
        return

    # Is there some storage_type that can find itself?
    for name in sorted(STORAGE_CLIENTS.iterkeys()):
        if storage_type is not None and storage_type != name:
            continue
        impl = STORAGE_CLIENTS[name]
        if not hasattr(impl, 'discover_config'):
            continue
        sub_config = config.get(name, {})
        impl.discover_config(sub_config, prefix + '.' + name)
        if 'storage_addresses' not in sub_config:
            continue
        # Again, we found something!
        config[name] = sub_config
        if storage_type is None:
            config['storage_type'] = name
        return
Ejemplo n.º 2
0
def run_all_perftests(redis_address=None, clientlist=None):
    '''Run all of the performance tests, on every backend.

    This is intended to be a fully-automated answer for
    :program:`kvlayer_test`.

    '''
    rc = 0
    for name in STORAGE_CLIENTS.iterkeys():
        if clientlist is not None:
            if name not in clientlist:
                logger.info('skipping backend %r', name)
                continue
        elif name in ['cassandra', 'accumulo', 'postgres', 'cborproxy']:
            logger.info('skipping backend %r', name)
            continue
        config = os.path.join(os.path.dirname(__file__),
                              'config_{0}.yaml'.format(name))
        if not os.path.exists(config):
            continue
        params = {'app_name': 'kvlayer_performance',
                  'namespace': 'ns' + uuid.uuid1().hex}
        if name == 'filestorage':
            params['kvlayer_filename'] = os.tmpnam()
        if name == 'redis' and redis_address is not None:
            params['storage_addresses'] = [redis_address]
        with yakonfig.defaulted_config(
                [kvlayer], filename=config, params=params):
            ret = run_perftests()
            if rc == 0:
                rc = ret
        if name == 'filestorage':
            os.unlink(params['kvlayer_filename'])
    return rc
Ejemplo n.º 3
0
            name = entry_point.name
            constructor = entry_point.load()
            _extension_test_configs[name] = constructor()
        except:
            logger.error('failed loading kvlayer test_config %r',
                         entry_point and entry_point.name, exc_info=True)


# run at global scope to ensure it's before we might possibly need
# STORAGE_CLIENTS in any context
load_entry_point_kvlayer_impls()
load_entry_point_kvlayer_test_configs()


@pytest.fixture(scope='module',
                params=STORAGE_CLIENTS.keys())
def backend(request):
    backend = request.param
    if backend == 'cassandra':
        pytest.skip('cassandra doesn\'t support non-UUID keys')
    if backend in _extension_test_configs:
        pass  # okay
    elif not request.fspath.dirpath('config_{0}.yaml'.format(backend)).exists():
        pytest.skip('no configuration file for backend {0}'.format(backend))
    #if backend not in ('local', 'redis', 'riak', 'accumulo', 'cborproxy'):
    #if backend != 'local':
    #    pytest.skip('TODO DELETE TEMPROARY SKIP ' + backend)
    return backend


@pytest.yield_fixture(scope='function')