def discover_config(config, prefix): '''Look out in the world to find kvlayer configuration. This will generally try to populate the `storage_type` and `storage_addresses` parameters, if a reasonable default for those can be found. ''' # If the caller specified a top-level storage_addresses, stop # (Even if they didn't specify storage_type) if 'storage_addresses' in config: return # Or, if they specified both storage_type and a subconfig with # storage_addresses, stop storage_type = config.get('storage_type', None) if (storage_type is not None and storage_type in config and 'storage_addresses' in config[storage_type]): return # Otherwise we can try to do discovery (either specifically on # storage_type, or on the alphabetically-first thing that has # anything) # Do we have a storage_addresses for some storage_type already? for name in sorted(STORAGE_CLIENTS.iterkeys()): if storage_type is not None and storage_type != name: continue if name not in config: continue if 'storage_addresses' not in config[name]: continue # We found something! if storage_type is None: config['storage_type'] = name return # Is there some storage_type that can find itself? for name in sorted(STORAGE_CLIENTS.iterkeys()): if storage_type is not None and storage_type != name: continue impl = STORAGE_CLIENTS[name] if not hasattr(impl, 'discover_config'): continue sub_config = config.get(name, {}) impl.discover_config(sub_config, prefix + '.' + name) if 'storage_addresses' not in sub_config: continue # Again, we found something! config[name] = sub_config if storage_type is None: config['storage_type'] = name return
def run_all_perftests(redis_address=None, clientlist=None): '''Run all of the performance tests, on every backend. This is intended to be a fully-automated answer for :program:`kvlayer_test`. ''' rc = 0 for name in STORAGE_CLIENTS.iterkeys(): if clientlist is not None: if name not in clientlist: logger.info('skipping backend %r', name) continue elif name in ['cassandra', 'accumulo', 'postgres', 'cborproxy']: logger.info('skipping backend %r', name) continue config = os.path.join(os.path.dirname(__file__), 'config_{0}.yaml'.format(name)) if not os.path.exists(config): continue params = {'app_name': 'kvlayer_performance', 'namespace': 'ns' + uuid.uuid1().hex} if name == 'filestorage': params['kvlayer_filename'] = os.tmpnam() if name == 'redis' and redis_address is not None: params['storage_addresses'] = [redis_address] with yakonfig.defaulted_config( [kvlayer], filename=config, params=params): ret = run_perftests() if rc == 0: rc = ret if name == 'filestorage': os.unlink(params['kvlayer_filename']) return rc