def test_helper_configuration(manager_with_spec, tempdir, monkeypatch): ''' Test writing an api with required user config to a config file and then running configure user_config with the flag --helper ''' # Create an insufficiently specified API api = DataAPI(username='******') # Attach a manager which requires username, contact api.attach_manager(manager_with_spec) # Export the API to a file config_file = os.path.join(tempdir, '.datafs.yml') to_config_file(api=api, config_file=config_file, profile='conftest') assert 'contact' not in api.user_config runner = CliRunner() prefix = ['--config-file', config_file, '--profile', 'conftest'] def get_user_email(*args, **kwargs): return "*****@*****.**" # override click.prompt monkeypatch.setattr('click.prompt', get_user_email) # Test the helper with the appropriate input stream result = runner.invoke(cli, prefix + ['configure', '--helper']) assert result.exit_code == 0 api2 = get_api(config_file=config_file, profile='conftest') assert api2.user_config['contact'] == "*****@*****.**"
def test_sufficient_configuration(manager_with_spec, tempdir): ''' Test writing an api with required user config to a config file and then running configure. ''' # Create an appropriately specified API api = DataAPI(username='******', contact='*****@*****.**') # Attach a manager which requires username, contact api.attach_manager(manager_with_spec) # Export the API to a file config_file = os.path.join(tempdir, '.datafs.yml') to_config_file(api=api, config_file=config_file, profile='conftest') runner = CliRunner() prefix = ['--config-file', config_file, '--profile', 'conftest'] # Test the configuration result = runner.invoke(cli, prefix + ['configure']) if result.exit_code != 0: traceback.print_exception(*result.exc_info) raise OSError('Errors encountered during execution')
def test_manual_configuration(manager_with_spec, tempdir): ''' Test writing an api with required user config to a config file and then running configure user_config specified as keyword arguments ''' # Create an insufficiently specified API api = DataAPI(username='******') # Attach a manager which requires username, contact api.attach_manager(manager_with_spec) # Export the API to a file config_file = os.path.join(tempdir, '.datafs.yml') to_config_file(api=api, config_file=config_file, profile='conftest') runner = CliRunner() prefix = ['--config-file', config_file, '--profile', 'conftest'] # Test the configuration and make sure an exception is raised result = runner.invoke( cli, prefix + ['configure', '--contact', '"*****@*****.**']) assert result.exit_code == 0
def api2(mgr_name): with prep_manager(mgr_name) as manager: api = DataAPI(username='******', contact='*****@*****.**') api.attach_manager(manager) yield api
def api1_module(): with prep_manager('mongo') as manager: api = DataAPI(username='******', contact='*****@*****.**') api.attach_manager(manager) yield api
def generate_api_from_config(config): for kw in ['user_config']: if kw not in config['api']: config['api'][kw] = {} api = DataAPI(**config['api']['user_config']) return api
def get_checker(service, filepath): if service.fs.isfile(filepath): checksum = DataAPI.hash_file( service.fs.getsyspath(filepath))['checksum'] else: checksum = None def check(chk): return chk['checksum'] == checksum return check
def api_with_pattern(manager_with_pattern, auth1): api = DataAPI() api.attach_manager(manager_with_pattern) api.attach_authority('auth', auth1) yield api
def api_with_spec(manager_with_spec, auth1): api = DataAPI(username='******', contact='*****@*****.**') api.attach_manager(manager_with_spec) api.attach_authority('auth', auth1) yield api
def api(mgr_name, fs_name): with prep_manager(mgr_name) as manager: api = DataAPI(username='******', contact='*****@*****.**') api.attach_manager(manager) with prep_filesystem(fs_name) as filesystem: api.attach_authority('filesys', filesystem) yield api
def api_with_diverse_archives(request): ITERATIONS = 7 VARS = 5 PARS = 5 CONF = 3 with prep_manager(request.param, table_name='diverse') as manager: api = DataAPI(username='******', contact='*****@*****.**') api.attach_manager(manager) def direct_create_archive_spec(archive_name): return api.manager._create_archive_metadata( archive_name=archive_name, authority_name='auth', archive_path='/'.join(archive_name.split('_')), versioned=True, raise_on_err=True, metadata={}, user_config={}, helper=False, tags=os.path.splitext(archive_name)[0].split('_')) with prep_filesystem('OSFS') as auth1: api.attach_authority('auth', auth1) archive_names = [] for indices in itertools.product(*(range(1, ITERATIONS + 1) for _ in range(VARS))): archive_name = ( 'team{}_project{}_task{}_variable{}_scenario{}.nc'.format( *indices)) archive_names.append(archive_name) for indices in itertools.product(*(range(1, ITERATIONS + 1) for _ in range(PARS))): archive_name = ('team{}_project{}_task{}_' + 'parameter{}_scenario{}.csv').format(*indices) archive_names.append(archive_name) for indices in itertools.product(*(range(1, ITERATIONS + 1) for _ in range(CONF))): archive_name = ('team{}_project{}_task{}_config.txt'.format( *indices)) archive_names.append(archive_name) batch_size = 500 for st_ind in range(0, len(archive_names), batch_size): current_batch = archive_names[st_ind:st_ind + batch_size] new_archives = list( map(direct_create_archive_spec, current_batch)) if request.param == 'mongo': api.manager.collection.insert_many(new_archives) elif request.param == 'dynamo': with api.manager._table.batch_writer() as batch: for item in new_archives: batch.put_item(Item=item) else: raise ValueError('Manager "{}" not recognized'.format( request.param)) api.TEST_ATTRS = { 'archives.variable': ITERATIONS**VARS, 'archives.parameter': ITERATIONS**PARS, 'archives.config': ITERATIONS**CONF, 'count.variable': ITERATIONS, 'count.parameter': ITERATIONS, 'count.config': 1 } yield api
def hasher(f): return DataAPI.hash_file(f)