def manager_with_pattern(mgr_name): with prep_manager(mgr_name, table_name='pattern-test') as manager: GCP_PATTERNS = [r'^(TLD1/(sub1|sub2|sub3)|TLD2/(sub1|sub2|sub3))'] manager.set_required_archive_patterns(GCP_PATTERNS) yield manager
def api2(mgr_name): with prep_manager(mgr_name) as manager: api = DataAPI(username='******', contact='*****@*****.**') api.attach_manager(manager) yield api
def api1_module(): with prep_manager('mongo') as manager: api = DataAPI(username='******', contact='*****@*****.**') api.attach_manager(manager) yield api
def test_api_locks(api, local_auth): api.lock_manager() api.lock_authorities() with pytest.raises((PermissionError, NameError)): with prep_manager('mongo') as manager: api.attach_manager(manager) with pytest.raises((PermissionError, NameError)): api.attach_authority('auth', local_auth)
def api(mgr_name, fs_name): with prep_manager(mgr_name) as manager: api = DataAPI(username='******', contact='*****@*****.**') api.attach_manager(manager) with prep_filesystem(fs_name) as filesystem: api.attach_authority('filesys', filesystem) yield api
def manager_with_spec(mgr_name): with prep_manager(mgr_name, table_name='spec-test') as manager: metadata_config = {'description': 'some metadata'} user_config = { 'username': '******', 'contact': '*****@*****.**' } manager.set_required_user_config(user_config) manager.set_required_archive_metadata(metadata_config) yield manager
def api_with_diverse_archives(request): ITERATIONS = 7 VARS = 5 PARS = 5 CONF = 3 with prep_manager(request.param, table_name='diverse') as manager: api = DataAPI(username='******', contact='*****@*****.**') api.attach_manager(manager) def direct_create_archive_spec(archive_name): return api.manager._create_archive_metadata( archive_name=archive_name, authority_name='auth', archive_path='/'.join(archive_name.split('_')), versioned=True, raise_on_err=True, metadata={}, user_config={}, helper=False, tags=os.path.splitext(archive_name)[0].split('_')) with prep_filesystem('OSFS') as auth1: api.attach_authority('auth', auth1) archive_names = [] for indices in itertools.product(*(range(1, ITERATIONS + 1) for _ in range(VARS))): archive_name = ( 'team{}_project{}_task{}_variable{}_scenario{}.nc'.format( *indices)) archive_names.append(archive_name) for indices in itertools.product(*(range(1, ITERATIONS + 1) for _ in range(PARS))): archive_name = ('team{}_project{}_task{}_' + 'parameter{}_scenario{}.csv').format(*indices) archive_names.append(archive_name) for indices in itertools.product(*(range(1, ITERATIONS + 1) for _ in range(CONF))): archive_name = ('team{}_project{}_task{}_config.txt'.format( *indices)) archive_names.append(archive_name) batch_size = 500 for st_ind in range(0, len(archive_names), batch_size): current_batch = archive_names[st_ind:st_ind + batch_size] new_archives = list( map(direct_create_archive_spec, current_batch)) if request.param == 'mongo': api.manager.collection.insert_many(new_archives) elif request.param == 'dynamo': with api.manager._table.batch_writer() as batch: for item in new_archives: batch.put_item(Item=item) else: raise ValueError('Manager "{}" not recognized'.format( request.param)) api.TEST_ATTRS = { 'archives.variable': ITERATIONS**VARS, 'archives.parameter': ITERATIONS**PARS, 'archives.config': ITERATIONS**CONF, 'count.variable': ITERATIONS, 'count.parameter': ITERATIONS, 'count.config': 1 } yield api