def manager_with_pattern(mgr_name):
with prep_manager(mgr_name, table_name='pattern-test') as manager:
GCP_PATTERNS = [r'^(TLD1/(sub1|sub2|sub3)|TLD2/(sub1|sub2|sub3))']
manager.set_required_archive_patterns(GCP_PATTERNS)
yield manager
def api2(mgr_name):
with prep_manager(mgr_name) as manager:
api = DataAPI(username='******', contact='*****@*****.**')
api.attach_manager(manager)
yield api
def api1_module():
with prep_manager('mongo') as manager:
api = DataAPI(username='******', contact='*****@*****.**')
api.attach_manager(manager)
yield api
def test_api_locks(api, local_auth):
api.lock_manager()
api.lock_authorities()
with pytest.raises((PermissionError, NameError)):
with prep_manager('mongo') as manager:
api.attach_manager(manager)
with pytest.raises((PermissionError, NameError)):
api.attach_authority('auth', local_auth)
def api(mgr_name, fs_name):
with prep_manager(mgr_name) as manager:
api = DataAPI(username='******', contact='*****@*****.**')
api.attach_manager(manager)
with prep_filesystem(fs_name) as filesystem:
api.attach_authority('filesys', filesystem)
yield api
def manager_with_spec(mgr_name):
with prep_manager(mgr_name, table_name='spec-test') as manager:
metadata_config = {'description': 'some metadata'}
user_config = {
'username': '******',
'contact': '*****@*****.**'
}
manager.set_required_user_config(user_config)
manager.set_required_archive_metadata(metadata_config)
yield manager
def api_with_diverse_archives(request):
ITERATIONS = 7
VARS = 5
PARS = 5
CONF = 3
with prep_manager(request.param, table_name='diverse') as manager:
api = DataAPI(username='******', contact='*****@*****.**')
api.attach_manager(manager)
def direct_create_archive_spec(archive_name):
return api.manager._create_archive_metadata(
archive_name=archive_name,
authority_name='auth',
archive_path='/'.join(archive_name.split('_')),
versioned=True,
raise_on_err=True,
metadata={},
user_config={},
helper=False,
tags=os.path.splitext(archive_name)[0].split('_'))
with prep_filesystem('OSFS') as auth1:
api.attach_authority('auth', auth1)
archive_names = []
for indices in itertools.product(*(range(1, ITERATIONS + 1)
for _ in range(VARS))):
archive_name = (
'team{}_project{}_task{}_variable{}_scenario{}.nc'.format(
*indices))
archive_names.append(archive_name)
for indices in itertools.product(*(range(1, ITERATIONS + 1)
for _ in range(PARS))):
archive_name = ('team{}_project{}_task{}_' +
'parameter{}_scenario{}.csv').format(*indices)
archive_names.append(archive_name)
for indices in itertools.product(*(range(1, ITERATIONS + 1)
for _ in range(CONF))):
archive_name = ('team{}_project{}_task{}_config.txt'.format(
*indices))
archive_names.append(archive_name)
batch_size = 500
for st_ind in range(0, len(archive_names), batch_size):
current_batch = archive_names[st_ind:st_ind + batch_size]
new_archives = list(
map(direct_create_archive_spec, current_batch))
if request.param == 'mongo':
api.manager.collection.insert_many(new_archives)
elif request.param == 'dynamo':
with api.manager._table.batch_writer() as batch:
for item in new_archives:
batch.put_item(Item=item)
else:
raise ValueError('Manager "{}" not recognized'.format(
request.param))
api.TEST_ATTRS = {
'archives.variable': ITERATIONS**VARS,
'archives.parameter': ITERATIONS**PARS,
'archives.config': ITERATIONS**CONF,
'count.variable': ITERATIONS,
'count.parameter': ITERATIONS,
'count.config': 1
}
yield api