def test_helper_configuration(manager_with_spec, tempdir, monkeypatch):
'''
Test writing an api with required user config to a config file and then
running configure user_config with the flag --helper
'''
# Create an insufficiently specified API
api = DataAPI(username='******')
# Attach a manager which requires username, contact
api.attach_manager(manager_with_spec)
# Export the API to a file
config_file = os.path.join(tempdir, '.datafs.yml')
to_config_file(api=api, config_file=config_file, profile='conftest')
assert 'contact' not in api.user_config
runner = CliRunner()
prefix = ['--config-file', config_file, '--profile', 'conftest']
def get_user_email(*args, **kwargs):
return "*****@*****.**"
# override click.prompt
monkeypatch.setattr('click.prompt', get_user_email)
# Test the helper with the appropriate input stream
result = runner.invoke(cli, prefix + ['configure', '--helper'])
assert result.exit_code == 0
api2 = get_api(config_file=config_file, profile='conftest')
assert api2.user_config['contact'] == "*****@*****.**"
def test_sufficient_configuration(manager_with_spec, tempdir):
'''
Test writing an api with required user config to a config file and then
running configure.
'''
# Create an appropriately specified API
api = DataAPI(username='******', contact='*****@*****.**')
# Attach a manager which requires username, contact
api.attach_manager(manager_with_spec)
# Export the API to a file
config_file = os.path.join(tempdir, '.datafs.yml')
to_config_file(api=api, config_file=config_file, profile='conftest')
runner = CliRunner()
prefix = ['--config-file', config_file, '--profile', 'conftest']
# Test the configuration
result = runner.invoke(cli, prefix + ['configure'])
if result.exit_code != 0:
traceback.print_exception(*result.exc_info)
raise OSError('Errors encountered during execution')
def test_manual_configuration(manager_with_spec, tempdir):
'''
Test writing an api with required user config to a config file and then
running configure user_config specified as keyword arguments
'''
# Create an insufficiently specified API
api = DataAPI(username='******')
# Attach a manager which requires username, contact
api.attach_manager(manager_with_spec)
# Export the API to a file
config_file = os.path.join(tempdir, '.datafs.yml')
to_config_file(api=api, config_file=config_file, profile='conftest')
runner = CliRunner()
prefix = ['--config-file', config_file, '--profile', 'conftest']
# Test the configuration and make sure an exception is raised
result = runner.invoke(
cli, prefix + ['configure', '--contact', '"*****@*****.**'])
assert result.exit_code == 0
def api2(mgr_name):
with prep_manager(mgr_name) as manager:
api = DataAPI(username='******', contact='*****@*****.**')
api.attach_manager(manager)
yield api
def api1_module():
with prep_manager('mongo') as manager:
api = DataAPI(username='******', contact='*****@*****.**')
api.attach_manager(manager)
yield api
def generate_api_from_config(config):
for kw in ['user_config']:
if kw not in config['api']:
config['api'][kw] = {}
api = DataAPI(**config['api']['user_config'])
return api
def get_checker(service, filepath):
if service.fs.isfile(filepath):
checksum = DataAPI.hash_file(
service.fs.getsyspath(filepath))['checksum']
else:
checksum = None
def check(chk):
return chk['checksum'] == checksum
return check
def api_with_pattern(manager_with_pattern, auth1):
api = DataAPI()
api.attach_manager(manager_with_pattern)
api.attach_authority('auth', auth1)
yield api
def api_with_spec(manager_with_spec, auth1):
api = DataAPI(username='******', contact='*****@*****.**')
api.attach_manager(manager_with_spec)
api.attach_authority('auth', auth1)
yield api
def api(mgr_name, fs_name):
with prep_manager(mgr_name) as manager:
api = DataAPI(username='******', contact='*****@*****.**')
api.attach_manager(manager)
with prep_filesystem(fs_name) as filesystem:
api.attach_authority('filesys', filesystem)
yield api
def api_with_diverse_archives(request):
ITERATIONS = 7
VARS = 5
PARS = 5
CONF = 3
with prep_manager(request.param, table_name='diverse') as manager:
api = DataAPI(username='******', contact='*****@*****.**')
api.attach_manager(manager)
def direct_create_archive_spec(archive_name):
return api.manager._create_archive_metadata(
archive_name=archive_name,
authority_name='auth',
archive_path='/'.join(archive_name.split('_')),
versioned=True,
raise_on_err=True,
metadata={},
user_config={},
helper=False,
tags=os.path.splitext(archive_name)[0].split('_'))
with prep_filesystem('OSFS') as auth1:
api.attach_authority('auth', auth1)
archive_names = []
for indices in itertools.product(*(range(1, ITERATIONS + 1)
for _ in range(VARS))):
archive_name = (
'team{}_project{}_task{}_variable{}_scenario{}.nc'.format(
*indices))
archive_names.append(archive_name)
for indices in itertools.product(*(range(1, ITERATIONS + 1)
for _ in range(PARS))):
archive_name = ('team{}_project{}_task{}_' +
'parameter{}_scenario{}.csv').format(*indices)
archive_names.append(archive_name)
for indices in itertools.product(*(range(1, ITERATIONS + 1)
for _ in range(CONF))):
archive_name = ('team{}_project{}_task{}_config.txt'.format(
*indices))
archive_names.append(archive_name)
batch_size = 500
for st_ind in range(0, len(archive_names), batch_size):
current_batch = archive_names[st_ind:st_ind + batch_size]
new_archives = list(
map(direct_create_archive_spec, current_batch))
if request.param == 'mongo':
api.manager.collection.insert_many(new_archives)
elif request.param == 'dynamo':
with api.manager._table.batch_writer() as batch:
for item in new_archives:
batch.put_item(Item=item)
else:
raise ValueError('Manager "{}" not recognized'.format(
request.param))
api.TEST_ATTRS = {
'archives.variable': ITERATIONS**VARS,
'archives.parameter': ITERATIONS**PARS,
'archives.config': ITERATIONS**CONF,
'count.variable': ITERATIONS,
'count.parameter': ITERATIONS,
'count.config': 1
}
yield api
def hasher(f):
return DataAPI.hash_file(f)
