def _prepare_spatial_weights_data(weights_file=WEIGHTS_FILE):
'''
Rescales the pix_cent_x colum values
Parameters
----------
weights_file: str
location of file used for weighting
.. note:: unnecessary if we can standardize our input
'''
api = datafs.get_api()
archive = api.get_archive(weights_file)
with archive.open('r') as f:
df = pd.read_csv(f)
# Re-label out-of-bounds pixel centers
df.set_value((df['pix_cent_x'] == 180.125), 'pix_cent_x', -179.875)
#probably totally unnecessary
df.drop_duplicates()
df.index.names = ['reshape_index']
df.rename(columns={'pix_cent_x': 'lon', 'pix_cent_y': 'lat'}, inplace=True)
return df
def test_helper_configuration(manager_with_spec, tempdir, monkeypatch):
'''
Test writing an api with required user config to a config file and then
running configure user_config with the flag --helper
'''
# Create an insufficiently specified API
api = DataAPI(username='******')
# Attach a manager which requires username, contact
api.attach_manager(manager_with_spec)
# Export the API to a file
config_file = os.path.join(tempdir, '.datafs.yml')
to_config_file(api=api, config_file=config_file, profile='conftest')
assert 'contact' not in api.user_config
runner = CliRunner()
prefix = ['--config-file', config_file, '--profile', 'conftest']
def get_user_email(*args, **kwargs):
return "*****@*****.**"
# override click.prompt
monkeypatch.setattr('click.prompt', get_user_email)
# Test the helper with the appropriate input stream
result = runner.invoke(cli, prefix + ['configure', '--helper'])
assert result.exit_code == 0
api2 = get_api(config_file=config_file, profile='conftest')
assert api2.user_config['contact'] == "*****@*****.**"
def retrieve(cls, archive_name, api=None):
if archive_name in cls._data:
return cls._data[archive_name]
if api is None:
if cls._api is None:
cls._api = datafs.get_api()
api = cls._api
archive = api.get_archive(archive_name)
if archive_name.endswith('.csv'):
with archive.open() as f:
data = metacsv.read_csv(f)
elif os.path.splitext(archive_name)[-1][:3] == '.nc':
with archive.get_local_path() as fp:
data = xr.open_dataset(fp).load()
else:
raise ValueError(
'file type not recognized: "{}"'.format(archive_name))
cls._data[archive_name] = data
return data
def test_cli_search(test_config):
profile, config_file = test_config
api = get_api(profile=profile, config_file=config_file)
runner = CliRunner()
prefix = ['--config-file', config_file, '--profile', 'myapi']
# Test the helper with the appropriate input stream
result = runner.invoke(
cli,
prefix + ['search', 'team2', 'var2', 'archive2']
)
assert result.exit_code == 0
assert 'team2_archive2_var2' in result.output.strip().split('\n')[0]
res = list(api.search('team2', 'var3', 'archive1'))
assert 'team2_archive1_var3' in res
# Test the helper with the appropriate input stream
result = runner.invoke(
cli,
prefix + ['search', 'var2', 'team2', 'archive2']
)
assert result.exit_code == 0
assert 'team2_archive2_var2' in result.output.strip().split('\n')[0]
def run(self):
api = datafs.get_api()
dataset_spec = ast.literal_eval(self.datasets)
datasets = {}
for var_name, arch_name in dataset_spec.items():
archive = api.get_archive(arch_name)
with archive.get_local_path() as f:
with xr.open_dataset(f) as ds:
datasets[var_name] = ds.load()
ds.close()
param_set = pd.read_csv(self.paramfile, header=None)
for i in range(len(param_set)):
parameters = param_set.iloc[i].values
outputter = self.action(parameters=parameters, **datasets)
dest = api.create('{}_{}.csv'.format(
self.output_var, '_'.join(list(map(str, parameters)))),
metadata=dict(description='Example outputs'),
tags=self.output_var.split('_'),
raise_if_exists=False)
with dest.open('wb+') as f:
outputter(f)
def main(dry_run=False, recreate=False, cache=False):
api = datafs.get_api()
upload_files(api,
PATTERN,
ADDITIONAL_METADATA,
dry_run=dry_run,
recreate=recreate,
cache=cache)
def test_update_metadata(test_config, monkeypatch):
'''
Update archive metadata with a description from the CLI
'''
profile, temp_file = test_config
api = get_api(profile=profile, config_file=temp_file)
runner = CliRunner()
prefix = ['--config-file', '{}'.format(temp_file), '--profile', 'myapi']
arch1 = api.create('my_next_archive')
with runner.isolated_filesystem():
with open('my_new_test_file.txt', 'w+') as to_update:
to_update.write(u'test test test')
def get_input_file(*args, **kwargs):
return 'my_new_test_file.txt'
# override click.prompt
monkeypatch.setattr('click.prompt', get_input_file)
result = runner.invoke(cli, prefix + ['update', 'my_next_archive'])
assert result.exit_code == 0
result = runner.invoke(
cli, prefix + [
'update_metadata', 'my_next_archive', '--description',
'some_description'
])
assert result.exit_code == 0
assert arch1.get_metadata() == {'description': 'some_description'}
os.remove('my_new_test_file.txt')
result = runner.invoke(cli,
prefix +
['update', 'my_next_archive', '--string'],
input='my new contents\ncan be piped in')
assert result.exit_code == 0
result = runner.invoke(cli, prefix + ['cat', 'my_next_archive'])
assert result.exit_code == 0
assert 'my new contents' in result.output.strip().split('\n')
assert 'can be piped in' in result.output.strip().split('\n')
arch1.delete()
def test_versioned_logging(preloaded_config):
'''
Test logging cli features
'''
profile, temp_file = preloaded_config
# Create a requirements file and
runner = CliRunner()
prefix = ['--config-file', '{}'.format(temp_file), '--profile', 'myapi']
with runner.isolated_filesystem():
# Download req_1 with version from requirements file
result = runner.invoke(cli, prefix + ['log', '/req/arch1'])
if result.exit_code != 0:
traceback.print_exception(*result.exc_info)
raise OSError('Errors encountered during execution')
log = result.output.strip()
verstr_matcher = r'(version [0-9]+(\.[0-9]+){1,2} \(md5 \w+\))'
versions = re.finditer((r'(?P<ver>' + verstr_matcher + r'\n([^\n]*(\n(?!' +
verstr_matcher + r'))?)+)'), log)
api = get_api(config_file=temp_file, profile='myapi')
arch = api.get_archive('/req/arch1')
hist = arch.get_history()
for i, vermatch in enumerate(reversed(list(versions))):
verstr = vermatch.group('ver')
verhist = hist[i]
assert verhist['checksum'] in verstr
assert verhist['message'] in verstr
for attr, val in verhist['user_config'].items():
assert attr in verstr
assert val in verstr
def population_weighted_mean(ds,
level='state',
dim='fips',
year=2012,
api=None,
pop=None):
if pop is None and api is None:
api = datafs.get_api()
if pop is None:
pop = _prep_pop_data(api)
if dim != 'fips':
pop = pop.rename({'fips': dim})
return (((ds * pop[str(year)]).groupby(level).sum(dim=dim)) /
((pop[str(year)]).groupby(level).sum(dim=dim)))
def setup_runner_resource(config_file, table_name):
# setup
runner = CliRunner()
api = get_api(config_file=config_file)
prefix = ['--config-file', config_file]
try:
api.manager.delete_table(table_name)
except KeyError:
pass
api.manager.create_archive_table(table_name)
# yield fixture
yield runner, api, config_file, prefix
# teardown
api.manager.delete_table(table_name)
def preloaded_config(test_config):
'''
Prepare a manager/auth config with 3 archives, each having 3 versions
.. note::
To save on test runtime, scope == module. Tests should not modify
these archives.
'''
profile, temp_file = test_config
api = get_api(profile=profile, config_file=temp_file)
# Set up a couple archives with multiple versions
arch1 = api.create('req_1')
arch2 = api.create('req_2')
arch3 = api.create('req_3')
with arch1.open('w+', bumpversion='minor') as f:
f.write(u'this is archive req_1 version 0.1')
with arch1.open('w+', bumpversion='major') as f:
f.write(u'this is archive req_1 version 1.0')
with arch1.open('w+', bumpversion='minor') as f:
f.write(u'this is archive req_1 version 1.1')
arch1_versions = arch1.get_versions()
assert '0.1' in arch1_versions
assert '1.0' in arch1_versions
assert '1.1' in arch1_versions
with arch2.open('w+', prerelease='alpha') as f:
f.write(u'this is archive req_2 version 0.0.1a1')
with arch2.open('w+', prerelease='alpha') as f:
f.write(u'this is archive req_2 version 0.0.1a2')
with arch2.open('w+', bumpversion='patch') as f:
f.write(u'this is archive req_2 version 0.0.1')
arch2_versions = arch2.get_versions()
assert '0.0.1a1' in arch2_versions
assert '0.0.1a2' in arch2_versions
assert '0.0.1' in arch2_versions
with arch3.open('w+', bumpversion='major') as f:
f.write(u'this is archive req_3 version 1.0')
with arch3.open('w+', bumpversion='minor', prerelease='alpha') as f:
f.write(u'this is archive req_3 version 1.1a1')
with arch3.open('w+', bumpversion='minor') as f:
f.write(u'this is archive req_3 version 1.1')
arch3_versions = arch3.get_versions()
assert '1.0' in arch3_versions
assert '1.1a1' in arch3_versions
assert '1.1' in arch3_versions
try:
yield profile, temp_file
finally:
arch1.delete()
arch2.delete()
arch3.delete()
def test_dependency_parsing(test_config):
'''
Update archive dependencies across versions from the CLI
'''
profile, temp_file = test_config
api = get_api(profile=profile, config_file=temp_file)
runner = CliRunner()
prefix = ['--config-file', '{}'.format(temp_file), '--profile', 'myapi']
arch1 = api.create('dep_archive')
with runner.isolated_filesystem():
with open('my_new_test_file.txt', 'w+') as to_update:
to_update.write(u'test test test')
result = runner.invoke(
cli, prefix + [
'update', 'dep_archive', 'my_new_test_file.txt',
'--bumpversion', 'minor', '--dependency', 'arch1==0.1.0',
'--dependency', 'arch2'
])
assert result.exit_code == 0
assert arch1.get_latest_version() == '0.1.0'
with arch1.open('r') as f:
assert f.read() == u'test test test'
assert arch1.get_dependencies(version='0.1.0') == {
'arch1': '0.1.0',
'arch2': None
}
result = runner.invoke(
cli, prefix + [
'set_dependencies', 'dep_archive', '--dependency',
'arch1==0.2.0', '--dependency', 'arch2==0.0.1'
])
assert result.exit_code == 0
assert arch1.get_dependencies(version='0.1.0') == {
'arch1': '0.2.0',
'arch2': '0.0.1'
}
with open('my_new_test_file.txt', 'w+') as to_update:
to_update.write(u'test test test two three four')
result = runner.invoke(
cli, prefix + [
'update', 'dep_archive', 'my_new_test_file.txt',
'--bumpversion', 'minor', '--dependency', 'arch1==0.2.0',
'--dependency', 'arch2==0.0.1'
])
assert result.exit_code == 0
assert arch1.get_dependencies(version='0.1.0') == {
'arch1': '0.2.0',
'arch2': '0.0.1'
}
result = runner.invoke(cli,
prefix + ['get_dependencies', 'dep_archive'])
assert result.exit_code == 0
assert 'arch1==0.2.0' in result.output
assert 'arch2==0.0.1' in result.output
result = runner.invoke(
cli,
prefix + ['get_dependencies', 'dep_archive', '--version', '0.1'])
assert result.exit_code == 0
assert 'arch1==0.2.0' in result.output
assert 'arch2==0.0.1' in result.output
result = runner.invoke(
cli, prefix + [
'set_dependencies', 'dep_archive', '--dependency',
'arch1==0.2.0', '--dependency', 'arch2'
])
assert result.exit_code == 0
result = runner.invoke(cli,
prefix + ['get_dependencies', 'dep_archive'])
assert result.exit_code == 0
assert 'arch1==0.2.0' in result.output
assert 'arch2' in result.output
assert 'arch2==' not in result.output
os.remove('my_new_test_file.txt')
api.delete_archive('dep_archive')
def test_cli_unversioned(test_config):
profile, temp_file = test_config
prefix = ['--config-file', temp_file, '--profile', 'myapi']
api2 = get_api(profile=profile, config_file=temp_file)
runner = CliRunner()
# test for configure and create archive
result = runner.invoke(
cli, prefix + ['create', 'unversioned', '--not-versioned'])
assert result.exit_code == 0
res = 'created archive <DataArchive local://unversioned>'
assert result.output.strip() == res
result = runner.invoke(cli, prefix + ['filter'])
assert result.exit_code == 0
assert ['unversioned'] == [result.output.strip()]
# test the actual creation of the object from the api side
assert len(list(api2.filter())) == 1
archive = api2.get_archive('unversioned')
assert archive.archive_name == 'unversioned'
with runner.isolated_filesystem():
with open('hello.txt', 'w') as f:
f.write('un-versioned data')
# update using CLI
result = runner.invoke(
cli, prefix + [
'update', 'unversioned', 'hello.txt', '--dependency', 'arch1',
'--dependency', 'arch2'
])
assert result.exit_code == 0
# assert that we get update feedback
expected = 'uploaded data to <DataArchive local://unversioned>.'
assert expected == result.output.strip()
# Try re-upload
result = runner.invoke(
cli, prefix + ['update', 'unversioned', '--string', 'new content'])
assert result.exit_code == 0
# assert that we get update feedback
intended_output = 'uploaded data to <DataArchive local://unversioned>.'
assert intended_output == result.output.strip()
with runner.isolated_filesystem():
# test download
result = runner.invoke(
cli, prefix + ['download', 'unversioned', 'here.txt'])
assert result.exit_code == 0
with open('here.txt', 'r') as downloaded:
assert downloaded.read() == 'new content'
# test download with 'latest' version argument'
result = runner.invoke(
cli, prefix +
['download', 'unversioned', 'here.txt', '--version', 'latest'])
assert result.exit_code != 0
# test download with incorrect version argument
result = runner.invoke(
cli, prefix +
['download', 'unversioned', 'here.txt', '--version', '0.0.1'])
assert result.exit_code != 0
os.remove('here.txt')
# teardown
result = runner.invoke(cli, prefix + ['delete', 'unversioned'])
result = runner.invoke(cli, prefix + ['filter'])
assert result.exit_code == 0
assert result.output.strip() == ''
assert len(list(api2.filter())) == 0
def test_cli_local(test_config):
profile, temp_file = test_config
prefix = ['--config-file', temp_file, '--profile', 'myapi']
api2 = get_api(profile=profile, config_file=temp_file)
runner = CliRunner()
# test for configure and create archive
result = runner.invoke(
cli, prefix + [
'create', 'my_first_archive', '--description',
'My test data archive'
])
assert result.exit_code == 0
res = 'created versioned archive <DataArchive local://my_first_archive>'
assert result.output.strip() == res
result = runner.invoke(cli, prefix + ['filter'])
assert result.exit_code == 0
assert 'my_first_archive' in result.output.strip().split('\n')
assert len(result.output.strip().split('\n')) == 1
# test the actual creation of the object from the api side
assert len(list(api2.filter())) == 1
archive = api2.get_archive('my_first_archive')
assert archive.archive_name == 'my_first_archive'
# testing the `metadata` option
result = runner.invoke(cli, prefix + ['metadata', 'my_first_archive'])
assert result.exit_code == 0
metadata = ast.literal_eval(result.output)
assert metadata['description'] == 'My test data archive'
# test the api side of the operation
assert u'My test data archive' == archive.get_metadata()['description']
with runner.isolated_filesystem():
with open('hello.txt', 'w') as f:
f.write('Hoo Yah! Stay Stoked!')
# update using CLI
result = runner.invoke(
cli, prefix + [
'update', 'my_first_archive', 'hello.txt', '--source',
'Surfers Journal'
])
assert result.exit_code == 0
# assert that we get update feedback
expected = 'uploaded data to <DataArchive local://my_first_archive>'
assert expected in result.output
# lets read the file to make sure it remains unchanged
with open('hello.txt', 'r') as f:
data = f.read()
assert data == 'Hoo Yah! Stay Stoked!'
# Try re-upload
result = runner.invoke(
cli, prefix + [
'update', 'my_first_archive', 'hello.txt', '--source',
'Surfers Journal'
])
assert result.exit_code == 0
# assert that we get update feedback
intended_output = ('uploaded data to <DataArchive '
'local://my_first_archive>. version remains 0.0.1.')
assert intended_output == result.output.strip()
# this is testing the feed through on the api
with api2.get_archive(list(api2.filter())[0]).open('r') as f:
data = f.read()
assert data == 'Hoo Yah! Stay Stoked!'
# lets check to make sure our metadata update also passed through
assert 'Surfers Journal' == api2.get_archive(list(
api2.filter())[0]).get_metadata()['source']
# test to assert metadata update
# test to assert file content change
with runner.isolated_filesystem():
result = runner.invoke(
cli, prefix + [
'update', 'my_first_archive', '--bumpversion', 'minor',
'--string', 'new version data'
])
assert result.exit_code == 0
result = runner.invoke(cli, prefix + ['cat', 'my_first_archive'])
assert result.exit_code == 0
'new version data' in result.output
result = runner.invoke(
cli, prefix +
['download', 'my_first_archive', 'here.txt', '--version', '0.0.1'])
assert result.exit_code == 0
'Hoo Yah! Stay Stoked!' in result.output
# test download of previous version
result = runner.invoke(
cli, prefix +
['download', 'my_first_archive', 'here.txt', '--version', '0.0.1'])
assert result.exit_code == 0
with open('here.txt', 'r') as downloaded:
assert downloaded.read() == 'Hoo Yah! Stay Stoked!'
# test download of nonexistant version (should fail without overwriting
# file)
result = runner.invoke(
cli, prefix +
['download', 'my_first_archive', 'here.txt', '--version', '3.0'])
assert result.exit_code != 0
with open('here.txt', 'r') as downloaded:
assert downloaded.read() == 'Hoo Yah! Stay Stoked!'
os.remove('here.txt')
# teardown
result = runner.invoke(cli, prefix + ['delete', 'my_first_archive'])
result = runner.invoke(cli, prefix + ['filter'])
assert result.exit_code == 0
assert result.output.strip() == ''
assert len(list(api2.filter())) == 0
def test_cli_log_with_various_messages(sample_config):
'''
Test CLI command ``log`` stability
Addresses :issue:`232` - log fails on versions with no message
'''
profile, temp_file = sample_config
api = get_api(profile=profile, config_file=temp_file)
runner = CliRunner()
prefix = ['--config-file', '{}'.format(temp_file), '--profile', 'myapi']
arch = api.create('test/archive1.txt')
def run_test():
# Test log on an empty archive
result = runner.invoke(cli, prefix + ['log', 'test/archive1.txt'])
if result.exit_code != 0:
traceback.print_exception(*result.exc_info)
raise OSError('Errors encountered during execution')
return result.output.strip()
try:
# Test log on an empty archive
log = run_test()
assert log == ''
# Test log with no message -- should still have info
with arch.open('w+') as f:
f.write(u('hello 1'))
log = run_test()
assert len(log.split('\n')) > 0
# Test log with message -- should appear in log
with arch.open('w+', message='a message') as f:
f.write(u('hello 2'))
log = run_test()
assert 'a message' in log
# Test log with numeric message -- should appear in log
with arch.open('w+', message=4.3) as f:
f.write(u('hello 3'))
log = run_test()
assert '4.3' in log
with arch.open('w+', message=lambda x: x**2) as f:
f.write(u('hello 4'))
log = run_test()
assert 'lambda' in log
# Test log with explicit None as message -- should not appear in log,
# but the version should be present
with arch.open('w+', message=None) as f:
f.write(u('hello 5'))
log = run_test()
assert str(arch.get_latest_version()) in log
assert 'None' not in log
finally:
arch.delete()
def main(dry_run=False):
api = datafs.get_api()
upload_files(api, PATTERN, ADDITIONAL_METADATA, dry_run=dry_run)
def preloaded_config(sample_config):
'''
Prepare a manager/auth config with 3 archives, each having 3 versions
.. note::
To save on test runtime, scope == module. Tests should not modify
these archives.
'''
profile, temp_file = sample_config
api = get_api(profile=profile, config_file=temp_file)
# Set up a couple archives with multiple versions
arch1 = api.create('/req/arch1')
arch2 = api.create('/req/arch2')
arch3 = api.create('/req/arch3')
with arch1.open('w+', bumpversion='minor', message='bumping to 0.1') as f:
f.write(u'this is archive /req/arch1 version 0.1')
with arch1.open('w+', bumpversion='major', message='bumping to 1.0') as f:
f.write(u'this is archive /req/arch1 version 1.0')
with arch1.open('w+', bumpversion='minor', message='bumping to 1.1') as f:
f.write(u'this is archive /req/arch1 version 1.1')
arch1_versions = arch1.get_versions()
assert '0.1' in arch1_versions
assert '1.0' in arch1_versions
assert '1.1' in arch1_versions
with arch2.open('w+', prerelease='alpha') as f:
f.write(u'this is archive /req/arch2 version 0.0.1a1')
with arch2.open('w+', prerelease='alpha') as f:
f.write(u'this is archive /req/arch2 version 0.0.1a2')
with arch2.open('w+', bumpversion='patch') as f:
f.write(u'this is archive /req/arch2 version 0.0.1')
arch2_versions = arch2.get_versions()
assert '0.0.1a1' in arch2_versions
assert '0.0.1a2' in arch2_versions
assert '0.0.1' in arch2_versions
with arch3.open('w+', bumpversion='major') as f:
f.write(u'this is archive /req/arch3 version 1.0')
with arch3.open('w+', bumpversion='minor', prerelease='alpha') as f:
f.write(u'this is archive /req/arch3 version 1.1a1')
with arch3.open('w+', bumpversion='minor') as f:
f.write(u'this is archive /req/arch3 version 1.1')
arch3_versions = arch3.get_versions()
assert '1.0' in arch3_versions
assert '1.1a1' in arch3_versions
assert '1.1' in arch3_versions
# Set up an unversioned archive with multiple versions
arch_uver = api.create('uver1', versioned=False)
with arch_uver.open('w+', message='bumping to 0.1') as f:
f.write(u'this is archive uver1 version 0.1')
with arch_uver.open('w+', message='bumping to 1.0') as f:
f.write(u'this is archive uver1 version 1.0')
with arch_uver.open('w+', message='bumping to 1.1') as f:
f.write(u'this is archive uver1 version 1.1')
arch_uver_versions = arch_uver.get_history()
assert len(arch_uver_versions) == 3
try:
yield profile, temp_file
finally:
arch1.delete()
arch2.delete()
arch3.delete()
arch_uver.delete()
