def _prepare_spatial_weights_data(weights_file=WEIGHTS_FILE): ''' Rescales the pix_cent_x colum values Parameters ---------- weights_file: str location of file used for weighting .. note:: unnecessary if we can standardize our input ''' api = datafs.get_api() archive = api.get_archive(weights_file) with archive.open('r') as f: df = pd.read_csv(f) # Re-label out-of-bounds pixel centers df.set_value((df['pix_cent_x'] == 180.125), 'pix_cent_x', -179.875) #probably totally unnecessary df.drop_duplicates() df.index.names = ['reshape_index'] df.rename(columns={'pix_cent_x': 'lon', 'pix_cent_y': 'lat'}, inplace=True) return df
def test_helper_configuration(manager_with_spec, tempdir, monkeypatch): ''' Test writing an api with required user config to a config file and then running configure user_config with the flag --helper ''' # Create an insufficiently specified API api = DataAPI(username='******') # Attach a manager which requires username, contact api.attach_manager(manager_with_spec) # Export the API to a file config_file = os.path.join(tempdir, '.datafs.yml') to_config_file(api=api, config_file=config_file, profile='conftest') assert 'contact' not in api.user_config runner = CliRunner() prefix = ['--config-file', config_file, '--profile', 'conftest'] def get_user_email(*args, **kwargs): return "*****@*****.**" # override click.prompt monkeypatch.setattr('click.prompt', get_user_email) # Test the helper with the appropriate input stream result = runner.invoke(cli, prefix + ['configure', '--helper']) assert result.exit_code == 0 api2 = get_api(config_file=config_file, profile='conftest') assert api2.user_config['contact'] == "*****@*****.**"
def retrieve(cls, archive_name, api=None): if archive_name in cls._data: return cls._data[archive_name] if api is None: if cls._api is None: cls._api = datafs.get_api() api = cls._api archive = api.get_archive(archive_name) if archive_name.endswith('.csv'): with archive.open() as f: data = metacsv.read_csv(f) elif os.path.splitext(archive_name)[-1][:3] == '.nc': with archive.get_local_path() as fp: data = xr.open_dataset(fp).load() else: raise ValueError( 'file type not recognized: "{}"'.format(archive_name)) cls._data[archive_name] = data return data
def test_cli_search(test_config): profile, config_file = test_config api = get_api(profile=profile, config_file=config_file) runner = CliRunner() prefix = ['--config-file', config_file, '--profile', 'myapi'] # Test the helper with the appropriate input stream result = runner.invoke( cli, prefix + ['search', 'team2', 'var2', 'archive2'] ) assert result.exit_code == 0 assert 'team2_archive2_var2' in result.output.strip().split('\n')[0] res = list(api.search('team2', 'var3', 'archive1')) assert 'team2_archive1_var3' in res # Test the helper with the appropriate input stream result = runner.invoke( cli, prefix + ['search', 'var2', 'team2', 'archive2'] ) assert result.exit_code == 0 assert 'team2_archive2_var2' in result.output.strip().split('\n')[0]
def run(self): api = datafs.get_api() dataset_spec = ast.literal_eval(self.datasets) datasets = {} for var_name, arch_name in dataset_spec.items(): archive = api.get_archive(arch_name) with archive.get_local_path() as f: with xr.open_dataset(f) as ds: datasets[var_name] = ds.load() ds.close() param_set = pd.read_csv(self.paramfile, header=None) for i in range(len(param_set)): parameters = param_set.iloc[i].values outputter = self.action(parameters=parameters, **datasets) dest = api.create('{}_{}.csv'.format( self.output_var, '_'.join(list(map(str, parameters)))), metadata=dict(description='Example outputs'), tags=self.output_var.split('_'), raise_if_exists=False) with dest.open('wb+') as f: outputter(f)
def main(dry_run=False, recreate=False, cache=False): api = datafs.get_api() upload_files(api, PATTERN, ADDITIONAL_METADATA, dry_run=dry_run, recreate=recreate, cache=cache)
def test_update_metadata(test_config, monkeypatch): ''' Update archive metadata with a description from the CLI ''' profile, temp_file = test_config api = get_api(profile=profile, config_file=temp_file) runner = CliRunner() prefix = ['--config-file', '{}'.format(temp_file), '--profile', 'myapi'] arch1 = api.create('my_next_archive') with runner.isolated_filesystem(): with open('my_new_test_file.txt', 'w+') as to_update: to_update.write(u'test test test') def get_input_file(*args, **kwargs): return 'my_new_test_file.txt' # override click.prompt monkeypatch.setattr('click.prompt', get_input_file) result = runner.invoke(cli, prefix + ['update', 'my_next_archive']) assert result.exit_code == 0 result = runner.invoke( cli, prefix + [ 'update_metadata', 'my_next_archive', '--description', 'some_description' ]) assert result.exit_code == 0 assert arch1.get_metadata() == {'description': 'some_description'} os.remove('my_new_test_file.txt') result = runner.invoke(cli, prefix + ['update', 'my_next_archive', '--string'], input='my new contents\ncan be piped in') assert result.exit_code == 0 result = runner.invoke(cli, prefix + ['cat', 'my_next_archive']) assert result.exit_code == 0 assert 'my new contents' in result.output.strip().split('\n') assert 'can be piped in' in result.output.strip().split('\n') arch1.delete()
def test_versioned_logging(preloaded_config): ''' Test logging cli features ''' profile, temp_file = preloaded_config # Create a requirements file and runner = CliRunner() prefix = ['--config-file', '{}'.format(temp_file), '--profile', 'myapi'] with runner.isolated_filesystem(): # Download req_1 with version from requirements file result = runner.invoke(cli, prefix + ['log', '/req/arch1']) if result.exit_code != 0: traceback.print_exception(*result.exc_info) raise OSError('Errors encountered during execution') log = result.output.strip() verstr_matcher = r'(version [0-9]+(\.[0-9]+){1,2} \(md5 \w+\))' versions = re.finditer((r'(?P<ver>' + verstr_matcher + r'\n([^\n]*(\n(?!' + verstr_matcher + r'))?)+)'), log) api = get_api(config_file=temp_file, profile='myapi') arch = api.get_archive('/req/arch1') hist = arch.get_history() for i, vermatch in enumerate(reversed(list(versions))): verstr = vermatch.group('ver') verhist = hist[i] assert verhist['checksum'] in verstr assert verhist['message'] in verstr for attr, val in verhist['user_config'].items(): assert attr in verstr assert val in verstr
def population_weighted_mean(ds, level='state', dim='fips', year=2012, api=None, pop=None): if pop is None and api is None: api = datafs.get_api() if pop is None: pop = _prep_pop_data(api) if dim != 'fips': pop = pop.rename({'fips': dim}) return (((ds * pop[str(year)]).groupby(level).sum(dim=dim)) / ((pop[str(year)]).groupby(level).sum(dim=dim)))
def setup_runner_resource(config_file, table_name): # setup runner = CliRunner() api = get_api(config_file=config_file) prefix = ['--config-file', config_file] try: api.manager.delete_table(table_name) except KeyError: pass api.manager.create_archive_table(table_name) # yield fixture yield runner, api, config_file, prefix # teardown api.manager.delete_table(table_name)
def preloaded_config(test_config): ''' Prepare a manager/auth config with 3 archives, each having 3 versions .. note:: To save on test runtime, scope == module. Tests should not modify these archives. ''' profile, temp_file = test_config api = get_api(profile=profile, config_file=temp_file) # Set up a couple archives with multiple versions arch1 = api.create('req_1') arch2 = api.create('req_2') arch3 = api.create('req_3') with arch1.open('w+', bumpversion='minor') as f: f.write(u'this is archive req_1 version 0.1') with arch1.open('w+', bumpversion='major') as f: f.write(u'this is archive req_1 version 1.0') with arch1.open('w+', bumpversion='minor') as f: f.write(u'this is archive req_1 version 1.1') arch1_versions = arch1.get_versions() assert '0.1' in arch1_versions assert '1.0' in arch1_versions assert '1.1' in arch1_versions with arch2.open('w+', prerelease='alpha') as f: f.write(u'this is archive req_2 version 0.0.1a1') with arch2.open('w+', prerelease='alpha') as f: f.write(u'this is archive req_2 version 0.0.1a2') with arch2.open('w+', bumpversion='patch') as f: f.write(u'this is archive req_2 version 0.0.1') arch2_versions = arch2.get_versions() assert '0.0.1a1' in arch2_versions assert '0.0.1a2' in arch2_versions assert '0.0.1' in arch2_versions with arch3.open('w+', bumpversion='major') as f: f.write(u'this is archive req_3 version 1.0') with arch3.open('w+', bumpversion='minor', prerelease='alpha') as f: f.write(u'this is archive req_3 version 1.1a1') with arch3.open('w+', bumpversion='minor') as f: f.write(u'this is archive req_3 version 1.1') arch3_versions = arch3.get_versions() assert '1.0' in arch3_versions assert '1.1a1' in arch3_versions assert '1.1' in arch3_versions try: yield profile, temp_file finally: arch1.delete() arch2.delete() arch3.delete()
def test_dependency_parsing(test_config): ''' Update archive dependencies across versions from the CLI ''' profile, temp_file = test_config api = get_api(profile=profile, config_file=temp_file) runner = CliRunner() prefix = ['--config-file', '{}'.format(temp_file), '--profile', 'myapi'] arch1 = api.create('dep_archive') with runner.isolated_filesystem(): with open('my_new_test_file.txt', 'w+') as to_update: to_update.write(u'test test test') result = runner.invoke( cli, prefix + [ 'update', 'dep_archive', 'my_new_test_file.txt', '--bumpversion', 'minor', '--dependency', 'arch1==0.1.0', '--dependency', 'arch2' ]) assert result.exit_code == 0 assert arch1.get_latest_version() == '0.1.0' with arch1.open('r') as f: assert f.read() == u'test test test' assert arch1.get_dependencies(version='0.1.0') == { 'arch1': '0.1.0', 'arch2': None } result = runner.invoke( cli, prefix + [ 'set_dependencies', 'dep_archive', '--dependency', 'arch1==0.2.0', '--dependency', 'arch2==0.0.1' ]) assert result.exit_code == 0 assert arch1.get_dependencies(version='0.1.0') == { 'arch1': '0.2.0', 'arch2': '0.0.1' } with open('my_new_test_file.txt', 'w+') as to_update: to_update.write(u'test test test two three four') result = runner.invoke( cli, prefix + [ 'update', 'dep_archive', 'my_new_test_file.txt', '--bumpversion', 'minor', '--dependency', 'arch1==0.2.0', '--dependency', 'arch2==0.0.1' ]) assert result.exit_code == 0 assert arch1.get_dependencies(version='0.1.0') == { 'arch1': '0.2.0', 'arch2': '0.0.1' } result = runner.invoke(cli, prefix + ['get_dependencies', 'dep_archive']) assert result.exit_code == 0 assert 'arch1==0.2.0' in result.output assert 'arch2==0.0.1' in result.output result = runner.invoke( cli, prefix + ['get_dependencies', 'dep_archive', '--version', '0.1']) assert result.exit_code == 0 assert 'arch1==0.2.0' in result.output assert 'arch2==0.0.1' in result.output result = runner.invoke( cli, prefix + [ 'set_dependencies', 'dep_archive', '--dependency', 'arch1==0.2.0', '--dependency', 'arch2' ]) assert result.exit_code == 0 result = runner.invoke(cli, prefix + ['get_dependencies', 'dep_archive']) assert result.exit_code == 0 assert 'arch1==0.2.0' in result.output assert 'arch2' in result.output assert 'arch2==' not in result.output os.remove('my_new_test_file.txt') api.delete_archive('dep_archive')
def test_cli_unversioned(test_config): profile, temp_file = test_config prefix = ['--config-file', temp_file, '--profile', 'myapi'] api2 = get_api(profile=profile, config_file=temp_file) runner = CliRunner() # test for configure and create archive result = runner.invoke( cli, prefix + ['create', 'unversioned', '--not-versioned']) assert result.exit_code == 0 res = 'created archive <DataArchive local://unversioned>' assert result.output.strip() == res result = runner.invoke(cli, prefix + ['filter']) assert result.exit_code == 0 assert ['unversioned'] == [result.output.strip()] # test the actual creation of the object from the api side assert len(list(api2.filter())) == 1 archive = api2.get_archive('unversioned') assert archive.archive_name == 'unversioned' with runner.isolated_filesystem(): with open('hello.txt', 'w') as f: f.write('un-versioned data') # update using CLI result = runner.invoke( cli, prefix + [ 'update', 'unversioned', 'hello.txt', '--dependency', 'arch1', '--dependency', 'arch2' ]) assert result.exit_code == 0 # assert that we get update feedback expected = 'uploaded data to <DataArchive local://unversioned>.' assert expected == result.output.strip() # Try re-upload result = runner.invoke( cli, prefix + ['update', 'unversioned', '--string', 'new content']) assert result.exit_code == 0 # assert that we get update feedback intended_output = 'uploaded data to <DataArchive local://unversioned>.' assert intended_output == result.output.strip() with runner.isolated_filesystem(): # test download result = runner.invoke( cli, prefix + ['download', 'unversioned', 'here.txt']) assert result.exit_code == 0 with open('here.txt', 'r') as downloaded: assert downloaded.read() == 'new content' # test download with 'latest' version argument' result = runner.invoke( cli, prefix + ['download', 'unversioned', 'here.txt', '--version', 'latest']) assert result.exit_code != 0 # test download with incorrect version argument result = runner.invoke( cli, prefix + ['download', 'unversioned', 'here.txt', '--version', '0.0.1']) assert result.exit_code != 0 os.remove('here.txt') # teardown result = runner.invoke(cli, prefix + ['delete', 'unversioned']) result = runner.invoke(cli, prefix + ['filter']) assert result.exit_code == 0 assert result.output.strip() == '' assert len(list(api2.filter())) == 0
def test_cli_local(test_config): profile, temp_file = test_config prefix = ['--config-file', temp_file, '--profile', 'myapi'] api2 = get_api(profile=profile, config_file=temp_file) runner = CliRunner() # test for configure and create archive result = runner.invoke( cli, prefix + [ 'create', 'my_first_archive', '--description', 'My test data archive' ]) assert result.exit_code == 0 res = 'created versioned archive <DataArchive local://my_first_archive>' assert result.output.strip() == res result = runner.invoke(cli, prefix + ['filter']) assert result.exit_code == 0 assert 'my_first_archive' in result.output.strip().split('\n') assert len(result.output.strip().split('\n')) == 1 # test the actual creation of the object from the api side assert len(list(api2.filter())) == 1 archive = api2.get_archive('my_first_archive') assert archive.archive_name == 'my_first_archive' # testing the `metadata` option result = runner.invoke(cli, prefix + ['metadata', 'my_first_archive']) assert result.exit_code == 0 metadata = ast.literal_eval(result.output) assert metadata['description'] == 'My test data archive' # test the api side of the operation assert u'My test data archive' == archive.get_metadata()['description'] with runner.isolated_filesystem(): with open('hello.txt', 'w') as f: f.write('Hoo Yah! Stay Stoked!') # update using CLI result = runner.invoke( cli, prefix + [ 'update', 'my_first_archive', 'hello.txt', '--source', 'Surfers Journal' ]) assert result.exit_code == 0 # assert that we get update feedback expected = 'uploaded data to <DataArchive local://my_first_archive>' assert expected in result.output # lets read the file to make sure it remains unchanged with open('hello.txt', 'r') as f: data = f.read() assert data == 'Hoo Yah! Stay Stoked!' # Try re-upload result = runner.invoke( cli, prefix + [ 'update', 'my_first_archive', 'hello.txt', '--source', 'Surfers Journal' ]) assert result.exit_code == 0 # assert that we get update feedback intended_output = ('uploaded data to <DataArchive ' 'local://my_first_archive>. version remains 0.0.1.') assert intended_output == result.output.strip() # this is testing the feed through on the api with api2.get_archive(list(api2.filter())[0]).open('r') as f: data = f.read() assert data == 'Hoo Yah! Stay Stoked!' # lets check to make sure our metadata update also passed through assert 'Surfers Journal' == api2.get_archive(list( api2.filter())[0]).get_metadata()['source'] # test to assert metadata update # test to assert file content change with runner.isolated_filesystem(): result = runner.invoke( cli, prefix + [ 'update', 'my_first_archive', '--bumpversion', 'minor', '--string', 'new version data' ]) assert result.exit_code == 0 result = runner.invoke(cli, prefix + ['cat', 'my_first_archive']) assert result.exit_code == 0 'new version data' in result.output result = runner.invoke( cli, prefix + ['download', 'my_first_archive', 'here.txt', '--version', '0.0.1']) assert result.exit_code == 0 'Hoo Yah! Stay Stoked!' in result.output # test download of previous version result = runner.invoke( cli, prefix + ['download', 'my_first_archive', 'here.txt', '--version', '0.0.1']) assert result.exit_code == 0 with open('here.txt', 'r') as downloaded: assert downloaded.read() == 'Hoo Yah! Stay Stoked!' # test download of nonexistant version (should fail without overwriting # file) result = runner.invoke( cli, prefix + ['download', 'my_first_archive', 'here.txt', '--version', '3.0']) assert result.exit_code != 0 with open('here.txt', 'r') as downloaded: assert downloaded.read() == 'Hoo Yah! Stay Stoked!' os.remove('here.txt') # teardown result = runner.invoke(cli, prefix + ['delete', 'my_first_archive']) result = runner.invoke(cli, prefix + ['filter']) assert result.exit_code == 0 assert result.output.strip() == '' assert len(list(api2.filter())) == 0
def test_cli_log_with_various_messages(sample_config): ''' Test CLI command ``log`` stability Addresses :issue:`232` - log fails on versions with no message ''' profile, temp_file = sample_config api = get_api(profile=profile, config_file=temp_file) runner = CliRunner() prefix = ['--config-file', '{}'.format(temp_file), '--profile', 'myapi'] arch = api.create('test/archive1.txt') def run_test(): # Test log on an empty archive result = runner.invoke(cli, prefix + ['log', 'test/archive1.txt']) if result.exit_code != 0: traceback.print_exception(*result.exc_info) raise OSError('Errors encountered during execution') return result.output.strip() try: # Test log on an empty archive log = run_test() assert log == '' # Test log with no message -- should still have info with arch.open('w+') as f: f.write(u('hello 1')) log = run_test() assert len(log.split('\n')) > 0 # Test log with message -- should appear in log with arch.open('w+', message='a message') as f: f.write(u('hello 2')) log = run_test() assert 'a message' in log # Test log with numeric message -- should appear in log with arch.open('w+', message=4.3) as f: f.write(u('hello 3')) log = run_test() assert '4.3' in log with arch.open('w+', message=lambda x: x**2) as f: f.write(u('hello 4')) log = run_test() assert 'lambda' in log # Test log with explicit None as message -- should not appear in log, # but the version should be present with arch.open('w+', message=None) as f: f.write(u('hello 5')) log = run_test() assert str(arch.get_latest_version()) in log assert 'None' not in log finally: arch.delete()
def main(dry_run=False): api = datafs.get_api() upload_files(api, PATTERN, ADDITIONAL_METADATA, dry_run=dry_run)
def preloaded_config(sample_config): ''' Prepare a manager/auth config with 3 archives, each having 3 versions .. note:: To save on test runtime, scope == module. Tests should not modify these archives. ''' profile, temp_file = sample_config api = get_api(profile=profile, config_file=temp_file) # Set up a couple archives with multiple versions arch1 = api.create('/req/arch1') arch2 = api.create('/req/arch2') arch3 = api.create('/req/arch3') with arch1.open('w+', bumpversion='minor', message='bumping to 0.1') as f: f.write(u'this is archive /req/arch1 version 0.1') with arch1.open('w+', bumpversion='major', message='bumping to 1.0') as f: f.write(u'this is archive /req/arch1 version 1.0') with arch1.open('w+', bumpversion='minor', message='bumping to 1.1') as f: f.write(u'this is archive /req/arch1 version 1.1') arch1_versions = arch1.get_versions() assert '0.1' in arch1_versions assert '1.0' in arch1_versions assert '1.1' in arch1_versions with arch2.open('w+', prerelease='alpha') as f: f.write(u'this is archive /req/arch2 version 0.0.1a1') with arch2.open('w+', prerelease='alpha') as f: f.write(u'this is archive /req/arch2 version 0.0.1a2') with arch2.open('w+', bumpversion='patch') as f: f.write(u'this is archive /req/arch2 version 0.0.1') arch2_versions = arch2.get_versions() assert '0.0.1a1' in arch2_versions assert '0.0.1a2' in arch2_versions assert '0.0.1' in arch2_versions with arch3.open('w+', bumpversion='major') as f: f.write(u'this is archive /req/arch3 version 1.0') with arch3.open('w+', bumpversion='minor', prerelease='alpha') as f: f.write(u'this is archive /req/arch3 version 1.1a1') with arch3.open('w+', bumpversion='minor') as f: f.write(u'this is archive /req/arch3 version 1.1') arch3_versions = arch3.get_versions() assert '1.0' in arch3_versions assert '1.1a1' in arch3_versions assert '1.1' in arch3_versions # Set up an unversioned archive with multiple versions arch_uver = api.create('uver1', versioned=False) with arch_uver.open('w+', message='bumping to 0.1') as f: f.write(u'this is archive uver1 version 0.1') with arch_uver.open('w+', message='bumping to 1.0') as f: f.write(u'this is archive uver1 version 1.0') with arch_uver.open('w+', message='bumping to 1.1') as f: f.write(u'this is archive uver1 version 1.1') arch_uver_versions = arch_uver.get_history() assert len(arch_uver_versions) == 3 try: yield profile, temp_file finally: arch1.delete() arch2.delete() arch3.delete() arch_uver.delete()