def test_search(): with config.set({'database-directory': './tests/test_collections'}): col = intake.open_esm_metadatastore(collection_name='cafe_test') cat = col.search( frequency='day', variable_short_name=['ucomp', 'wt'], start_date=['1988-03-01', ] ) assert isinstance(cat.query_results, pd.DataFrame) assert not cat.query_results.empty
def test_search(): with config.set({'database-directory': './tests/test_collections'}): c = intake.open_esm_metadatastore( collection_name='cmip6_test_collection') cat = c.search(source_id=['BCC-ESM1', 'CNRM-CM6-1', 'CNRM-ESM2-1']) assert isinstance(cat.ds, xr.Dataset) assert len(cat.ds.index) > 0 assert isinstance(cat.df, pd.DataFrame) assert isinstance(cat.nunique(), pd.Series) assert isinstance(cat.unique(), dict)
def test_to_xarray_cesm(chunks, expected_chunks): with config.set({'database-directory': './tests/test_collections'}): c = intake.open_esm_metadatastore(collection_name='cesm1-le') cat = c.search( variable=['STF_O2', 'SHF'], ensemble=[1, 3, 9], experiment=['20C', 'RCP85'], direct_access=True, ) dset = cat.to_xarray(chunks=chunks) ds = dset['pop.h.ocn'] assert ds['SHF'].data.chunksize == expected_chunks
def test_to_xarray_cmip(chunks, expected_chunks): with config.set({'database-directory': './tests/test_collections'}): c = intake.open_esm_metadatastore( collection_name='cmip6_test_collection') # Test for data from multiple models cat = c.search(source_id=['CNRM-ESM2-1', 'CNRM-CM6-1', 'BCC-ESM1'], variable_id=['tasmax']) ds = cat.to_xarray(decode_times=False, chunks=chunks) print(ds) assert isinstance(ds, dict) _, dset = ds.popitem() assert dset['tasmax'].data.chunksize == expected_chunks
def test_build_collection_file(): with config.set({'database-directory': './tests/test_collections'}): col = intake.open_esm_metadatastore(collection_input_definition=cdef, overwrite_existing=True) col = intake.open_esm_metadatastore( collection_name='cmip6_test_collection') assert isinstance(col.ds, xr.Dataset) assert isinstance(col.nunique(), pd.Series) assert isinstance(col.unique(), dict) assert set(col.df.grid_label.unique()) == set(['gr', 'gn']) assert set(col.df.variable_id.unique()) == set([ 'prsn', 'prra', 'tasmax', 'evspsblveg', 'landCoverFrac', 'mrso', 'co3', 'gpp', 'residualFrac', 'mrfso', ])
def test_to_xarray_cmip_empty(): with config.set({'database-directory': './tests/test_collections'}): c = intake.open_esm_metadatastore(collection_name='cmip5_test_collection') cat = c.search( model='CanESM2', experiment='rcp85', frequency='mon', modeling_realm='atmos', ensemble_member='r2i1p1', ) with pytest.raises(ValueError): cat.to_xarray()
def test_to_xarray_cmip(chunks, expected_chunks): with config.set({'database-directory': './tests/test_collections'}): c = intake.open_esm_metadatastore( collection_name='cmip6_test_collection') # Test for data from multiple institutions cat = c.search(source_id=['CNRM-ESM2-1', 'GISS-E2-1-G'], variable_id=['prra', 'tasmax']) ds = cat.to_xarray(decode_times=False, chunks=chunks) print(ds) assert isinstance(ds, dict) nasa_dset = ds['NASA-GISS.GISS-E2-1-G.amip.Omon.gn'] assert nasa_dset['prra'].data.chunksize == expected_chunks
def test_to_xarray_cesm(chunks, expected_chunks): with config.set({'database-directory': './tests/test_collections'}): c = intake.open_esm_metadatastore(collection_name='cesm1-le') query = { 'variable': ['STF_O2', 'SHF'], 'member_id': [1, 3, 9], 'experiment': ['20C', 'RCP85'], 'direct_access': True, } cat = c.search(**query) dset = cat.to_xarray(chunks=chunks) _, ds = dset.popitem() assert ds['SHF'].data.chunksize == expected_chunks
def test_to_xarray(): with config.set({'database-directory': './tests/test_collections'}): col = intake.open_esm_metadatastore(collection_name='mpige_test') cat = col.search(component='mpiom', stream='monitoring_ym') with pytest.warns(UserWarning): ds = cat.to_xarray() assert isinstance(ds, dict) cat = col.search( component=['mpiom', 'hamocc'], stream='monitoring_ym', experiment=['hist', 'rcp85'], ensemble=[2, 3], ) ds = cat.to_xarray(merge_exp=False) assert 'experiment_id' in ds.coords
def test_to_xarray_cesm_hsi(): data_cache_dir = f'{TMPDIR}/intake-esm-tests/transferred-data' with config.set({ 'database-directory': './tests/test_collections', 'data-cache-directory': data_cache_dir }): collection_input_definition = os.path.join( here, '../ensure-file-hsi-transfer-collection-input.yml') col = intake.open_esm_metadatastore( collection_input_definition=collection_input_definition, overwrite_existing=False) cat = col.search(variable=['SST']) dset = cat.to_xarray(chunks={'time': 365}) _, ds = dset.popitem() assert isinstance(ds['SST'], xr.DataArray)
def test_to_xarray_cmip(chunks, expected_chunks): with config.set({'database-directory': './tests/test_collections'}): c = intake.open_esm_metadatastore(collection_name='cmip5_test_collection') cat = c.search( variable=['hfls'], frequency='mon', modeling_realm='atmos', model=['CNRM-CM5'] ) dset = cat.to_xarray(decode_times=True, chunks=chunks) ds = dset['CNRM-CERFACS.CNRM-CM5.historical.mon.atmos'] assert ds['hfls'].data.chunksize == expected_chunks # Test for data from multiple institutions cat = c.search(variable=['hfls'], frequency='mon', modeling_realm='atmos') ds = cat.to_xarray(decode_times=False, chunks=chunks) assert isinstance(ds, dict) assert 'CCCma.CanCM4.historical.mon.atmos' in ds.keys()
def test_file_transfer_symlink(): data_cache_dir = f'{TMPDIR}/intake-esm-tests/transferred-data' with config.set( {'database-directory': './tests/test_collections', 'data-cache-directory': data_cache_dir} ): collection_input_definition = os.path.join(here, 'copy-to-cache-collection-input.yml') col = intake.open_esm_metadatastore( collection_input_definition=collection_input_definition, overwrite_existing=True ) cat = col.search(variable=['STF_O2', 'SHF']) query_results = _ensure_file_access(cat.ds) local_urlpaths = query_results['file_fullpath'].tolist() assert isinstance(local_urlpaths, list) assert len(local_urlpaths) > 0 shutil.rmtree(data_cache_dir)
def test_build_collection_dict(): with config.set({'database-directory': './tests/test_collections'}): collection_definition = { 'name': 'cmip6_test_collection', 'collection_type': 'cmip6', 'data_sources': { 'root_dir': { 'name': 'GLADE', 'loc_type': 'posix', 'direct_access': True, 'urlpath': './tests/sample_data/cmip/CMIP6', } }, } col = intake.open_esm_metadatastore( collection_input_definition=collection_definition, overwrite_existing=True) assert isinstance(col.df, pd.DataFrame)
def test_search_regex(): with config.set({'database-directory': './tests/test_collections'}): col = intake.open_esm_metadatastore(collection_name='cesm1-le') cases = sorted( col.search(case='b.e11.B20TRC5CNBDRD.f09_g16.10*').query_results. case.unique()) expected = sorted([ 'b.e11.B20TRC5CNBDRD.f09_g16.101', 'b.e11.B20TRC5CNBDRD.f09_g16.102', 'b.e11.B20TRC5CNBDRD.f09_g16.103', 'b.e11.B20TRC5CNBDRD.f09_g16.104', 'b.e11.B20TRC5CNBDRD.f09_g16.105', ]) assert cases == expected date_ranges = set( col.search( date_range=['192001-']).query_results.date_range.unique()) assert date_ranges == set(['192001-200512'])
import esmlab import intake import intake_esm intake_esm.config.set(dict(database_directory='tests/data')) col = intake.open_esm_metadatastore( collection_input_file='tests/data/collection_input.yml')
def test_build_collection_file(): with config.set({'database-directory': './tests/test_collections'}): col = intake.open_esm_metadatastore(collection_input_definition=cdef, overwrite_existing=True) assert isinstance(col.ds, xr.Dataset)
def test_search(): with config.set({'database-directory': './tests/test_collections'}): c = intake.open_esm_metadatastore(collection_name='cmip5_test_collection') cat = c.search(model=['CanESM2', 'CSIRO-Mk3-6-0']) assert isinstance(cat.query_results, pd.DataFrame) assert not cat.query_results.empty
def test_constructor(collection): with config.set({'database-directory': './tests/test_collections'}): c = intake.open_esm_metadatastore(collection_name=collection) assert isinstance(c, ESMMetadataStoreCatalog)