def test_validate_up():
up = UserParameter('name', default=1, type='int')
e = LocalCatalogEntry('',
'',
driver,
args={'arg1': "{{name}}"},
parameters=[up],
getenv=False)
s = e() # OK
assert s.kwargs['arg1'] == '1'
with pytest.raises(ValueError):
e(name='oi')
up = UserParameter('name', type='int')
e = LocalCatalogEntry('',
'',
driver,
args={'arg1': "{{name}}"},
parameters=[up],
getenv=False)
s = e() # OK
# arg1 is a string: real int gets rendered by jinja
assert s.kwargs['arg1'] == '0' # default default for int
s = e(arg1='something')
assert s.kwargs['arg1'] == 'something'
def test_maybe_default_from_env():
# maybe fill in parameter default from the env, depending on getenv
up = UserParameter('name', default='env(INTAKE_TEST_VAR)')
e = LocalCatalogEntry('',
'',
driver,
args={'arg1': "{{name}}"},
parameters=[up],
getenv=False)
s = e()
assert s.kwargs['arg1'] == 'env(INTAKE_TEST_VAR)'
os.environ['INTAKE_TEST_VAR'] = 'oi'
# Clear the cached source so we can (not) pick up the changed environment variable.
e.clear_cached_default_source()
s = e()
assert s.kwargs['arg1'] == 'env(INTAKE_TEST_VAR)'
up = UserParameter('name', default='env(INTAKE_TEST_VAR)')
e = LocalCatalogEntry('',
'',
driver,
args={'arg1': "{{name}}"},
parameters=[up],
getenv=True)
s = e()
assert s.kwargs['arg1'] == 'oi'
del os.environ['INTAKE_TEST_VAR']
# Clear the cached source so we can pick up the changed environment variable.
e.clear_cached_default_source()
s = e()
assert s.kwargs['arg1'] == ''
def test_maybe_default_from_env():
# maybe fill in parameter default from the env, depending on getenv
up = UserParameter('name', default='env(INTAKE_TEST_VAR)')
e = LocalCatalogEntry('',
'',
driver,
args={'arg1': "{{name}}"},
parameters=[up],
getenv=False)
s = e()
assert s.kwargs['arg1'] == 'env(INTAKE_TEST_VAR)'
os.environ['INTAKE_TEST_VAR'] = 'oi'
s = e()
assert s.kwargs['arg1'] == 'env(INTAKE_TEST_VAR)'
up = UserParameter('name', default='env(INTAKE_TEST_VAR)')
e = LocalCatalogEntry('',
'',
driver,
args={'arg1': "{{name}}"},
parameters=[up],
getenv=True)
s = e()
assert s.kwargs['arg1'] == 'oi'
del os.environ['INTAKE_TEST_VAR']
s = e()
assert s.kwargs['arg1'] == ''
def test_validate_par():
up = UserParameter('arg1', type='int')
e = LocalCatalogEntry('',
'',
driver,
args={'arg1': "oi"},
parameters=[up],
getenv=False)
with pytest.raises(ValueError):
e()
e = LocalCatalogEntry('',
'',
driver,
args={'arg1': 1},
parameters=[up],
getenv=False)
e() # OK
e = LocalCatalogEntry('',
'',
driver,
args={'arg1': "1"},
parameters=[up],
getenv=False)
s = e() # OK
assert s.kwargs['arg1'] == 1 # a number, not str
def test_parameter_default():
up = UserParameter('name', default='oi')
e = LocalCatalogEntry('',
'',
driver,
args={'arg1': "{{name}}"},
parameters=[up])
s = e()
assert s.kwargs['arg1'] == 'oi'
def test_up_override_and_render():
up = UserParameter('name', default='env(INTAKE_TEST_VAR)')
e = LocalCatalogEntry('',
'',
driver,
args={'arg1': "{{name}}"},
parameters=[up],
getenv=False)
s = e(name='other')
assert s.kwargs['arg1'] == 'other'
def test_user_explicit_override():
up = UserParameter('name', default='env(INTAKE_TEST_VAR)')
e = LocalCatalogEntry('',
'',
driver,
args={'arg1': "{{name}}"},
parameters=[up],
getenv=False)
# override wins over up
s = e(arg1='other')
assert s.kwargs['arg1'] == 'other'
def test_unknown():
e = LocalCatalogEntry('', '', driver, args={'arg1': "{{name}}"})
s = e()
assert s.kwargs['arg1'] == ""
# parameter has no default
up = UserParameter('name')
e = LocalCatalogEntry('',
'',
driver,
args={'arg1': "{{name}}"},
parameters=[up])
s = e()
assert s.kwargs['arg1'] == ""
def test_cache_default_source():
# If the user provides parameters, don't allow default caching
up = UserParameter('name', default='oi')
e = LocalCatalogEntry('', '', driver, getenv=False, parameters=[up])
s1 = e(name="oioi")
s2 = e()
assert s1 is not s2
s1 = e()
s2 = e(name="oioi")
assert s1 is not s2
# Otherwise, we can cache the default source
e = LocalCatalogEntry('', '', driver, getenv=False)
s1 = e()
s2 = e()
assert s1 is s2
def test_dict_save_complex():
from intake.catalog.base import Catalog
fn = os.path.join(tempfile.mkdtemp(), 'mycat.yaml')
cat = Catalog()
entry = LocalCatalogEntry(name='trial', description='get this back',
driver='csv', cache=[], catalog=cat,
parameters=[UserParameter(name='par1', description='desc', type='int')],
args={'urlpath': 'none'})
cat._entries = {'trial': entry}
cat.save(fn)
cat2 = open_catalog(fn)
assert 'trial' in cat2
assert cat2.name == 'mycat'
assert cat2.trial.describe()['plugin'][0] == 'csv'
def test_explicit_overrides():
e = LocalCatalogEntry('', '', driver, args={'arg1': "oi"})
s = e(arg1='hi')
assert s.kwargs['arg1'] == 'hi'
e = LocalCatalogEntry('', '', driver, args={'arg1': "{{name}}"})
s = e(name='hi')
assert s.kwargs['arg1'] == 'hi'
os.environ['INTAKE_TEST_VAR'] = 'another'
e = LocalCatalogEntry('', '', driver, args={'arg1': "oi"}, getenv=True)
s = e(arg1='{{env(INTAKE_TEST_VAR)}}')
assert s.kwargs['arg1'] == 'another'
up = UserParameter('arg1', type='int')
e = LocalCatalogEntry('', '', driver, args={'arg1': 1}, parameters=[up])
with pytest.raises(ValueError):
e(arg1='oi')
s = e(arg1='1')
assert s.kwargs['arg1'] == 1
def test_default_expansions():
try:
os.environ['INTAKE_INT_TEST'] = '1'
par = UserParameter('', '', 'int', default='env(INTAKE_INT_TEST)')
par.expand_defaults()
assert par.expanded_default == 1
finally:
del os.environ['INTAKE_INT_TEST']
par = UserParameter('', '', 'str', default='env(USER)')
par.expand_defaults(getenv=False)
assert par.expanded_default == 'env(USER)'
par.expand_defaults()
assert par.expanded_default == os.getenv('USER', '')
par = UserParameter('', '', 'str', default='client_env(USER)')
par.expand_defaults()
assert par.expanded_default == 'client_env(USER)'
par.expand_defaults(client=True)
assert par.expanded_default == os.getenv('USER', '')
par = UserParameter('', '', 'str', default='shell(echo success)')
par.expand_defaults(getshell=False)
assert par.expanded_default == 'shell(echo success)'
par.expand_defaults()
assert par.expanded_default == 'success'
par = UserParameter('', '', 'str', default='client_shell(echo success)')
par.expand_defaults(client=True)
assert par.expanded_default == 'success'
par = UserParameter('', '', 'int', default=1)
par.expand_defaults() # no error from string ops
def test_mlist_parameter():
up = UserParameter("", "", "mlist", allowed=["a", "b"])
up.validate([])
up.validate(['b'])
up.validate(['b', 'a'])
with pytest.raises(ValueError):
up.validate(["c"])
with pytest.raises(ValueError):
up.validate(["a", "c"])
with pytest.raises(ValueError):
up.validate("hello")
def test_default_expansions():
try:
os.environ['INTAKE_INT_TEST'] = '1'
par = UserParameter('', '', 'int', default='env(INTAKE_INT_TEST)')
par.expand_defaults()
assert par.default == 1
finally:
del os.environ['INTAKE_INT_TEST']
par = UserParameter('', '', 'str', default='env(USER)')
par.expand_defaults(getenv=False)
assert par.default == 'env(USER)'
par.expand_defaults()
assert par.default == os.getenv('USER', '')
par = UserParameter('', '', 'str', default='client_env(USER)')
par.expand_defaults()
assert par.default == 'client_env(USER)'
par.expand_defaults(client=True)
assert par.default == os.getenv('USER', '')
par = UserParameter('', '', 'str', default='shell(echo success)')
par.expand_defaults(getshell=False)
assert par.default == 'shell(echo success)'
par.expand_defaults()
assert par.default == 'success'
par = UserParameter('', '', 'str', default='client_shell(echo success)')
par.expand_defaults(client=True)
assert par.default == 'success'
par = UserParameter('', '', 'int', default=1)
par.expand_defaults() # no error from string ops
def build_catalog(
data_source,
path_pattern,
source_type,
grp_paths=None,
grp_path_params=None,
extra_engine_kwargs=None,
**kwargs
):
"""
Build a general Intake catalog for reading in ICESat-2 data.
This function is used by the read class object to create catalogs from lists of ICESat-2 variables.
Parameters
----------
data_source : string
A string with a full file path or full directory path to ICESat-2 hdf5 (.h5) format files.
Files within a directory must have a consistent filename pattern that includes the "ATL??" data product name.
Files must all be within a single directory.
path_pattern : string
String that shows the filename pattern as required for Intake's path_as_pattern argument.
source_type : string
String to use as the Local Catalog Entry name.
grp_paths : str, default None
Variable paths to load.
Can include general parameter names, which must be contained within double curly brackets and further
described in `grp_path_params`.
Default list based on data product of provided files.
If multiple data products are included in the files, the default list will be for the product of the first file.
This may result in errors during read-in if all files do not have the same variable paths.
grp_path_params : [dict], default None
List of dictionaries with a keyword for each parameter name specified in the `grp_paths` string.
Each parameter keyword should contain a dictionary with the acceptable keyword-value pairs for the driver being used.
**kwargs :
Keyword arguments to be passed through to `intake.catalog.Catalog.from_dict()`.
Keywords needed to override default inputs include:
- `source_args_dict` # highest level source information; keys include: "urlpath", "path_as_pattern", driver-specific ("xarray_kwargs" is default)
- `metadata_dict`
- `source_dict` # individual source entry information (default is supplied by data object; "name", "description", "driver", "args")
- `defaults_dict` # catalog "name", "description", "metadata", "entries", etc.
Returns
-------
intake.catalog.Catalog object
See Also
--------
read.Read
"""
from intake.catalog.local import LocalCatalogEntry, UserParameter
import intake_xarray
import icepyx.core.APIformatting as apifmt
assert (
grp_paths
), "You must enter a variable path or you will not be able to read in any data."
# generalize this/make it so the [engine] values can be entered as kwargs...
engine_key = "xarray_kwargs"
xarray_kwargs_dict = {"engine": "h5netcdf", "group": grp_paths}
if extra_engine_kwargs:
for key in extra_engine_kwargs.keys():
xarray_kwargs_dict[key] = extra_engine_kwargs[key]
source_args_dict = {
"urlpath": data_source,
"path_as_pattern": path_pattern,
engine_key: xarray_kwargs_dict,
}
metadata_dict = {"version": 1}
source_dict = {
"name": source_type,
"description": "",
"driver": "intake_xarray.netcdf.NetCDFSource", # NOTE: this must be a string or the catalog cannot be imported after saving
"args": source_args_dict,
}
if grp_path_params:
source_dict = apifmt.combine_params(
source_dict,
{"parameters": [UserParameter(**params) for params in grp_path_params]},
)
# NOTE: LocalCatalogEntry has some required positional args (name, description, driver)
# I tried doing this generally with *source_dict after the positional args (instead of as part of the if)
# but apparently I don't quite get something about passing dicts with * and ** and couldn't make it work
local_cat_source = {
source_type: LocalCatalogEntry(
name=source_dict.pop("name"),
description=source_dict.pop("description"),
driver=source_dict.pop("driver"),
parameters=source_dict.pop("parameters"),
args=source_dict.pop("args"),
)
}
else:
local_cat_source = {
source_type: LocalCatalogEntry(
name=source_dict.pop("name"),
description=source_dict.pop("description"),
driver=source_dict.pop("driver"),
args=source_dict.pop("args"),
)
}
defaults_dict = {
"name": "IS2-hdf5-icepyx-intake-catalog",
"description": "an icepyx-generated catalog for creating local ICESat-2 intake entries",
"metadata": metadata_dict,
"entries": local_cat_source,
}
build_cat_dict = apifmt.combine_params(defaults_dict, kwargs)
return Catalog.from_dict(**build_cat_dict)