def test_1(self):
drs_fs = CordexFileSystem(self.tmpdir)
drs_tree = DRSTree(drs_fs)
json_obj = json.load(open(op.join(test_dir, 'cordex_1.json')))
drs_tree.discover_incoming_fromjson(json_obj, activity='cordex')
assert len(drs_tree.pub_trees) == 3
def test_2(self):
drs_fs = SpecsFileSystem(self.tmpdir)
drs_tree = DRSTree(drs_fs)
with open(op.join(test_dir, 'specs_cedacc.json')) as fh:
json_obj = [json.loads(line) for line in fh]
drs_tree.discover_incoming_fromjson(json_obj, activity='specs')
# This id will not be present if realm is not correctly split on space
drs_id = 'specs.output.IPSL.IPSL-CM5A-LR.decadal.S20130101.mon.seaIce.OImon.sic.r3i1p1'
assert drs_id in drs_tree.pub_trees
p = drs_tree.pub_trees.values()[0]
p_vars = set(drs.variable for (drs_str, drs) in p._todo)
# All DRS objects should be for the same variable
assert len(p_vars) == 1
class Command(object):
def __init__(self, op, opts, args):
self.op = op
self.opts = opts
self.args = args
self.shelve_dir = None
self.p_cmip5_config = None
self.drs_root = None
self.drs_tree = None
self.make_drs_tree()
def _config_p_cmip5(self):
"""
Ensure self.shelve_dir is set. This is required for InitCommand
and any command that uses p_cmip5.
"""
self.shelve_dir = self.opts.shelve_dir
if self.shelve_dir is None:
try:
self.shelve_dir = config.config.get("p_cmip5", "shelve-dir")
except NoSectionError:
raise Exception(
"Shelve directory not specified. Please use --shelve-dir or set shelve_dir via metaconfig"
)
def _setup_p_cmip5(self):
"""
Instantiate the p_cmip5.cmip5_product object ready for deducing
the product component.
"""
shelves = p_cmip5.init._find_shelves(self.shelve_dir)
self.p_cmip5_config = self.opts.p_cmip5_config
if self.p_cmip5_config is None:
try:
self.p_cmip5_config = config.config.get("p_cmip5", "config")
except (NoSectionError, NoOptionError):
raise Exception(
"p_cmip5 configuration file not specified. Please use --p-cmip5-config or set via metaconfig"
)
self.drs_tree.set_p_cmip5(
p_cmip5.product.cmip5_product(
mip_table_shelve=shelves["stdo_mip"],
template=shelves["template"],
stdo=shelves["stdo"],
config=self.p_cmip5_config,
not_ok_excpt=True,
)
)
def make_drs_tree(self):
if self.opts.root:
self.drs_root = self.opts.root
else:
try:
self.drs_root = config.drs_defaults["root"]
except KeyError:
raise Exception("drs-root not defined")
if self.opts.incoming:
incoming = self.opts.incoming
else:
try:
incoming = config.drs_defaults["incoming"]
except KeyError:
incoming = os.path.join(self.drs_root, config.DEFAULT_INCOMING)
if self.opts.json_drs:
json_drs = self.opts.json_drs
else:
json_drs = None
drs_root = os.path.normpath(os.path.abspath(self.drs_root))
if self.opts.scheme:
scheme = self.opts.scheme
else:
scheme = config.default_drs_scheme
try:
fs_cls = config.get_drs_scheme(scheme)
except KeyError:
raise ValueError("Unrecognised DRS scheme %s" % scheme)
self.drs_fs = fs_cls(drs_root)
self.drs_tree = DRSTree(self.drs_fs)
if self.opts.move_cmd:
self.drs_tree.set_move_cmd(self.opts.move_cmd)
# This code is specifically for the deprecated DRS setting options
# Generic DRS component setting is handled below
kwargs = {}
for attr in ["activity", "product", "institute", "model", "experiment", "frequency", "realm", "ensemble"]:
try:
val = getattr(self.opts, attr)
# val may be there but None
if val is None:
raise AttributeError
warn("Option --%s is deprecated. Use --component instead" % attr)
except AttributeError:
val = config.drs_defaults.get(attr)
# Only add this component if it is valid for the DRS scheme
if attr in self.drs_fs.drs_cls.DRS_ATTRS:
log.info("Setting DRS component %s=%s" % (attr, val))
kwargs[attr] = val
try:
component_dict = self.opts.component_dict
except AttributeError:
component_dict = {}
for component in self.drs_fs.drs_cls._iter_components(to_publish_level=True):
if component in component_dict:
val = component_dict.get(component)
log.info("Setting DRS component %s=%s" % (component, val))
kwargs[component] = self.drs_fs.drs_cls._decode_component(component, val)
del component_dict[component]
# Error for any components not valid
for component in component_dict:
self.op.error("Unrecognised component %s for scheme %s" % (component, scheme))
# Get the template DRS from args
if self.args:
dataset_id = self.args[0]
drs = self.drs_fs.drs_cls.from_dataset_id(dataset_id, **kwargs)
else:
drs = self.drs_fs.drs_cls(**kwargs)
# Product detection
if self.opts.detect_product:
self._config_p_cmip5()
self._setup_p_cmip5()
# If JSON file selected use that, otherwise discover from filesystem
if json_drs:
with open(json_drs) as fh:
#!TODO: Remove json-array case
# This is a work-around until we have a stable json format
# The file might be a json array or it might be a series
# of json files, 1 per line
json_str = fh.readline()
if json_str[0] == "[":
json_obj = json.loads(json_str)
else:
json_obj = []
while json_str:
json_obj.append(json.loads(json_str))
json_str = fh.readline()
self.drs_tree.discover_incoming_fromjson(json_obj, **drs)
else:
self.drs_tree.discover(incoming, **drs)
def do(self):
raise NotImplementedError("Unimplemented command")
def print_header(self):
print """\
==============================================================================
DRS Tree at %s
------------------------------------------------------------------------------\
""" % self.drs_root
def print_sep(self):
print """\
------------------------------------------------------------------------------\
"""
def print_footer(self):
print """\
