def test_1(self): drs_fs = CordexFileSystem(self.tmpdir) drs_tree = DRSTree(drs_fs) json_obj = json.load(open(op.join(test_dir, 'cordex_1.json'))) drs_tree.discover_incoming_fromjson(json_obj, activity='cordex') assert len(drs_tree.pub_trees) == 3
def test_2(self): drs_fs = SpecsFileSystem(self.tmpdir) drs_tree = DRSTree(drs_fs) with open(op.join(test_dir, 'specs_cedacc.json')) as fh: json_obj = [json.loads(line) for line in fh] drs_tree.discover_incoming_fromjson(json_obj, activity='specs') # This id will not be present if realm is not correctly split on space drs_id = 'specs.output.IPSL.IPSL-CM5A-LR.decadal.S20130101.mon.seaIce.OImon.sic.r3i1p1' assert drs_id in drs_tree.pub_trees p = drs_tree.pub_trees.values()[0] p_vars = set(drs.variable for (drs_str, drs) in p._todo) # All DRS objects should be for the same variable assert len(p_vars) == 1
class Command(object): def __init__(self, op, opts, args): self.op = op self.opts = opts self.args = args self.shelve_dir = None self.p_cmip5_config = None self.drs_root = None self.drs_tree = None self.make_drs_tree() def _config_p_cmip5(self): """ Ensure self.shelve_dir is set. This is required for InitCommand and any command that uses p_cmip5. """ self.shelve_dir = self.opts.shelve_dir if self.shelve_dir is None: try: self.shelve_dir = config.config.get("p_cmip5", "shelve-dir") except NoSectionError: raise Exception( "Shelve directory not specified. Please use --shelve-dir or set shelve_dir via metaconfig" ) def _setup_p_cmip5(self): """ Instantiate the p_cmip5.cmip5_product object ready for deducing the product component. """ shelves = p_cmip5.init._find_shelves(self.shelve_dir) self.p_cmip5_config = self.opts.p_cmip5_config if self.p_cmip5_config is None: try: self.p_cmip5_config = config.config.get("p_cmip5", "config") except (NoSectionError, NoOptionError): raise Exception( "p_cmip5 configuration file not specified. Please use --p-cmip5-config or set via metaconfig" ) self.drs_tree.set_p_cmip5( p_cmip5.product.cmip5_product( mip_table_shelve=shelves["stdo_mip"], template=shelves["template"], stdo=shelves["stdo"], config=self.p_cmip5_config, not_ok_excpt=True, ) ) def make_drs_tree(self): if self.opts.root: self.drs_root = self.opts.root else: try: self.drs_root = config.drs_defaults["root"] except KeyError: raise Exception("drs-root not defined") if self.opts.incoming: incoming = self.opts.incoming else: try: incoming = config.drs_defaults["incoming"] except KeyError: incoming = os.path.join(self.drs_root, config.DEFAULT_INCOMING) if self.opts.json_drs: json_drs = self.opts.json_drs else: json_drs = None drs_root = os.path.normpath(os.path.abspath(self.drs_root)) if self.opts.scheme: scheme = self.opts.scheme else: scheme = config.default_drs_scheme try: fs_cls = config.get_drs_scheme(scheme) except KeyError: raise ValueError("Unrecognised DRS scheme %s" % scheme) self.drs_fs = fs_cls(drs_root) self.drs_tree = DRSTree(self.drs_fs) if self.opts.move_cmd: self.drs_tree.set_move_cmd(self.opts.move_cmd) # This code is specifically for the deprecated DRS setting options # Generic DRS component setting is handled below kwargs = {} for attr in ["activity", "product", "institute", "model", "experiment", "frequency", "realm", "ensemble"]: try: val = getattr(self.opts, attr) # val may be there but None if val is None: raise AttributeError warn("Option --%s is deprecated. Use --component instead" % attr) except AttributeError: val = config.drs_defaults.get(attr) # Only add this component if it is valid for the DRS scheme if attr in self.drs_fs.drs_cls.DRS_ATTRS: log.info("Setting DRS component %s=%s" % (attr, val)) kwargs[attr] = val try: component_dict = self.opts.component_dict except AttributeError: component_dict = {} for component in self.drs_fs.drs_cls._iter_components(to_publish_level=True): if component in component_dict: val = component_dict.get(component) log.info("Setting DRS component %s=%s" % (component, val)) kwargs[component] = self.drs_fs.drs_cls._decode_component(component, val) del component_dict[component] # Error for any components not valid for component in component_dict: self.op.error("Unrecognised component %s for scheme %s" % (component, scheme)) # Get the template DRS from args if self.args: dataset_id = self.args[0] drs = self.drs_fs.drs_cls.from_dataset_id(dataset_id, **kwargs) else: drs = self.drs_fs.drs_cls(**kwargs) # Product detection if self.opts.detect_product: self._config_p_cmip5() self._setup_p_cmip5() # If JSON file selected use that, otherwise discover from filesystem if json_drs: with open(json_drs) as fh: #!TODO: Remove json-array case # This is a work-around until we have a stable json format # The file might be a json array or it might be a series # of json files, 1 per line json_str = fh.readline() if json_str[0] == "[": json_obj = json.loads(json_str) else: json_obj = [] while json_str: json_obj.append(json.loads(json_str)) json_str = fh.readline() self.drs_tree.discover_incoming_fromjson(json_obj, **drs) else: self.drs_tree.discover(incoming, **drs) def do(self): raise NotImplementedError("Unimplemented command") def print_header(self): print """\ ============================================================================== DRS Tree at %s ------------------------------------------------------------------------------\ """ % self.drs_root def print_sep(self): print """\ ------------------------------------------------------------------------------\ """ def print_footer(self): print """\