def test_dmf_related(dmf_context, runner): create_foo_workspace(runner) # add the fully-connected 4 resources dmf = DMF() rlist = [ resource.Resource(value={ "desc": ltr, "aliases": [ltr], "tags": ["graph"] }) for ltr in "ABCD" ] A_id = rlist[0].id # root resource id, used in testcode relation = resource.Predicates.uses for r in rlist: for r2 in rlist: if r is r2: continue resource.create_relation(r, relation, r2) for r in rlist: dmf.add(r) # result = runner.invoke(related, [A_id, "--no-unicode", "--no-color"], catch_exceptions=False) assert result.exit_code == 0 rlines = result.output.split("\n") nrelations = sum( 1 for _ in filter(lambda s: resource.Predicates.uses in s, rlines)) assert nrelations == 12 # 3 blocks of (1 + 3)
def test_create_relation_in_resource(): a = resource.Resource() b = resource.Resource() resource.create_relation_args(a, "contains", b) assert len(a.v["relations"]) == 1 assert len(b.v["relations"]) == 1 # bad type with pytest.raises(TypeError): resource.create_relation("foo", "contains", b)
def test_create_relation_in_resource(): a = resource.Resource() b = resource.Resource() resource.create_relation_args(a, 'contains', b) assert len(a.v['relations']) == 1 assert len(b.v['relations']) == 1 # bad type with pytest.raises(TypeError): resource.create_relation('foo', 'contains', b)
def link(self, subj, predicate=Predicates.contains, obj=None): """Add and update relation triple in DMF. Args: subj (resource.Resource): Subject predicate (str): Predicate obj (resource.Resource): Object Returns: None """ if obj is None: obj = self resource.create_relation(subj, predicate, obj) self._dmf.update(subj) self._dmf.update(obj)
def add(self, rsrc): """Add a resource to an experiment. This does two things: 1. Establishes an "experiment" type of relationship between the new resource and the experiment. 2. Adds the resource to the DMF Args: rsrc (resource.Resource): The resource to add. Returns: resource.Resource: Added (input) resource, for chaining calls. """ resource.create_relation(self, Predicates.contains, rsrc) self._dmf.update(rsrc, upsert=True) self._dmf.update(self)
def test_create_relation(default_resource, example_resource): r1, r2 = default_resource, example_resource relation = resource.Triple(r1, Predicates.uses, r2) resource.create_relation(relation) with pytest.raises(ValueError): # duplicate will raise ValueError resource.create_relation(relation) assert len(r1.v["relations"]) == 1 assert len(r2.v["relations"]) == 1 assert r1.v["relations"][0]["role"] == "subject" assert r2.v["relations"][0]["role"] == "object" assert r1.v["relations"][0]["identifier"] == r2.v[r2.ID_FIELD] assert r2.v["relations"][0]["identifier"] == r1.v[r2.ID_FIELD] # some errors with pytest.raises(ValueError): resource.create_relation("foo", "bad predicate", "bar") # delete relation from subject to test duplicate check for object r1.v["relations"] = [] with pytest.raises(ValueError): # dup raises ValueError resource.create_relation(relation)
def test_create_relation(default_resource, example_resource): r1, r2 = default_resource, example_resource relation = resource.Triple(r1, resource.PR_USES, r2) resource.create_relation(relation) with pytest.raises(ValueError): # duplicate will raise ValueError resource.create_relation(relation) assert len(r1.v['relations']) == 1 assert len(r2.v['relations']) == 1 assert r1.v['relations'][0]['role'] == 'subject' assert r2.v['relations'][0]['role'] == 'object' assert r1.v['relations'][0]['identifier'] == r2.v[r2.ID_FIELD] assert r2.v['relations'][0]['identifier'] == r1.v[r2.ID_FIELD] # some errors with pytest.raises(ValueError): resource.create_relation_args('foo', 'bad predicate', 'bar') # delete relation from subject to test duplicate check for object r1.v['relations'] = [] with pytest.raises(ValueError): # dup raises ValueError resource.create_relation(relation)
def test_circular(): # # r0 -> derived -> r1 -> derived >- r2 -+ # ^ | # +------------------------------------+ # uses tmp_dir = scratch_path / "circular" dmf = DMF(path=tmp_dir, create=True) r = [resource.Resource({"name": "r{}".format(i)}) for i in range(3)] resource.create_relation(r[0], Predicates.derived, r[1]) resource.create_relation(r[1], Predicates.derived, r[2]) resource.create_relation(r[2], Predicates.uses, r[0]) for rr in r: dmf.add(rr) # outgoing from r0 names = [] for d, rr, m in dmf.find_related(r[0], meta=["aliases"]): names.append(m["aliases"][0]) names.sort() assert names == ["r0", "r1", "r2"] # incoming to r1 names = [] for d, rr, m in dmf.find_related(r[0], meta=["aliases"], outgoing=False): names.append(m["aliases"][0]) names.sort() assert names == ["r0", "r1", "r2"] # reducing depth shortens output names = [] for d, rr, m in dmf.find_related(r[0], meta=["aliases"], maxdepth=2): names.append(m["aliases"][0]) names.sort() assert names == ["r1", "r2"] names = [] for d, rr, m in dmf.find_related(r[0], meta=["aliases"], maxdepth=1): names.append(m["aliases"][0]) names.sort() assert names == ["r1"]
def visit_metadata(self, obj, meta): """Called for each property class encountered during the "walk" initiated by `index_property_metadata()`. Args: obj (property_base.PropertyParameterBase): Property class instance meta (property_base.PropertyClassMetadata): Associated metadata Returns: None Raises: AttributeError: if """ _log.debug("Adding resource to DMF that indexes the property package " '"{}"'.format(".".join([obj.__module__, obj.__name__]))) r = resource.Resource(type_=resource.ResourceTypes.code) r.data = {"units": meta.default_units, "properties": meta.properties} containing_module = obj.__module__ if hasattr(containing_module, "__version__"): obj_ver = resource.version_list(containing_module.__version__) elif self._defver is None: raise AttributeError("No __version__ for module {}, and no " "default".format(containing_module)) else: obj_ver = self._defver r.v["codes"].append({ "type": "class", "language": "python", "name": ".".join([obj.__module__, obj.__name__]), "version": obj_ver, }) r.v["tags"].append(self.INDEXED_PROPERTY_TAG) # Search for existing indexed codes. # A match exists if all 3 of these are the same: # codes.type == class # codes.language == python # codes.name == <module>.<class> info = {k: r.v["codes"][0][k] for k in ("type", "language", "name")} rsrc_list, dup_rsrc = [], None # Loop through all the right kind of resources for rsrc in self._dmf.find({ r.TYPE_FIELD: resource.ResourceTypes.code, "tags": ["indexed-property"] }): # skip any resources without one code if len(rsrc.v["codes"]) != 1: continue code = rsrc.v["codes"][0] # skip any resource of wrong code type, name, lang. skip = False for k in info: if code[k] != info[k]: skip = True break if skip: continue # skip any resources missing the recorded metadata skip = False for data_key in r.data.keys(): if data_key not in rsrc.data: skip = True break if skip: continue # If the version of the found code is the same as the # version of the one to be added, then it is a duplicate if code["version"] == obj_ver: dup_rsrc = rsrc break rsrc_list.append(rsrc) if dup_rsrc: # This is considered a normal, non-exceptional situation _log.debug("DMFVisitor: Not adding duplicate index for " "{}v{}".format(info["name"], obj_ver)) else: # add the resource r.validate() _log.debug( 'DMFVisitor: Adding resource for code "{}"v{} type={}'.format( r.v["codes"][0]["name"], r.v["codes"][0]["version"], r.v["codes"][0]["type"], )) self._dmf.add(r) if rsrc_list: # Connect to most recent (highest) version rsrc_list.sort(key=lambda rs: rs.v["codes"][0]["version"]) # for rsrc in rsrc_list: rsrc = rsrc_list[-1] rel = resource.Triple(r, resource.Predicates.version, rsrc) resource.create_relation(rel) self._dmf.update(rsrc) self._dmf.update(r)
def register( resource_type, url, info, copy, strict, unique, contained, derived, used, prev, is_subject, version, ): _log.debug(f"Register object type='{resource_type}' url/path='{url.path}'") # process url if url.scheme in ("file", ""): path = url.path else: click.echo("Currently, URL must be a file") sys.exit(Code.NOT_SUPPORTED.value) # create the resource _log.debug("create resource") try: rsrc = resource.Resource.from_file( path, as_type=resource_type, strict=strict, do_copy=copy ) except resource.Resource.InferResourceTypeError as err: click.echo(f"Failed to infer resource: {err}") sys.exit(Code.IMPORT_RESOURCE.value) except resource.Resource.LoadResourceError as err: click.echo(f"Failed to load resource: {err}") sys.exit(Code.IMPORT_RESOURCE.value) # connect to DMF try: dmf = DMF() except errors.WorkspaceError as err: click.echo(f"Failed to connect to DMF: {err}") sys.exit(Code.WORKSPACE_NOT_FOUND.value) except errors.DMFError as err: click.echo(f"Failed to connect to DMF: {err}") sys.exit(Code.DMF.value) # check uniqueness if unique: df = rsrc.v["datafiles"][0] # file info for this upload query = {"datafiles": [{"sha1": df["sha1"]}]} query_result, dup_ids = dmf.find(query), [] for dup in query_result: dup_df = dup.v["datafiles"][0] if dup_df["path"] in df["path"]: dup_ids.append(dup.id) n_dup = len(dup_ids) if n_dup > 0: click.echo( f"This file is already in {n_dup} resource(s): " f"{' '.join(dup_ids)}" ) sys.exit(Code.DMF_OPER.value) # process relations _log.debug("add relations") rel_to_add = { # translate into standard relation names Predicates.contains: contained, Predicates.derived: derived, Predicates.uses: used, Predicates.version: prev, } target_resources = {} # keep target resources in dict, update at end for rel_name, rel_ids in rel_to_add.items(): for rel_id in rel_ids: if rel_id in target_resources: rel_subj = target_resources[rel_id] else: rel_subj = dmf.fetch_one(rel_id) target_resources[rel_id] = rel_subj if rel_subj is None: click.echo(f"Relation {rel_name} target not found: {rel_id}") sys.exit(Code.DMF_OPER.value) if is_subject == "yes": resource.create_relation(rsrc, rel_name, rel_subj) else: resource.create_relation(rel_subj, rel_name, rsrc) _log.debug(f"added relation {rsrc.id} <-- {rel_name} -- {rel_id}") _log.debug("update resource relations") for rel_rsrc in target_resources.values(): dmf.update(rel_rsrc) # add metadata if version: try: vlist = resource.version_list(version) except ValueError: click.echo(f"Invalid version `{version}`") sys.exit(Code.INPUT_VALUE.value) else: rsrc.v["version_info"]["version"] = vlist # add the resource _log.debug("add resource begin") try: new_id = dmf.add(rsrc) except errors.DuplicateResourceError as err: click.echo(f"Failed to add resource: {err}") sys.exit(Code.DMF_OPER.value) _log.debug(f"added resource: {new_id}") if info == "yes": pfxlen = len(new_id) si = _ShowInfo("term", pfxlen) for rsrc in dmf.find_by_id(new_id): si.show(rsrc) else: click.echo(new_id)