def test_dmf_related(dmf_context, runner):
create_foo_workspace(runner)
# add the fully-connected 4 resources
dmf = DMF()
rlist = [
resource.Resource(value={
"desc": ltr,
"aliases": [ltr],
"tags": ["graph"]
}) for ltr in "ABCD"
]
A_id = rlist[0].id # root resource id, used in testcode
relation = resource.Predicates.uses
for r in rlist:
for r2 in rlist:
if r is r2:
continue
resource.create_relation(r, relation, r2)
for r in rlist:
dmf.add(r)
#
result = runner.invoke(related, [A_id, "--no-unicode", "--no-color"],
catch_exceptions=False)
assert result.exit_code == 0
rlines = result.output.split("\n")
nrelations = sum(
1 for _ in filter(lambda s: resource.Predicates.uses in s, rlines))
assert nrelations == 12 # 3 blocks of (1 + 3)
def test_create_relation_in_resource():
a = resource.Resource()
b = resource.Resource()
resource.create_relation_args(a, "contains", b)
assert len(a.v["relations"]) == 1
assert len(b.v["relations"]) == 1
# bad type
with pytest.raises(TypeError):
resource.create_relation("foo", "contains", b)
def test_create_relation_in_resource():
a = resource.Resource()
b = resource.Resource()
resource.create_relation_args(a, 'contains', b)
assert len(a.v['relations']) == 1
assert len(b.v['relations']) == 1
# bad type
with pytest.raises(TypeError):
resource.create_relation('foo', 'contains', b)
def link(self, subj, predicate=Predicates.contains, obj=None):
"""Add and update relation triple in DMF.
Args:
subj (resource.Resource): Subject
predicate (str): Predicate
obj (resource.Resource): Object
Returns:
None
"""
if obj is None:
obj = self
resource.create_relation(subj, predicate, obj)
self._dmf.update(subj)
self._dmf.update(obj)
def add(self, rsrc):
"""Add a resource to an experiment.
This does two things:
1. Establishes an "experiment" type of relationship between the
new resource and the experiment.
2. Adds the resource to the DMF
Args:
rsrc (resource.Resource): The resource to add.
Returns:
resource.Resource: Added (input) resource, for chaining calls.
"""
resource.create_relation(self, Predicates.contains, rsrc)
self._dmf.update(rsrc, upsert=True)
self._dmf.update(self)
def test_create_relation(default_resource, example_resource):
r1, r2 = default_resource, example_resource
relation = resource.Triple(r1, Predicates.uses, r2)
resource.create_relation(relation)
with pytest.raises(ValueError): # duplicate will raise ValueError
resource.create_relation(relation)
assert len(r1.v["relations"]) == 1
assert len(r2.v["relations"]) == 1
assert r1.v["relations"][0]["role"] == "subject"
assert r2.v["relations"][0]["role"] == "object"
assert r1.v["relations"][0]["identifier"] == r2.v[r2.ID_FIELD]
assert r2.v["relations"][0]["identifier"] == r1.v[r2.ID_FIELD]
# some errors
with pytest.raises(ValueError):
resource.create_relation("foo", "bad predicate", "bar")
# delete relation from subject to test duplicate check for object
r1.v["relations"] = []
with pytest.raises(ValueError): # dup raises ValueError
resource.create_relation(relation)
def test_create_relation(default_resource, example_resource):
r1, r2 = default_resource, example_resource
relation = resource.Triple(r1, resource.PR_USES, r2)
resource.create_relation(relation)
with pytest.raises(ValueError): # duplicate will raise ValueError
resource.create_relation(relation)
assert len(r1.v['relations']) == 1
assert len(r2.v['relations']) == 1
assert r1.v['relations'][0]['role'] == 'subject'
assert r2.v['relations'][0]['role'] == 'object'
assert r1.v['relations'][0]['identifier'] == r2.v[r2.ID_FIELD]
assert r2.v['relations'][0]['identifier'] == r1.v[r2.ID_FIELD]
# some errors
with pytest.raises(ValueError):
resource.create_relation_args('foo', 'bad predicate', 'bar')
# delete relation from subject to test duplicate check for object
r1.v['relations'] = []
with pytest.raises(ValueError): # dup raises ValueError
resource.create_relation(relation)
def test_circular():
#
# r0 -> derived -> r1 -> derived >- r2 -+
# ^ |
# +------------------------------------+
# uses
tmp_dir = scratch_path / "circular"
dmf = DMF(path=tmp_dir, create=True)
r = [resource.Resource({"name": "r{}".format(i)}) for i in range(3)]
resource.create_relation(r[0], Predicates.derived, r[1])
resource.create_relation(r[1], Predicates.derived, r[2])
resource.create_relation(r[2], Predicates.uses, r[0])
for rr in r:
dmf.add(rr)
# outgoing from r0
names = []
for d, rr, m in dmf.find_related(r[0], meta=["aliases"]):
names.append(m["aliases"][0])
names.sort()
assert names == ["r0", "r1", "r2"]
# incoming to r1
names = []
for d, rr, m in dmf.find_related(r[0], meta=["aliases"], outgoing=False):
names.append(m["aliases"][0])
names.sort()
assert names == ["r0", "r1", "r2"]
# reducing depth shortens output
names = []
for d, rr, m in dmf.find_related(r[0], meta=["aliases"], maxdepth=2):
names.append(m["aliases"][0])
names.sort()
assert names == ["r1", "r2"]
names = []
for d, rr, m in dmf.find_related(r[0], meta=["aliases"], maxdepth=1):
names.append(m["aliases"][0])
names.sort()
assert names == ["r1"]
def visit_metadata(self, obj, meta):
"""Called for each property class encountered during the "walk"
initiated by `index_property_metadata()`.
Args:
obj (property_base.PropertyParameterBase): Property class instance
meta (property_base.PropertyClassMetadata): Associated metadata
Returns:
None
Raises:
AttributeError: if
"""
_log.debug("Adding resource to DMF that indexes the property package "
'"{}"'.format(".".join([obj.__module__, obj.__name__])))
r = resource.Resource(type_=resource.ResourceTypes.code)
r.data = {"units": meta.default_units, "properties": meta.properties}
containing_module = obj.__module__
if hasattr(containing_module, "__version__"):
obj_ver = resource.version_list(containing_module.__version__)
elif self._defver is None:
raise AttributeError("No __version__ for module {}, and no "
"default".format(containing_module))
else:
obj_ver = self._defver
r.v["codes"].append({
"type": "class",
"language": "python",
"name": ".".join([obj.__module__, obj.__name__]),
"version": obj_ver,
})
r.v["tags"].append(self.INDEXED_PROPERTY_TAG)
# Search for existing indexed codes.
# A match exists if all 3 of these are the same:
# codes.type == class
# codes.language == python
# codes.name == <module>.<class>
info = {k: r.v["codes"][0][k] for k in ("type", "language", "name")}
rsrc_list, dup_rsrc = [], None
# Loop through all the right kind of resources
for rsrc in self._dmf.find({
r.TYPE_FIELD: resource.ResourceTypes.code,
"tags": ["indexed-property"]
}):
# skip any resources without one code
if len(rsrc.v["codes"]) != 1:
continue
code = rsrc.v["codes"][0]
# skip any resource of wrong code type, name, lang.
skip = False
for k in info:
if code[k] != info[k]:
skip = True
break
if skip:
continue
# skip any resources missing the recorded metadata
skip = False
for data_key in r.data.keys():
if data_key not in rsrc.data:
skip = True
break
if skip:
continue
# If the version of the found code is the same as the
# version of the one to be added, then it is a duplicate
if code["version"] == obj_ver:
dup_rsrc = rsrc
break
rsrc_list.append(rsrc)
if dup_rsrc:
# This is considered a normal, non-exceptional situation
_log.debug("DMFVisitor: Not adding duplicate index for "
"{}v{}".format(info["name"], obj_ver))
else:
# add the resource
r.validate()
_log.debug(
'DMFVisitor: Adding resource for code "{}"v{} type={}'.format(
r.v["codes"][0]["name"],
r.v["codes"][0]["version"],
r.v["codes"][0]["type"],
))
self._dmf.add(r)
if rsrc_list:
# Connect to most recent (highest) version
rsrc_list.sort(key=lambda rs: rs.v["codes"][0]["version"])
# for rsrc in rsrc_list:
rsrc = rsrc_list[-1]
rel = resource.Triple(r, resource.Predicates.version, rsrc)
resource.create_relation(rel)
self._dmf.update(rsrc)
self._dmf.update(r)
def register(
resource_type,
url,
info,
copy,
strict,
unique,
contained,
derived,
used,
prev,
is_subject,
version,
):
_log.debug(f"Register object type='{resource_type}' url/path='{url.path}'")
# process url
if url.scheme in ("file", ""):
path = url.path
else:
click.echo("Currently, URL must be a file")
sys.exit(Code.NOT_SUPPORTED.value)
# create the resource
_log.debug("create resource")
try:
rsrc = resource.Resource.from_file(
path, as_type=resource_type, strict=strict, do_copy=copy
)
except resource.Resource.InferResourceTypeError as err:
click.echo(f"Failed to infer resource: {err}")
sys.exit(Code.IMPORT_RESOURCE.value)
except resource.Resource.LoadResourceError as err:
click.echo(f"Failed to load resource: {err}")
sys.exit(Code.IMPORT_RESOURCE.value)
# connect to DMF
try:
dmf = DMF()
except errors.WorkspaceError as err:
click.echo(f"Failed to connect to DMF: {err}")
sys.exit(Code.WORKSPACE_NOT_FOUND.value)
except errors.DMFError as err:
click.echo(f"Failed to connect to DMF: {err}")
sys.exit(Code.DMF.value)
# check uniqueness
if unique:
df = rsrc.v["datafiles"][0] # file info for this upload
query = {"datafiles": [{"sha1": df["sha1"]}]}
query_result, dup_ids = dmf.find(query), []
for dup in query_result:
dup_df = dup.v["datafiles"][0]
if dup_df["path"] in df["path"]:
dup_ids.append(dup.id)
n_dup = len(dup_ids)
if n_dup > 0:
click.echo(
f"This file is already in {n_dup} resource(s): " f"{' '.join(dup_ids)}"
)
sys.exit(Code.DMF_OPER.value)
# process relations
_log.debug("add relations")
rel_to_add = { # translate into standard relation names
Predicates.contains: contained,
Predicates.derived: derived,
Predicates.uses: used,
Predicates.version: prev,
}
target_resources = {} # keep target resources in dict, update at end
for rel_name, rel_ids in rel_to_add.items():
for rel_id in rel_ids:
if rel_id in target_resources:
rel_subj = target_resources[rel_id]
else:
rel_subj = dmf.fetch_one(rel_id)
target_resources[rel_id] = rel_subj
if rel_subj is None:
click.echo(f"Relation {rel_name} target not found: {rel_id}")
sys.exit(Code.DMF_OPER.value)
if is_subject == "yes":
resource.create_relation(rsrc, rel_name, rel_subj)
else:
resource.create_relation(rel_subj, rel_name, rsrc)
_log.debug(f"added relation {rsrc.id} <-- {rel_name} -- {rel_id}")
_log.debug("update resource relations")
for rel_rsrc in target_resources.values():
dmf.update(rel_rsrc)
# add metadata
if version:
try:
vlist = resource.version_list(version)
except ValueError:
click.echo(f"Invalid version `{version}`")
sys.exit(Code.INPUT_VALUE.value)
else:
rsrc.v["version_info"]["version"] = vlist
# add the resource
_log.debug("add resource begin")
try:
new_id = dmf.add(rsrc)
except errors.DuplicateResourceError as err:
click.echo(f"Failed to add resource: {err}")
sys.exit(Code.DMF_OPER.value)
_log.debug(f"added resource: {new_id}")
if info == "yes":
pfxlen = len(new_id)
si = _ShowInfo("term", pfxlen)
for rsrc in dmf.find_by_id(new_id):
si.show(rsrc)
else:
click.echo(new_id)
