def test_index_property_metadata():
tmp_dir = scratch_path / "index_property_metadata"
dmf = DMF(path=tmp_dir, create=True)
propindex.index_property_metadata(
dmf, pkg=idaes.dmf, expr=".*IndexMePlease[0-9]", exclude_testdirs=False
)
# Check the resource
for rsrc in dmf.find():
assert rsrc.v[rsrc.TYPE_FIELD] == resource.ResourceTypes.code
def ls(color, show, sort_by, reverse, prefix):
d = DMF()
if not show:
show = ["type", "desc", "modified"] # note: 'id' is always first
else:
try:
show = _split_and_validate_fields(show)
except ValueError as err:
click.echo(f"Bad fields for --show option: {err}")
sys.exit(Code.INPUT_VALUE.value)
reverse = bool(reverse == "yes")
if not sort_by:
sort_by = ["id"]
resources = list(d.find())
_print_resource_table(resources, show, sort_by, reverse, prefix, color)
def find(
output_format,
color,
show,
sort_by,
prefix,
reverse,
by,
created,
filedesc,
modified,
name,
datatype,
):
d = DMF()
if output_format == "list":
if not show:
show = ["type", "desc", "modified"] # note: 'id' is always first
else:
try:
show = _split_and_validate_fields(show)
except ValueError as err:
click.echo(f"Bad fields for --show option: {err}")
sys.exit(Code.INPUT_VALUE.value)
reverse = bool(reverse == "yes")
if not sort_by:
sort_by = ["id"]
# Build query
query = {}
if by:
query["creator.name"] = by
if created:
try:
query["created"] = _date_query(created)
except ValueError as err:
click.echo(f"bad date for 'created': {err}")
sys.exit(Code.INPUT_VALUE.value)
if filedesc:
query["datafiles"] = [{"desc": filedesc}]
if modified:
try:
query["modified"] = _date_query(modified)
except ValueError as err:
click.echo(f"bad date for 'modified': {err}")
sys.exit(Code.INPUT_VALUE.value)
if name:
query["aliases"] = [name]
if datatype:
query["type"] = datatype
# Execute query
_log.info(f"find: query = '{query}'")
_log.debug("find.begin")
resources = list(d.find(query))
_log.debug("find.end")
# Print result
if output_format == "list":
# print resources like `ls`
_print_resource_table(resources, show, sort_by, reverse, prefix, color)
elif output_format == "info":
# print resources one by one
si = _ShowInfo("term", 32, color=color)
for rsrc in resources:
si.show(rsrc)
elif output_format == "json":
# print resources as JSON
for r in resources:
print(json.dumps(r.v, indent=2))
def register(
resource_type,
url,
info,
copy,
strict,
unique,
contained,
derived,
used,
prev,
is_subject,
version,
):
_log.debug(f"Register object type='{resource_type}' url/path='{url.path}'")
# process url
if url.scheme in ("file", ""):
path = url.path
else:
click.echo("Currently, URL must be a file")
sys.exit(Code.NOT_SUPPORTED.value)
# create the resource
_log.debug("create resource")
try:
rsrc = resource.Resource.from_file(path,
as_type=resource_type,
strict=strict,
do_copy=copy)
except resource.Resource.InferResourceTypeError as err:
click.echo(f"Failed to infer resource: {err}")
sys.exit(Code.IMPORT_RESOURCE.value)
except resource.Resource.LoadResourceError as err:
click.echo(f"Failed to load resource: {err}")
sys.exit(Code.IMPORT_RESOURCE.value)
# connect to DMF
try:
dmf = DMF()
except errors.WorkspaceError as err:
click.echo(f"Failed to connect to DMF: {err}")
sys.exit(Code.WORKSPACE_NOT_FOUND.value)
except errors.DMFError as err:
click.echo(f"Failed to connect to DMF: {err}")
sys.exit(Code.DMF.value)
# check uniqueness
if unique:
df = rsrc.v["datafiles"][0] # file info for this upload
query = {"datafiles": [{"sha1": df["sha1"]}]}
query_result, dup_ids = dmf.find(query), []
for dup in query_result:
dup_df = dup.v["datafiles"][0]
if dup_df["path"] in df["path"]:
dup_ids.append(dup.id)
n_dup = len(dup_ids)
if n_dup > 0:
click.echo(f"This file is already in {n_dup} resource(s): "
f"{' '.join(dup_ids)}")
sys.exit(Code.DMF_OPER.value)
# process relations
_log.debug("add relations")
rel_to_add = { # translate into standard relation names
resource.PR_CONTAINS: contained,
resource.PR_DERIVED: derived,
resource.PR_USES: used,
resource.PR_VERSION: prev,
}
target_resources = {} # keep target resources in dict, update at end
for rel_name, rel_ids in rel_to_add.items():
for rel_id in rel_ids:
if rel_id in target_resources:
rel_subj = target_resources[rel_id]
else:
rel_subj = dmf.fetch_one(rel_id)
target_resources[rel_id] = rel_subj
if rel_subj is None:
click.echo(f"Relation {rel_name} target not found: {rel_id}")
sys.exit(Code.DMF_OPER.value)
if is_subject == "yes":
resource.create_relation_args(rsrc, rel_name, rel_subj)
else:
resource.create_relation_args(rel_subj, rel_name, rsrc)
_log.debug(f"added relation {rsrc.id} <-- {rel_name} -- {rel_id}")
_log.debug("update resource relations")
for rel_rsrc in target_resources.values():
dmf.update(rel_rsrc)
# add metadata
if version:
try:
vlist = resource.version_list(version)
except ValueError:
click.echo(f"Invalid version `{version}`")
sys.exit(Code.INPUT_VALUE.value)
else:
rsrc.v["version_info"]["version"] = vlist
# add the resource
_log.debug("add resource begin")
try:
new_id = dmf.add(rsrc)
except errors.DuplicateResourceError as err:
click.echo(f"Failed to add resource: {err}")
sys.exit(Code.DMF_OPER.value)
_log.debug(f"added resource: {new_id}")
if info == "yes":
pfxlen = len(new_id)
si = _ShowInfo("term", pfxlen)
for rsrc in dmf.find_by_id(new_id):
si.show(rsrc)
else:
click.echo(new_id)
def test_index_multiple_versions():
tmp_dir = scratch_path / "index_multiple_versions"
dmf = DMF(path=tmp_dir, create=True)
v1, v2, v3 = "1.0.0", "6.6.6", "9.9.0"
# index initial version
propindex.index_property_metadata(
dmf,
pkg=idaes.dmf,
expr=".*IndexMePlease[0-9]",
exclude_testdirs=False,
default_version=v1,
)
# index again
propindex.index_property_metadata(
dmf,
pkg=idaes.dmf,
expr=".*IndexMePlease[0-9]",
exclude_testdirs=False,
default_version=v2,
)
# check that we now have two resources, and
# a relation between them
rlist = list(dmf.find({}))
assert len(rlist) == 2
rcodes = [r.v["codes"][0] for r in rlist]
if rcodes[0]["version"][:3] == ("6", "6", "6"):
first, second = 1, 0
else:
first, second = 0, 1
# Debugging only
# print('CODES:')
# print(' - first -')
# print(rcodes[first])
# print(rlist[first].v['relations'])
# print(' - second -')
# print(rcodes[second])
# print(rlist[second].v['relations'])
# Each resource has 1 relation
assert len(rlist[first].v["relations"]) == 1
assert len(rlist[second].v["relations"]) == 1
first_rel = rlist[first].v["relations"][0]
second_rel = rlist[second].v["relations"][0]
# First resource is pointed at by second
assert first_rel[resource.RR_ROLE] == resource.RR_OBJ
assert first_rel[resource.RR_PRED] == resource.Predicates.version
assert first_rel[resource.RR_ID] == rlist[second].id
# Second resource points at first
assert second_rel[resource.RR_ROLE] == resource.RR_SUBJ
assert second_rel[resource.RR_PRED] == resource.Predicates.version
assert second_rel[resource.RR_ID] == rlist[first].id
# Add the same version
propindex.index_property_metadata(
dmf,
pkg=idaes.dmf,
expr=".*IndexMePlease[0-9]",
exclude_testdirs=False,
default_version=v2,
)
# check that we still have two resources
rlist = list(dmf.find({}))
assert len(rlist) == 2
# Now add another version
propindex.index_property_metadata(
dmf,
pkg=idaes.dmf,
expr=".*IndexMePlease[0-9]",
exclude_testdirs=False,
default_version=v3,
)
# check that we now have three resources
rlist = list(dmf.find({}))
assert len(rlist) == 3
# check that we have 0 <--> 1 <--> 2
# first sort by version and save that in the 'indexes' array
indexes = [(r.v["codes"][0]["version"], i) for i, r in enumerate(rlist)]
indexes.sort()
# pull out relations into 'rel' array, in version order
rel = [rlist[indexes[i][1]].v["relations"] for i in range(3)]
# check first resource's relations
assert len(rel[0]) == 1
# 0 <-- 1
assert rel[0][0][resource.RR_ID] == rlist[indexes[1][1]].id
assert rel[0][0][resource.RR_ROLE] == resource.RR_OBJ
# check second resource's relations
assert len(rel[1]) == 2
for j in range(2):
if rel[1][j][resource.RR_ROLE] == resource.RR_SUBJ:
# 1 --> 0
assert rel[1][j][resource.RR_ID] == rlist[indexes[0][1]].id
else:
# 1 <-- 2
assert rel[1][j][resource.RR_ID] == rlist[indexes[2][1]].id
assert rel[1][j][resource.RR_ROLE] == resource.RR_OBJ
# check third resource's relations
# check third resource's relations
assert len(rel[2]) == 1
# 2 --> 1
assert rel[2][0][resource.RR_ID] == rlist[indexes[1][1]].id
assert rel[2][0][resource.RR_ROLE] == resource.RR_SUBJ