def test_circular(tmp_dmf):
#
# r0 -> derived -> r1 -> derived >- r2 -+
# ^ |
# +------------------------------------+
# uses
r = [resource.Resource({'name': 'r{}'.format(i)}) for i in range(3)]
resource.create_relation_args(r[0], resource.PR_DERIVED, r[1])
resource.create_relation_args(r[1], resource.PR_DERIVED, r[2])
resource.create_relation_args(r[2], resource.PR_USES, r[0])
for rr in r:
tmp_dmf.add(rr)
# outgoing from r0
names = []
for d, rr, m in tmp_dmf.find_related(r[0], meta=['name']):
names.append(m['name'])
names.sort()
assert (names == ['r0', 'r1', 'r2'])
# incoming to r1
names = []
for d, rr, m in tmp_dmf.find_related(r[0], meta=['name'], outgoing=False):
names.append(m['name'])
names.sort()
assert (names == ['r0', 'r1', 'r2'])
# reducing depth shortens output
names = []
for d, rr, m in tmp_dmf.find_related(r[0], meta=['name'], maxdepth=2):
names.append(m['name'])
names.sort()
assert (names == ['r1', 'r2'])
names = []
for d, rr, m in tmp_dmf.find_related(r[0], meta=['name'], maxdepth=1):
names.append(m['name'])
names.sort()
assert (names == ['r1'])
def test_relation_in_experiment():
tmp_dir = scratch_path / "relation_in_experiment"
dmf = DMF(path=tmp_dir, create=True)
e1 = experiment.Experiment(dmf, name="1")
a = resource.Resource(value={"name": "foo"})
e1.add(a)
assert len(a.v["relations"]) == 1
assert len(e1.v["relations"]) == 1
def test_property_data_resource():
r = resource.Resource()
r.data = {"data": _propd, "meta": _propm}
tbl = propdata.PropertyTable(data=r.data["data"], metadata=r.data["meta"])
assert tbl.data.properties[0]
assert tbl.data.properties[0]["values"] == _propd[0]["values"]
assert tbl.data.properties[0]["errors"] == _propd[0]["errors"]
assert tbl.metadata[0].title == _propm["title"]
def add_resources(dmf_obj, num=3, **attrs):
ids = []
for i in range(num):
r = resource.Resource(value=attrs, type_="test")
r.data = {"i": i}
dmf_obj.add(r)
ids.append(r.id)
return ids
def test_property_data_resource():
r = resource.Resource()
r.data = {'data': _propd, 'meta': _propm}
tbl = propdata.PropertyTable(data=r.data['data'], metadata=r.data['meta'])
assert tbl.data.properties[0]
assert tbl.data.properties[0]['values'] == _propd[0]['values']
assert tbl.data.properties[0]['errors'] == _propd[0]['errors']
assert tbl.metadata[0].title == _propm['title']
def to_dmf(self, dmf):
rsrc = resource.Resource(value={
'name': 'convergence_results',
'desc': 'statistics returned from run_convergence_evaluation',
'creator': {
'name': getpass.getuser()
},
'data': stats.to_dict()
},
type_=resource.ResourceTypes.data)
dmf.add(rsrc)
def test_is_resource_json():
with TempDir() as d:
f = open(os.path.join(d, "sample.txt"), "w")
assert not util.is_resource_json(f.name)
f = open(os.path.join(d, "sample.json"), "w")
assert not util.is_resource_json(f.name)
r = resource.Resource()
json.dump(r.v, f)
f.close()
assert util.is_resource_json(f.name)
assert not util.is_resource_json(f.name, max_bytes=1)
def test_find_propertydata(tmp_dmf):
# populate DMF with some property data resources
pj = prop_json[0]
n = 10
for i in range(n):
pd = resource.Resource(value={'data': pj}, type_=resource.TY_PROPERTY)
tmp_dmf.add(pd)
# get them back again
filter_ = {'type': resource.TY_PROPERTY}
pdata = list(tmp_dmf.find(filter_dict=filter_))
assert len(pdata) == n
def _add_data(self,
data,
dtype='',
x_colnames=None,
z_colname=None,
metadata=None):
"""Add the data as a DMF resource.
Args:
data (propdata.PropertyData|DataFrame): Data values
metadata (propdata.Metadata): If present, metadata
dtype (str): Either 'input' or 'validation', the type of data
x_colnames (List[str]): List of x (variable) column names
z_colname (str): z (response) column name
"""
if isinstance(data, propdata.PropertyData):
pdata = data.as_arr()
else:
# create property data from dataframe
assert x_colnames is not None
assert z_colname is not None
pdata = []
for col in x_colnames:
values = list(data[col])
pdata.append({
"name": "{} x:{}".format(dtype, col),
"units": "",
"values": values,
"errors": [0] * len(values),
"error_type": "absolute",
"type": "property"
})
values = list(data[z_colname])
pdata.append({
'name': '{} z:{}'.format(dtype, z_colname),
'units': '',
'values': values,
'errors': [0] * len(values),
'error_type': 'absolute',
'type': 'property'
})
# create resource
r = resource.Resource(type_=resource.ResourceTypes.property)
r.data = {
'data': pdata,
'meta': metadata.as_dict() if metadata else {}
}
r.v['aliases'].append(dtype)
_log.debug('adding resource dtype={}'.format(dtype))
# add resource to experiment
self._exp.add(r)
# add link between resource and the surrogate-model resource
self._exp.link(r, Predicates.uses, self._rsrc)
def test_property_data_file(tmp_dmf, tmp_propdata_file):
tmpf, prop = tmp_propdata_file, tmp_propdata()
# Add the resource
r = resource.Resource(type_="property_data")
r.v["datafiles"].append({"path": tmpf.name})
rid = tmp_dmf.add(r)
assert rid is not None
r2 = tmp_dmf.fetch_one(rid)
path = r2.v["datafiles"][0]["path"]
f2 = open(path, "r")
j2 = json.load(f2)
assert j2 == prop
def test_property_data_file(tmp_dmf, tmp_propdata_file):
tmpf, prop = tmp_propdata_file, tmp_propdata()
# Add the resource
r = resource.Resource(type_='property_data')
r.v['datafiles'].append({'path': tmpf.name})
rid = tmp_dmf.add(r)
assert rid is not None
r2 = tmp_dmf.fetch_one(rid)
path = r2.v['datafiles'][0]['path']
f2 = open(path, 'r')
j2 = json.load(f2)
assert j2 == prop
def test_is_resource_json():
tmp_dir = scratch_path / "is_resource_json"
tmp_dir.mkdir()
f = (tmp_dir / "sample.txt").open("w")
assert not util.is_resource_json(f.name)
f = (tmp_dir / "sample.json").open("w")
assert not util.is_resource_json(f.name)
r = resource.Resource()
json.dump(r.v, f)
f.close()
assert util.is_resource_json(f.name)
assert not util.is_resource_json(f.name, max_bytes=1)
def example_resource():
r = resource.Resource()
r.v["version_info"]["version"] = test_version
r.v["collaborators"] = [
{"name": "Clark Kent", "email": "*****@*****.**"},
{"name": "Superman", "email": "*****@*****.**"},
]
r.v["datafiles"].append({"path": "/etc/passwd"})
r.v["aliases"] = ["test_resource_full"]
r.v["tags"] = ["test", "resource"]
# r.relations -- deal with this separately
r.data = {"arbitrary": {"values": [1, 2, 3]}}
return r
def test_find_propertydata():
# populate DMF with some property data resources
pj = prop_json[0]
n, resource_ids = 10, []
for i in range(n):
pd = resource.Resource(value={"data": pj},
type_=resource.ResourceTypes.property)
resource_ids.append(scratch_dmf.add(pd))
# get them back again
filter_ = {"type": resource.ResourceTypes.property}
pdata = list(scratch_dmf.find(filter_dict=filter_))
assert len(pdata) == n
for rid in resource_ids:
scratch_dmf.remove(identifier=rid)
def test_resource_roundtrip(example_resource):
"""Build up a resource with all attributes,
then make sure it serializes and deserializes.
"""
r = example_resource # alias
# make sure we can serialize it
r_json = json.dumps(r.v, indent=2)
# make sure we can deserialize its JSON
r2_value = json.loads(r_json)
# reconstruct from the deserialized JSON
r2 = resource.Resource(value=r2_value)
# compare them
assert r2.v["version_info"] == r.v["version_info"]
assert r2.data["arbitrary"]["values"][0] == 1
def test_dmf_add_filesystem_err(tmp_dmf):
r = resource.Resource(value={'desc': 'test resource'})
# create datafile
tmpf1 = tempfile.NamedTemporaryFile(delete=False)
tmpf1.close()
r.v['datafiles'].append({'path': tmpf1.name})
# now, to get an error, make the DMF datafile path unwritable
path = os.path.join(tmp_dmf.root, tmp_dmf.datafile_dir)
os.chmod(path, 000)
# then try to add the resource, which includes copying the file into
# the (now unwritable) directory
pytest.raises(errors.DMFError, tmp_dmf.add, r)
# make the directory writable again so we can remove it
os.chmod(path, 0o777)
def test_dmf_add_tmp_no_unlink():
tmp_dir = Path(scratch_dir) / "dmf_add_tmp_no_unlink"
dmf = DMF(path=tmp_dir, create=True)
r = resource.Resource(value={"desc": "test resource"})
# create datafile, with temporary-file flag turned on
tmp_file = (tmp_dir / "foo").open("w")
r.v["datafiles"].append({
"path": str(tmp_file),
"is_tmp": True,
"do_copy": True
})
# we want an error trying to COPY this file; to get this,
# change the permissions of the directory
os.chmod(tmp_dir, 0o500)
pytest.raises(Exception, dmf.add, r)
def test_property_data_file():
tmp_propdata_file = (Path(scratch_dir) /
"property_data_file.json").open("w")
json.dump(prop_json[0], tmp_propdata_file)
tmp_propdata_file.close()
# Add the resource
r = resource.Resource(type_="property_data")
r.v["datafiles"].append({"path": tmp_propdata_file.name})
rid = scratch_dmf.add(r)
assert rid is not None
r2 = scratch_dmf.fetch_one(rid)
path = r2.v["datafiles"][0]["path"]
f2 = open(path, "r")
j2 = json.load(f2)
assert j2 == prop_json[0]
def to_dmf(self, dmf):
# PYLINT-TODO-FIX fix error due to undefined variable "stats"
rsrc = resource.Resource(
value={
'name': 'convergence_results',
'desc': 'statistics returned from run_convergence_evaluation',
'creator': {
'name': getpass.getuser()
},
# pylint: disable=undefined-variable
'data': stats.to_dict()
},
type_=resource.ResourceTypes.data)
# pylint: enable=undefined-variable
dmf.add(rsrc)
def test_relation_with_remove(tmp_dmf):
e1 = experiment.Experiment(tmp_dmf, name="1")
n, added = 10, []
for i in range(n):
a = resource.Resource({"name": "foo"})
e1.add(a)
added.append(a)
assert len(e1.v["relations"]) == n
# remove, then update e1
for a in added:
tmp_dmf.remove(identifier=a.id)
e1.update()
# relation to removed 'a' should be gone
n -= 1
assert (len(e1.v["relations"])) == n
def test_dmf_add_filesystem_err():
tmp_dir = Path(scratch_dir) / "dmf_add_filesystem_err"
dmf = DMF(path=tmp_dir, create=True)
r = resource.Resource(value={"desc": "test resource"})
# create datafile
tmpf1 = NamedTemporaryFile(delete=False)
tmpf1.close()
r.v["datafiles"].append({"path": tmpf1.name})
# now, to get an error, make the DMF datafile path unwritable
path = os.path.join(dmf.root, dmf.datafile_dir)
# this file permissions stuff is problematic in Windows, so this test is Linux-only
os.chmod(path, 000)
# then try to add the resource, which includes copying the file into
# the (now unwritable) directory
pytest.raises(errors.DMFError, dmf.add, r)
# make the directory writable again so we can remove it
os.chmod(path, 0o777)
def test_dmf_add_tmp_no_unlink(tmp_dmf):
r = resource.Resource(value={'desc': 'test resource'})
# create datafile, with temporary-file flag turned on
tmpdir = tempfile.mkdtemp()
tmpfile = os.path.join(tmpdir, 'foo')
open(tmpfile, 'w')
r.v['datafiles'].append({'path': tmpfile, 'is_tmp': True, 'do_copy': True})
# we want an error trying to UNLINK this file; to get this,
# change the permissions of the dir read-only
os.chmod(tmpdir, 0o500)
try:
tmp_dmf.add(r)
finally:
# change it back and clean up
os.chmod(tmpdir, 0o700)
os.unlink(tmpfile)
os.rmdir(tmpdir)
def test_dmf_add_tmp_no_unlink(tmp_dmf):
r = resource.Resource(value={"desc": "test resource"})
# create datafile, with temporary-file flag turned on
tmpdir = tempfile.mkdtemp()
tmpfile = os.path.join(tmpdir, "foo")
open(tmpfile, "w")
r.v["datafiles"].append({"path": tmpfile, "is_tmp": True, "do_copy": True})
# we want an error trying to UNLINK this file; to get this,
# change the permissions of the dir read-only
os.chmod(tmpdir, 0o500)
try:
tmp_dmf.add(r)
finally:
# change it back and clean up
os.chmod(tmpdir, 0o700)
os.unlink(tmpfile)
os.rmdir(tmpdir)
def test_relation_with_remove():
tmp_dir = scratch_path / "relation_with_remove"
dmf = DMF(path=tmp_dir, create=True)
e1 = experiment.Experiment(dmf, name="1")
n, added = 10, []
for i in range(n):
a = resource.Resource({"name": "foo"})
e1.add(a)
added.append(a)
assert len(e1.v["relations"]) == n
# remove, then update e1
for a in added:
dmf.remove(identifier=a.id)
e1.update()
# relation to removed 'a' should be gone
n -= 1
assert (len(e1.v["relations"])) == n
def test_dmf_add_tmp_no_copy():
tmp_dir = Path(scratch_dir) / "dmf_add_tmp_no_copy"
dmf = DMF(path=tmp_dir, create=True)
r = resource.Resource(value={"desc": "test resource"})
# create datafile, with temporary-file flag turned on
tmp_file = (tmp_dir / "foo").open("w")
r.v["datafiles"].append({
"path": str(tmp_file),
"is_tmp": True,
"do_copy": True
})
# we want an error trying to COPY this file; to get this,
# change the permissions of the directory
os.chmod(tmp_dir, 0o400)
ok = False
try:
dmf.add(r)
except errors.DMFError:
ok = True
if not ok:
assert False, "DMFError expected"
def test_add_property_data(tmp_dmf, tmp_propdata_file):
tmpf, prop = tmp_propdata_file, tmp_propdata()
# Add the resource
r = resource.Resource(type_='property_data')
r.set_id()
r.v['creator'] = {'name': 'Dan Gunter'}
m = prop['meta']
work = '{authors}, "{title}". {info}, {date}'.format(**m)
r.v['sources'].append({'source': work, 'date': m['date']})
r.data = {'notes': m['notes']}
r.v['tags'].append('MEA')
r.v['datafiles'].append({'path': tmpf.name})
rid = tmp_dmf.add(r)
assert rid is not None
# Retrieve the resource
r2 = tmp_dmf.fetch_one(rid)
# Validate the resource
assert r2.type == 'property_data'
assert 'MEA' in r2.v['tags']
# Remove the resource
tmp_dmf.remove(identifier=rid)
def test_add_property_data(tmp_dmf, tmp_propdata_file):
tmpf, prop = tmp_propdata_file, tmp_propdata()
# Add the resource
r = resource.Resource(type_="property_data")
r.set_id()
r.v["creator"] = {"name": "Dan Gunter"}
m = prop["meta"]
work = '{authors}, "{title}". {info}, {date}'.format(**m)
r.v["sources"].append({"source": work, "date": m["date"]})
r.data = {"notes": m["notes"]}
r.v["tags"].append("MEA")
r.v["datafiles"].append({"path": tmpf.name})
rid = tmp_dmf.add(r)
assert rid is not None
# Retrieve the resource
r2 = tmp_dmf.fetch_one(rid)
# Validate the resource
assert r2.type == "property_data"
assert "MEA" in r2.v["tags"]
# Remove the resource
tmp_dmf.remove(identifier=rid)
def test_find_related():
#
# r0
# | uses
# v
# r1
# | version
# v
# r2
# /\
# / \ derived
# v v
# r3 r4
#
tmp_dir = scratch_path / "find_related"
dmf = DMF(path=tmp_dir, create=True)
r = [resource.Resource({"name": "r{}".format(i)}) for i in range(5)]
# r3 <-- derived <-- r2 <-- version <-- r1
cr = resource.create_relation # shortcut
cr(r[2], Predicates.derived, r[3])
cr(r[1], Predicates.version, r[2])
# r4 <-- derived <-- r2
cr(r[2], Predicates.derived, r[4])
# r0 -- Uses --> r1
cr(r[0], Predicates.uses, r[1])
# add to dmf
for i in range(5):
dmf.add(r[i])
# outgoing from r0 should include 1,2,3,4
names = []
for d, rr, m in dmf.find_related(r[0], meta=["aliases"]):
names.append(m["aliases"][0])
names.sort()
assert names == ["r1", "r2", "r3", "r4"]
# incoming to r4 should include r0, r1, r2
names = []
for d, rr, m in dmf.find_related(r[4], meta=["aliases"], outgoing=False):
names.append(m["aliases"][0])
names.sort()
assert names == ["r0", "r1", "r2"]
def test_dmf_add_tmp_no_copy(tmp_dmf):
r = resource.Resource(value={'desc': 'test resource'})
# create datafile, with temporary-file flag turned on
tmpdir = tempfile.mkdtemp()
tmpfile = os.path.join(tmpdir, 'foo')
open(tmpfile, 'w')
r.v['datafiles'].append({'path': tmpfile, 'is_tmp': True, 'do_copy': True})
# we want an error trying to COPY this file; to get this,
# change the permissions of the directory
os.chmod(tmpdir, 0o400)
ok = False
try:
tmp_dmf.add(r)
except errors.DMFError:
ok = True
finally:
# change it back and clean up
os.chmod(tmpdir, 0o700)
os.unlink(tmpfile)
os.rmdir(tmpdir)
if not ok:
assert False, "DMFError expected"
def test_find_related(tmp_dmf):
#
# r0
# | uses
# v
# r1
# | version
# v
# r2
# /\
# / \ derived
# v v
# r3 r4
#
r = [resource.Resource({"name": "r{}".format(i)}) for i in range(5)]
# r3 <-- derived <-- r2 <-- version <-- r1
cr = resource.create_relation_args # shortcut
cr(r[2], resource.PR_DERIVED, r[3])
cr(r[1], resource.PR_VERSION, r[2])
# r4 <-- derived <-- r2
cr(r[2], resource.PR_DERIVED, r[4])
# r0 -- Uses --> r1
cr(r[0], resource.PR_USES, r[1])
# add to dmf
for i in range(5):
tmp_dmf.add(r[i])
# outgoing from r0 should include 1,2,3,4
names = []
for d, rr, m in tmp_dmf.find_related(r[0], meta=["name"]):
names.append(m["name"])
names.sort()
assert names == ["r1", "r2", "r3", "r4"]
# incoming to r4 should include r0, r1, r2
names = []
for d, rr, m in tmp_dmf.find_related(r[4], meta=["name"], outgoing=False):
names.append(m["name"])
names.sort()
assert names == ["r0", "r1", "r2"]
