def test_circular(tmp_dmf): # # r0 -> derived -> r1 -> derived >- r2 -+ # ^ | # +------------------------------------+ # uses r = [resource.Resource({'name': 'r{}'.format(i)}) for i in range(3)] resource.create_relation_args(r[0], resource.PR_DERIVED, r[1]) resource.create_relation_args(r[1], resource.PR_DERIVED, r[2]) resource.create_relation_args(r[2], resource.PR_USES, r[0]) for rr in r: tmp_dmf.add(rr) # outgoing from r0 names = [] for d, rr, m in tmp_dmf.find_related(r[0], meta=['name']): names.append(m['name']) names.sort() assert (names == ['r0', 'r1', 'r2']) # incoming to r1 names = [] for d, rr, m in tmp_dmf.find_related(r[0], meta=['name'], outgoing=False): names.append(m['name']) names.sort() assert (names == ['r0', 'r1', 'r2']) # reducing depth shortens output names = [] for d, rr, m in tmp_dmf.find_related(r[0], meta=['name'], maxdepth=2): names.append(m['name']) names.sort() assert (names == ['r1', 'r2']) names = [] for d, rr, m in tmp_dmf.find_related(r[0], meta=['name'], maxdepth=1): names.append(m['name']) names.sort() assert (names == ['r1'])
def test_relation_in_experiment(): tmp_dir = scratch_path / "relation_in_experiment" dmf = DMF(path=tmp_dir, create=True) e1 = experiment.Experiment(dmf, name="1") a = resource.Resource(value={"name": "foo"}) e1.add(a) assert len(a.v["relations"]) == 1 assert len(e1.v["relations"]) == 1
def test_property_data_resource(): r = resource.Resource() r.data = {"data": _propd, "meta": _propm} tbl = propdata.PropertyTable(data=r.data["data"], metadata=r.data["meta"]) assert tbl.data.properties[0] assert tbl.data.properties[0]["values"] == _propd[0]["values"] assert tbl.data.properties[0]["errors"] == _propd[0]["errors"] assert tbl.metadata[0].title == _propm["title"]
def add_resources(dmf_obj, num=3, **attrs): ids = [] for i in range(num): r = resource.Resource(value=attrs, type_="test") r.data = {"i": i} dmf_obj.add(r) ids.append(r.id) return ids
def test_property_data_resource(): r = resource.Resource() r.data = {'data': _propd, 'meta': _propm} tbl = propdata.PropertyTable(data=r.data['data'], metadata=r.data['meta']) assert tbl.data.properties[0] assert tbl.data.properties[0]['values'] == _propd[0]['values'] assert tbl.data.properties[0]['errors'] == _propd[0]['errors'] assert tbl.metadata[0].title == _propm['title']
def to_dmf(self, dmf): rsrc = resource.Resource(value={ 'name': 'convergence_results', 'desc': 'statistics returned from run_convergence_evaluation', 'creator': { 'name': getpass.getuser() }, 'data': stats.to_dict() }, type_=resource.ResourceTypes.data) dmf.add(rsrc)
def test_is_resource_json(): with TempDir() as d: f = open(os.path.join(d, "sample.txt"), "w") assert not util.is_resource_json(f.name) f = open(os.path.join(d, "sample.json"), "w") assert not util.is_resource_json(f.name) r = resource.Resource() json.dump(r.v, f) f.close() assert util.is_resource_json(f.name) assert not util.is_resource_json(f.name, max_bytes=1)
def test_find_propertydata(tmp_dmf): # populate DMF with some property data resources pj = prop_json[0] n = 10 for i in range(n): pd = resource.Resource(value={'data': pj}, type_=resource.TY_PROPERTY) tmp_dmf.add(pd) # get them back again filter_ = {'type': resource.TY_PROPERTY} pdata = list(tmp_dmf.find(filter_dict=filter_)) assert len(pdata) == n
def _add_data(self, data, dtype='', x_colnames=None, z_colname=None, metadata=None): """Add the data as a DMF resource. Args: data (propdata.PropertyData|DataFrame): Data values metadata (propdata.Metadata): If present, metadata dtype (str): Either 'input' or 'validation', the type of data x_colnames (List[str]): List of x (variable) column names z_colname (str): z (response) column name """ if isinstance(data, propdata.PropertyData): pdata = data.as_arr() else: # create property data from dataframe assert x_colnames is not None assert z_colname is not None pdata = [] for col in x_colnames: values = list(data[col]) pdata.append({ "name": "{} x:{}".format(dtype, col), "units": "", "values": values, "errors": [0] * len(values), "error_type": "absolute", "type": "property" }) values = list(data[z_colname]) pdata.append({ 'name': '{} z:{}'.format(dtype, z_colname), 'units': '', 'values': values, 'errors': [0] * len(values), 'error_type': 'absolute', 'type': 'property' }) # create resource r = resource.Resource(type_=resource.ResourceTypes.property) r.data = { 'data': pdata, 'meta': metadata.as_dict() if metadata else {} } r.v['aliases'].append(dtype) _log.debug('adding resource dtype={}'.format(dtype)) # add resource to experiment self._exp.add(r) # add link between resource and the surrogate-model resource self._exp.link(r, Predicates.uses, self._rsrc)
def test_property_data_file(tmp_dmf, tmp_propdata_file): tmpf, prop = tmp_propdata_file, tmp_propdata() # Add the resource r = resource.Resource(type_="property_data") r.v["datafiles"].append({"path": tmpf.name}) rid = tmp_dmf.add(r) assert rid is not None r2 = tmp_dmf.fetch_one(rid) path = r2.v["datafiles"][0]["path"] f2 = open(path, "r") j2 = json.load(f2) assert j2 == prop
def test_property_data_file(tmp_dmf, tmp_propdata_file): tmpf, prop = tmp_propdata_file, tmp_propdata() # Add the resource r = resource.Resource(type_='property_data') r.v['datafiles'].append({'path': tmpf.name}) rid = tmp_dmf.add(r) assert rid is not None r2 = tmp_dmf.fetch_one(rid) path = r2.v['datafiles'][0]['path'] f2 = open(path, 'r') j2 = json.load(f2) assert j2 == prop
def test_is_resource_json(): tmp_dir = scratch_path / "is_resource_json" tmp_dir.mkdir() f = (tmp_dir / "sample.txt").open("w") assert not util.is_resource_json(f.name) f = (tmp_dir / "sample.json").open("w") assert not util.is_resource_json(f.name) r = resource.Resource() json.dump(r.v, f) f.close() assert util.is_resource_json(f.name) assert not util.is_resource_json(f.name, max_bytes=1)
def example_resource(): r = resource.Resource() r.v["version_info"]["version"] = test_version r.v["collaborators"] = [ {"name": "Clark Kent", "email": "*****@*****.**"}, {"name": "Superman", "email": "*****@*****.**"}, ] r.v["datafiles"].append({"path": "/etc/passwd"}) r.v["aliases"] = ["test_resource_full"] r.v["tags"] = ["test", "resource"] # r.relations -- deal with this separately r.data = {"arbitrary": {"values": [1, 2, 3]}} return r
def test_find_propertydata(): # populate DMF with some property data resources pj = prop_json[0] n, resource_ids = 10, [] for i in range(n): pd = resource.Resource(value={"data": pj}, type_=resource.ResourceTypes.property) resource_ids.append(scratch_dmf.add(pd)) # get them back again filter_ = {"type": resource.ResourceTypes.property} pdata = list(scratch_dmf.find(filter_dict=filter_)) assert len(pdata) == n for rid in resource_ids: scratch_dmf.remove(identifier=rid)
def test_resource_roundtrip(example_resource): """Build up a resource with all attributes, then make sure it serializes and deserializes. """ r = example_resource # alias # make sure we can serialize it r_json = json.dumps(r.v, indent=2) # make sure we can deserialize its JSON r2_value = json.loads(r_json) # reconstruct from the deserialized JSON r2 = resource.Resource(value=r2_value) # compare them assert r2.v["version_info"] == r.v["version_info"] assert r2.data["arbitrary"]["values"][0] == 1
def test_dmf_add_filesystem_err(tmp_dmf): r = resource.Resource(value={'desc': 'test resource'}) # create datafile tmpf1 = tempfile.NamedTemporaryFile(delete=False) tmpf1.close() r.v['datafiles'].append({'path': tmpf1.name}) # now, to get an error, make the DMF datafile path unwritable path = os.path.join(tmp_dmf.root, tmp_dmf.datafile_dir) os.chmod(path, 000) # then try to add the resource, which includes copying the file into # the (now unwritable) directory pytest.raises(errors.DMFError, tmp_dmf.add, r) # make the directory writable again so we can remove it os.chmod(path, 0o777)
def test_dmf_add_tmp_no_unlink(): tmp_dir = Path(scratch_dir) / "dmf_add_tmp_no_unlink" dmf = DMF(path=tmp_dir, create=True) r = resource.Resource(value={"desc": "test resource"}) # create datafile, with temporary-file flag turned on tmp_file = (tmp_dir / "foo").open("w") r.v["datafiles"].append({ "path": str(tmp_file), "is_tmp": True, "do_copy": True }) # we want an error trying to COPY this file; to get this, # change the permissions of the directory os.chmod(tmp_dir, 0o500) pytest.raises(Exception, dmf.add, r)
def test_property_data_file(): tmp_propdata_file = (Path(scratch_dir) / "property_data_file.json").open("w") json.dump(prop_json[0], tmp_propdata_file) tmp_propdata_file.close() # Add the resource r = resource.Resource(type_="property_data") r.v["datafiles"].append({"path": tmp_propdata_file.name}) rid = scratch_dmf.add(r) assert rid is not None r2 = scratch_dmf.fetch_one(rid) path = r2.v["datafiles"][0]["path"] f2 = open(path, "r") j2 = json.load(f2) assert j2 == prop_json[0]
def to_dmf(self, dmf): # PYLINT-TODO-FIX fix error due to undefined variable "stats" rsrc = resource.Resource( value={ 'name': 'convergence_results', 'desc': 'statistics returned from run_convergence_evaluation', 'creator': { 'name': getpass.getuser() }, # pylint: disable=undefined-variable 'data': stats.to_dict() }, type_=resource.ResourceTypes.data) # pylint: enable=undefined-variable dmf.add(rsrc)
def test_relation_with_remove(tmp_dmf): e1 = experiment.Experiment(tmp_dmf, name="1") n, added = 10, [] for i in range(n): a = resource.Resource({"name": "foo"}) e1.add(a) added.append(a) assert len(e1.v["relations"]) == n # remove, then update e1 for a in added: tmp_dmf.remove(identifier=a.id) e1.update() # relation to removed 'a' should be gone n -= 1 assert (len(e1.v["relations"])) == n
def test_dmf_add_filesystem_err(): tmp_dir = Path(scratch_dir) / "dmf_add_filesystem_err" dmf = DMF(path=tmp_dir, create=True) r = resource.Resource(value={"desc": "test resource"}) # create datafile tmpf1 = NamedTemporaryFile(delete=False) tmpf1.close() r.v["datafiles"].append({"path": tmpf1.name}) # now, to get an error, make the DMF datafile path unwritable path = os.path.join(dmf.root, dmf.datafile_dir) # this file permissions stuff is problematic in Windows, so this test is Linux-only os.chmod(path, 000) # then try to add the resource, which includes copying the file into # the (now unwritable) directory pytest.raises(errors.DMFError, dmf.add, r) # make the directory writable again so we can remove it os.chmod(path, 0o777)
def test_dmf_add_tmp_no_unlink(tmp_dmf): r = resource.Resource(value={'desc': 'test resource'}) # create datafile, with temporary-file flag turned on tmpdir = tempfile.mkdtemp() tmpfile = os.path.join(tmpdir, 'foo') open(tmpfile, 'w') r.v['datafiles'].append({'path': tmpfile, 'is_tmp': True, 'do_copy': True}) # we want an error trying to UNLINK this file; to get this, # change the permissions of the dir read-only os.chmod(tmpdir, 0o500) try: tmp_dmf.add(r) finally: # change it back and clean up os.chmod(tmpdir, 0o700) os.unlink(tmpfile) os.rmdir(tmpdir)
def test_dmf_add_tmp_no_unlink(tmp_dmf): r = resource.Resource(value={"desc": "test resource"}) # create datafile, with temporary-file flag turned on tmpdir = tempfile.mkdtemp() tmpfile = os.path.join(tmpdir, "foo") open(tmpfile, "w") r.v["datafiles"].append({"path": tmpfile, "is_tmp": True, "do_copy": True}) # we want an error trying to UNLINK this file; to get this, # change the permissions of the dir read-only os.chmod(tmpdir, 0o500) try: tmp_dmf.add(r) finally: # change it back and clean up os.chmod(tmpdir, 0o700) os.unlink(tmpfile) os.rmdir(tmpdir)
def test_relation_with_remove(): tmp_dir = scratch_path / "relation_with_remove" dmf = DMF(path=tmp_dir, create=True) e1 = experiment.Experiment(dmf, name="1") n, added = 10, [] for i in range(n): a = resource.Resource({"name": "foo"}) e1.add(a) added.append(a) assert len(e1.v["relations"]) == n # remove, then update e1 for a in added: dmf.remove(identifier=a.id) e1.update() # relation to removed 'a' should be gone n -= 1 assert (len(e1.v["relations"])) == n
def test_dmf_add_tmp_no_copy(): tmp_dir = Path(scratch_dir) / "dmf_add_tmp_no_copy" dmf = DMF(path=tmp_dir, create=True) r = resource.Resource(value={"desc": "test resource"}) # create datafile, with temporary-file flag turned on tmp_file = (tmp_dir / "foo").open("w") r.v["datafiles"].append({ "path": str(tmp_file), "is_tmp": True, "do_copy": True }) # we want an error trying to COPY this file; to get this, # change the permissions of the directory os.chmod(tmp_dir, 0o400) ok = False try: dmf.add(r) except errors.DMFError: ok = True if not ok: assert False, "DMFError expected"
def test_add_property_data(tmp_dmf, tmp_propdata_file): tmpf, prop = tmp_propdata_file, tmp_propdata() # Add the resource r = resource.Resource(type_='property_data') r.set_id() r.v['creator'] = {'name': 'Dan Gunter'} m = prop['meta'] work = '{authors}, "{title}". {info}, {date}'.format(**m) r.v['sources'].append({'source': work, 'date': m['date']}) r.data = {'notes': m['notes']} r.v['tags'].append('MEA') r.v['datafiles'].append({'path': tmpf.name}) rid = tmp_dmf.add(r) assert rid is not None # Retrieve the resource r2 = tmp_dmf.fetch_one(rid) # Validate the resource assert r2.type == 'property_data' assert 'MEA' in r2.v['tags'] # Remove the resource tmp_dmf.remove(identifier=rid)
def test_add_property_data(tmp_dmf, tmp_propdata_file): tmpf, prop = tmp_propdata_file, tmp_propdata() # Add the resource r = resource.Resource(type_="property_data") r.set_id() r.v["creator"] = {"name": "Dan Gunter"} m = prop["meta"] work = '{authors}, "{title}". {info}, {date}'.format(**m) r.v["sources"].append({"source": work, "date": m["date"]}) r.data = {"notes": m["notes"]} r.v["tags"].append("MEA") r.v["datafiles"].append({"path": tmpf.name}) rid = tmp_dmf.add(r) assert rid is not None # Retrieve the resource r2 = tmp_dmf.fetch_one(rid) # Validate the resource assert r2.type == "property_data" assert "MEA" in r2.v["tags"] # Remove the resource tmp_dmf.remove(identifier=rid)
def test_find_related(): # # r0 # | uses # v # r1 # | version # v # r2 # /\ # / \ derived # v v # r3 r4 # tmp_dir = scratch_path / "find_related" dmf = DMF(path=tmp_dir, create=True) r = [resource.Resource({"name": "r{}".format(i)}) for i in range(5)] # r3 <-- derived <-- r2 <-- version <-- r1 cr = resource.create_relation # shortcut cr(r[2], Predicates.derived, r[3]) cr(r[1], Predicates.version, r[2]) # r4 <-- derived <-- r2 cr(r[2], Predicates.derived, r[4]) # r0 -- Uses --> r1 cr(r[0], Predicates.uses, r[1]) # add to dmf for i in range(5): dmf.add(r[i]) # outgoing from r0 should include 1,2,3,4 names = [] for d, rr, m in dmf.find_related(r[0], meta=["aliases"]): names.append(m["aliases"][0]) names.sort() assert names == ["r1", "r2", "r3", "r4"] # incoming to r4 should include r0, r1, r2 names = [] for d, rr, m in dmf.find_related(r[4], meta=["aliases"], outgoing=False): names.append(m["aliases"][0]) names.sort() assert names == ["r0", "r1", "r2"]
def test_dmf_add_tmp_no_copy(tmp_dmf): r = resource.Resource(value={'desc': 'test resource'}) # create datafile, with temporary-file flag turned on tmpdir = tempfile.mkdtemp() tmpfile = os.path.join(tmpdir, 'foo') open(tmpfile, 'w') r.v['datafiles'].append({'path': tmpfile, 'is_tmp': True, 'do_copy': True}) # we want an error trying to COPY this file; to get this, # change the permissions of the directory os.chmod(tmpdir, 0o400) ok = False try: tmp_dmf.add(r) except errors.DMFError: ok = True finally: # change it back and clean up os.chmod(tmpdir, 0o700) os.unlink(tmpfile) os.rmdir(tmpdir) if not ok: assert False, "DMFError expected"
def test_find_related(tmp_dmf): # # r0 # | uses # v # r1 # | version # v # r2 # /\ # / \ derived # v v # r3 r4 # r = [resource.Resource({"name": "r{}".format(i)}) for i in range(5)] # r3 <-- derived <-- r2 <-- version <-- r1 cr = resource.create_relation_args # shortcut cr(r[2], resource.PR_DERIVED, r[3]) cr(r[1], resource.PR_VERSION, r[2]) # r4 <-- derived <-- r2 cr(r[2], resource.PR_DERIVED, r[4]) # r0 -- Uses --> r1 cr(r[0], resource.PR_USES, r[1]) # add to dmf for i in range(5): tmp_dmf.add(r[i]) # outgoing from r0 should include 1,2,3,4 names = [] for d, rr, m in tmp_dmf.find_related(r[0], meta=["name"]): names.append(m["name"]) names.sort() assert names == ["r1", "r2", "r3", "r4"] # incoming to r4 should include r0, r1, r2 names = [] for d, rr, m in tmp_dmf.find_related(r[4], meta=["name"], outgoing=False): names.append(m["name"]) names.sort() assert names == ["r0", "r1", "r2"]