def test_dmf_related(dmf_context, runner):
create_foo_workspace(runner)
# add the fully-connected 4 resources
dmf = DMF()
rlist = [
resource.Resource(value={
"desc": ltr,
"aliases": [ltr],
"tags": ["graph"]
}) for ltr in "ABCD"
]
A_id = rlist[0].id # root resource id, used in testcode
relation = resource.Predicates.uses
for r in rlist:
for r2 in rlist:
if r is r2:
continue
resource.create_relation(r, relation, r2)
for r in rlist:
dmf.add(r)
#
result = runner.invoke(related, [A_id, "--no-unicode", "--no-color"],
catch_exceptions=False)
assert result.exit_code == 0
rlines = result.output.split("\n")
nrelations = sum(
1 for _ in filter(lambda s: resource.Predicates.uses in s, rlines))
assert nrelations == 12 # 3 blocks of (1 + 3)
def test_dmf_change_traits():
with TempDir() as tmpdir:
open(os.path.join(tmpdir, DMF.WORKSPACE_CONFIG), "w").write(
"_id: this-is-a-temporary-config"
)
d = DMF(path=tmpdir, name="my workspace", desc="It is a great place to work")
assert d.db_file
d.db_file = "newdb.json"
assert d.db_file == "newdb.json"
def test_dmf_change_traits():
with TempDir() as tmpdir:
open(os.path.join(tmpdir, DMF.WORKSPACE_CONFIG),
'w').write('_id: this-is-a-temporary-config')
d = DMF(path=tmpdir,
name='my workspace',
desc='It is a great place to work')
assert d.db_file
d.db_file = 'newdb.json'
assert d.db_file == 'newdb.json'
def test_dmf_change_traits():
tmp_dir = Path(scratch_dir) / "dmf_change_traits"
tmp_dir.mkdir()
(tmp_dir /
DMF.WORKSPACE_CONFIG).open("w").write("_id: this-is-a-temporary-config")
d = DMF(path=tmp_dir,
name="my workspace",
desc="It is a great place to work")
assert d.db_file
d.db_file = "newdb.json"
assert d.db_file == "newdb.json"
def test_dmf_update_newtype():
tmp_dir = Path(scratch_dir) / "dmf_update_newtype"
dmf = DMF(path=tmp_dir, create=True)
ids = add_resources(dmf, 1)
r1 = dmf.fetch_one(ids[0])
r1.v[r1.TYPE_FIELD] = "this type is different"
try:
dmf.update(r1)
except errors.DMFError:
pass
else:
assert False, "DMFError expected for update() with new type"
def test_dmf_init_workspace_name():
with TempDir() as tmpdir:
open(os.path.join(tmpdir, DMF.WORKSPACE_CONFIG),
'w').write('_id: this-is-a-temporary-config')
d = DMF(path=tmpdir,
name='my workspace',
desc='It is a great place to work')
def test_dmf_init_logconf():
with TempDir() as tmpdir:
open(os.path.join(tmpdir, DMF.WORKSPACE_CONFIG), "w").write(
"""
_id: this-is-a-temporary-config
logging:
idaes.dmf.dmfbase:
level: debug
output: _stderr_
root:
output: _stdout_
dmf:
output: _stdout_
.dmf.experiment:
output: _stdout_
# equivalent to previous
idaes.dmf.experiment:
output: /tmp/experiment.log
# user
crazy.little.logger:
level: error
output: _stderr_
"""
)
d = DMF(path=tmpdir)
def test_dmf_add_tmp_no_copy():
tmp_dir = Path(scratch_dir) / "dmf_add_tmp_no_copy"
dmf = DMF(path=tmp_dir, create=True)
r = resource.Resource(value={"desc": "test resource"})
# create datafile, with temporary-file flag turned on
tmp_file = (tmp_dir / "foo").open("w")
r.v["datafiles"].append({
"path": str(tmp_file),
"is_tmp": True,
"do_copy": True
})
# we want an error trying to COPY this file; to get this,
# change the permissions of the directory
os.chmod(tmp_dir, 0o400)
ok = False
try:
dmf.add(r)
except errors.DMFError:
ok = True
if not ok:
assert False, "DMFError expected"
def test_dmf_find():
tmp_dir = Path(scratch_dir) / "dmf_find"
dmf = DMF(path=tmp_dir, create=True)
# populate with batches of records
# they all have the tag 'all', each batch has 'batch<N>' as well
# All resources in a batch are given version 1.0.<N>
# Individual resources will have data of {i: 0..<batchsz-1>}
batchsz, numbatches = 10, 9
all_ids = []
for i in range(numbatches):
n = batchsz
batch = "batch{:d}".format(i + 1)
version = resource.version_list([1, 0, i + 1])
ids = add_resources(dmf,
num=n,
tags=["all", batch],
version_info={"version": version})
all_ids.extend(ids)
if _log.isEnabledFor(logging.DEBUG):
r = dmf.fetch_one(all_ids[0])
_log.debug("First resource:\n{}".format(r))
# Find all records, 2 ways
total_num = batchsz * numbatches
result = list(dmf.find())
assert len(result) == total_num
result = list(dmf.find({"tags": ["all"]}))
assert len(result) == total_num
# Find with 'all'
result = list(dmf.find({"tags!": ["all", "batch1"]}))
assert len(result) == batchsz
def test_dmf_add():
tmp_dir = Path(scratch_dir) / "dmf_add"
dmf = DMF(path=tmp_dir, create=True)
r = resource.Resource(value={"desc": "test resource"})
r.do_copy = True # copy by default
# (1) Copy, and don't remove {default behavior}
tmpf1 = NamedTemporaryFile(delete=False)
tmpf1.close()
r.v["datafiles"].append({"path": tmpf1.name})
# (2) Copy, and remove original
tmpf2 = NamedTemporaryFile(delete=False)
tmpf2.close()
r.v["datafiles"].append({"path": tmpf2.name, "is_tmp": True})
# (3) Do not copy (or remove)
tmpf3 = NamedTemporaryFile()
r.v["datafiles"].append({"path": tmpf3.name, "do_copy": False})
dmf.add(r)
os.unlink(tmpf1.name)
try:
os.unlink(tmpf2.name)
assert False, "Expected error"
except Exception as err:
pass
os.unlink(tmpf3.name)
# This is ignored. It makes no sense to ask the file
# to be removed, but not copied (just a file delete?!)
r = resource.Resource(value={"desc": "test resource"})
r.v["datafiles"].append({"path": "foo", "do_copy": False, "is_tmp": True})
dmf.add(r)
def test_dmf_update():
tmp_dir = Path(scratch_dir) / "dmf_update"
dmf = DMF(path=tmp_dir, create=True)
ids = add_resources(dmf, 2)
r1 = dmf.fetch_one(ids[0])
r1.v[r1.TYPE_FIELD] = "test"
r1.v["desc"] = "Updated description"
dmf.update(r1)
r1b = dmf.fetch_one(ids[0])
assert r1b.v["desc"] == "Updated description"
r2 = dmf.fetch_one(ids[1])
assert r2.v["desc"] != "Updated description"
def test_dmf_add_tmp_no_unlink():
tmp_dir = Path(scratch_dir) / "dmf_add_tmp_no_unlink"
dmf = DMF(path=tmp_dir, create=True)
r = resource.Resource(value={"desc": "test resource"})
# create datafile, with temporary-file flag turned on
tmp_file = (tmp_dir / "foo").open("w")
r.v["datafiles"].append({
"path": str(tmp_file),
"is_tmp": True,
"do_copy": True
})
# we want an error trying to COPY this file; to get this,
# change the permissions of the directory
os.chmod(tmp_dir, 0o500)
pytest.raises(Exception, dmf.add, r)
def test_dmf_remove():
tmp_dir = Path(scratch_dir) / "dmf_remove"
dmf = DMF(path=tmp_dir, create=True)
n = 10
ids = add_resources(dmf, num=n)
assert dmf.count() == n
while n > 0:
n = n - 1
dmf.remove(ids[n])
assert dmf.count() == n
def test_dmf_add_filesystem_err():
tmp_dir = Path(scratch_dir) / "dmf_add_filesystem_err"
dmf = DMF(path=tmp_dir, create=True)
r = resource.Resource(value={"desc": "test resource"})
# create datafile
tmpf1 = NamedTemporaryFile(delete=False)
tmpf1.close()
r.v["datafiles"].append({"path": tmpf1.name})
# now, to get an error, make the DMF datafile path unwritable
path = os.path.join(dmf.root, dmf.datafile_dir)
# this file permissions stuff is problematic in Windows, so this test is Linux-only
os.chmod(path, 000)
# then try to add the resource, which includes copying the file into
# the (now unwritable) directory
pytest.raises(errors.DMFError, dmf.add, r)
# make the directory writable again so we can remove it
os.chmod(path, 0o777)
def test_dmf_init_logconf():
tmp_dir = Path(scratch_dir) / "dmf_init_logconf"
tmp_dir.mkdir()
(tmp_dir /
DMF.WORKSPACE_CONFIG).open("w").write(f"""_id: this-is-a-temporary-config
logging:
idaes.dmf.dmfbase:
level: debug
output: _stderr_
root:
output: _stdout_
dmf:
output: _stdout_
.dmf.experiment:
output: _stdout_
# equivalent to previous
idaes.dmf.experiment:
output: {tmp_dir / "experiment.log"}
# user
crazy.little.logger:
level: error
output: _stderr_
""")
DMF(path=str(tmp_dir))
def test_dmf_remove_filter():
tmp_dir = Path(scratch_dir) / "dmf_remove_filter"
dmf = DMF(path=tmp_dir, create=True)
n = 10
ids = add_resources(dmf, num=n)
assert dmf.count() == n
# remove half of the added resources
# print("@@ remove half")
dmf.remove(filter_dict={"data.i": {"$lt": n / 2}})
n2 = dmf.count()
assert n2 == n / 2
# try to remove the same group (should do nothing
# print("@@ remove more")
dmf.remove(filter_dict={"data.i": {"$lt": n / 2}})
n2 = dmf.count()
assert dmf.count() == n / 2
# remove the rest
# print("@@ remove the rest")
dmf.remove(filter_dict={"data.i": {"$ge": n / 2}})
assert dmf.count() == 0
def test_dmf_init_workspace_name():
tmp_dir = Path(scratch_dir) / "dmf_init_workspace_name"
tmp_dir.mkdir()
(tmp_dir /
DMF.WORKSPACE_CONFIG).open("w").write("_id: this-is-a-temporary-config")
d = DMF(path=tmp_dir, name="my workspace", desc="It is a space to work")
def test_dmf_init_workspace_name():
with TempDir() as tmpdir:
open(os.path.join(tmpdir, DMF.WORKSPACE_CONFIG), "w").write(
"_id: this-is-a-temporary-config"
)
d = DMF(path=tmpdir, name="my workspace", desc="It is a great place to work")
def test_dmf_str():
tmp_dir = Path(scratch_dir) / "dmf_str"
dmf = DMF(path=tmp_dir, create=True)
s = str(dmf)
assert len(s) > 0
def setup_module(module):
global scratch_dir, scratch_dmf
scratch_dir = TemporaryDirectory(prefix="idaes_dmf_").name
scratch_dmf = DMF(path=scratch_dir, create=True)
def test_dmf_add_duplicate():
tmp_dir = Path(scratch_dir) / "dmf_add_duplicate"
dmf = DMF(path=tmp_dir, create=True)
r = resource.Resource(value={"desc": "test resource"})
dmf.add(r)
pytest.raises(errors.DuplicateResourceError, dmf.add, r)
