def test_wrap_hdf(self):
"""DataContainer should be able to be initialized by HDF objects."""
h = self.project.create_hdf(self.project.path, "wrap_test")
h["foo"] = 42
h.create_group("bar")["test"] = 23
h["bar"].create_group("nested")["test"] = 23
d = DataContainer(h)
self.assertTrue(isinstance(d.bar, DataContainer),
"HDF group not wrapped from ProjectHDFio.")
self.assertTrue(isinstance(d.bar.nested, DataContainer),
"Nested HDF group not wrapped from ProjectHDFio.")
self.assertEqual(
d.foo, 42,
"Top-level node not correctly wrapped from ProjectHDFio.")
self.assertEqual(
d.bar.test, 23,
"Nested node not correctly wrapped from ProjectHDFio.")
self.assertEqual(
d.bar.nested.test, 23,
"Nested node not correctly wrapped from ProjectHDFio.")
h = h5py.File(h.file_name)
d = DataContainer(h)
self.assertTrue(isinstance(d.wrap_test.bar, DataContainer),
"HDF group not wrapped from h5py.File.")
self.assertTrue(isinstance(d.wrap_test.bar.nested, DataContainer),
"Nested HDF group not wrapped from h5py.File.")
self.assertEqual(
d.wrap_test.foo, h["wrap_test/foo"],
"Top-level node not correctly wrapped from h5py.File.")
self.assertEqual(d.wrap_test.bar.test, h["wrap_test/bar/test"],
"Nested node not correctly wrapped from h5py.File.")
self.assertEqual(d.wrap_test.bar.nested.test,
h["wrap_test/bar/nested/test"],
"Nested node not correctly wrapped from h5py.File.")
def test_init_dict(self):
d = {"foo": 23, "test case": "bar"}
pl = DataContainer(d)
self.assertEqual(tuple(pl.items()), tuple(d.items()),
"source dict items not preserved")
with self.assertRaises(ValueError,
msg="no ValueError on invalid initializer"):
DataContainer({2: 0, 1: 1})
def setUpClass(cls):
super().setUpClass()
cls.pl = DataContainer([{
"foo": "bar"
}, 2, 42, {
"next": [0, {
"depth": 23
}]
}],
table_name="input")
cls.pl["tail"] = DataContainer([2, 4, 8])
cls.hdf = cls.project.create_hdf(cls.project.path, "test")
def setUpClass(cls):
super().setUpClass()
cls.hdf = cls.project.create_hdf(cls.project.path, "hdf")
cls.hdf["number"] = 42
cls.hdf["array"] = np.arange(100)
cls.data = DataContainer([1, 2, "three", 4.0])
cls.data.to_hdf(cls.hdf, "data")
def test_hdf_pandas(self):
"""Values that implement to_hdf/from_hdf, should write themselves to the HDF file correctly."""
pl = DataContainer(table_name="pandas")
pl.append(pd.DataFrame({"a": [1, 2], "b": ["x", "y"]}))
pl.to_hdf(hdf=self.hdf)
pl2 = self.hdf["pandas"].to_object()
self.assertEqual(type(pl[0]), type(pl2[0]))
def test_update_blacklist(self):
"""Wrapping nested mapping should only apply to types not in the blacklist."""
pl = DataContainer()
pl.update([{
"a": 1,
"b": 2
}, [{
"c": 3,
"d": 4
}]],
wrap=True,
blacklist=(dict, ))
self.assertTrue(isinstance(pl[0], dict),
"nested dict wrapped, even if black listed")
self.assertTrue(isinstance(pl[1][0], dict),
"nested dict wrapped, even if black listed")
pl.clear()
pl.update({
"a": [1, 2, 3],
"b": {
"c": [4, 5, 6]
}
},
wrap=True,
blacklist=(list, ))
self.assertTrue(isinstance(pl.a, list),
"nested list wrapped, even if black listed")
self.assertTrue(isinstance(pl.b.c, list),
"nested list wrapped, even if black listed")
pl.clear()
def test_init_tuple(self):
t = (1, 2, 3, 4)
pl = DataContainer(t)
self.assertEqual(len(pl), len(t),
"not the same length as source tuple")
self.assertEqual(tuple(pl.values()), t,
"conversion to tuple not the same as source tuple")
def __init__(self, project, job_name):
super(ScriptJob, self).__init__(project, job_name)
self.__version__ = "0.1"
self.__hdf_version__ = "0.2.0"
self.__name__ = "Script"
self._script_path = None
self.input = DataContainer(table_name="custom_dict")
def store_custom_output_dict(output_dict):
folder = Path(".").cwd().parts[-1]
hdf_file = Path(".").cwd().parents[1] / folder
hdf_file = str(hdf_file) + ".h5"
hdf = FileHDFio(hdf_file)
hdf[folder].create_group("output")
obj = DataContainer(output_dict)
obj.to_hdf(hdf[folder + "/output"])
def test_get_nested(self):
n = [{"foo": "bar"}, 2, 42, {"next": [0, {"depth": 23}]}]
pl = DataContainer(n)
self.assertEqual(type(pl[0]), DataContainer,
"nested dict not converted to DataContainer")
self.assertEqual(type(pl["3/next"]), DataContainer,
"nested list not converted to DataContainer")
self.assertEqual(type(pl["0/foo"]), str,
"nested str converted to DataContainer")
def test_read_write_consistency(self):
"""Writing a datacontainer, then reading it back in, should leave it unchanged."""
fn = "pl.yml"
self.pl.write(fn)
pl = DataContainer()
pl.read(fn)
self.assertEqual(
self.pl, pl,
"Read container from yaml, is not the same as written.")
os.remove(fn)
def test_hdf_empty_group(self):
"""Writing a list without table_name or group_name should only work if the HDF group is empty."""
l = DataContainer([1, 2, 3])
with self.assertRaises(
ValueError,
msg="No exception when writing to full hdf group."):
l.to_hdf(self.hdf)
h = self.hdf.create_group("empty_group")
l.to_hdf(h)
self.assertEqual(l, h.to_object())
def test_subclass_preservation(self):
self.pl.subclass = Sub(table_name='subclass')
self.pl.to_hdf(hdf=self.hdf)
loaded = DataContainer(table_name="input")
loaded.from_hdf(hdf=self.hdf)
self.assertIsInstance(
loaded.subclass, Sub, f"Subclass not preserved on loading. "
f"Expected {Sub.__name__} but got {type(loaded.subclass).__name__}."
)
self.pl.pop('subclass')
def test_mark(self):
pl = DataContainer([1, 2, 3])
pl.mark(1, "foo")
self.assertEqual(pl[1], pl.foo,
"marked element does not refer to correct element")
pl.mark(2, "foo")
self.assertEqual(pl[2], pl.foo, "marking with existing key broken")
with self.assertRaises(IndexError,
msg="no IndexError on invalid index"):
pl.mark(10, "foo")
def test_insert(self):
pl = DataContainer([1, 2, 3])
pl.insert(1, 42, key="foo")
self.assertTrue(pl[0] == 1 and pl[1] == 42 and pl[2] == 2,
"insert does not properly set value")
pl.insert(1, 24, key="bar")
self.assertTrue(pl[0] == 1 and pl.bar == 24 and pl.foo == 42,
"insert does not properly update keys")
pl.insert(10, 4)
self.assertEqual(pl[-1], 4,
"insert does not handle out of bounds gracefully")
def test_del_item(self):
pl = DataContainer({0: 1, "a": 2, "foo": 3})
with self.assertRaises(ValueError,
msg="no ValueError on invalid index type"):
del pl[{}]
del pl["a"]
self.assertTrue("a" not in pl, "delitem does not delete with str key")
del pl[0]
self.assertTrue(pl[0] != 1, "delitem does not delete with index")
def test_from_hdf_readonly(self):
"""Reading from HDF should restore the read-only property."""
self.pl.to_hdf(hdf=self.hdf, group_name="read_only_from")
pl = DataContainer()
pl.from_hdf(self.hdf, group_name="read_only_from")
self.assertEqual(pl.read_only, self.hdf["read_only_from/READ_ONLY"],
"read-only parameter not correctly read from HDF")
self.hdf["read_only_from/READ_ONLY"] = True
pl.from_hdf(self.hdf, group_name="read_only_from")
self.assertEqual(pl.read_only, self.hdf["read_only_from/READ_ONLY"],
"read-only parameter not correctly read from HDF")
def test_hdf_complex_members(self):
"""Values that implement to_hdf/from_hdf, should write themselves to the HDF file correctly."""
pl = DataContainer(table_name="complex")
pl.append(
self.project.create_job(self.project.job_type.ScriptJob, "dummy1"))
pl.append(
self.project.create_job(self.project.job_type.ScriptJob, "dummy2"))
pl.append(42)
pl["foo"] = "bar"
pl.to_hdf(hdf=self.hdf)
pl2 = self.hdf["complex"].to_object()
self.assertEqual(type(pl[0]), type(pl2[0]))
self.assertEqual(type(pl[1]), type(pl2[1]))
def test_hdf_no_wrap(self):
"""Nested mappings should not be wrapped as DataContainer after reading."""
l = DataContainer(table_name="mappings")
l.append({"foo": "bar"})
l.append([1, 2, 3])
l.to_hdf(self.hdf)
m = l.copy()
m.from_hdf(self.hdf, group_name="mappings")
self.assertEqual(l, m,
"List with nested mappings not restored from HDF.")
self.assertTrue(isinstance(m[0], dict),
"dicts wrapped after reading from HDF.")
self.assertTrue(isinstance(m[1], list),
"lists wrapped after reading from HDF.")
def test_set_append(self):
pl = DataContainer()
# should not raise and exception
pl[0] = 1
pl[1] = 2
self.assertEqual(pl[0], 1, "append via index broken on empty list")
self.assertEqual(pl[1], 2, "append via index broken on non-empty list")
pl.append([])
self.assertTrue(
isinstance(pl[-1], list),
"append wraps sequences as DataContainer, but should not")
pl.append({})
self.assertTrue(
isinstance(pl[-1], dict),
"append wraps mappings as DataContainer, but should not")
def test_to_hdf_type(self):
"""Should write correct type information."""
self.pl.to_hdf(hdf=self.hdf)
self.assertEqual(self.hdf["input/NAME"], "DataContainer")
self.assertEqual(self.hdf["input/OBJECT"], "DataContainer")
self.assertEqual(
self.hdf["input/TYPE"],
"<class 'pyiron_base.generic.datacontainer.DataContainer'>")
h = self.hdf.open('nested')
pl = DataContainer(self.pl)
pl.to_hdf(hdf=h)
self.assertEqual(h["NAME"], "DataContainer")
self.assertEqual(h["OBJECT"], "DataContainer")
self.assertEqual(
h["TYPE"],
"<class 'pyiron_base.generic.datacontainer.DataContainer'>")
def test_create_group(self):
"""create_group should not erase existing groups."""
cont = DataContainer()
sub1 = cont.create_group("sub")
self.assertTrue(isinstance(sub1, DataContainer),
"create_group doesn't return DataContainer")
sub1.foo = 42
sub2 = cont.create_group("sub")
self.assertEqual(sub1.foo, sub2.foo,
"create_group overwrites existing data.")
self.assertTrue(
sub1 is sub2,
"create_group return new DataContainer group instead of existing one."
)
with self.assertRaises(
ValueError, msg="No ValueError on existing data in Container"):
sub1.create_group("foo")
def test_from_hdf_readonly(self):
"""Reading from HDF should restore the read-only property."""
self.pl.to_hdf(hdf=self.hdf, group_name="read_only_from")
pl = DataContainer()
pl.from_hdf(self.hdf, group_name="read_only_from")
self.assertEqual(pl.read_only, self.hdf["read_only_from/READ_ONLY"],
"read-only parameter not correctly read from HDF")
self.hdf["read_only_from/READ_ONLY"] = True
with warnings.catch_warnings(record=True) as w:
pl.from_hdf(self.hdf, group_name="read_only_from")
self.assertEqual(
len(w), 0,
"from_hdf on read_only DataContainer should not call _read_only_error."
)
self.assertEqual(pl.read_only, self.hdf["read_only_from/READ_ONLY"],
"read-only parameter not correctly read from HDF")
def get_custom_dict():
folder = Path(".").cwd().parts[-1]
project_folder = Path(".").cwd().parents[1]
hdf_file = project_folder / folder
hdf_file = str(hdf_file).replace("\\", "/") + ".h5"
if Path(hdf_file).exists():
obj = DataContainer()
obj.from_hdf(
hdf=FileHDFio(hdf_file), group_name=folder + "/input/custom_dict"
)
obj["project_dir"] = str(project_folder)
return obj
elif Path("input.json").exists():
with open("input.json") as f:
return json.load(f)
else:
warnings.warn("{} not found".format(hdf_file))
return None
def test_update(self):
pl = DataContainer()
d = self.pl.to_builtin()
pl.update(d, wrap=True)
self.assertEqual(pl, self.pl,
"update from to_builtin does not restore list")
with self.assertRaises(ValueError,
msg="no ValueError on invalid initializer"):
pl.update("asdf")
pl = self.pl.copy()
pl.update({}, pyiron="yes", test="case")
self.assertEqual((pl.pyiron, pl.test), ("yes", "case"),
"update via kwargs does not set values")
pl.clear()
d = {"a": 0, "b": 1, "c": 2}
pl.update(d)
self.assertEqual(
dict(pl), d, "update without options does not call generic method")
def test_from_hdf_group(self):
"""Reading from HDF should give back the same list as written even with custom group name."""
self.pl.to_hdf(hdf=self.hdf, group_name="test_group")
l = DataContainer(table_name="input")
l.from_hdf(hdf=self.hdf, group_name="test_group")
self.assertEqual(self.pl, l)
def test_init_set(self):
s = {1, 2, 3, 4}
pl = DataContainer(s)
self.assertEqual(len(pl), len(s), "not the same length as source set")
self.assertEqual(set(pl.values()), s,
"conversion to set not the same as source set")
def test_init_list(self):
l = [1, 2, 3, 4]
pl = DataContainer(l)
self.assertEqual(len(pl), len(l), "not the same length as source list")
self.assertEqual(list(pl.values()), l,
"conversion to list not the same as source list")
def test_hdf_empty_list(self):
"""Writing and reading an empty list should work."""
l = DataContainer(table_name="empty_list")
l.to_hdf(self.hdf)
l.from_hdf(self.hdf)
self.assertEqual(len(l), 0, "Empty list read from HDF not empty.")
def test_recursive_append(self):
input_tmp = DataContainer()
input_tmp['some/argument/inside/another/argument'] = 3
self.assertEqual(input_tmp['some/argument/inside/another/argument'], 3)
self.assertEqual(input_tmp.some.argument.inside.another.argument, 3)
self.assertEqual(type(input_tmp.some), DataContainer)