def test_hdf_chunklength_one(self):
"""Reading a storage with all chunks of length one should give back exactly what was written!"""
# Regression test if all stored chunks are of length 1: there used to be a bug that read all arrays as per
# element in this case
store = FlattenedStorage()
store.add_array('foo', dtype=np.int64, shape=(), per="element")
store.add_array('bar', dtype=np.int64, shape=(), per="chunk")
for i in range(5):
store.add_chunk(1, foo=i, bar=i**2)
hdf = self.project.create_hdf(self.project.path, "test")
store.to_hdf(hdf)
read = FlattenedStorage()
try:
read.from_hdf(hdf)
except RuntimeError as e:
self.fail(f"Reading storage from HDF failed with {e}")
self.assertEqual(len(store), len(read),
"Length not equal after reading from HDF!")
for i in range(5):
store_foo = store.get_array("foo", i)
read_foo = read.get_array("foo", i)
self.assertTrue(
np.array_equal(store_foo, read_foo),
f"per element values not equal after reading from HDF! {store_foo} != {read_foo}"
)
self.assertEqual(
store.get_array("bar", i), read.get_array("bar", i),
"per chunk values not equal after reading from HDF!")
def test_init(self):
"""Adding arrays via __init__ should be equivalent to adding them via add_chunks manually."""
store = FlattenedStorage(even=self.even, odd=self.odd)
self.assertEqual(
len(store), 3,
"Length of storage doesn't match length of initializer!")
self.assertTrue((store.get_array("even", 1) == np.array([2, 4])).all(),
"Values added via init don't match expected values!")
self.assertTrue((store.get_array("odd", 2) == np.array([7, 9,
11])).all(),
"Values added via init don't match expected values!")
all_sum = [sum(e + o) for e, o in zip(self.even, self.odd)]
try:
FlattenedStorage(even=self.even, odd=self.odd, sum=all_sum)
except ValueError:
self.fail(
"Adding per chunk values to initializers raises error, but shouldn't!"
)
with self.assertRaises(ValueError,
msg="No error on inconsistent initializers!"):
odd = self.odd.copy()
odd[1] = [1, 3, 4]
FlattenedStorage(even=self.even, odd=odd)
with self.assertRaises(
ValueError,
msg="No error on initializers of different length!"):
FlattenedStorage(foo=[[1]], bar=[[2], [2, 3]])
def test_get_array_full(self):
"""get_array should return full array for all chunks if not given frame."""
store = FlattenedStorage(elem=[[1], [2, 3], [4, 5, 6]],
chunk=[-1, -2, -3])
elem = store.get_array("elem")
self.assertTrue(
np.array_equal(elem, [1, 2, 3, 4, 5, 6]),
f"get_array return did not return correct flat array, but {elem}.")
chunk = store.get_array("chunk")
self.assertTrue(
np.array_equal(chunk, [-1, -2, -3]),
f"get_array return did not return correct flat array, but {chunk}."
)
def test_add_chunk_add_array(self):
"""Adding arrays via add_chunk and add_array should be equivalent."""
cont = FlattenedStorage()
cont.add_array("perchunk", shape=(3, ), dtype=float, per="chunk")
val = np.array([1, 2, 3])
try:
cont.add_chunk(1, perchunk=val[np.newaxis, :])
except ValueError:
# both checks below are regression tests for https://github.com/pyiron/pyiron_contrib/pull/197
self.fail(
"add_chunk should not raise an exception when passed a value for an existing per-chunk array."
)
self.assertTrue(
np.array_equal(val, cont.get_array("perchunk", 0)),
"add_chunk did not remove first axis on a per chunk array!")
# test the same, but now let the array be created by add_chunk, instead of doing it on our own
cont.add_chunk(2, perelem=[1, 1], perchunk2=val[np.newaxis, :])
self.assertTrue(
np.array_equal(val, cont.get_array("perchunk2", 1)),
"add_chunk did not remove first axis on a per chunk array!")
def test_get_array(self):
"""get_array should return the arrays for the correct structures."""
store = FlattenedStorage()
for n, e, o in zip(("first", None, "third"), self.even, self.odd):
store.add_chunk(len(e),
identifier=n,
even=e,
odd=o,
sum=sum(e + o))
self.assertTrue(
np.array_equal(store.get_array("even", 0), self.even[0]),
"get_array returns wrong array for numeric index!")
self.assertTrue(
np.array_equal(store.get_array("even", "first"), self.even[0]),
"get_array returns wrong array for string identifier!")
self.assertTrue(
np.array_equal(store.get_array("even", "1"), self.even[1]),
"get_array returns wrong array for automatic identifier!")
self.assertTrue(
np.array_equal(store.get_array("sum", 0),
sum(self.even[0] + self.odd[0])),
"get_array returns wrong array for numeric index!")
self.assertTrue(
np.array_equal(store.get_array("sum", "first"),
sum(self.even[0] + self.odd[0])),
"get_array returns wrong array for string identifier!")
self.assertTrue(
np.array_equal(store.get_array("sum", "1"),
sum(self.even[1] + self.odd[1])),
"get_array returns wrong array for automatic identifier!")
with self.assertRaises(KeyError, msg="Non-existing identifier!"):
store.get_array("even", "foo")
def test_get_array_ragged(self):
"""get_array_ragged should return a raggend array of all elements in the storage."""
store = FlattenedStorage(elem=[[1], [2, 3], [4, 5, 6]],
chunk=[-1, -2, -3])
val = store.get_array_ragged("elem")
self.assertEqual(val.shape, (3, ), "shape not correct!")
for i, v in enumerate(val):
self.assertEqual(len(v), store._per_chunk_arrays["length"][i],
f"array {i} has incorrect length!")
self.assertTrue(np.array_equal(v, [[1], [2, 3], [4, 5, 6]][i]),
f"array {i} has incorrect values, {v}!")
self.assertTrue(
np.array_equal(
store.get_array("chunk"),
store.get_array_ragged("chunk"),
),
"get_array_ragged does not give same result as get_array for per chunk array"
)
def test_get_array_filled(self):
"""get_array_filled should return a padded array of all elements in the storage."""
store = FlattenedStorage(elem=[[1], [2, 3], [4, 5, 6]],
chunk=[-1, -2, -3])
store.add_array("fill", fill=23.42)
store.set_array("fill", 0, [-1])
store.set_array("fill", 1, [-2, -3])
store.set_array("fill", 2, [-4, -5, -6])
store.add_array("complex", shape=(3, ), dtype=np.float64)
store.set_array("complex", 0, [[1, 1, 1]])
store.set_array("complex", 1, [
[2, 2, 2],
[2, 2, 2],
])
store.set_array("complex", 2, [
[3, 3, 3],
[3, 3, 3],
[3, 3, 3],
])
val = store.get_array_filled("elem")
self.assertEqual(val.shape, (3, 3), "shape not correct!")
self.assertTrue(
np.array_equal(val, [[1, -1, -1], [2, 3, -1], [4, 5, 6]]),
"values in returned array not the same as in original array!")
self.assertEqual(
store.get_array_filled("fill")[0, 1], 23.42,
"incorrect fill value!")
val = store.get_array_filled("complex")
self.assertEqual(val.shape, (3, 3, 3), "shape not correct!")
self.assertTrue(
np.array_equal(
store.get_array("chunk"),
store.get_array_filled("chunk"),
),
"get_array_filled does not give same result as get_array for per chunk array"
)
def test_getitem_setitem(self):
"""Using __getitem__/__setitem__ should be equivalent to using get_array/set_array."""
store = FlattenedStorage(even=self.even, odd=self.odd, mylen=[1, 2, 3])
for i in range(len(store)):
self.assertTrue(
np.array_equal(
store["even", i],
store.get_array("even", i),
),
f"getitem returned different value ({store['even', i]}) than get_array ({store.get_array('even', i)}) for chunk {i}"
)
self.assertEqual(
store["mylen", i], store.get_array("mylen", i),
f"getitem returned different value ({store['mylen', i]}) than get_array ({store.get_array('mylen', i)}) for chunk {i}"
)
self.assertTrue(
np.array_equal(store["even"], store.get_array("even")),
f"getitem returned different value ({store['even']}) than get_array ({store.get_array('even')})"
)
self.assertTrue(
np.array_equal(store["mylen"], store.get_array("mylen")),
f"getitem returned different value ({store['mylen']}) than get_array ({store.get_array('mylen')})"
)
store["even", 0] = [4]
store["even", 1] = [2, 0]
store["mylen", 0] = 4
self.assertEqual(store.get_array("mylen", 0), 4,
"setitem did not set item correctly.")
self.assertTrue(np.array_equal(store.get_array("even", 0), [4]),
"setitem did not set item correctly.")
self.assertTrue(np.array_equal(store.get_array("even", 1), [2, 0]),
"setitem did not set item correctly.")
with self.assertRaises(
IndexError,
msg="Calling setitem with out index doesn't raise Error!"):
store["mylen"] = [1, 2, 3]