def test_multi_read():
# Makes a random dict of random paths and variables (random number
# of randomized paths with random numpy arrays as values).
data = dict()
for i in range(0, random.randint(min_dict_keys, \
max_dict_keys)):
name = random_name()
data[name] = \
random_numpy(random_numpy_shape( \
dict_value_subarray_dimensions, \
max_dict_value_subarray_axis_length), \
dtype=random.choice(dtypes))
paths = data.keys()
# Write it item by item and then read it back in one unit.
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
for p in paths:
hdf5storage.write(data=data[p], path=p, filename=filename)
out = hdf5storage.reads(paths=list(data.keys()), filename=filename)
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
# Compare data and out.
for i, p in enumerate(paths):
assert_equal(out[i], data[p])
def check_numpy_matrix(self, dtype):
# Makes a random numpy array of the given type, converts it to
# a matrix, writes it and reads it back, and then compares it.
shape = random_numpy_shape(2, max_array_axis_length)
data = np.matrix(random_numpy(shape, dtype))
out = self.write_readback(data, random_name(), self.options)
self.assert_equal(out, data)
def test_multi_write():
# Makes a random dict of random paths and variables (random number
# of randomized paths with random numpy arrays as values).
data = dict()
for i in range(0, random.randint(min_dict_keys, \
max_dict_keys)):
name = random_name()
data[name] = \
random_numpy(random_numpy_shape( \
dict_value_subarray_dimensions, \
max_dict_value_subarray_axis_length), \
dtype=random.choice(dtypes))
# Write it and then read it back item by item.
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
hdf5storage.writes(mdict=data, filename=filename)
out = dict()
for p in data:
out[p] = hdf5storage.read(path=p, filename=filename)
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
# Compare data and out.
assert_equal(out, data)
def check_numpy_chararray(self, dimensions):
# Makes a random numpy array of bytes, converts it to a
# chararray, writes it and reads it back, and then compares it.
shape = random_numpy_shape(dimensions, max_array_axis_length)
data = random_numpy(shape, 'S').view(np.chararray).copy()
out = self.write_readback(data, random_name(), self.options)
self.assert_equal(out, data)
def check_numpy_array(self, dtype, dimensions):
# Makes a random numpy array of the given type, writes it and
# reads it back, and then compares it.
shape = random_numpy_shape(dimensions, max_array_axis_length)
data = random_numpy(shape, dtype)
out = self.write_readback(data, random_name(), self.options)
self.assert_equal(out, data)
def check_write_filters(filters):
# Read out the filter arguments.
filts = {'compression': 'gzip',
'shuffle': True,
'fletcher32': True,
'gzip_level': 7}
for k, v in filters.items():
filts[k] = v
# Make some random data. The dtype must be restricted so that it can
# be read back reliably.
dims = random.randint(1, 4)
dts = tuple(set(dtypes) - set(['U', 'S', 'bool', 'complex64', \
'complex128']))
data = random_numpy(shape=random_numpy_shape(dims,
max_array_axis_length),
dtype=random.choice(dts))
# Make a random name.
name = random_name()
# Write the data to the proper file with the given name with the
# provided filters and read it backt. The file needs to be deleted
# after to keep junk from building up.
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
hdf5storage.write(data, path=name, filename=filename, \
store_python_metadata=False, matlab_compatible=False, \
compress=True, compress_size_threshold=0, \
compression_algorithm=filts['compression'], \
gzip_compression_level=filts['gzip_level'], \
shuffle_filter=filts['shuffle'], \
compressed_fletcher32_filter=filts['fletcher32'])
with h5py.File(filename, mode='r') as f:
d = f[name]
fletcher32 = d.fletcher32
shuffle = d.shuffle
compression = d.compression
gzip_level = d.compression_opts
out = d[...]
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
# Check the filters
assert_equal_nose(fletcher32, filts['fletcher32'])
assert_equal_nose(shuffle, filts['shuffle'])
assert_equal_nose(compression, filts['compression'])
if filts['compression'] == 'gzip':
assert_equal_nose(gzip_level, filts['gzip_level'])
# Compare
assert_equal(out, data)
def check_numpy_structured_array_empty(self, dimensions):
# Makes a random structured ndarray of the given type, writes it
# and reads it back, and then compares it.
shape = random_numpy_shape(dimensions, \
max_structured_ndarray_axis_length)
data = random_structured_numpy_array(shape, (1, 0))
out = self.write_readback(data, random_name(), self.options)
self.assert_equal(out, data)
def test_numpy_recarray_unicode_fields(self):
# Makes a random 1d structured ndarray with non-ascii characters
# in its fields, converts it to a recarray, writes it and reads
# it back, and then compares it.
shape = random_numpy_shape(1, \
max_structured_ndarray_axis_length)
data = random_structured_numpy_array(shape, nonascii_fields=True)
out = self.write_readback(data, random_name(), self.options)
self.assert_equal(out, data)
def check_numpy_recarray(self, dimensions):
# Makes a random structured ndarray of the given type, converts
# it to a recarray, writes it and reads it back, and then
# compares it.
shape = random_numpy_shape(dimensions, \
max_structured_ndarray_axis_length)
data = random_structured_numpy_array(shape).view(np.recarray).copy()
out = self.write_readback(data, random_name(), self.options)
self.assert_equal(out, data)
def test_dtype_structured(self):
base_dtypes = tuple((set(self.dtypes) | {'U2', 'S3'}) - {'U', 'S'})
for i in range(10):
names = []
for _ in range(random.randint(1, 5)):
s = random_str_ascii(random.randint(1, 10))
while s in names and s[0].isdigit():
s = random_str_ascii(random.randint(1, 10))
names.append(s)
desc = [(v, random.choice(base_dtypes),
random_numpy_shape(random.randint(1, 4), 10))
for v in names]
self.check_dtype(np.dtype(desc))
self.check_dtype(np.dtype(desc, align=True))
def check_numpy_recarray_field_special_char(self, ch, leading=False):
# Makes a random 1d structured ndarray with the character
# in one field, converts it to a recarray, writes it and reads
# it back, and then compares it.
field_names = [random_str_ascii(max_dict_key_length) for i in range(2)]
if leading:
field_names[1] = ch + field_names[1]
else:
field_names[1] = field_names[1][0] + ch + field_names[1][1:]
shape = random_numpy_shape(1, \
max_structured_ndarray_axis_length)
data = random_structured_numpy_array(shape, names=field_names).view(
np.recarray).copy()
out = self.write_readback(data, random_name(), self.options)
self.assert_equal(out, data)
def check_read_filters(filters):
# Read out the filter arguments.
filts = {'compression': 'gzip',
'shuffle': True,
'fletcher32': True,
'gzip_level': 7}
for k, v in filters.items():
filts[k] = v
if filts['compression'] == 'gzip':
filts['compression_opts'] = filts['gzip_level']
del filts['gzip_level']
# Make some random data.
dims = random.randint(1, 4)
data = random_numpy(shape=random_numpy_shape(dims,
max_array_axis_length),
dtype=random.choice(tuple(
set(dtypes) - set(['U']))))
# Make a random name.
name = random_name()
# Write the data to the proper file with the given name with the
# provided filters and read it backt. The file needs to be deleted
# after to keep junk from building up.
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
with h5py.File(filename, mode='w') as f:
f.create_dataset(name, data=data, chunks=True, **filts)
out = hdf5storage.read(path=name, filename=filename,
matlab_compatible=False)
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
# Compare
assert_equal(out, data)
def test_dtype_structured_with_offsets_titles(self):
base_dtypes = tuple((set(self.dtypes) | {'U2', 'S3'}) - {'U', 'S'})
for i in range(10):
names = []
for _ in range(random.randint(1, 5)):
s = random_str_ascii(random.randint(1, 10))
while s in names and s[0].isdigit():
s = random_str_ascii(random.randint(1, 10))
names.append(s)
formats = [(random.choice(base_dtypes),
random_numpy_shape(random.randint(1, 4), 10))
for _ in range(len(names))]
titles = [
random_str_some_unicode(random.randint(1, 10))
for _ in range(len(names))
]
offsets = [random.randint(0, 100) for _ in range(len(names))]
desc = {
'names': names,
'formats': formats,
'titles': titles,
'offsets': offsets
}
desc_with_itemsize = desc.copy()
desc_with_itemsize['itemsize'] = \
np.dtype(desc).itemsize + random.randint(1, 100)
# Make the dtypes in all combinations of the description and
# align. Note that if the type isn't valid at all, it is not
# tested.
dts = []
for d in (desc, desc_with_itemsize):
for align in (False, True):
try:
dts.append(np.dtype(d, align=align))
except:
pass
for dt in dts:
self.check_dtype(dt)
def check_python_collection(self, tp, same_dims):
# Makes a random collection of the specified type, writes it and
# reads it back, and then compares it.
if tp in (set, frozenset):
data = tp(random_list(max_list_length, python_or_numpy='python'))
else:
if same_dims == 'same-dims':
shape = random_numpy_shape(random.randrange(2, 4),
random.randrange(1, 4))
dtypes = ('uint8', 'uint16', 'uint32', 'uint64', 'int8',
'int16', 'int32', 'int64', 'float32', 'float64',
'complex64', 'complex128')
data = tp([
random_numpy(shape, random.choice(dtypes), allow_nan=True)
for i in range(random.randrange(2, 7))
])
elif same_dims == 'diff-dims':
data = tp(random_list(max_list_length,
python_or_numpy='numpy'))
else:
raise ValueError('invalid value of same_dims')
out = self.write_readback(data, random_name(), self.options)
self.assert_equal(out, data)
to_julia = dict()
# Julia MAT tends to squeeze extra singleton dimensions beyond 2,
# meaning a (1, 1, 1) goes to (1, 1). In addition, string conversions go
# on when going back and forth. Thus, string types will be excluded and
# the minimum length along each dimension will be 2.
dtypes_exclude = set(('S', 'U'))
dtypes_to_do = tuple(set(dtypes).difference(dtypes_exclude))
for dt in dtypes_to_do:
to_julia[dt] = random_numpy_scalar(dt)
for dm in (2, 3):
for dt in dtypes_to_do:
to_julia[dt + '_array_' + str(dm)] = \
random_numpy(random_numpy_shape(dm, 6, min_length=2), dt)
for dt in dtypes_to_do:
if dt in ('S', 'U'):
to_julia[dt + '_empty'] = np.array([], dtype=dt + str(6))
else:
to_julia[dt + '_empty'] = np.array([], dtype=dt)
to_julia['float32_nan'] = np.float32(np.NaN)
to_julia['float32_inf'] = np.float32(np.inf)
to_julia['float64_nan'] = np.float64(np.NaN)
to_julia['float64_inf'] = np.float64(-np.inf)
to_julia['object'] = random_numpy_scalar('object', \
object_element_dtypes=dtypes_to_do)
to_julia['object_array_2'] = random_numpy( \
random_numpy_shape(2, 6, min_length=2), \
def test_dtype_shaped(self):
base_dtypes = tuple((set(self.dtypes) | {'U2', 'S3'}) - {'U', 'S'})
for i in range(10):
desc = (random.choice(base_dtypes),
random_numpy_shape(random.randint(1, 4), 10))
self.check_dtype(np.dtype(desc))
