def test_dtype_structured(self):
base_dtypes = tuple((set(self.dtypes) | {'U2', 'S3'}) - {'U', 'S'})
for i in range(10):
names = []
for _ in range(random.randint(1, 5)):
s = random_str_ascii(random.randint(1, 10))
while s in names and s[0].isdigit():
s = random_str_ascii(random.randint(1, 10))
names.append(s)
desc = [(v, random.choice(base_dtypes),
random_numpy_shape(random.randint(1, 4), 10))
for v in names]
self.check_dtype(np.dtype(desc))
self.check_dtype(np.dtype(desc, align=True))
def check_dict_like_key_leading_periods(self, tp):
data = random_dict(tp)
prefix = '.' * random.randint(1, 10)
key = prefix + random_str_ascii(max_dict_key_length)
data[key] = random_int()
out = self.write_readback(data, random_name(), self.options)
self.assert_equal(out, data)
def test_str_with_null(self):
strs = [
random_str_ascii(random.randint(1, max_string_length))
for i in range(2)
]
data = '\x00'.join(strs)
out = self.write_readback(data, random_name(), self.options)
self.assert_equal(out, data)
def check_dict_like_key_back_slash(self, tp):
data = random_dict(tp)
ch = '\\'
key = ch.join(
[random_str_ascii(max_dict_key_length) for i in range(2)])
data[key] = random_int()
out = self.write_readback(data, random_name(), self.options)
self.assert_equal(out, data)
def test_dtype_structured_with_offsets_titles(self):
base_dtypes = tuple((set(self.dtypes) | {'U2', 'S3'}) - {'U', 'S'})
for i in range(10):
names = []
for _ in range(random.randint(1, 5)):
s = random_str_ascii(random.randint(1, 10))
while s in names and s[0].isdigit():
s = random_str_ascii(random.randint(1, 10))
names.append(s)
formats = [(random.choice(base_dtypes),
random_numpy_shape(random.randint(1, 4), 10))
for _ in range(len(names))]
titles = [
random_str_some_unicode(random.randint(1, 10))
for _ in range(len(names))
]
offsets = [random.randint(0, 100) for _ in range(len(names))]
desc = {
'names': names,
'formats': formats,
'titles': titles,
'offsets': offsets
}
desc_with_itemsize = desc.copy()
desc_with_itemsize['itemsize'] = \
np.dtype(desc).itemsize + random.randint(1, 100)
# Make the dtypes in all combinations of the description and
# align. Note that if the type isn't valid at all, it is not
# tested.
dts = []
for d in (desc, desc_with_itemsize):
for align in (False, True):
try:
dts.append(np.dtype(d, align=align))
except:
pass
for dt in dts:
self.check_dtype(dt)
def check_numpy_recarray_field_special_char(self, ch, leading=False):
# Makes a random 1d structured ndarray with the character
# in one field, converts it to a recarray, writes it and reads
# it back, and then compares it.
field_names = [random_str_ascii(max_dict_key_length) for i in range(2)]
if leading:
field_names[1] = ch + field_names[1]
else:
field_names[1] = field_names[1][0] + ch + field_names[1][1:]
shape = random_numpy_shape(1, \
max_structured_ndarray_axis_length)
data = random_structured_numpy_array(shape, names=field_names).view(
np.recarray).copy()
out = self.write_readback(data, random_name(), self.options)
self.assert_equal(out, data)
def test_int_key():
# generate some random keys_values_names
keys_values_names = [('keys', 'values')]
for i in range(3):
names = ('a', 'a')
while names[0] == names[1]:
names = [random_str_ascii(8) for i in range(2)]
keys_values_names.append(names)
for pyth_meta in (True, False):
for mat_meta in (True, False):
for tp in dict_like:
for names in keys_values_names:
options_keywords = { \
'store_python_metadata': pyth_meta, \
'matlab_compatible': mat_meta, \
'dict_like_keys_name': names[0], \
'dict_like_values_name': names[1]}
yield check_int_key, tp, options_keywords
def make_str_for_esc(include_escapes=None,
include_leading_periods=False,
no_unicode=False,
pack_digits=True):
sl = list(random_str_ascii(10))
if not no_unicode:
sl += list(random_str_some_unicode(10))
if pack_digits:
chars = b'0 1 2 3 4 5 6 7 8 9 a b c d e f A B C D E F'
sl += chars.decode('ascii').split(b' '.decode('ascii')) * 10
sl += [period] * 10
if include_escapes is not None:
for c in include_escapes:
sl += [c] * 3
random.shuffle(sl)
s = b''.decode('ascii').join(sl).lstrip(period)
if include_leading_periods:
s = period * random.randint(1, 10) + s
return s
def test_string_type_non_str_key():
# Set the other key types.
other_tps = ['bytes', 'numpy.bytes_', 'numpy.unicode_']
# generate some random keys_values_names
keys_values_names = [('keys', 'values')]
for i in range(1):
names = ('a', 'a')
while names[0] == names[1]:
names = [random_str_ascii(8) for i in range(2)]
keys_values_names.append(names)
for pyth_meta in (True, False):
for mat_meta in (True, False):
for tp in dict_like:
for other_tp in other_tps:
for names in keys_values_names:
options_keywords = { \
'store_python_metadata': pyth_meta, \
'matlab_compatible': mat_meta, \
'dict_like_keys_name': names[0], \
'dict_like_values_name': names[1]}
yield check_string_type_non_str_key, tp, other_tp, options_keywords
def check_str_key_previously_invalid_char(tp, ch, option_keywords):
options = hdf5storage.Options(**option_keywords)
key_value_names = (options.dict_like_keys_name,
options.dict_like_values_name)
data = random_dict(tp)
for k in key_value_names:
if k in data:
del data[k]
# Add a random invalid str key using the provided character
key = key_value_names[0]
while key in key_value_names:
key = ch.join(
[random_str_ascii(max_dict_key_length) for i in range(2)])
data[key] = random_int()
# Make a random name.
name = random_name()
# Write the data to the proper file with the given name with the
# provided options. The file needs to be deleted after to keep junk
# from building up.
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
hdf5storage.write(data, path=name, filename=filename, options=options)
with h5py.File(filename, mode='r') as f:
for k in key_value_names:
assert escape_path(k) not in f[name]
for k in data:
assert escape_path(k) in f[name]
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
def test_str_ascii(self):
data = random_str_ascii(random.randint(1, max_string_length))
out = self.write_readback(data, random_name(), self.options)
self.assert_equal(out, data)
