def test_dtype_structured(self): base_dtypes = tuple((set(self.dtypes) | {'U2', 'S3'}) - {'U', 'S'}) for i in range(10): names = [] for _ in range(random.randint(1, 5)): s = random_str_ascii(random.randint(1, 10)) while s in names and s[0].isdigit(): s = random_str_ascii(random.randint(1, 10)) names.append(s) desc = [(v, random.choice(base_dtypes), random_numpy_shape(random.randint(1, 4), 10)) for v in names] self.check_dtype(np.dtype(desc)) self.check_dtype(np.dtype(desc, align=True))
def check_dict_like_key_leading_periods(self, tp): data = random_dict(tp) prefix = '.' * random.randint(1, 10) key = prefix + random_str_ascii(max_dict_key_length) data[key] = random_int() out = self.write_readback(data, random_name(), self.options) self.assert_equal(out, data)
def test_str_with_null(self): strs = [ random_str_ascii(random.randint(1, max_string_length)) for i in range(2) ] data = '\x00'.join(strs) out = self.write_readback(data, random_name(), self.options) self.assert_equal(out, data)
def check_dict_like_key_back_slash(self, tp): data = random_dict(tp) ch = '\\' key = ch.join( [random_str_ascii(max_dict_key_length) for i in range(2)]) data[key] = random_int() out = self.write_readback(data, random_name(), self.options) self.assert_equal(out, data)
def test_dtype_structured_with_offsets_titles(self): base_dtypes = tuple((set(self.dtypes) | {'U2', 'S3'}) - {'U', 'S'}) for i in range(10): names = [] for _ in range(random.randint(1, 5)): s = random_str_ascii(random.randint(1, 10)) while s in names and s[0].isdigit(): s = random_str_ascii(random.randint(1, 10)) names.append(s) formats = [(random.choice(base_dtypes), random_numpy_shape(random.randint(1, 4), 10)) for _ in range(len(names))] titles = [ random_str_some_unicode(random.randint(1, 10)) for _ in range(len(names)) ] offsets = [random.randint(0, 100) for _ in range(len(names))] desc = { 'names': names, 'formats': formats, 'titles': titles, 'offsets': offsets } desc_with_itemsize = desc.copy() desc_with_itemsize['itemsize'] = \ np.dtype(desc).itemsize + random.randint(1, 100) # Make the dtypes in all combinations of the description and # align. Note that if the type isn't valid at all, it is not # tested. dts = [] for d in (desc, desc_with_itemsize): for align in (False, True): try: dts.append(np.dtype(d, align=align)) except: pass for dt in dts: self.check_dtype(dt)
def check_numpy_recarray_field_special_char(self, ch, leading=False): # Makes a random 1d structured ndarray with the character # in one field, converts it to a recarray, writes it and reads # it back, and then compares it. field_names = [random_str_ascii(max_dict_key_length) for i in range(2)] if leading: field_names[1] = ch + field_names[1] else: field_names[1] = field_names[1][0] + ch + field_names[1][1:] shape = random_numpy_shape(1, \ max_structured_ndarray_axis_length) data = random_structured_numpy_array(shape, names=field_names).view( np.recarray).copy() out = self.write_readback(data, random_name(), self.options) self.assert_equal(out, data)
def test_int_key(): # generate some random keys_values_names keys_values_names = [('keys', 'values')] for i in range(3): names = ('a', 'a') while names[0] == names[1]: names = [random_str_ascii(8) for i in range(2)] keys_values_names.append(names) for pyth_meta in (True, False): for mat_meta in (True, False): for tp in dict_like: for names in keys_values_names: options_keywords = { \ 'store_python_metadata': pyth_meta, \ 'matlab_compatible': mat_meta, \ 'dict_like_keys_name': names[0], \ 'dict_like_values_name': names[1]} yield check_int_key, tp, options_keywords
def make_str_for_esc(include_escapes=None, include_leading_periods=False, no_unicode=False, pack_digits=True): sl = list(random_str_ascii(10)) if not no_unicode: sl += list(random_str_some_unicode(10)) if pack_digits: chars = b'0 1 2 3 4 5 6 7 8 9 a b c d e f A B C D E F' sl += chars.decode('ascii').split(b' '.decode('ascii')) * 10 sl += [period] * 10 if include_escapes is not None: for c in include_escapes: sl += [c] * 3 random.shuffle(sl) s = b''.decode('ascii').join(sl).lstrip(period) if include_leading_periods: s = period * random.randint(1, 10) + s return s
def test_string_type_non_str_key(): # Set the other key types. other_tps = ['bytes', 'numpy.bytes_', 'numpy.unicode_'] # generate some random keys_values_names keys_values_names = [('keys', 'values')] for i in range(1): names = ('a', 'a') while names[0] == names[1]: names = [random_str_ascii(8) for i in range(2)] keys_values_names.append(names) for pyth_meta in (True, False): for mat_meta in (True, False): for tp in dict_like: for other_tp in other_tps: for names in keys_values_names: options_keywords = { \ 'store_python_metadata': pyth_meta, \ 'matlab_compatible': mat_meta, \ 'dict_like_keys_name': names[0], \ 'dict_like_values_name': names[1]} yield check_string_type_non_str_key, tp, other_tp, options_keywords
def check_str_key_previously_invalid_char(tp, ch, option_keywords): options = hdf5storage.Options(**option_keywords) key_value_names = (options.dict_like_keys_name, options.dict_like_values_name) data = random_dict(tp) for k in key_value_names: if k in data: del data[k] # Add a random invalid str key using the provided character key = key_value_names[0] while key in key_value_names: key = ch.join( [random_str_ascii(max_dict_key_length) for i in range(2)]) data[key] = random_int() # Make a random name. name = random_name() # Write the data to the proper file with the given name with the # provided options. The file needs to be deleted after to keep junk # from building up. fld = None try: fld = tempfile.mkstemp() os.close(fld[0]) filename = fld[1] hdf5storage.write(data, path=name, filename=filename, options=options) with h5py.File(filename, mode='r') as f: for k in key_value_names: assert escape_path(k) not in f[name] for k in data: assert escape_path(k) in f[name] except: raise finally: if fld is not None: os.remove(fld[1])
def test_str_ascii(self): data = random_str_ascii(random.randint(1, max_string_length)) out = self.write_readback(data, random_name(), self.options) self.assert_equal(out, data)