def test_multi_write():
    # Makes a random dict of random paths and variables (random number
    # of randomized paths with random numpy arrays as values).
    data = dict()
    for i in range(0, random.randint(min_dict_keys, \
            max_dict_keys)):
        name = random_name()
        data[name] = \
            random_numpy(random_numpy_shape( \
            dict_value_subarray_dimensions, \
            max_dict_value_subarray_axis_length), \
            dtype=random.choice(dtypes))

    # Write it and then read it back item by item.
    fld = None
    try:
        fld = tempfile.mkstemp()
        os.close(fld[0])
        filename = fld[1]
        hdf5storage.writes(mdict=data, filename=filename)
        out = dict()
        for p in data:
            out[p] = hdf5storage.read(path=p, filename=filename)
    except:
        raise
    finally:
        if fld is not None:
            os.remove(fld[1])

    # Compare data and out.
    assert_equal(out, data)
def test_multi_read():
    # Makes a random dict of random paths and variables (random number
    # of randomized paths with random numpy arrays as values).
    data = dict()
    for i in range(0, random.randint(min_dict_keys, \
            max_dict_keys)):
        name = random_name()
        data[name] = \
            random_numpy(random_numpy_shape( \
            dict_value_subarray_dimensions, \
            max_dict_value_subarray_axis_length), \
            dtype=random.choice(dtypes))

    paths = data.keys()
    # Write it item by item  and then read it back in one unit.
    fld = None
    try:
        fld = tempfile.mkstemp()
        os.close(fld[0])
        filename = fld[1]
        for p in paths:
            hdf5storage.write(data=data[p], path=p, filename=filename)
        out = hdf5storage.reads(paths=list(data.keys()), filename=filename)
    except:
        raise
    finally:
        if fld is not None:
            os.remove(fld[1])

    # Compare data and out.
    for i, p in enumerate(paths):
        assert_equal(out[i], data[p])
 def check_numpy_chararray(self, dimensions):
     # Makes a random numpy array of bytes, converts it to a
     # chararray, writes it and reads it back, and then compares it.
     shape = random_numpy_shape(dimensions, max_array_axis_length)
     data = random_numpy(shape, 'S').view(np.chararray).copy()
     out = self.write_readback(data, random_name(), self.options)
     self.assert_equal(out, data)
 def check_numpy_matrix(self, dtype):
     # Makes a random numpy array of the given type, converts it to
     # a matrix, writes it and reads it back, and then compares it.
     shape = random_numpy_shape(2, max_array_axis_length)
     data = np.matrix(random_numpy(shape, dtype))
     out = self.write_readback(data, random_name(), self.options)
     self.assert_equal(out, data)
 def check_numpy_array(self, dtype, dimensions):
     # Makes a random numpy array of the given type, writes it and
     # reads it back, and then compares it.
     shape = random_numpy_shape(dimensions, max_array_axis_length)
     data = random_numpy(shape, dtype)
     out = self.write_readback(data, random_name(), self.options)
     self.assert_equal(out, data)
Example #6
0
def check_write_filters(filters):
    # Read out the filter arguments.
    filts = {'compression': 'gzip',
             'shuffle': True,
             'fletcher32': True,
             'gzip_level': 7}
    for k, v in filters.items():
        filts[k] = v

    # Make some random data. The dtype must be restricted so that it can
    # be read back reliably.
    dims = random.randint(1, 4)
    dts = tuple(set(dtypes) - set(['U', 'S', 'bool', 'complex64', \
        'complex128']))

    data = random_numpy(shape=random_numpy_shape(dims,
                        max_array_axis_length),
                        dtype=random.choice(dts))
    # Make a random name.
    name = random_name()

    # Write the data to the proper file with the given name with the
    # provided filters and read it backt. The file needs to be deleted
    # after to keep junk from building up.
    fld = None
    try:
        fld = tempfile.mkstemp()
        os.close(fld[0])
        filename = fld[1]
        hdf5storage.write(data, path=name, filename=filename, \
            store_python_metadata=False, matlab_compatible=False, \
            compress=True, compress_size_threshold=0, \
            compression_algorithm=filts['compression'], \
            gzip_compression_level=filts['gzip_level'], \
            shuffle_filter=filts['shuffle'], \
            compressed_fletcher32_filter=filts['fletcher32'])

        with h5py.File(filename, mode='r') as f:
            d = f[name]
            fletcher32 = d.fletcher32
            shuffle = d.shuffle
            compression = d.compression
            gzip_level = d.compression_opts
            out = d[...]
    except:
        raise
    finally:
        if fld is not None:
            os.remove(fld[1])

    # Check the filters
    assert_equal_nose(fletcher32, filts['fletcher32'])
    assert_equal_nose(shuffle, filts['shuffle'])
    assert_equal_nose(compression, filts['compression'])
    if filts['compression'] == 'gzip':
        assert_equal_nose(gzip_level, filts['gzip_level'])

    # Compare
    assert_equal(out, data)
Example #7
0
def check_read_filters(filters):
    # Read out the filter arguments.
    filts = {'compression': 'gzip',
             'shuffle': True,
             'fletcher32': True,
             'gzip_level': 7}
    for k, v in filters.items():
        filts[k] = v
    if filts['compression'] == 'gzip':
        filts['compression_opts'] = filts['gzip_level']
    del filts['gzip_level']
    
    # Make some random data.
    dims = random.randint(1, 4)
    data = random_numpy(shape=random_numpy_shape(dims,
                        max_array_axis_length),
                        dtype=random.choice(tuple(
                        set(dtypes) - set(['U']))))
    # Make a random name.
    name = random_name()

    # Write the data to the proper file with the given name with the
    # provided filters and read it backt. The file needs to be deleted
    # after to keep junk from building up.
    fld = None
    try:
        fld = tempfile.mkstemp()
        os.close(fld[0])
        filename = fld[1]
        with h5py.File(filename, mode='w') as f:
            f.create_dataset(name, data=data, chunks=True, **filts)
        out = hdf5storage.read(path=name, filename=filename,
                               matlab_compatible=False)
    except:
        raise
    finally:
        if fld is not None:
            os.remove(fld[1])

    # Compare
    assert_equal(out, data)
    def check_python_collection(self, tp, same_dims):
        # Makes a random collection of the specified type, writes it and
        # reads it back, and then compares it.
        if tp in (set, frozenset):
            data = tp(random_list(max_list_length, python_or_numpy='python'))
        else:
            if same_dims == 'same-dims':
                shape = random_numpy_shape(random.randrange(2, 4),
                                           random.randrange(1, 4))
                dtypes = ('uint8', 'uint16', 'uint32', 'uint64', 'int8',
                          'int16', 'int32', 'int64', 'float32', 'float64',
                          'complex64', 'complex128')
                data = tp([
                    random_numpy(shape, random.choice(dtypes), allow_nan=True)
                    for i in range(random.randrange(2, 7))
                ])

            elif same_dims == 'diff-dims':
                data = tp(random_list(max_list_length,
                                      python_or_numpy='numpy'))
            else:
                raise ValueError('invalid value of same_dims')
        out = self.write_readback(data, random_name(), self.options)
        self.assert_equal(out, data)
to_julia = dict()

# Julia MAT tends to squeeze extra singleton dimensions beyond 2,
# meaning a (1, 1, 1) goes to (1, 1). In addition, string conversions go
# on when going back and forth. Thus, string types will be excluded and
# the minimum length along each dimension will be 2.

dtypes_exclude = set(('S', 'U'))
dtypes_to_do = tuple(set(dtypes).difference(dtypes_exclude))

for dt in dtypes_to_do:
    to_julia[dt] = random_numpy_scalar(dt)
for dm in (2, 3):
    for dt in dtypes_to_do:
        to_julia[dt + '_array_' + str(dm)] = \
            random_numpy(random_numpy_shape(dm, 6, min_length=2), dt)
for dt in dtypes_to_do:
    if dt in ('S', 'U'):
        to_julia[dt + '_empty'] = np.array([], dtype=dt + str(6))
    else:
        to_julia[dt + '_empty'] = np.array([], dtype=dt)

to_julia['float32_nan'] = np.float32(np.NaN)
to_julia['float32_inf'] = np.float32(np.inf)
to_julia['float64_nan'] = np.float64(np.NaN)
to_julia['float64_inf'] = np.float64(-np.inf)

to_julia['object'] = random_numpy_scalar('object', \
    object_element_dtypes=dtypes_to_do)
to_julia['object_array_2'] = random_numpy( \
    random_numpy_shape(2, 6, min_length=2), \