def make_h5(self, source_paths, h5_path, start=None, stop=None, filters=None):
print "Making h5 file for %s" % h5_path
h5file = tables.openFile(h5_path, mode='w', title="SNP Dataset")
if filters is None:
filters = tables.Filters(complib='blosc', complevel=5)
sizes = []
for c, source_path in enumerate(source_paths):
X = np.load(source_path)
if start is not None and stop is not None:
assert 0 <= start < stop
X = X[start:stop, :] / (2.0)
atom = (tables.Float32Atom() if config.floatX == "float32"
else tables.Float64Atom())
gcolumns = h5file.createGroup(h5file.root,
"Chr%d" % (c + 1),
"Chromosome %d" % (c + 1))
node = h5file.getNode("/", "Chr%d" % (c + 1))
h5file.createCArray(gcolumns, "X",
atom=atom,
shape=X.shape,
title="Chromosome %d" % (c + 1),
filters=filters)
PyTables.fill_hdf5(h5file, X, node=node)
sizes.append(X.shape[1])
h5file.createArray(h5file.root, "Sizes", sizes)
h5file.close()
def test_pytables():
"""
tests wether DenseDesignMatrixPyTables can be loaded and
initialize iterator
"""
# TODO more through test
x = np.ones((2, 3))
y = np.ones(2)
ds = DenseDesignMatrixPyTables(X=x, y=y)
it = ds.iterator(mode='sequential', batch_size=1)
it.next()
def test_pytables():
"""
tests wether DenseDesignMatrixPyTables can be loaded and
initialize iterator
"""
# TODO more through test
x = np.ones((2, 3))
y = np.ones(2)
ds = DenseDesignMatrixPyTables(X=x, y=y)
it = ds.iterator(mode='sequential', batch_size=1)
it.next()
def test_init_pytables_with_labels():
"""
Test whether DenseDesignMatrixPytables can be constructed with X_labels
and y_labels.
"""
rng = np.random.RandomState([34, 22, 89])
X = rng.randn(2, 3)
y = rng.randint(low=0, high=5, size=(2,))
ds = DenseDesignMatrixPyTables(
X=X,
y=y,
X_labels=len(np.unique(X).flat),
y_labels=np.max(y)+1
)
def read_hdf5_to_dataset(self, fn):
self.h5file = tables.openFile(fn, mode='r')
data = self.h5file.getNode('/', "Data")
dataset = DenseDesignMatrixPyTables(X=data.X, y=data.y)
return dataset, data.num_labels[0]