def convert(data, metadata):
"""
Convert raw mesoscope data by reshaping and merging rois.
Parameters
----------
data : numpy array or images
Raw image data as a numpy array or thunder images object.
metadata : dict
Dictionary with metadata.
"""
if isinstance(data, ndarray):
data = fromarray(data)
nchannels = metadata['nchannels']
nrois = metadata['nrois']
nlines = metadata['nlines']
order = metadata['order']
if metadata['merge']:
newdata = data.map(lambda x : reshape(x, nrois, nlines, order))
else:
newdata = data.map(lambda x : split(x, nrois, nlines))
if not metadata['fastZ'] and metadata['logFramesPerFile'] == metadata['framesPerSlice']:
newdata = fromarray(newdata.values.swapaxes(0,1))
newmetadata = merge(metadata)
newmetadata['shape'] = newdata.shape
return newdata, newmetadata
def convert(data, metadata):
"""
Convert raw mesoscope data by reshaping and merging rois.
Parameters
----------
data : numpy array or images
Raw image data as a numpy array or thunder images object.
metadata : dict
Dictionary with metadata.
"""
if isinstance(data, ndarray):
data = fromarray(data)
nchannels = metadata['nchannels']
nrois = metadata['nrois']
nlines = metadata['nlines']
order = metadata['order']
if metadata['merge']:
newdata = data.map(lambda x: reshape(x, nrois, nlines, order))
else:
newdata = data.map(lambda x: split(x, nrois, nlines))
if not metadata['fastZ'] and metadata['logFramesPerFile'] == metadata[
'framesPerSlice']:
newdata = fromarray(newdata.values.swapaxes(0, 1))
newmetadata = merge(metadata)
newmetadata['shape'] = newdata.shape
return newdata, newmetadata
def check_images(data):
"""
Check and reformat input images if needed
"""
if isinstance(data, ndarray):
data = fromarray(data)
if not isinstance(data, Images):
data = fromarray(asarray(data))
if len(data.shape) not in set([3, 4]):
raise Exception('Number of image dimensions %s must be 2 or 3' % (len(data.shape)))
return data
def check_images(data):
"""
Check and reformat input images if needed
"""
if isinstance(data, ndarray):
data = fromarray(data)
if not isinstance(data, Images):
data = fromarray(asarray(data))
if len(data.shape) not in set([3, 4]):
raise Exception('Number of image dimensions %s must be 2 or 3' % (len(data.shape)))
return data
def test_shaping_3_values(eng):
svd = lambda x: SVD(k=3, method='direct', seed=0).fit(x)
# baseline: ndarray (local) or BoltArray (spark)
x = make_low_rank_matrix(n_samples=10, n_features=10, random_state=0)
x = series.fromarray(x, engine=eng).values
u, s, v = svd(x)
# simple series
x1 = series.fromarray(x)
u1, s1, v1 = svd(x1)
assert allclose(u, u1)
assert allclose(s, s1)
assert allclose(v, v1)
# series with multiple dimensions
x1 = series.fromarray(x.reshape(2, 5, 10))
u1, s1, v1 = svd(x1)
u1 = u1.reshape(10, 3)
assert allclose(u, u1)
assert allclose(s, s1)
assert allclose(v, v1)
# images (must have multiple dimensions)
x1 = images.fromarray(x.reshape(10, 2, 5))
u1, s1, v1 = svd(x1)
v1 = v1.reshape(3, 10)
assert allclose(u, u1)
assert allclose(s, s1)
assert allclose(v, v1)
def test_shaping_2_values(eng):
pca = lambda x: PCA(k=3, svd_method='direct', seed=0).fit(x)
# baseline: ndarray (local) or BoltArray (spark)
x = make_low_rank_matrix(n_samples=10, n_features=10, random_state=0)
x = series.fromarray(x, engine=eng).values
t, w = pca(x)
# simple series
x1 = series.fromarray(x)
t1, w1 = pca(x1)
assert allclose(t, t1)
assert allclose(w, w1)
# series with multiple dimensions
x1 = series.fromarray(x.reshape(2, 5, 10))
t1, w1 = pca(x1)
t1 = t1.reshape(10, 3)
assert allclose(t, t1)
assert allclose(w, w1)
# images (must have multiple dimensions)
x1 = images.fromarray(x.reshape(10, 2, 5))
t1, w1 = pca(x1)
w1 = w1.reshape(3, 10)
assert allclose(t, t1)
assert allclose(w, w1)
def test_shaping_2_values(eng):
pca = lambda x: PCA(k=3, svd_method='direct', seed=0).fit(x)
# baseline: ndarray (local) or BoltArray (spark)
x = make_low_rank_matrix(n_samples=10, n_features=10, random_state=0)
x = series.fromarray(x, engine=eng).values
t, w = pca(x)
# simple series
x1 = series.fromarray(x)
t1, w1 = pca(x1)
assert allclose(t, t1)
assert allclose(w, w1)
# series with multiple dimensions
x1 = series.fromarray(x.reshape(2, 5, 10))
t1, w1 = pca(x1)
t1 = t1.reshape(10, 3)
assert allclose(t, t1)
assert allclose(w, w1)
# images (must have multiple dimensions)
x1 = images.fromarray(x.reshape(10, 2, 5))
t1, w1 = pca(x1)
w1 = w1.reshape(3, 10)
assert allclose(t, t1)
assert allclose(w, w1)
def test_shaping_3_values(eng):
svd= lambda x: SVD(k=3, method='direct', seed=0).fit(x)
# baseline: ndarray (local) or BoltArray (spark)
x = make_low_rank_matrix(n_samples=10, n_features=10, random_state=0)
x = series.fromarray(x, engine=eng).values
u, s, v = svd(x)
# simple series
x1 = series.fromarray(x)
u1, s1, v1 = svd(x1)
assert allclose(u, u1)
assert allclose(s, s1)
assert allclose(v, v1)
# series with multiple dimensions
x1 = series.fromarray(x.reshape(2, 5, 10))
u1, s1, v1 = svd(x1)
u1 = u1.reshape(10, 3)
assert allclose(u, u1)
assert allclose(s, s1)
assert allclose(v, v1)
# images (must have multiple dimensions)
x1 = images.fromarray(x.reshape(10, 2, 5))
u1, s1, v1 = svd(x1)
v1 = v1.reshape(3, 10)
assert allclose(u, u1)
assert allclose(s, s1)
assert allclose(v, v1)
def test_model_transform_single(eng):
regions = many([[[0, 0], [0, 1]]])
model = ExtractionModel(regions=regions)
im0 = [[0, 1], [1, 2]]
im1 = [[3, 4], [5, 6]]
im2 = [[7, 8], [9, 10]]
data = fromarray([im0, im1, im2], engine=eng)
transformed = model.transform(data)
assert allclose(transformed.toarray(), [[0.5, 3.5, 7.5]])
def test_svd(eng):
x_local = make_low_rank_matrix(n_samples=10, n_features=50, random_state=0)
x = fromarray(x_local.reshape(10, 10, 5), engine=eng)
x.cache()
x.count()
u1, s1, v1 = randomized_svd(x_local, n_components=2, random_state=0)
u2, v2, s2 = getSVD(x, k=2, getComponents=True, getS=True)
assert u1.shape == u2.shape
assert s1.shape == s2.shape
assert v1.shape == (2, 50)
assert v2.shape == (2, 10, 5)
u2, v2, s2 = getSVD(x,
k=2,
getComponents=True,
getS=True,
normalization='nanmean')
assert u1.shape == u2.shape
assert s1.shape == s2.shape
assert v1.shape == (2, 50)
assert v2.shape == (2, 10, 5)
u2, v2, s2 = getSVD(x,
k=2,
getComponents=True,
getS=True,
normalization='zscore')
assert u1.shape == u2.shape
assert s1.shape == s2.shape
assert v1.shape == (2, 50)
assert v2.shape == (2, 10, 5)
u2, v2, s2 = getSVD(x,
k=2,
getComponents=True,
getS=True,
normalization=None)
assert u1.shape == u2.shape
assert s1.shape == s2.shape
assert v1.shape == (2, 50)
assert v2.shape == (2, 10, 5)
with pytest.raises(ValueError) as ex:
u2, v2, s2 = getSVD(x,
k=2,
getComponents=True,
getS=True,
normalization='error')
assert 'Normalization should be one of' in str(ex.value)
