def test_toarray(eng):
original = [array([1, 2, 3]), array([4, 5, 6])]
data = series.fromlist(original, engine=eng)
assert allclose(data.toarray(), original)
original = [array([[1, 2], [3, 4]]), array([[5, 6], [7, 8]])]
data = images.fromlist(original, engine=eng)
assert allclose(data.toarray(), original)
def test_first(eng):
data = series.fromlist([array([1, 2, 3]), array([4, 5, 6])], engine=eng)
assert allclose(data.first(), [1, 2, 3])
data = images.fromlist([array([[1, 2], [3, 4]]),
array([[5, 6], [7, 8]])],
engine=eng)
assert allclose(data.first(), [[1, 2], [3, 4]])
def test_toarray(eng):
original = [array([1, 2, 3]), array([4, 5, 6])]
data = series.fromlist(original, engine=eng)
assert allclose(data.toarray(), original)
original = [array([[1, 2], [3, 4]]), array([[5, 6], [7, 8]])]
data = images.fromlist(original, engine=eng)
assert allclose(data.toarray(), original)
def prepare_images(files, context, median_filter_size, background_offset):
from thunder import images as tdims
from fish.util.fileio import read_image
images = tdims.fromlist(files, accessor=read_image, engine=context)
images = images.map(lambda v: (v - background_offset).clip(1, None))
images = images.median_filter(size=median_filter_size)
return images
def test_map_with_keys(eng):
data = series.fromlist([array([1, 2, 3]), array([4, 5, 6])], engine=eng)
mapped = data.map(lambda kv: kv[0] + kv[1], with_keys=True)
assert allclose(mapped.shape, [2, 3])
assert allclose(mapped.toarray(), [[1, 2, 3], [5, 6, 7]])
data = images.fromlist([array([[1, 1], [1, 1]]), array([[2, 2], [2, 2]])], engine=eng)
mapped = data.map(lambda kv: kv[0] + kv[1], with_keys=True)
assert allclose(mapped.shape, [2, 2, 2])
assert allclose(mapped.toarray(), [[[1, 1], [1, 1]], [[3, 3], [3, 3]]])
def test_map_with_keys(eng):
data = series.fromlist([array([1, 2, 3]), array([4, 5, 6])], engine=eng)
mapped = data.map(lambda kv: kv[0] + kv[1], with_keys=True)
assert allclose(mapped.shape, [2, 3])
assert allclose(mapped.toarray(), [[1, 2, 3], [5, 6, 7]])
data = images.fromlist([array([[1, 1], [1, 1]]),
array([[2, 2], [2, 2]])],
engine=eng)
mapped = data.map(lambda kv: kv[0] + kv[1], with_keys=True)
assert allclose(mapped.shape, [2, 2, 2])
assert allclose(mapped.toarray(), [[[1, 1], [1, 1]], [[3, 3], [3, 3]]])
def test_is_cached(eng):
if eng is not None:
data = images.fromlist([array([1, 1]), array([2, 2]), array([3, 3]), array([4, 4]),
array([5, 5]), array([6, 6]), array([7, 7]), array([8, 8]),
array([9, 9]), array([10, 10]), array([11, 11]), array([12, 12])],
engine=eng, npartitions=10)
assert data.iscached() is False
data.cache()
assert data.iscached()
data.uncache()
assert data.iscached() is False
def test_repartition(eng):
if eng is not None:
data = images.fromlist([
array([1, 1]),
array([2, 2]),
array([3, 3]),
array([4, 4]),
array([5, 5]),
array([6, 6]),
array([7, 7]),
array([8, 8]),
array([9, 9]),
array([10, 10]),
array([11, 11]),
array([12, 12])
],
engine=eng,
npartitions=10)
assert allclose(data.first(), array([1, 1]))
assert isinstance(data.first(), (ndarray, generic))
data = data.repartition(3)
assert allclose(data.first(), array([1, 1]))
data = series.fromlist([
array([1, 1]),
array([2, 2]),
array([3, 3]),
array([4, 4]),
array([5, 5]),
array([6, 6]),
array([7, 7]),
array([8, 8]),
array([9, 9]),
array([10, 10]),
array([11, 11]),
array([12, 12])
],
engine=eng,
npartitions=10)
assert allclose(data.first(), array([1, 1]))
data = data.repartition(3)
assert allclose(data.first(), array([1, 1]))
assert isinstance(data.first(), (ndarray, generic))
def test_repartition(eng):
if eng is not None:
data = images.fromlist([array([1, 1]), array([2, 2]), array([3, 3]), array([4, 4]),
array([5, 5]), array([6, 6]), array([7, 7]), array([8, 8]),
array([9, 9]), array([10, 10]), array([11, 11]), array([12, 12])],
engine=eng, npartitions=10)
assert allclose(data.first(), array([1, 1]))
assert isinstance(data.first(), (ndarray, generic))
data = data.repartition(3)
assert allclose(data.first(), array([1, 1]))
data = series.fromlist([array([1, 1]), array([2, 2]), array([3, 3]), array([4, 4]),
array([5, 5]), array([6, 6]), array([7, 7]), array([8, 8]),
array([9, 9]), array([10, 10]), array([11, 11]), array([12, 12])],
engine=eng, npartitions=10)
assert allclose(data.first(), array([1, 1]))
data = data.repartition(3)
assert allclose(data.first(), array([1, 1]))
assert isinstance(data.first(), (ndarray, generic))
def test_is_cached(eng):
if eng is not None:
data = images.fromlist([
array([1, 1]),
array([2, 2]),
array([3, 3]),
array([4, 4]),
array([5, 5]),
array([6, 6]),
array([7, 7]),
array([8, 8]),
array([9, 9]),
array([10, 10]),
array([11, 11]),
array([12, 12])
],
engine=eng,
npartitions=10)
assert data.iscached() is False
data.cache()
assert data.iscached()
data.uncache()
assert data.iscached() is False
def make_gaussian(shape=(100, 200), n=5, t=100, sd=3, noise=0.1, seed=None, engine=None, withparams=False):
"""
Generate random gaussian source data.
Uses a spatial mixture gaussians with time-varying amplitudes.
Parameters
----------
shape : tuple, optional, default = (100,200)
Shape of data.
n : int, optional, default = 5
Number of sources.
t : int, optional, default = 100
Number of time points.
sd : float, optional, default = 3.0
Standard deviation of gaussians.
noise : float, optional, default = 0.1
Random noise to add to result.
seed : int, optional, default = None
Random seed.
engine : computational backend, optional, default = None
Can be None (for local) or a SparkContext (for spark)
withparams : bool, optionla, default = False
If True, returns generating parameters along with data.
"""
from .model import ExtractionModel
random.seed(seed)
margin = [shape[0] * 0.1, shape[1] * 0.1]
xcenters = (shape[0] - margin[0]) * random.random_sample(n) + margin[0]/2
ycenters = (shape[1] - margin[1]) * random.random_sample(n) + margin[1]/2
centers = list(zip(xcenters, ycenters))
series = [random.randn(t) for i in range(0, n)]
series = clip(asarray([gaussian_filter1d(vec, 5) for vec in series]), 0, 1)
for ii, tt in enumerate(series):
series[ii] = (tt / (tt.max() + 0.01)) * 2
frames = []
for tt in range(t):
frame = zeros(shape)
for nn in range(n):
base = zeros(shape)
base[centers[nn][0], centers[nn][1]] = 1
img = gaussian_filter(base, sd)
img = img/max(img)
frame += img * series[nn][tt]
frame += clip(random.randn(shape[0], shape[1]) * noise, 0, inf)
frames.append(frame)
def point_to_circle(center, radius):
rr, cc = circle(center[0], center[1], radius)
return array(zip(rr, cc))
r = round(sd * 1.5)
model = ExtractionModel([point_to_circle(c, r) for c in centers])
data = fromlist(frames, engine=engine).astype('float')
if withparams is True:
return data, series, model
else:
return data
def test_first(eng):
data = series.fromlist([array([1, 2, 3]), array([4, 5, 6])], engine=eng)
assert allclose(data.first(), [1, 2, 3])
data = images.fromlist([array([[1, 2], [3, 4]]), array([[5, 6], [7, 8]])], engine=eng)
assert allclose(data.first(), [[1, 2], [3, 4]])
def make_gaussian(shape=(100, 200), n=5, t=100, sd=3, noise=0.1, seed=None, engine=None, withparams=False):
"""
Generate random gaussian source data.
Uses a spatial mixture gaussians with time-varying amplitudes.
Parameters
----------
shape : tuple, optional, default = (100,200)
Shape of data.
n : int, optional, default = 5
Number of sources.
t : int, optional, default = 100
Number of time points.
sd : float, optional, default = 3.0
Standard deviation of gaussians.
noise : float, optional, default = 0.1
Random noise to add to result.
seed : int, optional, default = None
Random seed.
engine : computational backend, optional, default = None
Can be None (for local) or a SparkContext (for spark)
withparams : bool, optionla, default = False
If True, returns generating parameters along with data.
"""
from .model import ExtractionModel
random.seed(seed)
margin = [shape[0] * 0.1, shape[1] * 0.1]
xcenters = (shape[0] - margin[0]) * random.random_sample(n) + margin[0]/2
ycenters = (shape[1] - margin[1]) * random.random_sample(n) + margin[1]/2
centers = list(zip(xcenters, ycenters))
series = [random.randn(t) for i in range(0, n)]
series = clip(asarray([gaussian_filter1d(vec, 5) for vec in series]), 0, 1)
for ii, tt in enumerate(series):
series[ii] = (tt / (tt.max() + 0.01)) * 2
frames = []
for tt in range(t):
frame = zeros(shape)
for nn in range(n):
base = zeros(shape)
base[int(centers[nn][0]), int(centers[nn][1])] = 1
img = gaussian_filter(base, sd)
img = img/max(img)
frame += img * series[nn][tt]
frame += clip(random.randn(shape[0], shape[1]) * noise, 0, inf)
frames.append(frame)
def point_to_circle(center, radius):
rr, cc = circle(center[0], center[1], radius)
return array(list(zip(rr, cc)))
r = round(sd * 1.5)
model = ExtractionModel([point_to_circle(c, r) for c in centers])
data = fromlist(frames, engine=engine).astype('float')
if withparams is True:
return data, series, model
else:
return data