def test_fromMultiTimepointStacks(self):
ary = arange(16, dtype=dtypeFunc('uint8')).reshape((4, 2, 2))
ary2 = arange(16, 32, dtype=dtypeFunc('uint8')).reshape((4, 2, 2))
ary.tofile(os.path.join(self.outputdir, "test01.stack"))
ary2.tofile(os.path.join(self.outputdir, "test02.stack"))
image = ImagesLoader(self.sc).fromStack(self.outputdir, dtype="uint8", dims=(2, 2, 4), nplanes=2)
collectedImage = image.collect()
# we don't expect to have nrecords cached, since we get an unknown number of images per file
assert_true(image._nrecords is None)
assert_equals(4, image.nrecords)
assert_equals(4, len(collectedImage))
# check keys:
assert_equals(0, collectedImage[0][0])
assert_equals(1, collectedImage[1][0])
assert_equals(2, collectedImage[2][0])
assert_equals(3, collectedImage[3][0])
# check values:
assert_true(array_equal(ary[:2].T, collectedImage[0][1]))
assert_true(array_equal(ary[2:].T, collectedImage[1][1]))
assert_true(array_equal(ary2[:2].T, collectedImage[2][1]))
assert_true(array_equal(ary2[2:].T, collectedImage[3][1]))
# 3 planes does not divide 4
assert_raises(ValueError, ImagesLoader(self.sc).fromStack, self.outputdir, dtype="uint8",
dims=(2, 2, 4), nplanes=3)
def test_fromMultiTimepointStacks(self):
ary = arange(16, dtype=dtypeFunc('uint8')).reshape((4, 2, 2))
ary2 = arange(16, 32, dtype=dtypeFunc('uint8')).reshape((4, 2, 2))
ary.tofile(os.path.join(self.outputdir, "test01.stack"))
ary2.tofile(os.path.join(self.outputdir, "test02.stack"))
image = ImagesLoader(self.sc).fromStack(self.outputdir, dtype="uint8", dims=(2, 2, 4), nplanes=2)
collectedImage = image.collect()
# we don't expect to have nrecords cached, since we get an unknown number of images per file
assert_true(image._nrecords is None)
assert_equals(4, image.nrecords)
assert_equals(4, len(collectedImage))
# check keys:
assert_equals(0, collectedImage[0][0])
assert_equals(1, collectedImage[1][0])
assert_equals(2, collectedImage[2][0])
assert_equals(3, collectedImage[3][0])
# check values:
assert_true(array_equal(ary[:2].T, collectedImage[0][1]))
assert_true(array_equal(ary[2:].T, collectedImage[1][1]))
assert_true(array_equal(ary2[:2].T, collectedImage[2][1]))
assert_true(array_equal(ary2[2:].T, collectedImage[3][1]))
# 3 planes does not divide 4
assert_raises(ValueError, ImagesLoader(self.sc).fromStack, self.outputdir, dtype="uint8",
dims=(2, 2, 4), nplanes=3)
def test_fromMultipleMultiTimepointTifs(self):
imagePath = os.path.join(self.testResourcesDir, "multilayer_tif",
"dotdotdot_lzw*.tif")
tiffImages = ImagesLoader(self.sc).fromTif(imagePath, nplanes=1)
assert_true(tiffImages._nrecords is None)
assert_equals(6, tiffImages.nrecords)
collectedTiffImages = tiffImages.collect()
assert_equals(6, len(collectedTiffImages),
"Expected 6 images, got %d" % len(collectedTiffImages))
expectedSums = [1140006, 1119161, 1098917, 1140006, 1119161, 1098917]
expectedIdx = 0
for idx, tiffAry in collectedTiffImages:
assert_equals((70, 75), tiffAry.shape)
assert_equals(expectedIdx, idx)
assert_equals(expectedSums[idx], tiffAry.ravel().sum())
expectedIdx += 1
# 3 pages / file is not evenly divisible by 2 planes
# note this will still log a big traceback, since the exception is in the executor,
# not the driver. But this is expected behavior.
assert_raises(
Exception,
ImagesLoader(self.sc).fromTif(imagePath, nplanes=2).count)
def _run_tst_roundtripThroughBlocks(self, strategy):
imagepath = findSourceTreeDir("utils/data/fish/tif-stack")
images = ImagesLoader(self.sc).fromTif(imagepath)
blockedimages = images.toBlocks(strategy)
recombinedimages = blockedimages.toImages()
collectedimages = images.collect()
roundtrippedimages = recombinedimages.collect()
for orig, roundtripped in zip(collectedimages, roundtrippedimages):
assert_true(array_equal(orig[1], roundtripped[1]))
def test_fromArrays(self):
ary = arange(8, dtype=dtypeFunc('int16')).reshape((2, 4))
image = ImagesLoader(self.sc).fromArrays(ary)
collectedImage = image.collect()
assert_equals(1, len(collectedImage))
assert_equals(ary.shape, image.dims.count)
assert_equals(0, collectedImage[0][0]) # check key
assert_true(array_equal(ary, collectedImage[0][1])) # check value
def _run_tst_roundtripThroughBlocks(self, strategy):
imagepath = findSourceTreeDir("utils/data/fish/images")
images = ImagesLoader(self.sc).fromTif(imagepath)
blockedimages = images.toBlocks(strategy)
recombinedimages = blockedimages.toImages()
collectedimages = images.collect()
roundtrippedimages = recombinedimages.collect()
for orig, roundtripped in zip(collectedimages, roundtrippedimages):
assert_true(array_equal(orig[1], roundtripped[1]))
def test_fromArrays(self):
ary = arange(8, dtype=dtypeFunc('int16')).reshape((2, 4))
image = ImagesLoader(self.sc).fromArrays(ary)
collectedImage = image.collect()
assert_equals(1, len(collectedImage))
assert_equals(ary.shape, image.dims.count)
assert_equals(0, collectedImage[0][0]) # check key
assert_true(array_equal(ary, collectedImage[0][1])) # check value
def test_fromStack(self):
ary = arange(8, dtype=dtypeFunc('int16')).reshape((2, 4))
filename = os.path.join(self.outputdir, "test.stack")
ary.tofile(filename)
image = ImagesLoader(self.sc).fromStack(filename, dims=(4, 2))
collectedImage = image.collect()
assert_equals(1, len(collectedImage))
assert_equals(0, collectedImage[0][0]) # check key
# assert that image shape *matches* that in image dimensions:
assert_equals(image.dims.count, collectedImage[0][1].shape)
assert_true(array_equal(ary.T, collectedImage[0][1])) # check value
def test_fromStack(self):
ary = arange(8, dtype=dtypeFunc('int16')).reshape((2, 4))
filename = os.path.join(self.outputdir, "test.stack")
ary.tofile(filename)
image = ImagesLoader(self.sc).fromStack(filename, dims=(4, 2))
collectedImage = image.collect()
assert_equals(1, len(collectedImage))
assert_equals(0, collectedImage[0][0]) # check key
# assert that image shape *matches* that in image dimensions:
assert_equals(image.dims.count, collectedImage[0][1].shape)
assert_true(array_equal(ary.T, collectedImage[0][1])) # check value
def test_fromSignedIntTif(self):
imagePath = os.path.join(self.testResourcesDir, "multilayer_tif", "test_signed.tif")
tiffImages = ImagesLoader(self.sc).fromTif(imagePath, nplanes=1)
assert_equals(2, tiffImages.nrecords)
assert_equals('int16', tiffImages.dtype)
collectedTiffImages = tiffImages.collect()
assert_equals(2, len(collectedTiffImages), "Expected 2 images, got %d" % len(collectedTiffImages))
expectedSums = [1973201, 2254767]
expectedIdx = 0
for idx, tiffAry in collectedTiffImages:
assert_equals((120, 120), tiffAry.shape)
assert_equals('int16', str(tiffAry.dtype))
assert_equals(expectedIdx, idx)
assert_equals(expectedSums[idx], tiffAry.ravel().sum())
expectedIdx += 1
def test_fromStacks(self):
ary = arange(8, dtype=dtypeFunc('int16')).reshape((2, 4))
ary2 = arange(8, 16, dtype=dtypeFunc('int16')).reshape((2, 4))
filename = os.path.join(self.outputdir, "test01.stack")
ary.tofile(filename)
filename = os.path.join(self.outputdir, "test02.stack")
ary2.tofile(filename)
image = ImagesLoader(self.sc).fromStack(self.outputdir, dims=(4, 2))
collectedImage = image.collect()
assert_equals(2, len(collectedImage))
assert_equals(0, collectedImage[0][0]) # check key
assert_equals(image.dims.count, collectedImage[0][1].shape)
assert_true(array_equal(ary.T, collectedImage[0][1])) # check value
assert_equals(1, collectedImage[1][0]) # check image 2
assert_true(array_equal(ary2.T, collectedImage[1][1]))
def test_fromSignedIntTif(self):
imagePath = os.path.join(self.testResourcesDir, "multilayer_tif", "test_signed.tif")
tiffImages = ImagesLoader(self.sc).fromTif(imagePath, nplanes=1)
assert_equals(2, tiffImages.nrecords)
assert_equals('int16', tiffImages.dtype)
collectedTiffImages = tiffImages.collect()
assert_equals(2, len(collectedTiffImages), "Expected 2 images, got %d" % len(collectedTiffImages))
expectedSums = [1973201, 2254767]
expectedIdx = 0
for idx, tiffAry in collectedTiffImages:
assert_equals((120, 120), tiffAry.shape)
assert_equals('int16', str(tiffAry.dtype))
assert_equals(expectedIdx, idx)
assert_equals(expectedSums[idx], tiffAry.ravel().sum())
expectedIdx += 1
def test_fromMultiTimepointTif(self):
imagePath = os.path.join(self.testResourcesDir, "multilayer_tif", "dotdotdot_lzw.tif")
tiffImages = ImagesLoader(self.sc).fromTif(imagePath, nplanes=1)
# we don't expect to have nrecords cached, since the driver doesn't know how many images there are per file
assert_true(tiffImages._nrecords is None)
assert_equals(3, tiffImages.nrecords)
collectedTiffImages = tiffImages.collect()
assert_equals(3, len(collectedTiffImages), "Expected 3 images, got %d" % len(collectedTiffImages))
expectedSums = [1140006, 1119161, 1098917]
expectedIdx = 0
for idx, tiffAry in collectedTiffImages:
assert_equals((70, 75), tiffAry.shape)
assert_equals(expectedIdx, idx)
assert_equals(expectedSums[idx], tiffAry.ravel().sum())
expectedIdx += 1
def test_fromMultiTimepointTif(self):
imagePath = os.path.join(self.testResourcesDir, "multilayer_tif", "dotdotdot_lzw.tif")
tiffImages = ImagesLoader(self.sc).fromTif(imagePath, nplanes=1)
# we don't expect to have nrecords cached, since the driver doesn't know how many images there are per file
assert_true(tiffImages._nrecords is None)
assert_equals(3, tiffImages.nrecords)
collectedTiffImages = tiffImages.collect()
assert_equals(3, len(collectedTiffImages), "Expected 3 images, got %d" % len(collectedTiffImages))
expectedSums = [1140006, 1119161, 1098917]
expectedIdx = 0
for idx, tiffAry in collectedTiffImages:
assert_equals((70, 75), tiffAry.shape)
assert_equals(expectedIdx, idx)
assert_equals(expectedSums[idx], tiffAry.ravel().sum())
expectedIdx += 1
def test_fromStacks(self):
ary = arange(8, dtype=dtypeFunc('int16')).reshape((2, 4))
ary2 = arange(8, 16, dtype=dtypeFunc('int16')).reshape((2, 4))
filename = os.path.join(self.outputdir, "test01.stack")
ary.tofile(filename)
filename = os.path.join(self.outputdir, "test02.stack")
ary2.tofile(filename)
image = ImagesLoader(self.sc).fromStack(self.outputdir, dims=(4, 2))
collectedImage = image.collect()
assert_equals(2, len(collectedImage))
assert_equals(0, collectedImage[0][0]) # check key
assert_equals(image.dims.count, collectedImage[0][1].shape)
assert_true(array_equal(ary.T, collectedImage[0][1])) # check value
assert_equals(1, collectedImage[1][0]) # check image 2
assert_true(array_equal(ary2.T, collectedImage[1][1]))
def test_fromMultipleMultiTimepointTifs(self):
imagePath = os.path.join(self.testResourcesDir, "multilayer_tif", "dotdotdot_lzw*.tif")
tiffImages = ImagesLoader(self.sc).fromTif(imagePath, nplanes=1)
assert_true(tiffImages._nrecords is None)
assert_equals(6, tiffImages.nrecords)
collectedTiffImages = tiffImages.collect()
assert_equals(6, len(collectedTiffImages), "Expected 6 images, got %d" % len(collectedTiffImages))
expectedSums = [1140006, 1119161, 1098917, 1140006, 1119161, 1098917]
expectedIdx = 0
for idx, tiffAry in collectedTiffImages:
assert_equals((70, 75), tiffAry.shape)
assert_equals(expectedIdx, idx)
assert_equals(expectedSums[idx], tiffAry.ravel().sum())
expectedIdx += 1
# 3 pages / file is not evenly divisible by 2 planes
# note this will still log a big traceback, since the exception is in the executor,
# not the driver. But this is expected behavior.
assert_raises(Exception, ImagesLoader(self.sc).fromTif(imagePath, nplanes=2).count)
def test_fromStacksWithConf(self):
ary = arange(8, dtype=dtypeFunc('int32')).reshape((2, 4))
ary2 = arange(8, 16, dtype=dtypeFunc('int32')).reshape((2, 4))
filename = os.path.join(self.outputdir, "test01.stack")
ary.tofile(filename)
filename = os.path.join(self.outputdir, "test02.stack")
ary2.tofile(filename)
conf = {"dims": [4, 2], "dtype": "int32"}
with open(os.path.join(self.outputdir, "conf.json"), 'w') as fp:
json.dump(conf, fp)
image = ImagesLoader(self.sc).fromStack(self.outputdir)
assert_equals("int32", image._dtype)
assert_equals(2, image._nrecords)
assert_equals((4, 2), image._dims.count)
collectedImage = image.collect()
assert_equals(2, len(collectedImage))
assert_equals(0, collectedImage[0][0]) # check key
assert_equals(image.dims.count, collectedImage[0][1].shape)
assert_true(array_equal(ary.T, collectedImage[0][1])) # check value
assert_equals(1, collectedImage[1][0]) # check image 2
assert_true(array_equal(ary2.T, collectedImage[1][1]))
def test_fromStacksWithConf(self):
ary = arange(8, dtype=dtypeFunc('int32')).reshape((2, 4))
ary2 = arange(8, 16, dtype=dtypeFunc('int32')).reshape((2, 4))
filename = os.path.join(self.outputdir, "test01.stack")
ary.tofile(filename)
filename = os.path.join(self.outputdir, "test02.stack")
ary2.tofile(filename)
conf = {"dims": [4, 2], "dtype": "int32"}
with open(os.path.join(self.outputdir, "conf.json"), 'w') as fp:
json.dump(conf, fp)
image = ImagesLoader(self.sc).fromStack(self.outputdir)
assert_equals("int32", image._dtype)
assert_equals(2, image._nrecords)
assert_equals((4, 2), image._dims.count)
collectedImage = image.collect()
assert_equals(2, len(collectedImage))
assert_equals(0, collectedImage[0][0]) # check key
assert_equals(image.dims.count, collectedImage[0][1].shape)
assert_true(array_equal(ary.T, collectedImage[0][1])) # check value
assert_equals(1, collectedImage[1][0]) # check image 2
assert_true(array_equal(ary2.T, collectedImage[1][1]))
