def test_fromMultiTimepointStacks(self): ary = arange(16, dtype=dtypeFunc('uint8')).reshape((4, 2, 2)) ary2 = arange(16, 32, dtype=dtypeFunc('uint8')).reshape((4, 2, 2)) ary.tofile(os.path.join(self.outputdir, "test01.stack")) ary2.tofile(os.path.join(self.outputdir, "test02.stack")) image = ImagesLoader(self.sc).fromStack(self.outputdir, dtype="uint8", dims=(2, 2, 4), nplanes=2) collectedImage = image.collect() # we don't expect to have nrecords cached, since we get an unknown number of images per file assert_true(image._nrecords is None) assert_equals(4, image.nrecords) assert_equals(4, len(collectedImage)) # check keys: assert_equals(0, collectedImage[0][0]) assert_equals(1, collectedImage[1][0]) assert_equals(2, collectedImage[2][0]) assert_equals(3, collectedImage[3][0]) # check values: assert_true(array_equal(ary[:2].T, collectedImage[0][1])) assert_true(array_equal(ary[2:].T, collectedImage[1][1])) assert_true(array_equal(ary2[:2].T, collectedImage[2][1])) assert_true(array_equal(ary2[2:].T, collectedImage[3][1])) # 3 planes does not divide 4 assert_raises(ValueError, ImagesLoader(self.sc).fromStack, self.outputdir, dtype="uint8", dims=(2, 2, 4), nplanes=3)
def test_fromMultiTimepointStacks(self): ary = arange(16, dtype=dtypeFunc('uint8')).reshape((4, 2, 2)) ary2 = arange(16, 32, dtype=dtypeFunc('uint8')).reshape((4, 2, 2)) ary.tofile(os.path.join(self.outputdir, "test01.stack")) ary2.tofile(os.path.join(self.outputdir, "test02.stack")) image = ImagesLoader(self.sc).fromStack(self.outputdir, dtype="uint8", dims=(2, 2, 4), nplanes=2) collectedImage = image.collect() # we don't expect to have nrecords cached, since we get an unknown number of images per file assert_true(image._nrecords is None) assert_equals(4, image.nrecords) assert_equals(4, len(collectedImage)) # check keys: assert_equals(0, collectedImage[0][0]) assert_equals(1, collectedImage[1][0]) assert_equals(2, collectedImage[2][0]) assert_equals(3, collectedImage[3][0]) # check values: assert_true(array_equal(ary[:2].T, collectedImage[0][1])) assert_true(array_equal(ary[2:].T, collectedImage[1][1])) assert_true(array_equal(ary2[:2].T, collectedImage[2][1])) assert_true(array_equal(ary2[2:].T, collectedImage[3][1])) # 3 planes does not divide 4 assert_raises(ValueError, ImagesLoader(self.sc).fromStack, self.outputdir, dtype="uint8", dims=(2, 2, 4), nplanes=3)
def test_fromMultipleMultiTimepointTifs(self): imagePath = os.path.join(self.testResourcesDir, "multilayer_tif", "dotdotdot_lzw*.tif") tiffImages = ImagesLoader(self.sc).fromTif(imagePath, nplanes=1) assert_true(tiffImages._nrecords is None) assert_equals(6, tiffImages.nrecords) collectedTiffImages = tiffImages.collect() assert_equals(6, len(collectedTiffImages), "Expected 6 images, got %d" % len(collectedTiffImages)) expectedSums = [1140006, 1119161, 1098917, 1140006, 1119161, 1098917] expectedIdx = 0 for idx, tiffAry in collectedTiffImages: assert_equals((70, 75), tiffAry.shape) assert_equals(expectedIdx, idx) assert_equals(expectedSums[idx], tiffAry.ravel().sum()) expectedIdx += 1 # 3 pages / file is not evenly divisible by 2 planes # note this will still log a big traceback, since the exception is in the executor, # not the driver. But this is expected behavior. assert_raises( Exception, ImagesLoader(self.sc).fromTif(imagePath, nplanes=2).count)
def _run_tst_roundtripThroughBlocks(self, strategy): imagepath = findSourceTreeDir("utils/data/fish/tif-stack") images = ImagesLoader(self.sc).fromTif(imagepath) blockedimages = images.toBlocks(strategy) recombinedimages = blockedimages.toImages() collectedimages = images.collect() roundtrippedimages = recombinedimages.collect() for orig, roundtripped in zip(collectedimages, roundtrippedimages): assert_true(array_equal(orig[1], roundtripped[1]))
def test_fromArrays(self): ary = arange(8, dtype=dtypeFunc('int16')).reshape((2, 4)) image = ImagesLoader(self.sc).fromArrays(ary) collectedImage = image.collect() assert_equals(1, len(collectedImage)) assert_equals(ary.shape, image.dims.count) assert_equals(0, collectedImage[0][0]) # check key assert_true(array_equal(ary, collectedImage[0][1])) # check value
def _run_tst_roundtripThroughBlocks(self, strategy): imagepath = findSourceTreeDir("utils/data/fish/images") images = ImagesLoader(self.sc).fromTif(imagepath) blockedimages = images.toBlocks(strategy) recombinedimages = blockedimages.toImages() collectedimages = images.collect() roundtrippedimages = recombinedimages.collect() for orig, roundtripped in zip(collectedimages, roundtrippedimages): assert_true(array_equal(orig[1], roundtripped[1]))
def test_fromArrays(self): ary = arange(8, dtype=dtypeFunc('int16')).reshape((2, 4)) image = ImagesLoader(self.sc).fromArrays(ary) collectedImage = image.collect() assert_equals(1, len(collectedImage)) assert_equals(ary.shape, image.dims.count) assert_equals(0, collectedImage[0][0]) # check key assert_true(array_equal(ary, collectedImage[0][1])) # check value
def test_fromStack(self): ary = arange(8, dtype=dtypeFunc('int16')).reshape((2, 4)) filename = os.path.join(self.outputdir, "test.stack") ary.tofile(filename) image = ImagesLoader(self.sc).fromStack(filename, dims=(4, 2)) collectedImage = image.collect() assert_equals(1, len(collectedImage)) assert_equals(0, collectedImage[0][0]) # check key # assert that image shape *matches* that in image dimensions: assert_equals(image.dims.count, collectedImage[0][1].shape) assert_true(array_equal(ary.T, collectedImage[0][1])) # check value
def test_fromStack(self): ary = arange(8, dtype=dtypeFunc('int16')).reshape((2, 4)) filename = os.path.join(self.outputdir, "test.stack") ary.tofile(filename) image = ImagesLoader(self.sc).fromStack(filename, dims=(4, 2)) collectedImage = image.collect() assert_equals(1, len(collectedImage)) assert_equals(0, collectedImage[0][0]) # check key # assert that image shape *matches* that in image dimensions: assert_equals(image.dims.count, collectedImage[0][1].shape) assert_true(array_equal(ary.T, collectedImage[0][1])) # check value
def test_fromSignedIntTif(self): imagePath = os.path.join(self.testResourcesDir, "multilayer_tif", "test_signed.tif") tiffImages = ImagesLoader(self.sc).fromTif(imagePath, nplanes=1) assert_equals(2, tiffImages.nrecords) assert_equals('int16', tiffImages.dtype) collectedTiffImages = tiffImages.collect() assert_equals(2, len(collectedTiffImages), "Expected 2 images, got %d" % len(collectedTiffImages)) expectedSums = [1973201, 2254767] expectedIdx = 0 for idx, tiffAry in collectedTiffImages: assert_equals((120, 120), tiffAry.shape) assert_equals('int16', str(tiffAry.dtype)) assert_equals(expectedIdx, idx) assert_equals(expectedSums[idx], tiffAry.ravel().sum()) expectedIdx += 1
def test_fromStacks(self): ary = arange(8, dtype=dtypeFunc('int16')).reshape((2, 4)) ary2 = arange(8, 16, dtype=dtypeFunc('int16')).reshape((2, 4)) filename = os.path.join(self.outputdir, "test01.stack") ary.tofile(filename) filename = os.path.join(self.outputdir, "test02.stack") ary2.tofile(filename) image = ImagesLoader(self.sc).fromStack(self.outputdir, dims=(4, 2)) collectedImage = image.collect() assert_equals(2, len(collectedImage)) assert_equals(0, collectedImage[0][0]) # check key assert_equals(image.dims.count, collectedImage[0][1].shape) assert_true(array_equal(ary.T, collectedImage[0][1])) # check value assert_equals(1, collectedImage[1][0]) # check image 2 assert_true(array_equal(ary2.T, collectedImage[1][1]))
def test_fromSignedIntTif(self): imagePath = os.path.join(self.testResourcesDir, "multilayer_tif", "test_signed.tif") tiffImages = ImagesLoader(self.sc).fromTif(imagePath, nplanes=1) assert_equals(2, tiffImages.nrecords) assert_equals('int16', tiffImages.dtype) collectedTiffImages = tiffImages.collect() assert_equals(2, len(collectedTiffImages), "Expected 2 images, got %d" % len(collectedTiffImages)) expectedSums = [1973201, 2254767] expectedIdx = 0 for idx, tiffAry in collectedTiffImages: assert_equals((120, 120), tiffAry.shape) assert_equals('int16', str(tiffAry.dtype)) assert_equals(expectedIdx, idx) assert_equals(expectedSums[idx], tiffAry.ravel().sum()) expectedIdx += 1
def test_fromMultiTimepointTif(self): imagePath = os.path.join(self.testResourcesDir, "multilayer_tif", "dotdotdot_lzw.tif") tiffImages = ImagesLoader(self.sc).fromTif(imagePath, nplanes=1) # we don't expect to have nrecords cached, since the driver doesn't know how many images there are per file assert_true(tiffImages._nrecords is None) assert_equals(3, tiffImages.nrecords) collectedTiffImages = tiffImages.collect() assert_equals(3, len(collectedTiffImages), "Expected 3 images, got %d" % len(collectedTiffImages)) expectedSums = [1140006, 1119161, 1098917] expectedIdx = 0 for idx, tiffAry in collectedTiffImages: assert_equals((70, 75), tiffAry.shape) assert_equals(expectedIdx, idx) assert_equals(expectedSums[idx], tiffAry.ravel().sum()) expectedIdx += 1
def test_fromMultiTimepointTif(self): imagePath = os.path.join(self.testResourcesDir, "multilayer_tif", "dotdotdot_lzw.tif") tiffImages = ImagesLoader(self.sc).fromTif(imagePath, nplanes=1) # we don't expect to have nrecords cached, since the driver doesn't know how many images there are per file assert_true(tiffImages._nrecords is None) assert_equals(3, tiffImages.nrecords) collectedTiffImages = tiffImages.collect() assert_equals(3, len(collectedTiffImages), "Expected 3 images, got %d" % len(collectedTiffImages)) expectedSums = [1140006, 1119161, 1098917] expectedIdx = 0 for idx, tiffAry in collectedTiffImages: assert_equals((70, 75), tiffAry.shape) assert_equals(expectedIdx, idx) assert_equals(expectedSums[idx], tiffAry.ravel().sum()) expectedIdx += 1
def test_fromStacks(self): ary = arange(8, dtype=dtypeFunc('int16')).reshape((2, 4)) ary2 = arange(8, 16, dtype=dtypeFunc('int16')).reshape((2, 4)) filename = os.path.join(self.outputdir, "test01.stack") ary.tofile(filename) filename = os.path.join(self.outputdir, "test02.stack") ary2.tofile(filename) image = ImagesLoader(self.sc).fromStack(self.outputdir, dims=(4, 2)) collectedImage = image.collect() assert_equals(2, len(collectedImage)) assert_equals(0, collectedImage[0][0]) # check key assert_equals(image.dims.count, collectedImage[0][1].shape) assert_true(array_equal(ary.T, collectedImage[0][1])) # check value assert_equals(1, collectedImage[1][0]) # check image 2 assert_true(array_equal(ary2.T, collectedImage[1][1]))
def test_fromMultipleMultiTimepointTifs(self): imagePath = os.path.join(self.testResourcesDir, "multilayer_tif", "dotdotdot_lzw*.tif") tiffImages = ImagesLoader(self.sc).fromTif(imagePath, nplanes=1) assert_true(tiffImages._nrecords is None) assert_equals(6, tiffImages.nrecords) collectedTiffImages = tiffImages.collect() assert_equals(6, len(collectedTiffImages), "Expected 6 images, got %d" % len(collectedTiffImages)) expectedSums = [1140006, 1119161, 1098917, 1140006, 1119161, 1098917] expectedIdx = 0 for idx, tiffAry in collectedTiffImages: assert_equals((70, 75), tiffAry.shape) assert_equals(expectedIdx, idx) assert_equals(expectedSums[idx], tiffAry.ravel().sum()) expectedIdx += 1 # 3 pages / file is not evenly divisible by 2 planes # note this will still log a big traceback, since the exception is in the executor, # not the driver. But this is expected behavior. assert_raises(Exception, ImagesLoader(self.sc).fromTif(imagePath, nplanes=2).count)
def test_fromStacksWithConf(self): ary = arange(8, dtype=dtypeFunc('int32')).reshape((2, 4)) ary2 = arange(8, 16, dtype=dtypeFunc('int32')).reshape((2, 4)) filename = os.path.join(self.outputdir, "test01.stack") ary.tofile(filename) filename = os.path.join(self.outputdir, "test02.stack") ary2.tofile(filename) conf = {"dims": [4, 2], "dtype": "int32"} with open(os.path.join(self.outputdir, "conf.json"), 'w') as fp: json.dump(conf, fp) image = ImagesLoader(self.sc).fromStack(self.outputdir) assert_equals("int32", image._dtype) assert_equals(2, image._nrecords) assert_equals((4, 2), image._dims.count) collectedImage = image.collect() assert_equals(2, len(collectedImage)) assert_equals(0, collectedImage[0][0]) # check key assert_equals(image.dims.count, collectedImage[0][1].shape) assert_true(array_equal(ary.T, collectedImage[0][1])) # check value assert_equals(1, collectedImage[1][0]) # check image 2 assert_true(array_equal(ary2.T, collectedImage[1][1]))
def test_fromStacksWithConf(self): ary = arange(8, dtype=dtypeFunc('int32')).reshape((2, 4)) ary2 = arange(8, 16, dtype=dtypeFunc('int32')).reshape((2, 4)) filename = os.path.join(self.outputdir, "test01.stack") ary.tofile(filename) filename = os.path.join(self.outputdir, "test02.stack") ary2.tofile(filename) conf = {"dims": [4, 2], "dtype": "int32"} with open(os.path.join(self.outputdir, "conf.json"), 'w') as fp: json.dump(conf, fp) image = ImagesLoader(self.sc).fromStack(self.outputdir) assert_equals("int32", image._dtype) assert_equals(2, image._nrecords) assert_equals((4, 2), image._dims.count) collectedImage = image.collect() assert_equals(2, len(collectedImage)) assert_equals(0, collectedImage[0][0]) # check key assert_equals(image.dims.count, collectedImage[0][1].shape) assert_true(array_equal(ary.T, collectedImage[0][1])) # check value assert_equals(1, collectedImage[1][0]) # check image 2 assert_true(array_equal(ary2.T, collectedImage[1][1]))