def test_partitioneq(self):
"""Check equivalence function for volumePartition.
"""
part1 = volumePartition(7, VolumeOffset(3, 1, 5))
part2 = volumePartition(7, VolumeOffset(3, 2, 5))
part3 = volumePartition(8, VolumeOffset(3, 1, 5))
# should be equal even with different offsets
self.assertEqual(part1, part2)
# should be different if index is different
self.assertNotEqual(part1, part3)
def test_partitionhash(self):
"""Check hashing function for volumePartition.
"""
part1 = volumePartition(7, VolumeOffset(3, 1, 5))
part2 = volumePartition(7, VolumeOffset(3, 2, 5))
part3 = volumePartition(8, VolumeOffset(3, 1, 5))
# hash should be equal even with different offsets
self.assertEqual(hash(part1), hash(part2))
# hash should be different if index is different
self.assertNotEqual(hash(part1), hash(part3))
def test_creatinglargevol(self):
"""Take small partitions and group into one large partition.
"""
arr = np.random.randint(1025, size=(4, 6, 4)).astype(np.uint16)
schema = partitionSchema(PartitionDims(2, 0, 0),
blank_delimiter=1111,
padding=2)
volpart = volumePartition(0, VolumeOffset(1, 1, 1))
res = schema.partition_data([(volpart, arr)])
self.assertEqual(len(res), 3) # 3 partitions with zsize=2 each
# make a new volume and pad
arrcomp = np.zeros((6, 8, 6), dtype=np.uint16)
arrcomp[:] = 1111
arrcomp[1, 1:7, 1:5] = arr[0, :, :]
arrcomp[2, 1:7, 1:5] = arr[1, :, :]
arrcomp[3, 1:7, 1:5] = arr[2, :, :]
arrcomp[4, 1:7, 1:5] = arr[3, :, :]
# reverse procedure should be same as the original
schemaglb = partitionSchema(PartitionDims(0, 0, 0))
res2 = schemaglb.partition_data(res)
self.assertEqual(len(res2), 1) # 3 partitions with zsize=2 each
match = np.array_equal(arrcomp, res2[0][1])
self.assertEqual(match, True)
def test_fixedpartitionandreloffset(self):
"""Test rigid partition sizes and relative offsets.
No padding should be needed and mask should be None.
"""
arr = np.random.randint(1025, size=(4, 6, 4)).astype(np.uint16)
schema = partitionSchema(PartitionDims(2, 8, 8),
blank_delimiter=1111,
padding=2,
enablemask=True)
volpart = volumePartition(0,
VolumeOffset(1, 1, 1),
reloffset=VolumeOffset(1, 1, 1))
res = schema.partition_data([[volpart, arr]])
self.assertEqual(len(res), 2) # 2 partitions with zsize=2 each
for partpair in res:
part, partvol = partpair
zidx = part.get_offset().z
# this mask should be all 1
self.assertEqual(part.get_reloffset().x, 2)
self.assertEqual(part.get_reloffset().y, 2)
self.assertEqual(part.get_reloffset().z, 0)
match = np.array_equal(arr[zidx - 2:zidx, :, :], partvol)
self.assertEqual(match, True)
self.assertEqual(part.mask, None)
def repartition_down(part_volume):
part, volume = part_volume
downsampled_offset = np.array(part.get_offset()) // 2
downsampled_reloffset = np.array(part.get_reloffset()) // 2
offsetnew = VolumeOffset(*downsampled_offset)
reloffsetnew = VolumeOffset(*downsampled_reloffset)
partnew = volumePartition((offsetnew.z, offsetnew.y, offsetnew.x), offsetnew, reloffset=reloffsetnew)
return partnew, volume
def repartition_down(part_volume):
part, volume = part_volume
downsampled_offset = np.array(part.get_offset()) // 2
downsampled_reloffset = np.array(
part.get_reloffset()) // 2
offsetnew = VolumeOffset(*downsampled_offset)
reloffsetnew = VolumeOffset(*downsampled_reloffset)
partnew = volumePartition(
(offsetnew.z, offsetnew.y, offsetnew.x),
offsetnew,
reloffset=reloffsetnew)
return partnew, volume
def test_createtiles(self):
"""Take a 3D volume and transform into a series of slices.
"""
arr = np.random.randint(255, size=(45, 25, 13)).astype(np.uint8)
#arr = np.random.randint(255, size=(2,3,4)).astype(np.uint8)
schema = partitionSchema(PartitionDims(1, 0, 0))
volpart = volumePartition(0, VolumeOffset(0, 0, 0))
res = schema.partition_data([[volpart, arr]])
self.assertEqual(len(res), 45)
for tilepair in res:
part, tile = tilepair
zplane = part.get_offset().z
match = np.array_equal(tile[0, :, :], arr[zplane, :, :])
self.assertEqual(match, True)
def test_dvidpadlabels(self):
"""Check padding data with DVID labels.
"""
service = DVIDServerService(dvidserver)
uuid = service.create_new_repo("foo", "bar")
ns = DVIDNodeService(dvidserver, uuid)
ns.create_labelblk("labels")
arr = np.random.randint(12442, size=(58, 58, 58)).astype(np.uint64)
arr2 = np.zeros((64, 64, 64), np.uint64)
arr2[0:58, 0:58, 0:58] = arr
# load gray data
ns.put_labels3D("labels", arr2, (0, 0, 0))
# load shifted data for comparison
arr2[6:64, 6:64, 6:64] = arr
# read and pad data
schema = partitionSchema(PartitionDims(32, 64, 64),
enablemask=True,
padding=8,
blank_delimiter=99999)
volpart = volumePartition(0, VolumeOffset(6, 6, 6))
partitions = schema.partition_data([(volpart, arr)])
# fetch with mask
dvidreader = dvidSrc(dvidserver, uuid, "labels", partitions)
newparts = dvidreader.extract_volume()
self.assertEqual(len(newparts), 2)
for (part, vol) in newparts:
if part.get_offset().z == 0:
match = np.array_equal(arr2[0:32, :, :], vol)
self.assertTrue(match)
else:
match = np.array_equal(arr2[32:64, :, :], vol)
self.assertTrue(match)
def test_createtilesshifted(self):
"""Take a 3d volume with offset and transform into a series of slices.
The size of each tile should be the same even with the shift.
"""
arr = np.random.randint(255, size=(45, 25, 13)).astype(np.uint8)
#arr = np.random.randint(255, size=(2,3,4)).astype(np.uint8)
schema = partitionSchema(PartitionDims(1, 0, 0))
volpart = volumePartition(0, VolumeOffset(4, 2, 3))
res = schema.partition_data([[volpart, arr]])
self.assertEqual(len(res), 45)
for tilepair in res:
part, tile = tilepair
zplane = part.get_offset().z - 4
self.assertEqual(part.get_offset().x, 0)
self.assertEqual(part.get_offset().y, 0)
match = np.array_equal(tile[0, :, :], arr[zplane, :, :])
self.assertEqual(match, True)
def test_createvolshiftandpad(self):
"""Take a 3d volume with offset and transform into a series of subvolumes with padding.
The data is moved to the proper partition and is padded so that the x, y, z
of the stored data is lined up to a grid determined by the padding specified.
Also, tests >8bit data and the data mask.
"""
arr = np.random.randint(1025, size=(4, 6, 4)).astype(np.uint16)
#arr = np.random.randint(255, size=(2,3,4)).astype(np.uint8)
schema = partitionSchema(PartitionDims(2, 0, 0),
blank_delimiter=1111,
padding=2,
enablemask=True)
volpart = volumePartition(0, VolumeOffset(1, 1, 1))
res = schema.partition_data([[volpart, arr]])
self.assertEqual(len(res), 3) # 3 partitions with zsize=2 each
arrcomp = np.zeros((6, 8, 6), dtype=np.uint16)
arrcomp[:] = 1111
arrcomp[1, 1:7, 1:5] = arr[0, :, :]
arrcomp[2, 1:7, 1:5] = arr[1, :, :]
arrcomp[3, 1:7, 1:5] = arr[2, :, :]
arrcomp[4, 1:7, 1:5] = arr[3, :, :]
for partpair in res:
part, partvol = partpair
zidx = part.get_offset().z
# make a mask
mask = arrcomp[zidx:zidx + 2, :, :].copy()
mask[mask != 1111] = 1
mask[mask == 1111] = 0
match = np.array_equal(arrcomp[zidx:zidx + 2, :, :], partvol)
matchmask = np.array_equal(mask, part.mask)
self.assertEqual(match, True)
self.assertEqual(matchmask, True)
def test_dvidfetchgray(self):
"""Check reading grayscale from DVID from partitions.
This also checks basic iteration and overwrite of
previous data.
"""
service = DVIDServerService(dvidserver)
uuid = service.create_new_repo("foo", "bar")
ns = DVIDNodeService(dvidserver, uuid)
ns.create_grayscale8("gray")
arr = np.random.randint(255, size=(64, 64, 64)).astype(np.uint8)
# load gray data
ns.put_gray3D("gray", arr, (0, 0, 0))
# read data
schema = partitionSchema(PartitionDims(32, 64, 64))
volpart = volumePartition(0, VolumeOffset(0, 0, 0))
overwrite = np.random.randint(255, size=(64, 64, 64)).astype(np.uint8)
partitions = schema.partition_data([(volpart, overwrite)])
dvidreader = dvidSrc(dvidserver,
uuid,
"gray",
partitions,
maskonly=False)
newparts = dvidreader.extract_volume()
self.assertEqual(len(newparts), 2)
for (part, vol) in newparts:
if part.get_offset().z == 0:
match = np.array_equal(arr[0:32, :, :], vol)
self.assertTrue(match)
else:
match = np.array_equal(arr[32:64, :, :], vol)
self.assertTrue(match)
# test iteration
dvidreader2 = dvidSrc(dvidserver,
uuid,
"gray",
partitions,
maskonly=False)
newparts2 = []
for newpart in dvidreader2:
self.assertEqual(len(newpart), 1)
newparts2.extend(newpart)
self.assertEqual(len(newparts2), 2)
for (part, vol) in newparts2:
if part.get_offset().z == 0:
match = np.array_equal(arr[0:32, :, :], vol)
self.assertTrue(match)
else:
match = np.array_equal(arr[32:64, :, :], vol)
self.assertTrue(match)