def test_iteration(self): """Reads 32 images at a time and checks the final result is equal to original. """ zplanes = 503 arr = np.random.randint(255, size=(zplanes, 233, 112)).astype(np.uint8) filterstr, formatstr, fnames = writeImages(arr) schema = partitionSchema(PartitionDims(32, 64, 64), padding=32) imgreader = imagefileSrc(schema, formatstr, minmaxplane=(0, zplanes), offset=VolumeOffset(35, 21, 55)) # use the iterator, the iteration size is determiend by the Z partition size partitions = [] for partitions_iter in imgreader: partitions.extend(partitions_iter) for fname in fnames: os.remove(fname) self.assertEqual(len(partitions), 192) schema = partitionSchema(PartitionDims(0, 0, 0)) res = schema.partition_data(partitions) self.assertEqual(len(res), 1) # data that comes back will be padded by 32 origvol = np.zeros((512, 256, 160), dtype=np.uint8) origvol[3:506, 21:254, 23:135] = arr finalvol = res[0][1] match = np.array_equal(origvol, finalvol) self.assertEqual(match, True)
def test_largeimportandreverse(self): """Imports a large volume shifted, partitions, then unpartitions. """ zplanes = 503 arr = np.random.randint(255, size=(zplanes, 233, 112)).astype(np.uint8) filterstr, formatstr, fnames = writeImages(arr) schema = partitionSchema(PartitionDims(32, 64, 64), padding=32) imgreader = imagefileSrc(schema, formatstr, minmaxplane=(0, zplanes), offset=VolumeOffset(35, 21, 55)) partitions = imgreader.extract_volume() for fname in fnames: os.remove(fname) self.assertEqual(len(partitions), 192) schema = partitionSchema(PartitionDims(0, 0, 0)) res = schema.partition_data(partitions) self.assertEqual(len(res), 1) # data that comes back will be padded by 32 origvol = np.zeros((512, 256, 160), dtype=np.uint8) origvol[3:506, 21:254, 23:135] = arr finalvol = res[0][1] match = np.array_equal(origvol, finalvol) self.assertEqual(match, True)
def test_creatinglargevol(self): """Take small partitions and group into one large partition. """ arr = np.random.randint(1025, size=(4, 6, 4)).astype(np.uint16) schema = partitionSchema(PartitionDims(2, 0, 0), blank_delimiter=1111, padding=2) volpart = volumePartition(0, VolumeOffset(1, 1, 1)) res = schema.partition_data([(volpart, arr)]) self.assertEqual(len(res), 3) # 3 partitions with zsize=2 each # make a new volume and pad arrcomp = np.zeros((6, 8, 6), dtype=np.uint16) arrcomp[:] = 1111 arrcomp[1, 1:7, 1:5] = arr[0, :, :] arrcomp[2, 1:7, 1:5] = arr[1, :, :] arrcomp[3, 1:7, 1:5] = arr[2, :, :] arrcomp[4, 1:7, 1:5] = arr[3, :, :] # reverse procedure should be same as the original schemaglb = partitionSchema(PartitionDims(0, 0, 0)) res2 = schemaglb.partition_data(res) self.assertEqual(len(res2), 1) # 3 partitions with zsize=2 each match = np.array_equal(arrcomp, res2[0][1]) self.assertEqual(match, True)
def test_fixedpartitionandreloffset(self): """Test rigid partition sizes and relative offsets. No padding should be needed and mask should be None. """ arr = np.random.randint(1025, size=(4, 6, 4)).astype(np.uint16) schema = partitionSchema(PartitionDims(2, 8, 8), blank_delimiter=1111, padding=2, enablemask=True) volpart = volumePartition(0, VolumeOffset(1, 1, 1), reloffset=VolumeOffset(1, 1, 1)) res = schema.partition_data([[volpart, arr]]) self.assertEqual(len(res), 2) # 2 partitions with zsize=2 each for partpair in res: part, partvol = partpair zidx = part.get_offset().z # this mask should be all 1 self.assertEqual(part.get_reloffset().x, 2) self.assertEqual(part.get_reloffset().y, 2) self.assertEqual(part.get_reloffset().z, 0) match = np.array_equal(arr[zidx - 2:zidx, :, :], partvol) self.assertEqual(match, True) self.assertEqual(part.mask, None)
def test_createtiles(self): """Take a 3D volume and transform into a series of slices. """ arr = np.random.randint(255, size=(45, 25, 13)).astype(np.uint8) #arr = np.random.randint(255, size=(2,3,4)).astype(np.uint8) schema = partitionSchema(PartitionDims(1, 0, 0)) volpart = volumePartition(0, VolumeOffset(0, 0, 0)) res = schema.partition_data([[volpart, arr]]) self.assertEqual(len(res), 45) for tilepair in res: part, tile = tilepair zplane = part.get_offset().z match = np.array_equal(tile[0, :, :], arr[zplane, :, :]) self.assertEqual(match, True)
def test_retrieve_shiftedpaddedimages(self): """Converts 3d numpy array and imports into shifted global space with padding. Note: Tests min/max plane format string as well. """ zplanes = 32 arr = np.random.randint(255, size=(zplanes, 25, 13)).astype(np.uint8) filterstr, formatstr, fnames = writeImages(arr, 5) schema = partitionSchema(PartitionDims(32, 0, 0), padding=8) imgreader = imagefileSrc(schema, formatstr, minmaxplane=(5, 5 + zplanes), offset=VolumeOffset(1, 3, 2)) partitions = imgreader.extract_volume() for fname in fnames: os.remove(fname) self.assertEqual(len(partitions), 2) origvol = np.zeros((40, 32, 16), dtype=np.uint8) origvol[1:33, 3:28, 2:15] = arr zoff = partitions[0][0].get_offset().z if zoff == 0: finalvol = partitions[0][1] match = np.array_equal(origvol[0:32, 0:32, 0:16], finalvol) self.assertEqual(match, True) finalvol = partitions[1][1] match = np.array_equal(origvol[32:40, 0:32, 0:16], finalvol) self.assertEqual(match, True) else: finalvol = partitions[1][1] match = np.array_equal(origvol[0:32, 0:32, 0:16], finalvol) self.assertEqual(match, True) finalvol = partitions[0][1] match = np.array_equal(origvol[32:40, 0:32, 0:16], finalvol) self.assertEqual(match, True)
def test_dvidpadlabels(self): """Check padding data with DVID labels. """ service = DVIDServerService(dvidserver) uuid = service.create_new_repo("foo", "bar") ns = DVIDNodeService(dvidserver, uuid) ns.create_labelblk("labels") arr = np.random.randint(12442, size=(58, 58, 58)).astype(np.uint64) arr2 = np.zeros((64, 64, 64), np.uint64) arr2[0:58, 0:58, 0:58] = arr # load gray data ns.put_labels3D("labels", arr2, (0, 0, 0)) # load shifted data for comparison arr2[6:64, 6:64, 6:64] = arr # read and pad data schema = partitionSchema(PartitionDims(32, 64, 64), enablemask=True, padding=8, blank_delimiter=99999) volpart = volumePartition(0, VolumeOffset(6, 6, 6)) partitions = schema.partition_data([(volpart, arr)]) # fetch with mask dvidreader = dvidSrc(dvidserver, uuid, "labels", partitions) newparts = dvidreader.extract_volume() self.assertEqual(len(newparts), 2) for (part, vol) in newparts: if part.get_offset().z == 0: match = np.array_equal(arr2[0:32, :, :], vol) self.assertTrue(match) else: match = np.array_equal(arr2[32:64, :, :], vol) self.assertTrue(match)
def test_createtilesshifted(self): """Take a 3d volume with offset and transform into a series of slices. The size of each tile should be the same even with the shift. """ arr = np.random.randint(255, size=(45, 25, 13)).astype(np.uint8) #arr = np.random.randint(255, size=(2,3,4)).astype(np.uint8) schema = partitionSchema(PartitionDims(1, 0, 0)) volpart = volumePartition(0, VolumeOffset(4, 2, 3)) res = schema.partition_data([[volpart, arr]]) self.assertEqual(len(res), 45) for tilepair in res: part, tile = tilepair zplane = part.get_offset().z - 4 self.assertEqual(part.get_offset().x, 0) self.assertEqual(part.get_offset().y, 0) match = np.array_equal(tile[0, :, :], arr[zplane, :, :]) self.assertEqual(match, True)
def test_extractpartitionaligned(self): """Imports images shifted into partitioned space that is padded entire Z size. """ zplanes = 33 arr = np.random.randint(255, size=(zplanes, 25, 13)).astype(np.uint8) filterstr, formatstr, fnames = writeImages(arr, 5) schema = partitionSchema(PartitionDims(32, 0, 0), padding=32) imgreader = imagefileSrc(schema, formatstr, minmaxplane=(5, 5 + zplanes), offset=VolumeOffset(1, 3, 2)) partitions = imgreader.extract_volume() for fname in fnames: os.remove(fname) self.assertEqual(len(partitions), 2) origvol = np.zeros((64, 32, 32), dtype=np.uint8) origvol[1:34, 3:28, 2:15] = arr zoff = partitions[0][0].get_offset().z if zoff == 0: finalvol = partitions[0][1] match = np.array_equal(origvol[0:32, :, :], finalvol) self.assertEqual(match, True) finalvol = partitions[1][1] match = np.array_equal(origvol[32:64, :, :], finalvol) self.assertEqual(match, True) else: finalvol = partitions[1][1] match = np.array_equal(origvol[0:32, :, :], finalvol) self.assertEqual(match, True) finalvol = partitions[0][1] match = np.array_equal(origvol[32:64, :, :], finalvol) self.assertEqual(match, True)
def test_createvolshiftandpad(self): """Take a 3d volume with offset and transform into a series of subvolumes with padding. The data is moved to the proper partition and is padded so that the x, y, z of the stored data is lined up to a grid determined by the padding specified. Also, tests >8bit data and the data mask. """ arr = np.random.randint(1025, size=(4, 6, 4)).astype(np.uint16) #arr = np.random.randint(255, size=(2,3,4)).astype(np.uint8) schema = partitionSchema(PartitionDims(2, 0, 0), blank_delimiter=1111, padding=2, enablemask=True) volpart = volumePartition(0, VolumeOffset(1, 1, 1)) res = schema.partition_data([[volpart, arr]]) self.assertEqual(len(res), 3) # 3 partitions with zsize=2 each arrcomp = np.zeros((6, 8, 6), dtype=np.uint16) arrcomp[:] = 1111 arrcomp[1, 1:7, 1:5] = arr[0, :, :] arrcomp[2, 1:7, 1:5] = arr[1, :, :] arrcomp[3, 1:7, 1:5] = arr[2, :, :] arrcomp[4, 1:7, 1:5] = arr[3, :, :] for partpair in res: part, partvol = partpair zidx = part.get_offset().z # make a mask mask = arrcomp[zidx:zidx + 2, :, :].copy() mask[mask != 1111] = 1 mask[mask == 1111] = 0 match = np.array_equal(arrcomp[zidx:zidx + 2, :, :], partvol) matchmask = np.array_equal(mask, part.mask) self.assertEqual(match, True) self.assertEqual(matchmask, True)
def test_retrieve_wholevolume(self): """Converts 3d numpy array to 2D slices and imports these slices as a 3D volume. Note: Also checks that volume offset works properly. """ zplanes = 5 arr = np.random.randint(255, size=(zplanes, 25, 13)).astype(np.uint8) filterstr, formatstr, fnames = writeImages(arr, 10) schema = partitionSchema(PartitionDims(0, 0, 0)) imgreader = imagefileSrc(schema, filterstr, offset=VolumeOffset(1, 0, 0)) partitions = imgreader.extract_volume() for fname in fnames: os.remove(fname) self.assertEqual(len(partitions), 1) finalvol = partitions[0][1] match = np.array_equal(arr, finalvol) self.assertEqual(match, True)
def execute(self): """ Execute spark workflow. """ self._sanitize_config() session = default_dvid_session() dvid_info = self.config_data["dvid-info"] options = self.config_data["options"] block_shape = 3 * (options["blocksize"], ) self.partition_size = options["blockwritelimit"] * options["blocksize"] # ?? num parallel requests might be really small at high levels of pyramids # xdim is unbounded or very large partition_dims = PartitionDims(options["blocksize"], options["blocksize"], self.partition_size) partition_schema = partitionSchema( partition_dims, blank_delimiter=options["blankdelimiter"], padding=options["blocksize"], enablemask=options["has-dvidmask"]) offset_zyx = np.array(options["offset"][::-1]) offset_zyx[0] += options["minslice"] imgreader = imagefileSrc(partition_schema, options["basename"], (options["minslice"], options["maxslice"]), VolumeOffset(*offset_zyx), self.sc) # !! hack: override iteration size that is set to partition size, TODO: add option # this just makes the downstream processing a little more convenient, and reduces # unnecessary DVID patching if that is enabled. # (must be a multiple of block size) imgreader.iteration_size = options["num-tasks"] # get dims from image (hackage) from PIL import Image import requests if '%' in options["basename"]: minslice_name = options["basename"] % options["minslice"] elif '{' in options["basename"]: minslice_name = options["basename"].format(options["minslice"]) else: raise RuntimeError( f"Unrecognized format string for image basename: {options['basename']}" ) img = Image.open(minslice_name) volume_shape = (1 + options["maxslice"] - options["minslice"], img.height, img.width) del img global_box_zyx = np.zeros((2, 3), dtype=int) global_box_zyx[0] = options["offset"] global_box_zyx[0] += (options["minslice"], 0, 0) global_box_zyx[1] = global_box_zyx[0] + volume_shape if options["create-pyramid"]: if is_datainstance(dvid_info["dvid-server"], dvid_info["uuid"], dvid_info["dataname"]): logger.info( "'{dataname}' already exists, skipping creation".format( **dvid_info)) else: # create data instance and disable dvidmask # !! assume if data instance exists and mask is set that all pyramid # !! also exits, meaning the mask should be used. options["has-dvidmask"] = False if options["disable-original"]: logger.info( "Not creating '{dataname}' due to 'disable-original' config setting" .format(**dvid_info)) elif 0 in options["skipped-pyramid-levels"]: logger.info( "Not creating '{dataname}' due to 'skipped-pyramid-levels' config setting" .format(**dvid_info)) else: if options["is-rawarray"]: create_rawarray8(dvid_info["dvid-server"], dvid_info["uuid"], dvid_info["dataname"], block_shape) else: create_label_instance(dvid_info["dvid-server"], dvid_info["uuid"], dvid_info["dataname"], 0, block_shape) if not options["disable-original"] and 0 not in options[ "skipped-pyramid-levels"]: update_extents(dvid_info["dvid-server"], dvid_info["uuid"], dvid_info["dataname"], global_box_zyx) # Bottom level of pyramid is listed as neuroglancer-compatible extend_list_value(dvid_info["dvid-server"], dvid_info["uuid"], '.meta', 'neuroglancer', [dvid_info["dataname"]]) # determine number of pyramid levels if not specified if options["create-pyramid"] or options["create-pyramid-jpeg"]: if options["pyramid-depth"] == -1: options["pyramid-depth"] = 0 zsize = options["maxslice"] - options["minslice"] + 1 while zsize > 512: options["pyramid-depth"] += 1 zsize /= 2 # NeuTu doesn't work well if there aren't at least a few pyramid levels. # Even for small volumes, use at least a few pyramid levels, # unless the depth was explicit in the config. options["pyramid-depth"] = max(options["pyramid-depth"], 4) # create pyramid data instances if options["create-pyramid-jpeg"]: dataname_jpeg = dvid_info["dataname"] + self.JPEGPYRAMID_NAME if 0 in options["skipped-pyramid-levels"]: logger.info( "Not creating '{}' due to 'skipped-pyramid-levels' config setting" .format(dataname_jpeg)) else: if is_datainstance(dvid_info["dvid-server"], dvid_info["uuid"], dataname_jpeg): logger.info( "'{}' already exists, skipping creation".format( dataname_jpeg)) else: create_rawarray8(dvid_info["dvid-server"], dvid_info["uuid"], dataname_jpeg, block_shape, Compression.JPEG) update_extents(dvid_info["dvid-server"], dvid_info["uuid"], dataname_jpeg, global_box_zyx) # Bottom level of pyramid is listed as neuroglancer-compatible extend_list_value(dvid_info["dvid-server"], dvid_info["uuid"], '.meta', 'neuroglancer', [dataname_jpeg]) if options["create-pyramid"]: for level in range(1, 1 + options["pyramid-depth"]): downsampled_box_zyx = global_box_zyx // (2**level) downname = dvid_info["dataname"] + "_%d" % level if level in options["skipped-pyramid-levels"]: logger.info( "Not creating '{}' due to 'skipped-pyramid-levels' config setting" .format(downname)) continue if is_datainstance(dvid_info["dvid-server"], dvid_info["uuid"], downname): logger.info( "'{}' already exists, skipping creation".format( downname)) else: if options["is-rawarray"]: create_rawarray8(dvid_info["dvid-server"], dvid_info["uuid"], downname, block_shape) else: create_label_instance(dvid_info["dvid-server"], dvid_info["uuid"], downname, 0, block_shape) update_extents(dvid_info["dvid-server"], dvid_info["uuid"], downname, downsampled_box_zyx) # Higher-levels of the pyramid should not appear in the DVID-lite console. extend_list_value(dvid_info["dvid-server"], dvid_info["uuid"], '.meta', 'restrictions', [downname]) if options["create-pyramid-jpeg"]: for level in range(1, 1 + options["pyramid-depth"]): downsampled_box_zyx = global_box_zyx // (2**level) downname = dvid_info[ "dataname"] + self.JPEGPYRAMID_NAME + "_%d" % level if level in options["skipped-pyramid-levels"]: logger.info( "Not creating '{}' due to 'skipped-pyramid-levels' config setting" .format(downname)) continue if is_datainstance(dvid_info["dvid-server"], dvid_info["uuid"], downname): logger.info( "'{}' already exists, skipping creation".format( downname)) else: create_rawarray8(dvid_info["dvid-server"], dvid_info["uuid"], downname, block_shape, Compression.JPEG) update_extents(dvid_info["dvid-server"], dvid_info["uuid"], downname, downsampled_box_zyx) # Higher-levels of the pyramid should not appear in the DVID-lite console. extend_list_value(dvid_info["dvid-server"], dvid_info["uuid"], '.meta', 'restrictions', [downname]) # create tiles if options["create-tiles"] or options["create-tiles-jpeg"]: MinTileCoord = global_box_zyx[0][::-1] // options["tilesize"] MaxTileCoord = global_box_zyx[1][::-1] // options["tilesize"] # get max level by just finding max tile coord maxval = max(MaxTileCoord) - min(MinTileCoord) + 1 import math self.maxlevel = int(math.log(maxval) / math.log(2)) tilemeta = {} tilemeta["MinTileCoord"] = MinTileCoord.tolist() tilemeta["MaxTileCoord"] = MaxTileCoord.tolist() tilemeta["Levels"] = {} currres = 8.0 # just use as placeholder for now for level in range(0, self.maxlevel + 1): tilemeta["Levels"][str(level)] = { "Resolution": 3 * [currres], "TileSize": 3 * [options["tilesize"]] } currres *= 2 if options["create-tiles"]: session.post("{dvid-server}/api/repo/{uuid}/instance".format( **dvid_info), json={ "typename": "imagetile", "dataname": dvid_info["dataname"] + self.TILENAME, "source": dvid_info["dataname"], "format": "png" }) session.post( "{dvid-server}/api/repo/{uuid}/{dataname}{tilename}/metadata" .format(tilename=self.TILENAME, **dvid_info), json=tilemeta) if options["create-tiles-jpeg"]: session.post("{dvid-server}/api/repo/{uuid}/instance".format( **dvid_info), json={ "typename": "imagetile", "dataname": dvid_info["dataname"] + self.JPEGTILENAME, "source": dvid_info["dataname"], "format": "jpg" }) session.post( "{dvid-server}/api/repo/{uuid}/{dataname_jpeg_tile}/metadata" .format(dataname_jpeg_tile=dvid_info["dataname"] + self.JPEGTILENAME, **dvid_info), json=tilemeta) if dvid_info["dvid-server"].startswith("http://127.0.0.1"): def reload_meta(): reload_server_metadata(dvid_info["dvid-server"]) self.run_on_each_worker(reload_meta) # TODO Validation: should verify syncs exist, should verify pyramid depth # TODO: set syncs for pyramids, tiles if base datatype exists # syncs should be removed before ingestion and added afterward levels_cache = {} # iterate through each partition for arraypartition in imgreader: # DVID pad if necessary if options["has-dvidmask"]: dvidsrc = dvidSrc(dvid_info["dvid-server"], dvid_info["uuid"], dvid_info["dataname"], arraypartition, resource_server=self.resource_server, resource_port=self.resource_port) arraypartition = dvidsrc.extract_volume() # potentially need for future iterations arraypartition.persist() # check for final layer finallayer = imgreader.curr_slice > imgreader.end_slice if not options["disable-original"]: # Write level-0 of the raw data, even if we aren't writing the rest of the pyramid. dataname = datanamelossy = None if options["create-pyramid"]: dataname = dvid_info["dataname"] if options["create-pyramid-jpeg"]: datanamelossy = dvid_info[ "dataname"] + self.JPEGPYRAMID_NAME if (dataname or datanamelossy ) and 0 not in options["skipped-pyramid-levels"]: self._write_blocks(arraypartition, dataname, datanamelossy) if options["create-tiles"] or options["create-tiles-jpeg"]: # repartition into tiles schema = partitionSchema(PartitionDims(1, 0, 0)) tilepartition = schema.partition_data(arraypartition) # write unpadded tilesize (will pad with delimiter if needed) self._writeimagepyramid(tilepartition) if options["create-pyramid"] or options["create-pyramid-jpeg"]: if 0 not in levels_cache: levels_cache[0] = [] levels_cache[0].append(arraypartition) curr_level = 1 downsample_factor = 2 # should be a multiple of Z blocks or the final fetch assert imgreader.curr_slice % options["blocksize"] == 0 while ((((imgreader.curr_slice // options["blocksize"]) % downsample_factor) == 0) or finallayer) and curr_level <= options["pyramid-depth"]: partlist = levels_cache[curr_level - 1] part = partlist[0] # union all RDDs from the same level for iter1 in range(1, len(partlist)): part = part.union(partlist[iter1]) # downsample map israw = options["is-rawarray"] def downsample(part_vol): part, vol = part_vol if not israw: vol = downsample_3Dlabels(vol)[0] else: vol = downsample_raw(vol)[0] return (part, vol) downsampled_array = part.map(downsample) # repart (vol and offset will always be power of two because of padding) def repartition_down(part_volume): part, volume = part_volume downsampled_offset = np.array(part.get_offset()) // 2 downsampled_reloffset = np.array( part.get_reloffset()) // 2 offsetnew = VolumeOffset(*downsampled_offset) reloffsetnew = VolumeOffset(*downsampled_reloffset) partnew = volumePartition( (offsetnew.z, offsetnew.y, offsetnew.x), offsetnew, reloffset=reloffsetnew) return partnew, volume downsampled_array = downsampled_array.map(repartition_down) # repartition downsample data partition_dims = PartitionDims(options["blocksize"], options["blocksize"], self.partition_size) schema = partitionSchema( partition_dims, blank_delimiter=options["blankdelimiter"], padding=options["blocksize"], enablemask=options["has-dvidmask"]) downsampled_array = schema.partition_data( downsampled_array) # persist before padding if there are more levels if curr_level < options["pyramid-depth"]: downsampled_array.persist() if curr_level not in levels_cache: levels_cache[curr_level] = [] levels_cache[curr_level].append(downsampled_array) # pad from DVID (move before persist will allow multi-ingest # but will lead to slightly non-optimal downsampling boundary # effects if using a lossy compression only. if options["has-dvidmask"]: padname = dvid_info["dataname"] if options[ "create-pyramid-jpeg"]: # !! should pad with orig if computing # pad with jpeg padname += self.JPEGPYRAMID_NAME padname += "_%d" % curr_level dvidsrc = dvidSrc(dvid_info["dvid-server"], dvid_info["uuid"], padname, downsampled_array, resource_server=self.resource_server, resource_port=self.resource_port) downsampled_array = dvidsrc.extract_volume() # write result downname = None downnamelossy = None if options["create-pyramid"]: downname = dvid_info["dataname"] + "_%d" % curr_level if options["create-pyramid-jpeg"]: downnamelossy = dvid_info[ "dataname"] + self.JPEGPYRAMID_NAME + "_%d" % curr_level if curr_level not in options["skipped-pyramid-levels"]: self._write_blocks(downsampled_array, downname, downnamelossy) # remove previous level del levels_cache[curr_level - 1] curr_level += 1 downsample_factor *= 2
def test_dvidfetchgray(self): """Check reading grayscale from DVID from partitions. This also checks basic iteration and overwrite of previous data. """ service = DVIDServerService(dvidserver) uuid = service.create_new_repo("foo", "bar") ns = DVIDNodeService(dvidserver, uuid) ns.create_grayscale8("gray") arr = np.random.randint(255, size=(64, 64, 64)).astype(np.uint8) # load gray data ns.put_gray3D("gray", arr, (0, 0, 0)) # read data schema = partitionSchema(PartitionDims(32, 64, 64)) volpart = volumePartition(0, VolumeOffset(0, 0, 0)) overwrite = np.random.randint(255, size=(64, 64, 64)).astype(np.uint8) partitions = schema.partition_data([(volpart, overwrite)]) dvidreader = dvidSrc(dvidserver, uuid, "gray", partitions, maskonly=False) newparts = dvidreader.extract_volume() self.assertEqual(len(newparts), 2) for (part, vol) in newparts: if part.get_offset().z == 0: match = np.array_equal(arr[0:32, :, :], vol) self.assertTrue(match) else: match = np.array_equal(arr[32:64, :, :], vol) self.assertTrue(match) # test iteration dvidreader2 = dvidSrc(dvidserver, uuid, "gray", partitions, maskonly=False) newparts2 = [] for newpart in dvidreader2: self.assertEqual(len(newpart), 1) newparts2.extend(newpart) self.assertEqual(len(newparts2), 2) for (part, vol) in newparts2: if part.get_offset().z == 0: match = np.array_equal(arr[0:32, :, :], vol) self.assertTrue(match) else: match = np.array_equal(arr[32:64, :, :], vol) self.assertTrue(match)
def doit(): partitionSchema(PartitionDims(2, 0, 0), blank_delimiter=1111, padding=3, enablemask=True)