def test_multiprocess(): from concurrent.futures import ProcessPoolExecutor, as_completed delete_layer() cv, _ = create_layer(size=(128, 64, 64, 1), offset=(0, 0, 0)) cv.commit_info() # "The ProcessPoolExecutor class has known (unfixable) # problems on Python 2 and should not be relied on # for mission critical work." # https://pypi.org/project/futures/ if sys.version_info[0] < 3: print(yellow("External multiprocessing not supported in Python 2.")) return futures = [] with ProcessPoolExecutor(max_workers=4) as ppe: for _ in range(0, 5): futures.append(ppe.submit(cv.refresh_info)) for future in as_completed(futures): # an error should be re-raised in one of the futures future.result() delete_layer()
def view(img, segmentation=False, resolution=None, offset=None, hostname="localhost", port=DEFAULT_PORT): from cloudvolume.volumecutout import VolumeCutout img = to3d(img) resolution = getresolution(img, resolution) offset = getoffset(img, offset) # Makes sense for viewing not segmentation # which requires uints currently. (Jan. 2019) if np.dtype(img.dtype).itemsize == 8 and not np.issubdtype( img.dtype, np.float64): print( yellow(""" Converting {} to float64 for display. Javascript does not support native 64-bit integer arrays. """.format(img.dtype))) img = img.astype(np.float64) cutout = VolumeCutout( buf=img, path=ExtractedPath('mem', hostname, '/', '', '', '', ''), cloudpath='IN MEMORY', resolution=resolution, mip=-1, layer_type=('segmentation' if segmentation else 'image'), bounds=Bbox(offset, offset + Vec(*(img.shape[:3]))), handle=None, ) return run([cutout], hostname=hostname, port=port)
def to_tif(self, file_name: str = None, compression: str = 'zlib'): if file_name is None: file_name = f'{self.bbox.to_filename()}.tif' logging.info(f'write chunk to file: {file_name}') if self.array.dtype == np.float32: # visualization in float32 is not working correctly in ImageJ # this might not work correctly if you want to save the image as it is! print(yellow('transforming data type from float32 to uint8')) img = self.array * 255 img = img.astype(np.uint8) else: img = self.array if self.ndim == 3: axes = 'ZYX' elif self.ndim == 4: axes = 'CZYX' metadata = {'spacing': 1, 'unit': 'nm', 'axes': axes} tifffile.imwrite( file_name, data=img, imagej=True, resolution=self.voxel_size, metadata=metadata, compression=compression, )
def operator_contact(): contact = '' try: contact = subprocess.check_output("git config user.email", shell=True) contact = str(contact.rstrip()) except: try: print( yellow( 'Unable to determine provenance contact email. Set "git config user.email". Using unix $USER instead.' )) contact = os.environ['USER'] except: print( yellow( '$USER was not set. The "owner" field of the provenance file will be blank.' )) contact = '' return contact
def _auto_convert_dtype(self, chunk, volume): """convert the data type to fit volume datatype""" if volume.dtype != chunk.dtype: print( yellow(f'converting chunk data type {chunk.dtype} ' + f'to volume data type: {volume.dtype}')) # float_chunk = chunk.astype(np.float64) # chunk = float_chunk / np.iinfo(chunk.dtype).max * np.iinfo(self.volume.dtype).max chunk = chunk / chunk.array.max() * np.iinfo(volume.dtype).max return chunk.astype(volume.dtype) else: return chunk
def emulate_eight_uint64(img): # Makes sense for viewing not segmentation # which requires uints currently. (Jan. 2019) if np.dtype(img.dtype).itemsize == 8 and not np.issubdtype( img.dtype, np.float64): print( yellow(""" Converting {} to float64 for display. Javascript does not support native 64-bit integer arrays. """.format(img.dtype))) return img.astype(np.float64) return img
def to_tif(self, file_name: str = None, global_offset: tuple = None): if file_name is None: file_name = f'{self.bbox.to_filename()}.tif' print('write chunk to file: ', file_name) if self.array.dtype == np.float32: # visualization in float32 is not working correctly in ImageJ # this might not work correctly if you want to save the image as it is! print(yellow('transforming data type from float32 to uint8')) img = self.array * 255 img = img.astype(np.uint8) else: img = self.array tifffile.imwrite(file_name, data=img)
def to_h5(self, file_name: str, with_offset: bool = True, chunk_size: Cartesian = Cartesian(64, 64, 64), with_unique: bool = True, compression="gzip", voxel_size: tuple = None): """ :param file_name: output file name. If it is not end with h5, the coordinate will be appended to the file name. :param with_offset: save the voxel offset or not :param with_unique: if this is a segmentation chunk, save the unique object ids or not. :param compression: use HDF5 compression or not. Options are gzip, lzf """ if chunk_size: assert len(chunk_size) == 3 if not file_name.endswith('.h5'): file_name += self.bbox.to_filename() + '.h5' logging.info(f'write chunk to file: {file_name}') if os.path.exists(file_name): print(yellow(f'deleting existing file: {file_name}')) os.remove(file_name) with h5py.File(file_name, 'w') as f: f.create_dataset('/main', data=self.array, chunks=chunk_size, compression=compression) if voxel_size is None and self.voxel_size is not None: voxel_size = self.voxel_size if voxel_size is not None: f.create_dataset('/voxel_size', data=voxel_size) if with_offset and self.voxel_offset is not None: f.create_dataset('/voxel_offset', data=self.voxel_offset) if with_unique and self.is_segmentation: unique = np.unique(self.array) if unique[0]: unique = unique[1:] f.create_dataset('/unique_nonzeros', data=unique) return file_name
def _auto_convert_dtype(self, chunk, volume): """convert the data type to fit volume datatype""" if np.issubdtype(volume.dtype, np.floating) and np.issubdtype( chunk.dtype, np.uint8): chunk = chunk.astype(volume.dtype) chunk /= 255. # chunk = chunk / chunk.array.max() * np.iinfo(volume.dtype).max elif np.issubdtype(volume.dtype, np.uint8) and np.issubdtype( chunk.dtype, np.floating): chunk.max() <= 1. chunk *= 255 if volume.dtype != chunk.dtype: print( yellow(f'converting chunk data type {chunk.dtype} ' + f'to volume data type: {volume.dtype}')) # float_chunk = chunk.astype(np.float64) # chunk = float_chunk / np.iinfo(chunk.dtype).max * np.iinfo(self.volume.dtype).max # chunk = chunk / chunk.array.max() * np.iinfo(volume.dtype).max return chunk.astype(volume.dtype) else: return chunk
MaskAffinitymapTask, InferenceTask) # from igneous.tasks import BigArrayTask # for provenance files OPERATOR_CONTACT = '' if __name__ == '__main__': try: OPERATOR_CONTACT = subprocess.check_output("git config user.email", shell=True) OPERATOR_CONTACT = str(OPERATOR_CONTACT.rstrip()) except: try: print( yellow( 'Unable to determine provenance contact email. Set "git config user.email". Using unix $USER instead.' )) OPERATOR_CONTACT = os.environ['USER'] except: print( yellow( '$USER was not set. The "owner" field of the provenance file will be blank.' )) OPERATOR_CONTACT = '' def get_bounds(vol, bounds, shape, mip, chunk_size=None): if bounds is None: bounds = vol.bounds.clone() else: bounds = Bbox.create(bounds)
def setup_environment(dry_run, volume_start, volume_stop, volume_size, layer_path, max_ram_size, output_patch_size, input_patch_size, channel_num, dtype, output_patch_overlap, crop_chunk_margin, mip, thumbnail_mip, max_mip, thumbnail, encoding, voxel_size, overwrite_info): """Prepare storage info files and produce tasks.""" assert not (volume_stop is None and volume_size is None) if isinstance(volume_start, tuple): volume_start = Vec(*volume_start) if isinstance(volume_stop, tuple): volume_stop = Vec(*volume_stop) if isinstance(volume_size, tuple): volume_size = Vec(*volume_size) if input_patch_size is None: input_patch_size = output_patch_size if volume_size is not None: assert len(volume_size) == 3 assert volume_stop is None volume_stop = volume_start + volume_size else: volume_size = volume_stop - volume_start print('\noutput volume start: ' + tuple2string(volume_start)) print('output volume stop: ' + tuple2string(volume_stop)) print('output volume size: ' + tuple2string(volume_size)) if output_patch_overlap is None: # use 50% patch overlap in default output_patch_overlap = tuple(s//2 for s in output_patch_size) assert output_patch_overlap[1] == output_patch_overlap[2] if crop_chunk_margin is None: crop_chunk_margin = output_patch_overlap assert crop_chunk_margin[1] == crop_chunk_margin[2] print('margin size: ' + tuple2string(crop_chunk_margin)) if thumbnail: # thumnail requires maximum mip level of 5 thumbnail_mip = max(thumbnail_mip, 5) block_size, output_chunk_size, factor = get_optimized_block_size( output_patch_size, output_patch_overlap, max_ram_size, channel_num, max_mip, crop_chunk_margin, input_patch_size, mip, thumbnail_mip, volume_start ) if not dry_run: storage = SimpleStorage(layer_path) thumbnail_layer_path = os.path.join(layer_path, 'thumbnail') thumbnail_storage = SimpleStorage(thumbnail_layer_path) if not overwrite_info: print('\ncheck that we are not overwriting existing info file.') assert storage.exists('info') assert thumbnail_storage.exists('info') if overwrite_info: print('create and upload info file to ', layer_path) # Note that cloudvolume use fortran order rather than C order info = CloudVolume.create_new_info(channel_num, layer_type='image', data_type=dtype, encoding=encoding, resolution=voxel_size[::-1], voxel_offset=volume_start[::-1], volume_size=volume_size[::-1], chunk_size=block_size[::-1], max_mip=mip) vol = CloudVolume(layer_path, info=info) vol.commit_info() if overwrite_info: thumbnail_factor = 2**thumbnail_mip thumbnail_block_size = (output_chunk_size[0]//factor, output_chunk_size[1]//thumbnail_factor, output_chunk_size[2]//thumbnail_factor) print('thumbnail block size: ' + tuple2string(thumbnail_block_size)) thumbnail_info = CloudVolume.create_new_info( 1, layer_type='image', data_type='uint8', encoding='raw', resolution=voxel_size[::-1], voxel_offset=volume_start[::-1], volume_size=volume_size[::-1], chunk_size=thumbnail_block_size[::-1], max_mip=thumbnail_mip) thumbnail_vol = CloudVolume(thumbnail_layer_path, info=thumbnail_info) thumbnail_vol.commit_info() print('create a list of bounding boxes...') roi_start = (volume_start[0], volume_start[1]//factor, volume_start[2]//factor) roi_size = (volume_size[0], volume_size[1]//factor, volume_size[2]//factor) roi_stop = tuple(s+z for s, z in zip(roi_start, roi_size)) # create bounding boxes and ingest to queue bboxes = BoundingBoxes.from_manual_setup( output_chunk_size, roi_start=roi_start, roi_stop=roi_stop) logging.info(f'total number of tasks: {len(bboxes)}') logging.debug(f'bounding boxes: {bboxes}') print(yellow( 'Note that you should reuse the printed out parameters in the production run.' + ' These parameters are not ingested to AWS SQS queue.')) return bboxes
def create_luminance_levels_tasks(layer_path, levels_path=None, coverage_factor=0.01, shape=None, offset=None, mip=0, bounds_mip=0, bounds=None): """ Compute per slice luminance level histogram and write them as $layer_path/levels/$z. Each z file looks like: { "levels": [ 0, 35122, 12, ... ], # 256 indices, index = luminance i.e. 0 is black, 255 is white "patch_size": [ sx, sy, sz ], # metadata on how large the patches were "num_patches": 20, # metadata on "coverage_ratio": 0.011, # actual sampled area on this slice normalized by ROI size. } Args: layer_path: source image to sample from levels_path: which path to write ./levels/ to (default: $layer_path) coverage_factor: what fraction of the image to sample offset & shape: Allows you to specify an ROI if much of the edges are black. Defaults to entire image. mip (int): which mip to work with, default maximum resolution bounds_mip (int): mip of the input bounds bounds (Bbox-like) """ if shape or offset: print( yellow( "Create Luminance Levels Tasks: Deprecation Notice: " "shape and offset parameters are deprecated in favor of the bounds argument." )) vol = CloudVolume(layer_path, mip=mip) if bounds is None: bounds = vol.bounds.clone() bounds = get_bounds(vol, bounds, mip, bounds_mip=bounds_mip) if shape is None and bounds is None: shape = Vec(*vol.shape) shape.z = 1 elif bounds is not None: shape = Vec(*bounds.size3()) shape.z = 1 if not offset: offset = bounds.minpt offset = Vec(*offset) zoffset = offset.clone() protocol = vol.meta.path.protocol class LuminanceLevelsTaskIterator(object): def __len__(self): return bounds.maxpt.z - bounds.minpt.z def __iter__(self): for z in range(bounds.minpt.z, bounds.maxpt.z + 1): zoffset.z = z yield LuminanceLevelsTask( src_path=layer_path, levels_path=levels_path, shape=shape, offset=zoffset, coverage_factor=coverage_factor, mip=mip, ) if protocol == 'boss': raise StopIteration() if levels_path: try: vol = CloudVolume(levels_path) except cloudvolume.exceptions.InfoUnavailableError: vol = CloudVolume(levels_path, info=vol.info) else: vol = CloudVolume(layer_path, mip=mip) vol.provenance.processing.append({ 'method': { 'task': 'LuminanceLevelsTask', 'src': layer_path, 'levels_path': levels_path, 'shape': Vec(*shape).tolist(), 'offset': Vec(*offset).tolist(), 'bounds': [bounds.minpt.tolist(), bounds.maxpt.tolist()], 'coverage_factor': coverage_factor, 'mip': mip, }, 'by': operator_contact(), 'date': strftime('%Y-%m-%d %H:%M %Z'), }) vol.commit_provenance() return LuminanceLevelsTaskIterator()