def test_multiprocess():
from concurrent.futures import ProcessPoolExecutor, as_completed
delete_layer()
cv, _ = create_layer(size=(128, 64, 64, 1), offset=(0, 0, 0))
cv.commit_info()
# "The ProcessPoolExecutor class has known (unfixable)
# problems on Python 2 and should not be relied on
# for mission critical work."
# https://pypi.org/project/futures/
if sys.version_info[0] < 3:
print(yellow("External multiprocessing not supported in Python 2."))
return
futures = []
with ProcessPoolExecutor(max_workers=4) as ppe:
for _ in range(0, 5):
futures.append(ppe.submit(cv.refresh_info))
for future in as_completed(futures):
# an error should be re-raised in one of the futures
future.result()
delete_layer()
def view(img,
segmentation=False,
resolution=None,
offset=None,
hostname="localhost",
port=DEFAULT_PORT):
from cloudvolume.volumecutout import VolumeCutout
img = to3d(img)
resolution = getresolution(img, resolution)
offset = getoffset(img, offset)
# Makes sense for viewing not segmentation
# which requires uints currently. (Jan. 2019)
if np.dtype(img.dtype).itemsize == 8 and not np.issubdtype(
img.dtype, np.float64):
print(
yellow("""
Converting {} to float64 for display.
Javascript does not support native 64-bit integer arrays.
""".format(img.dtype)))
img = img.astype(np.float64)
cutout = VolumeCutout(
buf=img,
path=ExtractedPath('mem', hostname, '/', '', '', '', ''),
cloudpath='IN MEMORY',
resolution=resolution,
mip=-1,
layer_type=('segmentation' if segmentation else 'image'),
bounds=Bbox(offset, offset + Vec(*(img.shape[:3]))),
handle=None,
)
return run([cutout], hostname=hostname, port=port)
def to_tif(self, file_name: str = None, compression: str = 'zlib'):
if file_name is None:
file_name = f'{self.bbox.to_filename()}.tif'
logging.info(f'write chunk to file: {file_name}')
if self.array.dtype == np.float32:
# visualization in float32 is not working correctly in ImageJ
# this might not work correctly if you want to save the image as it is!
print(yellow('transforming data type from float32 to uint8'))
img = self.array * 255
img = img.astype(np.uint8)
else:
img = self.array
if self.ndim == 3:
axes = 'ZYX'
elif self.ndim == 4:
axes = 'CZYX'
metadata = {'spacing': 1, 'unit': 'nm', 'axes': axes}
tifffile.imwrite(
file_name,
data=img,
imagej=True,
resolution=self.voxel_size,
metadata=metadata,
compression=compression,
)
def operator_contact():
contact = ''
try:
contact = subprocess.check_output("git config user.email", shell=True)
contact = str(contact.rstrip())
except:
try:
print(
yellow(
'Unable to determine provenance contact email. Set "git config user.email". Using unix $USER instead.'
))
contact = os.environ['USER']
except:
print(
yellow(
'$USER was not set. The "owner" field of the provenance file will be blank.'
))
contact = ''
return contact
def _auto_convert_dtype(self, chunk, volume):
"""convert the data type to fit volume datatype"""
if volume.dtype != chunk.dtype:
print(
yellow(f'converting chunk data type {chunk.dtype} ' +
f'to volume data type: {volume.dtype}'))
# float_chunk = chunk.astype(np.float64)
# chunk = float_chunk / np.iinfo(chunk.dtype).max * np.iinfo(self.volume.dtype).max
chunk = chunk / chunk.array.max() * np.iinfo(volume.dtype).max
return chunk.astype(volume.dtype)
else:
return chunk
def emulate_eight_uint64(img):
# Makes sense for viewing not segmentation
# which requires uints currently. (Jan. 2019)
if np.dtype(img.dtype).itemsize == 8 and not np.issubdtype(
img.dtype, np.float64):
print(
yellow("""
Converting {} to float64 for display.
Javascript does not support native 64-bit integer arrays.
""".format(img.dtype)))
return img.astype(np.float64)
return img
def to_tif(self, file_name: str = None, global_offset: tuple = None):
if file_name is None:
file_name = f'{self.bbox.to_filename()}.tif'
print('write chunk to file: ', file_name)
if self.array.dtype == np.float32:
# visualization in float32 is not working correctly in ImageJ
# this might not work correctly if you want to save the image as it is!
print(yellow('transforming data type from float32 to uint8'))
img = self.array * 255
img = img.astype(np.uint8)
else:
img = self.array
tifffile.imwrite(file_name, data=img)
def to_h5(self,
file_name: str,
with_offset: bool = True,
chunk_size: Cartesian = Cartesian(64, 64, 64),
with_unique: bool = True,
compression="gzip",
voxel_size: tuple = None):
"""
:param file_name: output file name. If it is not end with h5, the coordinate will be appended to the file name.
:param with_offset: save the voxel offset or not
:param with_unique: if this is a segmentation chunk, save the unique object ids or not.
:param compression: use HDF5 compression or not. Options are gzip, lzf
"""
if chunk_size:
assert len(chunk_size) == 3
if not file_name.endswith('.h5'):
file_name += self.bbox.to_filename() + '.h5'
logging.info(f'write chunk to file: {file_name}')
if os.path.exists(file_name):
print(yellow(f'deleting existing file: {file_name}'))
os.remove(file_name)
with h5py.File(file_name, 'w') as f:
f.create_dataset('/main',
data=self.array,
chunks=chunk_size,
compression=compression)
if voxel_size is None and self.voxel_size is not None:
voxel_size = self.voxel_size
if voxel_size is not None:
f.create_dataset('/voxel_size', data=voxel_size)
if with_offset and self.voxel_offset is not None:
f.create_dataset('/voxel_offset', data=self.voxel_offset)
if with_unique and self.is_segmentation:
unique = np.unique(self.array)
if unique[0]:
unique = unique[1:]
f.create_dataset('/unique_nonzeros', data=unique)
return file_name
def _auto_convert_dtype(self, chunk, volume):
"""convert the data type to fit volume datatype"""
if np.issubdtype(volume.dtype, np.floating) and np.issubdtype(
chunk.dtype, np.uint8):
chunk = chunk.astype(volume.dtype)
chunk /= 255.
# chunk = chunk / chunk.array.max() * np.iinfo(volume.dtype).max
elif np.issubdtype(volume.dtype, np.uint8) and np.issubdtype(
chunk.dtype, np.floating):
chunk.max() <= 1.
chunk *= 255
if volume.dtype != chunk.dtype:
print(
yellow(f'converting chunk data type {chunk.dtype} ' +
f'to volume data type: {volume.dtype}'))
# float_chunk = chunk.astype(np.float64)
# chunk = float_chunk / np.iinfo(chunk.dtype).max * np.iinfo(self.volume.dtype).max
# chunk = chunk / chunk.array.max() * np.iinfo(volume.dtype).max
return chunk.astype(volume.dtype)
else:
return chunk
MaskAffinitymapTask, InferenceTask)
# from igneous.tasks import BigArrayTask
# for provenance files
OPERATOR_CONTACT = ''
if __name__ == '__main__':
try:
OPERATOR_CONTACT = subprocess.check_output("git config user.email",
shell=True)
OPERATOR_CONTACT = str(OPERATOR_CONTACT.rstrip())
except:
try:
print(
yellow(
'Unable to determine provenance contact email. Set "git config user.email". Using unix $USER instead.'
))
OPERATOR_CONTACT = os.environ['USER']
except:
print(
yellow(
'$USER was not set. The "owner" field of the provenance file will be blank.'
))
OPERATOR_CONTACT = ''
def get_bounds(vol, bounds, shape, mip, chunk_size=None):
if bounds is None:
bounds = vol.bounds.clone()
else:
bounds = Bbox.create(bounds)
def setup_environment(dry_run, volume_start, volume_stop, volume_size, layer_path,
max_ram_size, output_patch_size,
input_patch_size, channel_num, dtype,
output_patch_overlap, crop_chunk_margin, mip, thumbnail_mip, max_mip,
thumbnail, encoding, voxel_size,
overwrite_info):
"""Prepare storage info files and produce tasks."""
assert not (volume_stop is None and volume_size is None)
if isinstance(volume_start, tuple):
volume_start = Vec(*volume_start)
if isinstance(volume_stop, tuple):
volume_stop = Vec(*volume_stop)
if isinstance(volume_size, tuple):
volume_size = Vec(*volume_size)
if input_patch_size is None:
input_patch_size = output_patch_size
if volume_size is not None:
assert len(volume_size) == 3
assert volume_stop is None
volume_stop = volume_start + volume_size
else:
volume_size = volume_stop - volume_start
print('\noutput volume start: ' + tuple2string(volume_start))
print('output volume stop: ' + tuple2string(volume_stop))
print('output volume size: ' + tuple2string(volume_size))
if output_patch_overlap is None:
# use 50% patch overlap in default
output_patch_overlap = tuple(s//2 for s in output_patch_size)
assert output_patch_overlap[1] == output_patch_overlap[2]
if crop_chunk_margin is None:
crop_chunk_margin = output_patch_overlap
assert crop_chunk_margin[1] == crop_chunk_margin[2]
print('margin size: ' + tuple2string(crop_chunk_margin))
if thumbnail:
# thumnail requires maximum mip level of 5
thumbnail_mip = max(thumbnail_mip, 5)
block_size, output_chunk_size, factor = get_optimized_block_size(
output_patch_size, output_patch_overlap, max_ram_size,
channel_num, max_mip, crop_chunk_margin,
input_patch_size, mip, thumbnail_mip, volume_start
)
if not dry_run:
storage = SimpleStorage(layer_path)
thumbnail_layer_path = os.path.join(layer_path, 'thumbnail')
thumbnail_storage = SimpleStorage(thumbnail_layer_path)
if not overwrite_info:
print('\ncheck that we are not overwriting existing info file.')
assert storage.exists('info')
assert thumbnail_storage.exists('info')
if overwrite_info:
print('create and upload info file to ', layer_path)
# Note that cloudvolume use fortran order rather than C order
info = CloudVolume.create_new_info(channel_num, layer_type='image',
data_type=dtype,
encoding=encoding,
resolution=voxel_size[::-1],
voxel_offset=volume_start[::-1],
volume_size=volume_size[::-1],
chunk_size=block_size[::-1],
max_mip=mip)
vol = CloudVolume(layer_path, info=info)
vol.commit_info()
if overwrite_info:
thumbnail_factor = 2**thumbnail_mip
thumbnail_block_size = (output_chunk_size[0]//factor,
output_chunk_size[1]//thumbnail_factor,
output_chunk_size[2]//thumbnail_factor)
print('thumbnail block size: ' + tuple2string(thumbnail_block_size))
thumbnail_info = CloudVolume.create_new_info(
1, layer_type='image',
data_type='uint8',
encoding='raw',
resolution=voxel_size[::-1],
voxel_offset=volume_start[::-1],
volume_size=volume_size[::-1],
chunk_size=thumbnail_block_size[::-1],
max_mip=thumbnail_mip)
thumbnail_vol = CloudVolume(thumbnail_layer_path, info=thumbnail_info)
thumbnail_vol.commit_info()
print('create a list of bounding boxes...')
roi_start = (volume_start[0],
volume_start[1]//factor,
volume_start[2]//factor)
roi_size = (volume_size[0],
volume_size[1]//factor,
volume_size[2]//factor)
roi_stop = tuple(s+z for s, z in zip(roi_start, roi_size))
# create bounding boxes and ingest to queue
bboxes = BoundingBoxes.from_manual_setup(
output_chunk_size,
roi_start=roi_start, roi_stop=roi_stop)
logging.info(f'total number of tasks: {len(bboxes)}')
logging.debug(f'bounding boxes: {bboxes}')
print(yellow(
'Note that you should reuse the printed out parameters in the production run.' +
' These parameters are not ingested to AWS SQS queue.'))
return bboxes
def create_luminance_levels_tasks(layer_path,
levels_path=None,
coverage_factor=0.01,
shape=None,
offset=None,
mip=0,
bounds_mip=0,
bounds=None):
"""
Compute per slice luminance level histogram and write them as
$layer_path/levels/$z. Each z file looks like:
{
"levels": [ 0, 35122, 12, ... ], # 256 indices, index = luminance i.e. 0 is black, 255 is white
"patch_size": [ sx, sy, sz ], # metadata on how large the patches were
"num_patches": 20, # metadata on
"coverage_ratio": 0.011, # actual sampled area on this slice normalized by ROI size.
}
Args:
layer_path: source image to sample from
levels_path: which path to write ./levels/ to (default: $layer_path)
coverage_factor: what fraction of the image to sample
offset & shape: Allows you to specify an ROI if much of
the edges are black. Defaults to entire image.
mip (int): which mip to work with, default maximum resolution
bounds_mip (int): mip of the input bounds
bounds (Bbox-like)
"""
if shape or offset:
print(
yellow(
"Create Luminance Levels Tasks: Deprecation Notice: "
"shape and offset parameters are deprecated in favor of the bounds argument."
))
vol = CloudVolume(layer_path, mip=mip)
if bounds is None:
bounds = vol.bounds.clone()
bounds = get_bounds(vol, bounds, mip, bounds_mip=bounds_mip)
if shape is None and bounds is None:
shape = Vec(*vol.shape)
shape.z = 1
elif bounds is not None:
shape = Vec(*bounds.size3())
shape.z = 1
if not offset:
offset = bounds.minpt
offset = Vec(*offset)
zoffset = offset.clone()
protocol = vol.meta.path.protocol
class LuminanceLevelsTaskIterator(object):
def __len__(self):
return bounds.maxpt.z - bounds.minpt.z
def __iter__(self):
for z in range(bounds.minpt.z, bounds.maxpt.z + 1):
zoffset.z = z
yield LuminanceLevelsTask(
src_path=layer_path,
levels_path=levels_path,
shape=shape,
offset=zoffset,
coverage_factor=coverage_factor,
mip=mip,
)
if protocol == 'boss':
raise StopIteration()
if levels_path:
try:
vol = CloudVolume(levels_path)
except cloudvolume.exceptions.InfoUnavailableError:
vol = CloudVolume(levels_path, info=vol.info)
else:
vol = CloudVolume(layer_path, mip=mip)
vol.provenance.processing.append({
'method': {
'task': 'LuminanceLevelsTask',
'src': layer_path,
'levels_path': levels_path,
'shape': Vec(*shape).tolist(),
'offset': Vec(*offset).tolist(),
'bounds': [bounds.minpt.tolist(),
bounds.maxpt.tolist()],
'coverage_factor': coverage_factor,
'mip': mip,
},
'by':
operator_contact(),
'date':
strftime('%Y-%m-%d %H:%M %Z'),
})
vol.commit_provenance()
return LuminanceLevelsTaskIterator()
