def reorder_axes(input_arr: numpy.ndarray, *, from_axes_tags: str, to_axes_tags: str):
if isinstance(from_axes_tags, AxisTags):
from_axes_tags = "".join(from_axes_tags.keys())
if isinstance(to_axes_tags, AxisTags):
to_axes_tags = "".join(to_axes_tags.keys())
op = OpReorderAxes(graph=Graph())
tagged_arr = vigra.VigraArray(input_arr, axistags=vigra.defaultAxistags(from_axes_tags))
op.Input.setValue(tagged_arr)
op.AxisOrder.setValue(to_axes_tags)
return op.Output([]).wait()
class ObjectExtractionTimeComparison(object):
def __init__(self):
# Set memory and number of threads here
#lazyflow.request.Request.reset_thread_pool(2)
#Memory.setAvailableRam(500*1024**2)
binary_img = binaryImage()
raw_img = rawImage()
g = Graph()
# Reorder axis operators
self.op5Raw = OpReorderAxes(graph=g)
self.op5Raw.AxisOrder.setValue("txyzc")
#self.op5Raw.Input.connect(self.opReaderRaw.OutputImage)#self.opReaderRaw.OutputImage)
self.op5Raw.Input.setValue(raw_img)
self.op5Binary = OpReorderAxes(graph=g)
self.op5Binary.AxisOrder.setValue("txyzc")
#self.op5Binary.Input.connect(self.opReaderBinary.OutputImage)
self.op5Binary.Input.setValue(binary_img)
# Cache operators
self.opCacheRaw = OpBlockedArrayCache(graph=g)
self.opCacheRaw.Input.connect(self.op5Raw.Output)
self.opCacheRaw.BlockShape.setValue((1, ) +
self.op5Raw.Output.meta.shape[1:])
self.opCacheBinary = OpBlockedArrayCache(graph=g)
self.opCacheBinary.Input.connect(self.op5Binary.Output)
self.opCacheBinary.BlockShape.setValue(
(1, ) + self.op5Binary.Output.meta.shape[1:])
# Label volume operator
self.opLabel = OpLabelVolume(graph=g)
self.opLabel.Input.connect(self.op5Binary.Output)
#self.opLabel.Input.connect(self.opCacheBinary.Output)
# Object extraction
self.opObjectExtraction = OpObjectExtraction(graph=g)
self.opObjectExtraction.RawImage.connect(self.op5Raw.Output)
self.opObjectExtraction.BinaryImage.connect(self.op5Binary.Output)
self.opObjectExtraction.Features.setValue(FEATURES)
# Simplified object features operator (No overhead)
self.opObjectFeaturesSimp = OpObjectFeaturesSimplified(graph=g)
self.opObjectFeaturesSimp.RawVol.connect(self.opCacheRaw.Output)
self.opObjectFeaturesSimp.BinaryVol.connect(self.opCacheBinary.Output)
def run(self):
# # Load caches beforehand (To remove overhead of reading frames)
# with Timer() as timerCaches:
# rawVol = self.opCacheRaw.Output([]).wait()
# binaryVol = self.opCacheBinary.Output([]).wait()
#
# print "Caches took {} secs".format(timerCaches.seconds())
#
# del rawVol
# del binaryVol
# Profile object extraction simplified
print(
"\nStarting object extraction simplified (single-thread, without cache)"
)
with Timer() as timerObjectFeaturesSimp:
featsObjectFeaturesSimp = self.opObjectFeaturesSimp.Features(
[]).wait()
print("Simplified object extraction took: {} seconds".format(
timerObjectFeaturesSimp.seconds()))
# Profile object extraction optimized
print("\nStarting object extraction (multi-thread, without cache)")
with Timer() as timerObjectExtraction:
featsObjectExtraction = self.opObjectExtraction.RegionFeatures(
[]).wait()
print("Object extraction took: {} seconds".format(
timerObjectExtraction.seconds()))
# Profile for basic multi-threaded feature computation
# just a multi-threaded loop that labels volumes and extract object features directly (No operators, no plugin system, no overhead, just a loop)
featsBasicFeatureComp = dict.fromkeys(
list(range(self.op5Raw.Output.meta.shape[0])), None)
print("\nStarting basic multi-threaded feature computation")
pool = RequestPool()
for t in range(0, self.op5Raw.Output.meta.shape[0], 1):
pool.add(
Request(
partial(self._computeObjectFeatures, t,
featsBasicFeatureComp)))
with Timer() as timerBasicFeatureComp:
pool.wait()
print(
"Basic multi-threaded feature extraction took: {} seconds".format(
timerBasicFeatureComp.seconds()))
# Compute object features for single frame
def _computeObjectFeatures(self, t, result):
roi = [slice(None) for i in range(len(self.op5Raw.Output.meta.shape))]
roi[0] = slice(t, t + 1)
roi = tuple(roi)
# rawVol = self.opCacheRaw.Output(roi).wait()
# binaryVol = self.opCacheBinary.Output(roi).wait()
rawVol = self.op5Raw.Output(roi).wait()
binaryVol = self.op5Binary.Output(roi).wait()
features = [
'Count', 'Coord<Minimum>', 'RegionCenter',
'Coord<Principal<Kurtosis>>', 'Coord<Maximum>'
]
for i in range(t, t + 1):
labelVol = vigra.analysis.labelImageWithBackground(
binaryVol[i - t].squeeze(), background_value=int(0))
res = vigra.analysis.extractRegionFeatures(
rawVol[i - t].squeeze().astype(np.float32),
labelVol.squeeze().astype(np.uint32),
features,
ignoreLabel=0)
# Cleanup results (as done in vigra_objfeats)
local_features = [x for x in features if "Global<" not in x]
nobj = res[local_features[0]].shape[0]
result[i] = cleanup(res, nobj, features)
class TikTorchLazyflowClassifier(LazyflowPixelwiseClassifierABC):
HDF5_GROUP_FILENAME = "pytorch_network_path"
def __init__(self,
tiktorch_net,
filename=None,
HALO_SIZE=32,
BATCH_SIZE=3):
"""
Args:
tiktorch_net (tiktorch): tiktorch object to be loaded into this
classifier object
filename (None, optional): Save file name for future reference
"""
self._filename = filename
if self._filename is None:
self._filename = ""
self.HALO_SIZE = HALO_SIZE
self.BATCH_SIZE = BATCH_SIZE
if tiktorch_net is None:
print(self._filename)
tiktorch_net = TikTorch.unserialize(self._filename)
# print (self._filename)
# assert tiktorch_net.return_hypercolumns == False
# print('blah')
self._tiktorch_net = tiktorch_net
self._opReorderAxes = OpReorderAxes(graph=Graph())
self._opReorderAxes.AxisOrder.setValue("zcyx")
def predict_probabilities_pixelwise(self,
feature_image,
roi,
axistags=None):
"""
Implicitly assumes that feature_image is includes the surrounding HALO!
roi must be chosen accordingly
"""
logger.info(
f"predicting using pytorch network for image of shape {feature_image.shape} and roi {roi}"
)
logger.info(
f"Stats of input: min={feature_image.min()}, max={feature_image.max()}, mean={feature_image.mean()}"
)
logger.info(
f"expected pytorch input shape is {self._tiktorch_net.expected_input_shape}"
)
logger.info(
f"expected pytorch output shape is {self._tiktorch_net.expected_output_shape}"
)
# print(self._tiktorch_net.expected_input_shape)
# print(self._tiktorch_net.expected_output_shape)
num_channels = len(self.known_classes)
expected_shape = [stop - start for start, stop in zip(roi[0], roi[1])
] + [num_channels]
self._opReorderAxes.Input.setValue(
vigra.VigraArray(feature_image, axistags=axistags))
self._opReorderAxes.AxisOrder.setValue("zcyx")
reordered_feature_image = self._opReorderAxes.Output([]).wait()
# normalizing patch
# reordered_feature_image = (reordered_feature_image - reordered_feature_image.mean()) / (reordered_feature_image.std() + 0.000001)
if len(self._tiktorch_net.get("window_size")) == 2:
exp_input_shape = numpy.array(
self._tiktorch_net.expected_input_shape)
exp_input_shape = tuple(numpy.append(1, exp_input_shape))
print(exp_input_shape)
else:
exp_input_shape = self._tiktorch_net.expected_input_shape
logger.info(
f"input axistags are {axistags}, "
f"Shape after reordering input is {reordered_feature_image.shape}, "
f"axistags are {self._opReorderAxes.Output.meta.axistags}")
slice_shape = list(reordered_feature_image.shape[1::]) # ignore z axis
# assuming [z, y, x]
result_roi = numpy.array(roi)
if slice_shape != list(exp_input_shape[1::]):
logger.info(f"Expected input shape is {exp_input_shape[1::]}, "
f"but got {slice_shape}, reshaping...")
# adding a zero border to images that have the specific shape
exp_shape = list(self._tiktorch_net.expected_input_shape[1::])
zero_img = numpy.zeros(exp_shape)
# diff shape: cyx
diff_shape = numpy.array(
exp_input_shape[1::]) - numpy.array(slice_shape)
# diff_shape = numpy.array(self._tiktorch_net.expected_input_shape) - numpy.array(slice_shape)
# offset shape z, y, x, c for easy indexing, with c = 0, z = 0
offset = numpy.array([0, 0, 0, 0])
logger.info(f"Diff_shape {diff_shape}")
# at least one of y, x (diff_shape[1], diff_shape[2]) should be off
# let's determine how to adjust the offset -> offset[2] and offset[3]
# caveat: this code assumes that image requests were tiled in a regular
# pattern starting from left upper corner.
# We use a blocked array-cache to achieve that
# y-offset:
if diff_shape[1] > 0:
# was the halo added to the upper side of the feature image?
# HACK: this only works because we assume the data to be in zyx!!!
if roi[0][1] == 0:
# no, doesn't seem like it
offset[1] = self.HALO_SIZE
# x-offsets:
if diff_shape[2] > 0:
# was the halo added to the upper side of the feature image?
# HACK: this only works because we assume the data to be in zyx!!!
if roi[0][2] == 0:
# no, doesn't seem like it
offset[2] = self.HALO_SIZE
# HACK: still assuming zyxc
result_roi[0] += offset[0:3]
result_roi[1] += offset[0:3]
reorder_feature_image_extents = numpy.array(
reordered_feature_image.shape)
# add the offset:
reorder_feature_image_extents[2:4] += offset[1:3]
# zero_img[:, :, offset[1]:reorder_feature_image_extents[2], offset[2]:reorder_feature_image_extents[3]] = \
# reordered_feature_image
# reordered_feature_image = zero_img
pad_img = numpy.pad(
reordered_feature_image,
[
(0, 0),
(0, 0),
(offset[1],
exp_input_shape[2] - reorder_feature_image_extents[2]),
(offset[2],
exp_input_shape[3] - reorder_feature_image_extents[3]),
],
"reflect",
)
reordered_feature_image = pad_img
logger.info(f"New Image shape {reordered_feature_image.shape}")
result = numpy.zeros([reordered_feature_image.shape[0], num_channels] +
list(reordered_feature_image.shape[2:]))
logger.info(f"forward")
# we always predict in 2D, per z-slice, so we loop over z
for z in range(0, reordered_feature_image.shape[0], self.BATCH_SIZE):
# logger.warning("Dumping to {}".format('"/Users/chaubold/Desktop/dump.h5"'))
# vigra.impex.writeHDF5(reordered_feature_image[z,...], "data", "/Users/chaubold/Desktop/dump.h5")
# create batch of desired num slices. Multiple slices can be processed on multiple GPUs!
batch = [
reordered_feature_image[zi:zi + 1, ...].reshape(
self._tiktorch_net.expected_input_shape)
for zi in range(
z,
min(z + self.BATCH_SIZE, reordered_feature_image.shape[0]))
]
logger.info(f"batch info: {[x.shape for x in batch]}")
print("batch info:", [x.shape for x in batch])
# if len(self._tiktorch_net.get('window_size')) == 2:
# print("BATTCHHHHH", batch.shape)
result_batch = self._tiktorch_net.forward(batch)
logger.info(
f"Resulting slices from {z} to {z + len(batch)} have shape {result_batch[0].shape}"
)
print("Resulting slices from ", z, " to ", z + len(batch),
" have shape ", result_batch[0].shape)
for i, zi in enumerate(range(z, (z + len(batch)))):
result[zi:(zi + 1), ...] = result_batch[i]
logger.info(f"Obtained a predicted block of shape {result.shape}")
print("Obtained a predicted block of shape ", result.shape)
self._opReorderAxes.Input.setValue(
vigra.VigraArray(result, axistags=vigra.makeAxistags("zcyx")))
# axistags is vigra.AxisTags, but opReorderAxes expects a string
self._opReorderAxes.AxisOrder.setValue("".join(axistags.keys()))
result = self._opReorderAxes.Output([]).wait()
logger.info(f"Reordered result to shape {result.shape}")
# FIXME: not needed for real neural net results:
logger.info(
f"Stats of result: min={result.min()}, max={result.max()}, mean={result.mean()}"
)
# cut out the required roi
logger.info(f"Roi shape {result_roi}")
# crop away halo and reorder axes to match "axistags"
# crop in X and Y:
cropped_result = result[roiToSlice(*result_roi)]
logger.info(
f"cropped the predicted block to shape {cropped_result.shape}")
return cropped_result
@property
def known_classes(self):
return list(range(self._tiktorch_net.expected_output_shape[0]))
@property
def feature_count(self):
return self._tiktorch_net.expected_input_shape[0]
def get_halo_shape(self, data_axes="zyxc"):
if len(data_axes) == 4:
return (0, self.HALO_SIZE, self.HALO_SIZE, 0)
# FIXME: assuming 'yxc' !
elif len(data_axes) == 3:
return (self.HALO_SIZE, self.HALO_SIZE, 0)
def serialize_hdf5(self, h5py_group):
logger.debug("Serializing")
h5py_group[self.HDF5_GROUP_FILENAME] = self._filename
h5py_group["pickled_type"] = pickle.dumps(type(self), 0)
# HACK: can this be done more elegantly?
with tempfile.TemporaryFile() as f:
self._tiktorch_net.serialize(f)
f.seek(0)
h5py_group["classifier"] = numpy.void(f.read())
@classmethod
def deserialize_hdf5(cls, h5py_group):
# TODO: load from HDF5 instead of hard coded path!
logger.debug("Deserializing")
# HACK:
# filename = PYTORCH_MODEL_FILE_PATH
filename = h5py_group[cls.HDF5_GROUP_FILENAME]
logger.debug("Deserializing from {}".format(filename))
with tempfile.TemporaryFile() as f:
f.write(h5py_group["classifier"].value)
f.seek(0)
loaded_pytorch_net = TikTorch.unserialize(f)
return TikTorchLazyflowClassifier(loaded_pytorch_net, filename)
class TestOpReorderAxes(unittest.TestCase):
def setUp(self):
self.array = None
self.axis = list('tzyxc')
self.tests = 20
graph = Graph()
self.operator = OpReorderAxes(graph=graph)
def prepareVolnOp(self, possible_axes='tzyxc', num=5):
tags = random.sample(possible_axes, random.randint(2, num))
tagStr = ''
for s in tags:
tagStr += s
axisTags = vigra.defaultAxistags(tagStr)
self.shape = []
for tag in axisTags:
self.shape.append(random.randint(20, 30))
self.array = (numpy.random.rand(*tuple(self.shape)) * 255)
self.array = (float(250) / 255 * self.array + 5).astype(int)
self.inArray = vigra.VigraArray(self.array, axistags=axisTags)
opProvider = OpArrayProvider(graph=self.operator.graph)
opProvider.Input.setValue(self.inArray)
self.operator.Input.connect(opProvider.Output)
def test_Full(self):
for i in range(self.tests):
self.prepareVolnOp()
result = self.operator.Output().wait()
logger.debug(
'------------------------------------------------------')
logger.debug("self.array.shape = " + str(self.array.shape))
logger.debug("type(input) == " +
str(type(self.operator.Input.value)))
logger.debug("input.shape == " +
str(self.operator.Input.meta.shape))
logger.debug("Input Tags:")
logger.debug(str(self.operator.Input.meta.axistags))
logger.debug("Output Tags:")
logger.debug(str(self.operator.Output.meta.axistags))
logger.debug("type(result) == " + str(type(result)))
logger.debug("result.shape == " + str(result.shape))
logger.debug(
'------------------------------------------------------')
# Check the shape
assert len(result.shape) == 5
assert not isinstance(result, vigra.VigraArray), \
"For compatibility with generic code, output should be provided as a plain numpy array."
# Ensure the result came out in default order
assert self.operator.Output.meta.axistags == vigra.defaultAxistags(
'tzyxc')
# Check the data
vresult = result.view(vigra.VigraArray)
vresult.axistags = self.operator.Output.meta.axistags
reorderedInput = self.inArray.withAxes(
*[tag.key for tag in vresult.axistags])
assert numpy.all(vresult == reorderedInput)
def test_Roi_default_order(self):
for i in range(self.tests):
self.prepareVolnOp()
shape = self.operator.Output.meta.shape
roi = [None, None]
roi[1] = [
numpy.random.randint(2, s) if s != 1 else 1 for s in shape
]
roi[0] = [
numpy.random.randint(0, roi[1][i]) if s != 1 else 0
for i, s in enumerate(shape)
]
roi[0] = TinyVector(roi[0])
roi[1] = TinyVector(roi[1])
result = self.operator.Output(roi[0], roi[1]).wait()
logger.debug(
'------------------------------------------------------')
logger.debug("self.array.shape = " + str(self.array.shape))
logger.debug("type(input) == " +
str(type(self.operator.Input.value)))
logger.debug("input.shape == " +
str(self.operator.Input.meta.shape))
logger.debug("Input Tags:")
logger.debug(str(self.operator.Input.meta.axistags))
logger.debug("Output Tags:")
logger.debug(str(self.operator.Output.meta.axistags))
logger.debug("roi= " + str(roi))
logger.debug("type(result) == " + str(type(result)))
logger.debug("result.shape == " + str(result.shape))
logger.debug(
'------------------------------------------------------')
# Check the shape
assert len(result.shape) == 5
assert not isinstance(result, vigra.VigraArray), \
"For compatibility with generic code, output should be provided as a plain numpy array."
# Ensure the result came out in volumina order
assert self.operator.Output.meta.axistags == vigra.defaultAxistags(
'tzyxc')
# Check the data
vresult = result.view(vigra.VigraArray)
vresult.axistags = self.operator.Output.meta.axistags
reorderedInput = self.inArray.withAxes(
*[tag.key for tag in self.operator.Output.meta.axistags])
assert numpy.all(
vresult == reorderedInput[roiToSlice(roi[0], roi[1])])
def test_Roi_custom_order(self):
self._impl_roi_custom_order('cztxy')
self._impl_roi_custom_order('xyz')
def _impl_roi_custom_order(self, axisorder):
for i in range(self.tests):
self.prepareVolnOp(axisorder, len(axisorder) - 1)
# Specify a strange order for the output axis tags
self.operator.AxisOrder.setValue(axisorder)
shape = self.operator.Output.meta.shape
roi = [None, None]
roi[1] = [
numpy.random.randint(2, s) if s != 1 else 1 for s in shape
]
roi[0] = [
numpy.random.randint(0, roi[1][i]) if s != 1 else 0
for i, s in enumerate(shape)
]
roi[0] = TinyVector(roi[0])
roi[1] = TinyVector(roi[1])
result = self.operator.Output(roi[0], roi[1]).wait()
logger.debug(
'------------------------------------------------------')
logger.debug("self.array.shape = " + str(self.array.shape))
logger.debug("type(input) == " +
str(type(self.operator.Input.value)))
logger.debug("input.shape == " +
str(self.operator.Input.meta.shape))
logger.debug("Input Tags:")
logger.debug(str(self.operator.Input.meta.axistags))
logger.debug("Output Tags:")
logger.debug(str(self.operator.Output.meta.axistags))
logger.debug("roi= " + str(roi))
logger.debug("type(result) == " + str(type(result)))
logger.debug("result.shape == " + str(result.shape))
logger.debug(
'------------------------------------------------------')
# Check the shape
assert len(result.shape) == len(axisorder)
assert not isinstance(result, vigra.VigraArray), \
"For compatibility with generic code, output should be provided as a plain numpy array."
# Ensure the result came out in the same strange order we asked for.
assert self.operator.Output.meta.axistags == vigra.defaultAxistags(
axisorder)
# Check the data
vresult = result.view(vigra.VigraArray)
vresult.axistags = self.operator.Output.meta.axistags
reorderedInput = self.inArray.withAxes(
*[tag.key for tag in self.operator.Output.meta.axistags])
assert numpy.all(
vresult == reorderedInput[roiToSlice(roi[0], roi[1])])
def test_insert_singleton_axis(self):
for i in range(self.tests):
self.prepareVolnOp('xyzc', 4)
# Specify a strange order for the output axis tags
self.operator.AxisOrder.setValue('yxtzc')
shape = self.operator.Output.meta.shape
roi = [None, None]
roi[1] = [
numpy.random.randint(2, s) if s != 1 else 1 for s in shape
]
roi[0] = [
numpy.random.randint(0, roi[1][i]) if s != 1 else 0
for i, s in enumerate(shape)
]
roi[0] = TinyVector(roi[0])
roi[1] = TinyVector(roi[1])
result = self.operator.Output(roi[0], roi[1]).wait()
logger.debug(
'------------------------------------------------------')
logger.debug("self.array.shape = " + str(self.array.shape))
logger.debug("type(input) == " +
str(type(self.operator.Input.value)))
logger.debug("input.shape == " +
str(self.operator.Input.meta.shape))
logger.debug("Input Tags:")
logger.debug(str(self.operator.Input.meta.axistags))
logger.debug("Output Tags:")
logger.debug(str(self.operator.Output.meta.axistags))
logger.debug("roi= " + str(roi))
logger.debug("type(result) == " + str(type(result)))
logger.debug("result.shape == " + str(result.shape))
logger.debug(
'------------------------------------------------------')
# Check the shape
assert len(result.shape) == 5
assert not isinstance(result, vigra.VigraArray), \
"For compatibility with generic code, output should be provided as a plain numpy array."
# Ensure the result came out in the same strange order we asked for.
assert self.operator.Output.meta.axistags == vigra.defaultAxistags(
'yxtzc')
# Check the data
vresult = result.view(vigra.VigraArray)
vresult.axistags = self.operator.Output.meta.axistags
reorderedInput = self.inArray.withAxes(
*[tag.key for tag in self.operator.Output.meta.axistags])
assert numpy.all(
vresult == reorderedInput[roiToSlice(roi[0], roi[1])])
def test_attempt_drop_nonsingleton_axis(self):
"""
Attempt to configure the operator with invalid settings by trying to drop a non-singleton axis.
The execute method should assert in that case.
"""
data = numpy.zeros((100, 100, 100), dtype=numpy.uint8)
data = vigra.taggedView(data, vigra.defaultAxistags('xyz'))
op = OpReorderAxes(graph=Graph())
op.Input.setValue(data)
# Attempt to drop some axes that can't be dropped.
op.AxisOrder.setValue('txc')
# Make sure this results in an error.
req = op.Output[:]
req.notify_failed(
lambda *args: None
) # We expect an exception here, so disable the default fail handler to hide the traceback
self.assertRaises(AssertionError, req.wait)
class OpExportMultipageTiff(Operator):
Input = InputSlot(
) # The last two non-singleton axes (except 'c') are the axes of the 'pages'.
# Re-order the axes yourself if you want an alternative slicing direction
Filepath = InputSlot()
DEFAULT_BATCH_SIZE = 4
def __init__(self, *args, **kwargs):
super(OpExportMultipageTiff, self).__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
self._opReorderAxes = OpReorderAxes(parent=self)
self._opReorderAxes.Input.connect(self.Input)
def setupOutputs(self):
# Always export in tzcyx order (but omit missing axes)
input_axes = self.Input.meta.getAxisKeys()
export_axes = "".join(filter(lambda k: k in input_axes, 'tzcyx'))
if not set("yx").issubset(set(export_axes)):
# This could potentially be fixed...
raise Exception(
"I don't know how to export data without both an X and Y axis")
self._opReorderAxes.AxisOrder.setValue(export_axes)
self._export_axes = export_axes
def run_export(self):
"""
Request the volume in slices (running in parallel), and write each slice to the correct page.
Note: We can't use BigRequestStreamer here, because the data for each slice wouldn't be
guaranteed to arrive in the correct order.
"""
# Delete existing image if present
image_path = self.Filepath.value
if os.path.exists(image_path):
os.remove(image_path)
tagged_shape = self.Input.meta.getTaggedShape()
export_shape = self._opReorderAxes.Output.meta.shape
shape_yx = export_shape[-2:]
stacked_axes_shape = export_shape[:-2]
num_pages = numpy.prod(stacked_axes_shape)
def create_slice_req():
for stacked_axes_ndindex in numpy.ndindex(*stacked_axes_shape):
roi = numpy.zeros((2, ) + (len(export_shape), ), dtype=int)
roi[:, :-2] = stacked_axes_ndindex
roi[1, :-2] += 1
roi[1, -2:] = shape_yx
yield self._opReorderAxes.Output(*roi)
iter_slice_requests = create_slice_req()
parallel_requests = self.DEFAULT_BATCH_SIZE
# If ram usage info is available, make a better guess about how many requests we can launch in parallel
ram_usage_per_requested_pixel = self.Input.meta.ram_usage_per_requested_pixel
if ram_usage_per_requested_pixel is not None:
pixels_per_slice = numpy.prod(shape_yx)
if 'c' in tagged_shape:
pixels_per_slice /= tagged_shape['c']
ram_usage_per_slice = pixels_per_slice * ram_usage_per_requested_pixel
# Fudge factor: Reduce RAM usage by a bit
available_ram = psutil.virtual_memory().available
available_ram *= 0.5
parallel_requests = int(available_ram / ram_usage_per_slice)
# Start with a batch of images
reqs = collections.deque()
for next_request_index in range(min(parallel_requests, num_pages)):
reqs.append(iter_slice_requests.next())
self.progressSignal(0)
pages_written = 0
while reqs:
self.progressSignal(100 * next_request_index / num_pages)
req = reqs.popleft()
slice_data = req.wait()
slice_data = vigra.taggedView(slice_data, self._export_axes)
next_request_index += 1
# Add a new request to the batch
if next_request_index < num_pages:
reqs.append(iter_slice_requests.next())
if pages_written == 0:
xml_description = OpExportMultipageTiff.generate_ome_xml_description(
self._opReorderAxes.Output.meta.getAxisKeys(),
self._opReorderAxes.Output.meta.shape,
self._opReorderAxes.Output.meta.dtype,
os.path.split(image_path)[1])
# Write the first slice with tifffile, which allows us to write the tags.
with tifffile.TiffWriter(image_path,
software='ilastik',
byteorder='<') as writer:
writer.save(slice_data.withAxes('yx'),
description=xml_description,
planarconfig='planar')
else:
# Append a slice to the multipage tiff file
vigra.impex.writeImage(slice_data.withAxes('yx'),
image_path,
dtype='',
compression='NONE',
mode='a')
pages_written += 1
self.progressSignal(100)
# No output slots...
def execute(self, slot, subindex, roi, result):
pass
def propagateDirty(self, slot, subindex, roi):
pass
@classmethod
def generate_ome_xml_description(cls, axes, shape, dtype, filename=''):
"""
Generate an OME XML description of the data we're exporting,
suitable for the image_description tag of the first page.
axes and shape should be provided in C-order (will be reversed in the XML)
"""
import uuid
import xml.etree.ElementTree as ET
# Normalize the inputs
axes = "".join(axes)
shape = tuple(shape)
if not isinstance(dtype, type):
dtype = dtype().type
ome = ET.Element('OME')
uuid_str = "urn:uuid:" + str(uuid.uuid1())
ome.set('UUID', uuid_str)
ome.set('xmlns:xsi', "http://www.w3.org/2001/XMLSchema-instance")
ome.set(
'xsi:schemaLocation',
"http://www.openmicroscopy.org/Schemas/OME/2015-01 "
"http://www.openmicroscopy.org/Schemas/OME/2015-01/ome.xsd")
image = ET.SubElement(ome, 'Image')
image.set('ID', 'Image:0')
image.set('Name', 'exported-data')
pixels = ET.SubElement(image, 'Pixels')
pixels.set('BigEndian', 'true')
pixels.set('ID', 'Pixels:0')
fortran_axes = "".join(reversed(axes)).upper()
pixels.set('DimensionOrder', fortran_axes)
for axis, dim in zip(axes.upper(), shape):
pixels.set('Size' + axis, str(dim))
types = {
numpy.uint8: 'uint8',
numpy.uint16: 'uint16',
numpy.uint32: 'uint32',
numpy.int8: 'int8',
numpy.int16: 'int16',
numpy.int32: 'int32',
numpy.float32: 'float',
numpy.float64: 'double',
numpy.complex64: 'complex',
numpy.complex128: 'double-complex'
}
pixels.set('Type', types[dtype])
# Omit channel information (is that okay?)
# channel0 = ET.SubElement(pixels, "Channel")
# channel0.set("ID", "Channel0:0")
# channel0.set("SamplesPerPixel", "1")
assert axes[-2:] == "yx"
for page_index, page_ndindex in enumerate(numpy.ndindex(*shape[:-2])):
tiffdata = ET.SubElement(pixels, "TiffData")
for axis, index in zip(axes[:-2].upper(), page_ndindex):
tiffdata.set("First" + axis, str(index))
tiffdata.set("PlaneCount", "1")
tiffdata.set("IFD", str(page_index))
uuid_tag = ET.SubElement(tiffdata, "UUID")
uuid_tag.text = uuid_str
uuid_tag.set('FileName', filename)
from textwrap import dedent
from StringIO import StringIO
xml_stream = StringIO()
comment = ET.Comment(
dedent(
'\
<!-- Warning: this comment is an OME-XML metadata block, which contains crucial '
'dimensional parameters and other important metadata. Please edit cautiously '
'(if at all), and back up the original data before doing so. For more information, '
'see the OME-TIFF web site: http://ome-xml.org/wiki/OmeTiff. -->'
))
tree = ET.ElementTree(ome)
tree.write(xml_stream, encoding='utf-8', xml_declaration=True)
if logger.isEnabledFor(logging.DEBUG):
import xml.dom.minidom
reparsed = xml.dom.minidom.parseString(xml_stream.getvalue())
logger.debug("Generated OME-TIFF metadata:\n" +
reparsed.toprettyxml())
return xml_stream.getvalue()
