def setInSlot(self, slot, subindex, roi, value):
key = roi.toSlice()
if slot == self.WriteSeeds:
with Timer() as timer:
logger.info("Writing seeds to label array")
self.opLabelArray.LabelSinkInput[roi.toSlice()] = value
logger.info(
"Writing seeds to label array took {} seconds".format(
timer.seconds()))
assert self._mst is not None
# Important: mst.seeds will requires erased values to be 255 (a.k.a -1)
#value[:] = numpy.where(value == 100, 255, value)
seedVal = value.max()
with Timer() as timer:
logger.info("Writing seeds to MST")
if hasattr(key, '__len__'):
self._mst.addSeeds(roi=roi, brushStroke=value.squeeze())
else:
raise RuntimeError("when is this part of the code called")
self._mst.seeds[key] = value
logger.info("Writing seeds to MST took {} seconds".format(
timer.seconds()))
self.has_seeds = True
else:
raise RuntimeError("unknown slots")
class EventPlayer(object):
def __init__(self, playback_speed=None, comment_display=None):
self._playback_speed = playback_speed
self._timer = Timer()
self._timer.unpause()
if comment_display is None:
self._comment_display = self._default_comment_display
else:
self._comment_display = comment_display
def play_script(self, path, finish_callback=None):
"""
Start execution of the given script in a separate thread and return immediately.
Note: You should handle any exceptions from the playback script via sys.execpthook.
"""
_globals = {}
_locals = {}
"""
Calls to events in the playback script like: player.post_event(obj,PyQt4.QtGui.QMouseEvent(...),t)
are/were responsible for the xcb-error on Ubuntu, because you may not use
a Gui-object from a thread other than the MainThread running the Gui
"""
execfile(path, _globals, _locals)
def run():
_locals['playback_events'](player=self)
if finish_callback is not None:
finish_callback()
th = threading.Thread( target=run )
th.daemon = True
th.start()
def post_event(self, obj, event, timestamp_in_seconds):
if self._playback_speed is not None:
self._timer.sleep_until(timestamp_in_seconds / self._playback_speed)
assert threading.current_thread().name != "MainThread"
event.spont = True
QApplication.postEvent(obj, event)
assert QApplication.instance().thread() == obj.thread()
flusher = EventFlusher()
flusher.moveToThread( obj.thread() )
flusher.setParent( QApplication.instance() )
signaler = Signaler()
signaler.sig.connect( flusher.set, Qt.QueuedConnection )
signaler.sig.emit()
flusher.wait()
flusher.clear()
def display_comment(self, comment):
self._comment_display(comment)
def _default_comment_display(self, comment):
print "--------------------------------------------------"
print comment
print "--------------------------------------------------"
def execute(self, slot, subindex, roi, result):
if tuple(roi.stop - roi.start) != self.Output.meta.shape:
raise ValueError("Blockwise Watershed must be run on the entire volume")
if self.Input.meta.getAxisKeys() != list("txyzc"):
raise ValueError(f"Unsupported input axis keys {self.Input.meta.getAxisKeys()}")
if self.Input.meta.getTaggedShape()["z"] > 1:
result_idx = numpy.s_[0, ..., 0]
else:
result_idx = numpy.s_[0, ..., 0, 0]
input_ = self.Input(roi.start, roi.stop).wait().squeeze()
if input_.ndim not in (2, 3):
raise ValueError(f"Input shape {input_.shape} has an invalid number of non-singleton dimensions")
with Timer() as timer:
logger.info("Run block-wise watershed in %dd", input_.ndim)
if self.DoAgglo.value:
result[result_idx], max_id = watershed_and_agglomerate(
input_,
max_workers=max(1, Request.global_thread_pool.num_workers),
size_regularizer=self.SizeRegularizer.value,
reduce_to=self.ReduceTo.value,
)
else:
result[result_idx], max_id = vigra.analysis.watershedsNew(input_)
logger.info("done %d", max_id)
logger.info("Blockwise Watershed took %f seconds", timer.seconds())
return result
def execute(self, slot, subindex, roi, result):
assert roi.stop - roi.start == self.Output.meta.shape, "Watershed must be run on the entire volume."
input_image = self.Input(roi.start, roi.stop).wait()
volume_feat = input_image[0, ..., 0]
result_view = result[0, ..., 0]
with Timer() as watershedTimer:
if self.Input.meta.getTaggedShape()['z'] > 1:
sys.stdout.write("Watershed...")
sys.stdout.flush()
#result_view[...] = vigra.analysis.watersheds(volume_feat[:,:])[0].astype(numpy.int32)
result_view[...] = vigra.analysis.watershedsNew(
volume_feat[:, :].astype(numpy.uint8))[0]
logger.info("done {}".format(numpy.max(result[...])))
else:
sys.stdout.write("Watershed...")
sys.stdout.flush()
labelVolume = vigra.analysis.watershedsNew(
volume_feat[:, :, 0])[0] #.view(dtype=numpy.int32)
result_view[...] = labelVolume[:, :, numpy.newaxis]
logger.info("done {}".format(numpy.max(labelVolume)))
logger.info("Watershed took {} seconds".format(
watershedTimer.seconds()))
return result
def _loadProject(self, hdf5File, projectFilePath, readOnly):
"""
Load the data from the given hdf5File (which should already be open).
:param hdf5File: An already-open h5py.File, usually created via ``ProjectManager.createBlankProjectFile``
:param projectFilePath: The path to the file represented in the ``hdf5File`` parameter.
:param readOnly: Set to True if the project file should NOT be modified.
"""
# We are about to create a LOT of tiny objects.
# Temporarily disable garbage collection while we do this.
gc.disable()
assert self.currentProjectFile is None
# Minor GUI nicety: Pre-activate the progress signals for all applets so
# the progress manager treats these tasks as a group instead of several sequential jobs.
for aplt in self._applets:
aplt.progressSignal.emit(0)
# Save this as the current project
self.currentProjectFile = hdf5File
self.currentProjectPath = projectFilePath
self.currentProjectIsReadOnly = readOnly
try:
# Applet serializable items are given the whole file (root group)
for aplt in self._applets:
with Timer() as timer:
for serializer in aplt.dataSerializers:
assert serializer.base_initialized, "AppletSerializer subclasses must call AppletSerializer.__init__ upon construction."
serializer.ignoreDirty = True
if serializer.caresOfHeadless:
serializer.deserializeFromHdf5(
self.currentProjectFile, projectFilePath,
self._headless)
else:
serializer.deserializeFromHdf5(
self.currentProjectFile, projectFilePath)
serializer.ignoreDirty = False
logger.debug(
'Deserializing applet "{}" took {} seconds'.format(
aplt.name, timer.seconds()))
self.closed = False
# Call the workflow's custom post-load initialization (if any)
self.workflow.onProjectLoaded(self)
self.workflow.handleAppletStateUpdateRequested()
except:
msg = "Project could not be loaded due to the exception shown above.\n"
msg += "Aborting Project Open Action"
log_exception(logger, msg)
self._closeCurrentProject()
raise
finally:
gc.enable()
for aplt in self._applets:
aplt.progressSignal.emit(100)
def __init__(self, playback_speed=None, comment_display=None):
self._playback_speed = playback_speed
self._timer = Timer()
self._timer.unpause()
if comment_display is None:
self._comment_display = self._default_comment_display
else:
self._comment_display = comment_display
def __init__(self, parent=None, ignore_parent_events=True):
QObject.__init__(self, parent=parent)
self._ignore_parent_events = False
if parent is not None and ignore_parent_events:
self._ignore_parent_events = True
self._parent_name = get_fully_qualified_name(parent)
self._captured_events = []
self._timer = Timer()
def read(self, view_roi, result_out):
"""
roi: (start, stop) tuples, ordered according to description.output_axes
roi should be relative to the view
"""
output_axes = self.description.output_axes
roi_transposed = list(zip(*view_roi))
roi_dict = dict(list(zip(output_axes, roi_transposed)))
view_roi = list(zip(*(roi_dict["z"], roi_dict["y"], roi_dict["x"])))
# First, normalize roi and result to zyx order
result_out = vigra.taggedView(result_out, output_axes)
result_out = result_out.withAxes(*"zyx")
assert numpy.array(view_roi).shape == (2, 3), "Invalid roi for 3D volume: {}".format(view_roi)
view_roi = numpy.array(view_roi)
assert (result_out.shape == (view_roi[1] - view_roi[0])).all()
# User gave roi according to the view output.
# Now offset it find global roi.
roi = view_roi + self.description.view_origin_zyx
tile_blockshape = (1,) + tuple(self.description.tile_shape_2d_yx)
tile_starts = getIntersectingBlocks(tile_blockshape, roi)
pool = RequestPool()
for tile_start in tile_starts:
tile_roi_in = getBlockBounds(self.description.bounds_zyx, tile_blockshape, tile_start)
tile_roi_in = numpy.array(tile_roi_in)
# This tile's portion of the roi
intersecting_roi = getIntersection(roi, tile_roi_in)
intersecting_roi = numpy.array(intersecting_roi)
# Compute slicing within destination array and slicing within this tile
destination_relative_intersection = numpy.subtract(intersecting_roi, roi[0])
tile_relative_intersection = intersecting_roi - tile_roi_in[0]
# Get a view to the output slice
result_region = result_out[roiToSlice(*destination_relative_intersection)]
rest_args = self._get_rest_args(tile_blockshape, tile_roi_in)
if self.description.tile_url_format.startswith("http"):
retrieval_fn = partial(self._retrieve_remote_tile, rest_args, tile_relative_intersection, result_region)
else:
retrieval_fn = partial(self._retrieve_local_tile, rest_args, tile_relative_intersection, result_region)
PARALLEL_REQ = True
if PARALLEL_REQ:
pool.add(Request(retrieval_fn))
else:
# execute serially (leave the pool empty)
retrieval_fn()
if PARALLEL_REQ:
with Timer() as timer:
pool.wait()
logger.info("Loading {} tiles took a total of {}".format(len(tile_starts), timer.seconds()))
def impl():
workflow = self.shell.projectManager.workflow
pixClassApplet = workflow.pcApplet
gui = pixClassApplet.getMultiLaneGui()
# Clear all the labels
while len(gui.currentGui()._labelControlUi.labelListModel) > 2:
gui.currentGui()._labelControlUi.labelListModel.removeRow(2)
# Re-add all labels
self.test_4_AddLabels()
# Make sure the entire slice is visible
viewMenu = gui.currentGui().menus()[0]
viewMenu.actionFitToScreen.trigger()
with Timer() as timer:
# Enable interactive mode
assert gui.currentGui(
)._labelControlUi.liveUpdateButton.isChecked() == False
gui.currentGui()._labelControlUi.liveUpdateButton.click()
# Do to the way we wait for the views to finish rendering, the GUI hangs while we wait.
self.waitForViews(gui.currentGui().editor.imageViews)
logger.debug("Interactive Mode Rendering Time: {}".format(
timer.seconds()))
# There should be a prediction layer for each label
labelNames = [
label.name for label in gui.currentGui().labelListData
]
labelColors = gui.currentGui()._colorTable16[1:5]
for i, labelName in enumerate(labelNames):
try:
index = gui.currentGui().layerstack.findMatchingIndex(
lambda layer: labelName in layer.name)
layer = gui.currentGui().layerstack[index]
# Check the color
assert isinstance(
layer,
AlphaModulatedLayer), "layer is {}".format(layer)
assert layer.tintColor.rgba(
) == labelColors[i], "Expected {}, got {}".format(
hex(labelColors[i]), hex(layer.tintColor.rgba()))
except ValueError:
assert False, "Could not find layer for label with name: {}".format(
labelName)
# Disable iteractive mode.
gui.currentGui()._labelControlUi.liveUpdateButton.click()
self.waitForViews(gui.currentGui().editor.imageViews)
def setup_class(cls):
# Base class first
super().setup_class()
# input files:
current_dir = os.path.split(__file__)[0]
cls.sample_data_raw = os.path.abspath(
os.path.join(current_dir, "../data/inputdata/3d.h5"))
# output files:
cls.temp_dir = tempfile.mkdtemp()
# uncomment for debugging
# cls.temp_dir = os.path.expanduser('~/tmp')
# if os.path.exists(cls.temp_dir):
# shutil.rmtree(cls.temp_dir)
# os.makedirs(cls.temp_dir)
cls.project_file = os.path.join(cls.temp_dir,
"test_project_carving.ilp")
cls.output_file = os.path.join(cls.temp_dir,
"out_carving_object_segmentation.h5")
cls.output_obj_file = os.path.join(cls.temp_dir,
"out_carving_object_1.obj")
# reference files
# unzip the zip-file ;)
cls.reference_zip_file = os.path.join(
current_dir, "../data/outputdata/testCarvingGuiReference.zip")
cls.reference_path = os.path.join(cls.temp_dir, "reference")
cls.reference_files = {
"output_obj_file":
os.path.join(cls.reference_path,
"testCarvingGuiReference/3d_carving_object_1.obj"),
"output_file":
os.path.join(
cls.reference_path,
"testCarvingGuiReference/3d_carving_completed_segments_1_object.h5"
),
"carving_label_file":
os.path.join(cls.reference_path,
"testCarvingGuiReference/3d_carving_labels.h5"),
}
os.makedirs(cls.reference_path)
with zipfile.ZipFile(cls.reference_zip_file, mode="r") as zip_file:
zip_file.extractall(path=cls.reference_path)
cls.unzipped_reference_files = [
os.path.join(cls.reference_path, fp) for fp in zip_file.namelist()
]
for file_name in cls.reference_files.values():
assert os.path.exists(file_name)
# Start the timer
cls.timer = Timer()
cls.timer.unpause()
def impl():
shell = self.shell
workflow = shell.projectManager.workflow
object_classification_applet = workflow.objectClassificationApplet
gui = object_classification_applet.getMultiLaneGui()
# activate the object classification applet
shell.setSelectedAppletDrawer(3)
# let the gui catch up
QApplication.processEvents()
with Timer() as timer:
# Enable interactive mode
assert gui.currentGui(
)._labelControlUi.liveUpdateButton.isChecked() is False
gui.currentGui()._labelControlUi.liveUpdateButton.click()
assert gui.currentGui(
)._labelControlUi.liveUpdateButton.isChecked() is True
# Do to the way we wait for the views to finish rendering, the GUI hangs while we wait.
self.waitForViews(gui.currentGui().editor.imageViews)
logger.debug(f"Interactive Mode Rendering Time: {timer.seconds()}")
# Disable iteractive mode.
gui.currentGui()._labelControlUi.liveUpdateButton.click()
assert gui.currentGui()._labelControlUi.liveUpdateButton.isChecked(
) is False
# There should be a prediction layer for each label
labelNames = [
label.name for label in gui.currentGui().labelListData
]
labelColors = gui.currentGui()._colorTable16[1:4]
for i, labelName in enumerate(labelNames):
try:
index = gui.currentGui().layerstack.findMatchingIndex(
lambda layer: labelName in layer.name)
layer = gui.currentGui().layerstack[index]
# Check the color
assert isinstance(layer,
AlphaModulatedLayer), f"layer is {layer}"
assert (
layer.tintColor.rgba() == labelColors[i]
), f"Expected {hex(labelColors[i])}, got {hex(layer.tintColor.rgba())}"
except ValueError:
assert False, "Could not find layer for label with name: {}".format(
labelName)
# Save the project
saveThread = self.shell.onSaveProjectActionTriggered()
saveThread.join()
self.waitForViews(gui.currentGui().editor.imageViews)
def execute(self, slot, subindex, ignored_roi, result):
configFilePath = self.ConfigFilePath.value
config = parseClusterConfigFile(configFilePath)
blockwiseFileset = self._primaryBlockwiseFileset
# Check axis compatibility
inputAxes = list(self.Input.meta.getTaggedShape().keys())
outputAxes = list(blockwiseFileset.description.axes)
assert set(inputAxes) == set(
outputAxes
), "Output dataset has the wrong set of axes. Input axes: {}, Output axes: {}".format(
"".join(inputAxes), "".join(outputAxes))
roiString = self.RoiString.value
roi = Roi.loads(roiString)
if len(roi.start) != len(self.Input.meta.shape):
assert (
False
), "Task roi: {} is not valid for this input. Did the master launch this task correctly?".format(
roiString)
logger.info("Executing for roi: {}".format(roi))
if config.use_node_local_scratch:
assert False, "FIXME."
assert (
blockwiseFileset.getEntireBlockRoi(roi.start)[1] == roi.stop
).all(
), "Each task must execute exactly one full block. ({},{}) is not a valid block roi.".format(
roi.start, roi.stop)
assert self.Input.ready()
with Timer() as computeTimer:
# Stream the data out to disk.
request_blockshape = (
self._primaryBlockwiseFileset.description.sub_block_shape
) # Could be None. That's okay.
streamer = BigRequestStreamer(self.Input, (roi.start, roi.stop),
request_blockshape)
streamer.progressSignal.subscribe(self.progressSignal)
streamer.resultSignal.subscribe(self._handlePrimaryResultBlock)
streamer.execute()
# Now the block is ready. Update the status.
blockwiseFileset.setBlockStatus(roi.start,
BlockwiseFileset.BLOCK_AVAILABLE)
logger.info("Finished task in {} seconds".format(
computeTimer.seconds()))
result[0] = True
return result
def execute(self, slot, subindex, roi, result):
assert all(roi.stop <= self.Input.meta.shape),\
"Requested roi {} is too large for this input image of shape {}.".format(roi, self.Input.meta.shape)
# Determine how much input data we'll need, and where the result will be
# relative to that input roi
# inputRoi is a 5d roi, computeRoi depends on the number of singletons
# in shape, but is at most 3d
inputRoi, computeRoi = self._getInputComputeRois(roi)
# Obtain the input data
with Timer() as resultTimer:
data = self.Input(*inputRoi).wait()
logger.debug("Obtaining input data took {} seconds for roi {}".format(
resultTimer.seconds(), inputRoi))
data = vigra.taggedView(data, axistags='txyzc')
# input is in txyzc order
tIndex = 0
cIndex = 4
# Must be float32
if data.dtype != numpy.float32:
data = data.astype(numpy.float32)
# we need to remove a singleton z axis, otherwise we get
# 'kernel longer than line' errors
ts = self.Input.meta.getTaggedShape()
tags = [k for k in 'xyz' if ts[k] > 1]
sigma = [self._sigmas[k] for k in tags]
# Check if we need to smooth
if any([x < 0.1 for x in sigma]):
# just pipe the input through
result[...] = data
return
for i, t in enumerate(xrange(roi.start[tIndex], roi.stop[tIndex])):
for j, c in enumerate(xrange(roi.start[cIndex], roi.stop[cIndex])):
# prepare the result as an argument
resview = vigra.taggedView(result[i, ..., j], axistags='xyz')
dataview = data[i, ..., j]
# TODO make this general, not just for z axis
resview = resview.withAxes(*tags)
dataview = dataview.withAxes(*tags)
# Smooth the input data
vigra.filters.gaussianSmoothing(dataview,
sigma,
window_size=2.0,
roi=computeRoi,
out=resview)
def impl():
workflow = self.shell.projectManager.workflow
pixClassApplet = workflow.pcApplet
gui = pixClassApplet.getMultiLaneGui()
with Timer() as timer:
gui.currentGui().editor.posModel.slicingPos = (0, 0, 1)
# Do to the way we wait for the views to finish rendering, the GUI hangs while we wait.
self.waitForViews(gui.currentGui().editor.imageViews)
logger.debug("New Slice Rendering Time: {}".format(
timer.seconds()))
def setup_class(cls):
# Base class first
super().setup_class()
# input files:
current_dir = os.path.split(__file__)[0]
cls.sample_data_raw = os.path.abspath(os.path.join(current_dir, "../data/inputdata/3d.h5"))
cls.sample_data_prob = os.path.abspath(os.path.join(current_dir, "../data/inputdata/3d_Probabilities.h5"))
# output files:
cls.temp_dir = tempfile.mkdtemp()
# uncomment for debugging
# cls.temp_dir = os.path.expanduser('~/tmp')
# if os.path.exists(cls.temp_dir):
# shutil.rmtree(cls.temp_dir)
# os.makedirs(cls.temp_dir)
cls.project_file = os.path.join(cls.temp_dir, "test_project_oc.ilp")
cls.output_file = os.path.join(cls.temp_dir, "out_object_prediction.h5")
cls.table_h5_file = os.path.join(cls.temp_dir, "table.h5")
cls.table_h5_file_exported = None # Will be filled in test_06
cls.table_csv_file = os.path.join(cls.temp_dir, "table.csv")
cls.table_csv_file_exported = None # Will be filled in test_06
# reference files
# unzip the zip-file ;)
cls.reference_zip_file = os.path.join(
current_dir, "../data/outputdata/testObjectClassificationGuiReference.zip"
)
cls.reference_path = os.path.join(cls.temp_dir, "reference")
cls.reference_files = {
"csv_table": os.path.join(
cls.reference_path, "testObjectClassificationGuiReference/table-test_data_table.csv"
),
"h5_table": os.path.join(cls.reference_path, "testObjectClassificationGuiReference/table-test_data.h5"),
"predictions_h5": os.path.join(
cls.reference_path, "testObjectClassificationGuiReference/reference_out_object_prediction.h5"
),
}
os.makedirs(cls.reference_path)
with zipfile.ZipFile(cls.reference_zip_file, mode="r") as zip_file:
zip_file.extractall(path=cls.reference_path)
cls.unzipped_reference_files = [os.path.join(cls.reference_path, fp) for fp in zip_file.namelist()]
for file_name in cls.reference_files.values():
assert os.path.exists(file_name)
# Start the timer
cls.timer = Timer()
cls.timer.unpause()
def setupClass(cls):
# Base class first
super(TestPixelClassificationGui, cls).setupClass()
if hasattr(cls, 'SAMPLE_DATA'):
cls.using_random_data = False
else:
cls.using_random_data = True
cls.SAMPLE_DATA = os.path.split(__file__)[0] + '/random_data.npy'
data = numpy.random.random((1,200,200,50,1))
data *= 256
numpy.save(cls.SAMPLE_DATA, data.astype(numpy.uint8))
# Start the timer
cls.timer = Timer()
cls.timer.unpause()
def _buildDone(self):
"""
Builds the done segmentation anew, for example after saving an object or
deleting an object.
"""
if self._mst is None:
return
with Timer() as timer:
self._done_lut = numpy.zeros(self._mst.numNodes+1, dtype=numpy.int32)
self._done_seg_lut = numpy.zeros(self._mst.numNodes+1, dtype=numpy.int32)
logger.info( "building 'done' luts" )
for name, objectSupervoxels in self._mst.object_lut.items():
if name == self._currObjectName:
continue
self._done_lut[objectSupervoxels] += 1
assert name in self._mst.object_names, "%s not in self._mst.object_names, keys are %r" % (name, list(self._mst.object_names.keys()))
self._done_seg_lut[objectSupervoxels] = self._mst.object_names[name]
logger.info( "building the 'done' luts took {} seconds".format( timer.seconds() ) )
def impl():
workflow = self.shell.projectManager.workflow
pixClassApplet = workflow.pcApplet
gui = pixClassApplet.getMultiLaneGui()
# Make sure the entire slice is visible
gui.currentGui().menuGui.actionFitToScreen.trigger()
with Timer() as timer:
# Enable interactive mode
assert gui.currentGui(
)._labelControlUi.liveUpdateButton.isChecked() == False
gui.currentGui()._labelControlUi.liveUpdateButton.click()
# Do to the way we wait for the views to finish rendering, the GUI hangs while we wait.
self.waitForViews(gui.currentGui().editor.imageViews)
logger.debug("Interactive Mode Rendering Time: {}".format(
timer.seconds()))
def setup_class(cls):
# Base class first
super(TestObjectCountingGui, cls).setup_class()
if hasattr(cls, "SAMPLE_DATA"):
cls.using_random_data = False
else:
cls.using_random_data = True
cls.SAMPLE_DATA = os.path.join(os.path.split(__file__)[0], "random_data.npy")
data = numpy.random.random((200, 200, 3))
data *= 256
numpy.save(cls.SAMPLE_DATA, data.astype(numpy.uint8))
# Sample Sigma value for OpCounting.opTrain (OpTrainCounter).
cls.COUNTING_SIGMA = 4.2
# Start the timer
cls.timer = Timer()
cls.timer.unpause()
def setupClass(cls):
# This test is useful for performance evaluation,
# but it takes too long to be useful as part of the normal test suite.
raise nose.SkipTest
# Base class first
super(TestPixelClassificationGuiBenchmarking, cls).setupClass()
if hasattr(cls, 'SAMPLE_DATA'):
cls.using_random_data = False
else:
cls.using_random_data = True
cls.SAMPLE_DATA = os.path.split(__file__)[0] + '/random_data.npy'
data = numpy.random.random((1, 512, 512, 128, 1))
data *= 256
numpy.save(cls.SAMPLE_DATA, data.astype(numpy.uint8))
# Start the timer
cls.timer = Timer()
cls.timer.start()
def setupClass(cls):
if 'TRAVIS' in os.environ:
# The counting workflow doesn't import correctly on Travis, so skip this test.
import nose
raise nose.SkipTest
# Base class first
super(TestObjectCountingGui, cls).setupClass()
if hasattr(cls, 'SAMPLE_DATA'):
cls.using_random_data = False
else:
cls.using_random_data = True
cls.SAMPLE_DATA = os.path.split(__file__)[0] + '/random_data.npy'
data = numpy.random.random((200,200,3))
data *= 256
numpy.save(cls.SAMPLE_DATA, data.astype(numpy.uint8))
# Start the timer
cls.timer = Timer()
cls.timer.unpause()
def associate_skeletons(self, skeleton_association_input, debug=False):
with Timer() as node_timer:
node_segmenter_outputs = associate_skeletons(
self.hdf5_path, self.opPixelClassification,
self.multicut_shell, self.catmaid, skeleton_association_input)
logging.getLogger(self.inner_logger.name + '.timing').info(
"TILE TIMER: {}".format(node_timer.seconds()))
# if debug:
# node_locations_arr = node_locations_to_array(segmentation_xy.shape, node_locations)
# path = os.path.join(
# self.paths.debug_synapse_dir, '{}_{}.hdf5'.format(synapse_object_id, roi_xyz[0, 2])
# )
# dump_images(
# path, roi_xyz, raw=raw_xy, synapse_cc=synapse_cc_xy, predictions=predictions_xyc,
# segmentation=segmentation_xy, node_locations=node_locations_arr
# )
return node_segmenter_outputs
def detect_synapses(self, tile_idx, debug=False):
self.inner_logger.debug("detect_synapses called")
with Timer() as timer:
output = detect_synapses(self.tile_size,
self.opPixelClassification, tile_idx)
logging.getLogger(self.inner_logger.name + '.timing').info(
"NODE TIMER: {}".format(timer.seconds()))
if debug:
roi_xyz = tile_index_to_bounds(tile_idx, self.tile_size)
path = os.path.join(
self.paths.debug_tile_dir,
'x{}-y{}-z{}.hdf5'.format(tile_idx.x_idx, tile_idx.y_idx,
tile_idx.z_idx))
dump_images(path,
roi_xyz,
synapse_cc=output.synapse_cc_xy,
predictions=output.predictions_xyc)
return output
def setup_class(cls):
# Base class first
super(TestPixelClassificationGuiBenchmarking, cls).setup_class()
# This test is useful for performance evaluation,
# but it takes too long to be useful as part of the normal test suite.
super().teardown_class()
pytest.skip("For benchmark purposes only")
if hasattr(cls, "SAMPLE_DATA"):
cls.using_random_data = False
else:
cls.using_random_data = True
cls.SAMPLE_DATA = os.path.split(__file__)[0] + "/random_data.npy"
data = numpy.random.random((1, 512, 512, 128, 1))
data *= 256
numpy.save(cls.SAMPLE_DATA, data.astype(numpy.uint8))
# Start the timer
cls.timer = Timer()
cls.timer.unpause()
def execute(self, slot, subindex, roi, result):
assert roi.stop - roi.start == self.Output.meta.shape, "Blockwise Watershed must be run on the entire volume."
input_image = self.Input(roi.start, roi.stop).wait()
volume_feat = input_image[0, ..., 0]
result_view = result[0, ..., 0]
with Timer() as watershedTimer:
if self.Input.meta.getTaggedShape()['z'] > 1:
sys.stdout.write("Blockwise Watershed 3D...")
sys.stdout.flush()
if not self.DoAgglo.value:
result_view[...] = vigra.analysis.watersheds(
volume_feat[...])[0].astype(numpy.int32)
else:
result_view[...] = simple_parallel_ws(
volume_feat,
max_workers=Request.global_thread_pool.num_workers,
size_regularizer=self.SizeRegularizer.value,
reduce_to=self.ReduceTo.value)
logger.info("done {}".format(numpy.max(result[...])))
else:
if not self.DoAgglo.value:
result_view[...] = vigra.analysis.watersheds(
volume_feat[:, :, 0])[0].astype(numpy.int32)
else:
sys.stdout.write("Blockwise Watershed...")
sys.stdout.flush()
labelVolume = simple_parallel_ws(
volume_feat[:, :, 0],
max_workers=Request.global_thread_pool.num_workers,
size_regularizer=self.SizeRegularizer.value,
reduce_to=self.ReduceTo.value)
result_view[...] = labelVolume[:, :, numpy.newaxis]
logger.info("done {}".format(numpy.max(labelVolume)))
logger.info("Blockwise Watershed took {} seconds".format(
watershedTimer.seconds()))
return result
def execute(self, slot, subindex, roi, result):
assert roi.stop - roi.start == self.Output.meta.shape, "Blockwise Watershed must be run on the entire volume."
input_image = self.Input(roi.start, roi.stop).wait()
volume_feat = input_image[0, ..., 0]
result_view = result[0, ..., 0]
# handle the special case of the Request threadpool not having any workers
max_workers = max(1, Request.global_thread_pool.num_workers)
with Timer() as watershedTimer:
# 3d watersheds
if self.Input.meta.getTaggedShape()['z'] > 1:
logger.info("Run block-wise watershed in 3d")
if self.DoAgglo.value:
result_view[...], max_id = watershed_and_agglomerate(
volume_feat,
max_workers=max_workers,
size_regularizer=self.SizeRegularizer.value,
reduce_to=self.ReduceTo.value)
else:
result_view[...], max_id = vigra.analysis.watershedsNew(
volume_feat)
# 2d watersheds
else:
logger.info("Run block-wise watershed in 2d")
if self.DoAgglo.value:
result_view[...], max_id = watershed_and_agglomerate(
volume_feat[:, :, 0],
max_workers=max_workers,
size_regularizer=self.SizeRegularizer.value,
reduce_to=self.ReduceTo.value)
else:
result_view[...], max_id = vigra.analysis.watersheds(
volume_feat[:, :, 0])
logger.info("done {}".format(max_id))
logger.info("Blockwise Watershed took {} seconds".format(
watershedTimer.seconds()))
return result
def execute(self, slot, subindex, roi, result):
assert all(
roi.stop <= self.Input.meta.shape
), "Requested roi {} is too large for this input image of shape {}.".format(
roi, self.Input.meta.shape)
# Determine how much input data we'll need, and where the result will be relative to that input roi
inputRoi, computeRoi = self._getInputComputeRois(roi)
# Obtain the input data
with Timer() as resultTimer:
data = self.Input(*inputRoi).wait()
logger.debug("Obtaining input data took {} seconds for roi {}".format(
resultTimer.seconds(), inputRoi))
xIndex = self.Input.meta.axistags.index('x')
yIndex = self.Input.meta.axistags.index('y')
zIndex = self.Input.meta.axistags.index(
'z') if self.Input.meta.axistags.index('z') < len(
self.Input.meta.shape) else None
cIndex = self.Input.meta.axistags.index(
'c') if self.Input.meta.axistags.index('c') < len(
self.Input.meta.shape) else None
# Must be float32
if data.dtype != numpy.float32:
data = data.astype(numpy.float32)
axiskeys = self.Input.meta.getAxisKeys()
spatialkeys = filter(lambda k: k in 'xyz', axiskeys)
# we need to remove a singleton z axis, otherwise we get
# 'kernel longer than line' errors
reskey = [slice(None, None, None)] * len(self.Input.meta.shape)
reskey[cIndex] = 0
if zIndex and self.Input.meta.shape[zIndex] == 1:
removedZ = True
data = data.reshape((data.shape[xIndex], data.shape[yIndex]))
reskey[zIndex] = 0
spatialkeys = filter(lambda k: k in 'xy', axiskeys)
else:
removedZ = False
sigma = map(self._sigmas.get, spatialkeys)
#Check if we need to smooth
if any([x < 0.1 for x in sigma]):
if removedZ:
resultXY = vigra.taggedView(result, axistags="".join(axiskeys))
resultXY = resultXY.withAxes(*'xy')
resultXY[:] = data
else:
result[:] = data
return result
# Smooth the input data
smoothed = vigra.filters.gaussianSmoothing(
data,
sigma,
window_size=2.0,
roi=computeRoi,
out=result[tuple(reskey)]) # FIXME: Assumes channel is last axis
expectedShape = tuple(
TinyVector(computeRoi[1]) - TinyVector(computeRoi[0]))
assert tuple(
smoothed.shape
) == expectedShape, "Smoothed data shape {} didn't match expected shape {}".format(
smoothed.shape, roi.stop - roi.start)
return result
def loadObject(self, name):
logger.info("want to load object with name = %s" % name)
if not self.hasObjectWithName(name):
logger.info(" --> no such object '%s'" % name)
return False
if self.hasCurrentObject():
self.saveCurrentObject()
self._clearLabels()
fgVoxels, bgVoxels = self.loadObject_impl(name)
fg_bounding_box_start = numpy.array(map(numpy.min, fgVoxels))
fg_bounding_box_stop = 1 + numpy.array(map(numpy.max, fgVoxels))
bg_bounding_box_start = numpy.array(map(numpy.min, bgVoxels))
bg_bounding_box_stop = 1 + numpy.array(map(numpy.max, bgVoxels))
bounding_box_start = numpy.minimum(fg_bounding_box_start,
bg_bounding_box_start)
bounding_box_stop = numpy.maximum(fg_bounding_box_stop,
bg_bounding_box_stop)
bounding_box_slicing = roiToSlice(bounding_box_start,
bounding_box_stop)
bounding_box_shape = tuple(bounding_box_stop - bounding_box_start)
dtype = self.opLabelArray.Output.meta.dtype
# Convert coordinates to be relative to bounding box
fgVoxels = numpy.array(fgVoxels)
fgVoxels = fgVoxels - numpy.array([bounding_box_start]).transpose()
fgVoxels = list(fgVoxels)
bgVoxels = numpy.array(bgVoxels)
bgVoxels = bgVoxels - numpy.array([bounding_box_start]).transpose()
bgVoxels = list(bgVoxels)
with Timer() as timer:
logger.info("Loading seeds....")
z = numpy.zeros(bounding_box_shape, dtype=dtype)
logger.info("Allocating seed array took {} seconds".format(
timer.seconds()))
z[fgVoxels] = 2
z[bgVoxels] = 1
self.WriteSeeds[(slice(0, 1), ) + bounding_box_slicing +
(slice(0, 1), )] = z[numpy.newaxis, :, :, :,
numpy.newaxis]
logger.info("Loading seeds took a total of {} seconds".format(
timer.seconds()))
#restore the correct parameter values
mst = self._mst
assert name in mst.object_lut
assert name in mst.object_seeds_fg_voxels
assert name in mst.object_seeds_bg_voxels
assert name in mst.bg_priority
assert name in mst.no_bias_below
assert name in mst.bg_priority
assert name in mst.no_bias_below
self.BackgroundPriority.setValue(mst.bg_priority[name])
self.NoBiasBelow.setValue(mst.no_bias_below[name])
#self.updatePreprocessing()
# The entire segmentation layer needs to be refreshed now.
self.Segmentation.setDirty()
return True
class EventRecorder( QObject ):
"""
Records spontaneous events from the UI and serializes them as strings that can be evaluated in Python.
"""
def __init__(self, parent=None, ignore_parent_events=True):
QObject.__init__(self, parent=parent)
self._ignore_parent_events = False
if parent is not None and ignore_parent_events:
self._ignore_parent_events = True
self._parent_name = get_fully_qualified_name(parent)
self._captured_events = []
self._timer = Timer()
@property
def paused(self):
return self._timer.paused
QEvent_Style = 91
IgnoredEventTypes = set( [ QEvent.Paint,
QEvent.KeyboardLayoutChange,
QEvent.WindowActivate,
QEvent.WindowDeactivate,
QEvent.ActivationChange,
# These event symbols are not exposed in pyqt, so we pull them from our own enum
EventTypes.Style,
EventTypes.ApplicationActivate,
EventTypes.ApplicationDeactivate,
EventTypes.NonClientAreaMouseMove,
EventTypes.NonClientAreaMouseButtonPress,
EventTypes.NonClientAreaMouseButtonRelease,
EventTypes.NonClientAreaMouseButtonDblClick
] )
IgnoredEventClasses = (QChildEvent, QTimerEvent, QGraphicsSceneMouseEvent, QWindowStateChangeEvent, QMoveEvent)
def captureEvent(self, watched, event):
if self._shouldSaveEvent(event):
try:
eventstr = event_to_string(event)
except KeyError:
logger.warn("Don't know how to record event: {}".format( str(event) ))
print "Don't know", str(event)
else:
timestamp_in_seconds = self._timer.seconds()
objname = str(get_fully_qualified_name(watched))
if not ( self._ignore_parent_events and objname.startswith(self._parent_name) ):
self._captured_events.append( (eventstr, objname, timestamp_in_seconds) )
return False
def insertComment(self, comment):
self._captured_events.append( (comment, "comment", None) )
def _shouldSaveEvent(self, event):
if isinstance(event, QMouseEvent):
# Ignore most mouse movement events if the user isn't pressing anything.
if event.type() == QEvent.MouseMove \
and int(event.button()) == 0 \
and int(event.buttons()) == 0 \
and int(event.modifiers()) == 0:
# Somewhat hackish (and slow), but we have to record mouse movements during combo box usage.
# Same for QMenu usage (on Mac, it doesn't seem to matter, but on Fedora it does matter.)
widgetUnderCursor = QApplication.instance().widgetAt( QCursor.pos() )
if widgetUnderCursor is not None and widgetUnderCursor.objectName() == "qt_scrollarea_viewport":
return has_ancestor(widgetUnderCursor, QComboBox)
if isinstance(widgetUnderCursor, QMenu):
return True
return False
else:
return True
# Ignore non-spontaneous events
if not event.spontaneous():
return False
if event.type() in self.IgnoredEventTypes:
return False
if isinstance(event, self.IgnoredEventClasses):
return False
return True
def unpause(self):
# Here, we use a special override of QApplication.notify() instead of using QApplication.instance().installEventFilter().
# That's because (contrary to the documentation), the QApplication eventFilter does NOT get to see every event in the application.
# Testing shows that events that were "filtered out" by a different event filter may not be seen by the QApplication event filter.
self._timer.unpause()
def _notify(receiver, event):
self.captureEvent(receiver, event)
return _orig_QApp_notify(receiver, event)
from ilastik.shell.gui.startShellGui import EventRecordingApp
assert isinstance( QApplication.instance(), EventRecordingApp )
QApplication.instance()._notify =_notify
def pause(self):
self._timer.pause()
QApplication.instance()._notify = _orig_QApp_notify
def writeScript(self, fileobj):
# Write header comments
fileobj.write(
"""
# Event Recording
# Started at: {}
""".format( str(self._timer.start_time) ) )
# Write playback function definition
fileobj.write(
"""
def playback_events(player):
import PyQt4.QtCore
from PyQt4.QtCore import Qt, QEvent, QPoint
import PyQt4.QtGui
from ilastik.utility.gui.eventRecorder.objectNameUtils import get_named_object
from ilastik.utility.gui.eventRecorder.eventRecorder import EventPlayer
from ilastik.shell.gui.startShellGui import shell
player.display_comment("SCRIPT STARTING")
""")
# Write all events and comments
for eventstr, objname, timestamp_in_seconds in self._captured_events:
if objname == "comment":
eventstr = eventstr.replace('\\', '\\\\')
eventstr = eventstr.replace('"', '\\"')
eventstr = eventstr.replace("'", "\\'")
fileobj.write(
"""
########################
player.display_comment(\"""{eventstr}\""")
########################
""".format( **locals() ) )
else:
fileobj.write(
"""
obj = get_named_object( '{objname}' )
player.post_event( obj, {eventstr}, {timestamp_in_seconds} )
""".format( **locals() )
)
fileobj.write(
"""
player.display_comment("SCRIPT COMPLETE")
""")
def execute(self, slot, subindex, roi, result):
#make sure raw data is 5D: t,{x,y,z},c
ax = self.Input.meta.axistags
sh = self.Input.meta.shape
assert len(ax) == 5
assert ax[0].key == "t" and sh[0] == 1
for i in range(1, 4):
assert ax[i].isSpatial()
assert ax[4].key == "c" and sh[4] == 1
volume5d = self.Input.value
sigma = self.Sigma.value
volume = volume5d[0, :, :, :, 0]
result_view = result[0, :, :, :, 0]
logger.info("input volume shape: %r" % (volume.shape, ))
logger.info("input volume size: %r MB",
(old_div(volume.nbytes, 1024**2), ))
fvol = numpy.asarray(volume, numpy.float32)
#Choose filter selected by user
volume_filter = self.Filter.value
filter_name = self.FILTER_NAMES[volume_filter]
logger.info("applying filter on shape = %r" % (fvol.shape, ))
with Timer() as filterTimer:
# check dimensionality of input and reduce to 2d volume
# if we have actual 2d input
if fvol.shape[2] == 1:
fvol = fvol[:, :, 0]
# we need to invert the input for filter mode RAW_INVERTED
if volume_filter == OpFilter.RAW_INVERTED:
fvol = -fvol
# for the hessian filters, we only need to keep one channel,
# and we discard the other channels during block-wise computation to save memory
if volume_filter == OpFilter.HESSIAN_BRIGHT: # HESSIAN_BRIGHT -> last eigenvalue
channel = fvol.ndim - 1
elif volume_filter == OpFilter.HESSIAN_DARK: # HESSIAN_DARK -> first eigenvalue
channel = 0
else:
channel = None
# handle the special case of the Request threadpool not having any workers
max_workers = max(1, Request.global_thread_pool.num_workers)
# compute the filter response block-wise
response = parallel_filter(filter_name,
fvol,
sigma,
max_workers=max_workers,
return_channel=channel)
# need to invert response for hessian bright
if volume_filter == OpFilter.HESSIAN_BRIGHT:
response = numpy.max(response) - response
# write the response to result view
if fvol.ndim == 2:
result_view[:, :, 0] = response
else:
result_view[...] = response
logger.info("Filter took {} seconds".format(filterTimer.seconds()))
return result
def execute(self, slot, subindex, roi, result):
#make sure raw data is 5D: t,{x,y,z},c
ax = self.Input.meta.axistags
sh = self.Input.meta.shape
assert len(ax) == 5
assert ax[0].key == "t" and sh[0] == 1
for i in range(1, 4):
assert ax[i].isSpatial()
assert ax[4].key == "c" and sh[4] == 1
volume5d = self.Input.value
sigma = self.Sigma.value
volume = volume5d[0, :, :, :, 0]
result_view = result[0, :, :, :, 0]
logger.info("input volume shape: %r" % (volume.shape, ))
logger.info("input volume size: %r MB", (volume.nbytes / 1024**2, ))
fvol = numpy.asarray(volume, numpy.float32)
#Choose filter selected by user
volume_filter = self.Filter.value
logger.info("applying filter on shape = %r" % (fvol.shape, ))
with Timer() as filterTimer:
if fvol.shape[2] > 1:
# true 3D volume
if volume_filter == OpFilter.HESSIAN_BRIGHT:
logger.info("lowest eigenvalue of Hessian of Gaussian")
options = vigra.blockwise.BlockwiseConvolutionOptions3D()
options.stdDev = (sigma, ) * 3
result_view[
...] = vigra.blockwise.hessianOfGaussianLastEigenvalue(
fvol, options)[:, :, :]
result_view[:] = numpy.max(result_view) - result_view
elif volume_filter == OpFilter.HESSIAN_DARK:
logger.info("greatest eigenvalue of Hessian of Gaussian")
options = vigra.blockwise.BlockwiseConvolutionOptions3D()
options.stdDev = (sigma, ) * 3
result_view[
...] = vigra.blockwise.hessianOfGaussianFirstEigenvalue(
fvol, options)[:, :, :]
elif volume_filter == OpFilter.STEP_EDGES:
logger.info("Gaussian Gradient Magnitude")
result_view[...] = vigra.filters.gaussianGradientMagnitude(
fvol, sigma)
elif volume_filter == OpFilter.RAW:
logger.info("Gaussian Smoothing")
result_view[...] = vigra.filters.gaussianSmoothing(
fvol, sigma)
elif volume_filter == OpFilter.RAW_INVERTED:
logger.info("negative Gaussian Smoothing")
result_view[...] = vigra.filters.gaussianSmoothing(
-fvol, sigma)
logger.info("Filter took {} seconds".format(
filterTimer.seconds()))
else:
# 2D Image
fvol = fvol[:, :, 0]
if volume_filter == OpFilter.HESSIAN_BRIGHT:
logger.info("lowest eigenvalue of Hessian of Gaussian")
volume_feat = vigra.filters.hessianOfGaussianEigenvalues(
fvol, sigma)[:, :, 1]
volume_feat[:] = numpy.max(volume_feat) - volume_feat
elif volume_filter == OpFilter.HESSIAN_DARK:
logger.info("greatest eigenvalue of Hessian of Gaussian")
volume_feat = vigra.filters.hessianOfGaussianEigenvalues(
fvol, sigma)[:, :, 0]
elif volume_filter == OpFilter.STEP_EDGES:
logger.info("Gaussian Gradient Magnitude")
volume_feat = vigra.filters.gaussianGradientMagnitude(
fvol, sigma)
elif volume_filter == OpFilter.RAW:
logger.info("Gaussian Smoothing")
volume_feat = vigra.filters.gaussianSmoothing(fvol, sigma)
elif volume_filter == OpFilter.RAW_INVERTED:
logger.info("negative Gaussian Smoothing")
volume_feat = vigra.filters.gaussianSmoothing(-fvol, sigma)
result_view[:, :, 0] = volume_feat
logger.info("Filter took {} seconds".format(
filterTimer.seconds()))
return result
def impl():
shell = self.shell
workflow = shell.projectManager.workflow
object_export_applet = workflow.dataExportApplet
gui = object_export_applet.getMultiLaneGui()
op_object_export = object_export_applet.topLevelOperator.getLane(0)
object_classification_applet = workflow.objectClassificationApplet
op_object_classification = object_classification_applet.topLevelOperator.getLane(
0)
op_object_export_tlo = object_export_applet.topLevelOperator
# activate the object information export applet
shell.setSelectedAppletDrawer(4)
# let the gui catch up
QApplication.processEvents()
op_object_export.OutputFilenameFormat.setValue(self.output_file)
op_object_export.OutputFormat.setValue("hdf5")
op_object_export.OutputInternalPath.setValue("exported_data")
initial_table_export_settings = {
"file type": "csv",
"file path": self.table_csv_file,
"normalize":
True, # self.ui.normalizeLabeling.checkState() == Qt.Checked,
"margin": 3,
"include raw": False,
# compression settings cannot be edited in the gui atm.
# values here are assumed defaults (taken from exportObjectInfoDialog.ui)
"compression": {
"compression": "gzip",
"shuffle": False,
"compression_opts": 9
},
}
table_export_settings, export_features = self.configure_export_dialog(
gui, initial_table_export_settings)
# here is some awkwardness of the csv output, which will alter the
# table name: some_name.csv -> some_name_test_data_table.csv
base, ext = os.path.splitext(self.table_csv_file)
csv_out = f"{base}_table{ext}"
TestObjectClassificationGui.table_csv_file_exported = csv_out
exporter = gui.get_exporting_operator()
exporter.configure_table_export_settings(table_export_settings,
export_features)
# self.configure_export_dialog(op_object_export_tlo)
with Timer() as timer:
# this will not properly wait for the export to finish.
# gui.drawer.exportAllButton.click()
gui.exportSync(op_object_export_tlo)
assert object_export_applet.busy is False
assert os.path.exists(csv_out), f"Could not find {csv_out}"
assert os.path.exists(self.output_file)
logger.debug(f"Export time (data + csv): {timer.seconds()}")
initial_table_export_settings.update({
"file type":
"h5",
"file path":
self.table_h5_file
})
table_export_settings, export_features = self.configure_export_dialog(
gui, initial_table_export_settings)
# here is some awkwardness of the h5 output, which will alter the
# table name: some_name.h5 -> some_name_test_data.h5
base, ext = os.path.splitext(self.table_h5_file)
h5_out = f"{base}{ext}"
TestObjectClassificationGui.table_h5_file_exported = h5_out
exporter.configure_table_export_settings(table_export_settings,
export_features)
with Timer() as timer:
# this will not properly wait for the export to finish.
# gui.drawer.exportAllButton.click()
gui.exportSync(op_object_export_tlo)
assert object_export_applet.busy is False
assert os.path.exists(h5_out), f"Could not find {h5_out}"
assert os.path.exists(self.output_file)
logger.debug(f"Export time (data + h5): {timer.seconds()}")
# Save the project
saveThread = self.shell.onSaveProjectActionTriggered()
saveThread.join()
