def get(self, roi):
"""This method is used to retrieve the actual content of a Slot.
:param roi: the region of interest, e.g. a subregion in the
case of an ArrayLike stype
:param destination: this may define a destination area for the
request, for example a ndarray into which the results should
be written in the case of an ArrayLike stype
Returns:
a request.Request object.
"""
if self._value is not None:
# this handles the case of an inputslot
# having a ._value
# --> construct cheaper request object for this case
result = self.stype.writeIntoDestination(None, self._value, roi)
return ValueRequest(result)
elif self.partner is not None:
# this handles the case of an inputslot
# --> just relay the request
return self.partner.get(roi)
else:
if not self.ready():
# Something is wrong. Are we cancelled?
Request.raise_if_cancelled()
msg = "Can't get data from slot {}.{} yet."\
" It isn't ready."\
"First upstream problem slot is: {}"
msg = msg.format( self.getRealOperator().__class__, self.name, Slot._findUpstreamProblemSlot(self) )
raise Slot.SlotNotReadyError(msg)
# If someone is asking for data from an inputslot that has
# no value and no partner, then something is wrong.
if self._type == "input":
# Something is wrong. Are we cancelled?
Request.raise_if_cancelled()
assert self._type != "input", "This inputSlot has no value and no partner. You can't ask for its data yet!"
# normal (outputslot) case
# --> construct heavy request object..
execWrapper = Slot.RequestExecutionWrapper(self, roi)
request = Request(execWrapper)
# We must decrement the execution count even if the
# request is cancelled
request.notify_cancelled(execWrapper.handleCancel)
return request
def __getitem__(self, key):
"""If level=0, emulate __call__ but with a slicing instead of
a roi.
If level>0, return the subslot corresponding to the key, which
may be a tuple
"""
if self.level > 0:
if isinstance(key, tuple):
assert len(key) > 0
assert len(key) <= self.level
if len(key) == 1:
return self._subSlots[key[0]]
else:
return self._subSlots[key[0]][key[1:]]
return self._subSlots[key]
else:
if self.meta.shape is None:
# Something is wrong. Are we cancelled?
Request.raise_if_cancelled()
if not self.ready():
msg = "This slot ({}.{}) isn't ready yet, which means " \
"you can't ask for its data. Is it connected?".format(self.getRealOperator().name, self.name)
self.logger.error(msg)
slotInfoMsg = "Can't get data from slot {}.{} yet."\
" It isn't ready."\
"First upstream problem slot is: {}"\
"".format( self.getRealOperator().__class__, self.name, Slot._findUpstreamProblemSlot(self) )
self.logger.error(slotInfoMsg)
raise Slot.SlotNotReadyError("Slot isn't ready. See error log.")
assert self.meta.shape is not None, \
("Can't ask for slices of this slot yet:"
" self.meta.shape is None!"
" (operator {} [self={}] slot: {}, key={}".format(
self.operator.name, self.operator, self.name, key))
return self(pslice=key)
def run_export(self, role_path_dict ):
"""
Run the export for each dataset listed in role_path_dict,
which must be a dict of {role_index : path_list}.
For each dataset:
1. Append a lane to the workflow
2. Configure the new lane's DataSelection inputs with the new file (or files, if there is more than one role).
3. Export the results from the new lane
4. Remove the lane from the workflow.
By appending/removing the batch lane for EACH dataset we process, we trigger the workflow's usual
prepareForNewLane() and connectLane() logic, which ensures that we get a fresh new lane that's
ready to process data.
After each lane is processed, the given post-processing callback will be executed.
signature: lane_postprocessing_callback(batch_lane_index)
"""
self.progressSignal.emit(0)
try:
assert isinstance(role_path_dict, OrderedDict)
template_infos = self._get_template_dataset_infos()
# Invert dict from [role][batch_index] -> path to a list-of-tuples, indexed by batch_index:
# [ (role-1-path, role-2-path, ...),
# (role-1-path, role-2-path,...) ]
paths_by_batch_index = zip( *role_path_dict.values() )
# Call customization hook
self.dataExportApplet.prepare_for_entire_export()
batch_lane_index = len(self.dataSelectionApplet.topLevelOperator)
for batch_dataset_index, role_input_paths in enumerate(paths_by_batch_index):
# Add a lane to the end of the workflow for batch processing
# (Expanding OpDataSelection by one has the effect of expanding the whole workflow.)
self.dataSelectionApplet.topLevelOperator.addLane( batch_lane_index )
try:
# The above setup can take a long time for a big workflow.
# If the user has ALREADY cancelled, quit now instead of waiting for the first request to begin.
Request.raise_if_cancelled()
def emit_progress(dataset_percent):
overall_progress = (batch_dataset_index + dataset_percent/100.0)/len(paths_by_batch_index)
self.progressSignal.emit(100*overall_progress)
# Now use the new lane to export the batch results for the current file.
self._run_export_with_empty_batch_lane( role_input_paths,
batch_lane_index,
template_infos,
emit_progress )
finally:
# Remove the batch lane. See docstring above for explanation.
try:
self.dataSelectionApplet.topLevelOperator.removeLane( batch_lane_index, batch_lane_index )
except Request.CancellationException:
log_exception(logger)
# If you see this, something went wrong in a graph setup operation.
raise RuntimeError("Encountered an unexpected CancellationException while removing the batch lane.")
assert len(self.dataSelectionApplet.topLevelOperator.DatasetGroup) == batch_lane_index
# Call customization hook
self.dataExportApplet.post_process_entire_export()
finally:
self.progressSignal.emit(100)
def run_export(self,
role_data_dict,
input_axes=None,
export_to_array=False,
sequence_axis=None):
"""
Run the export for each dataset listed in role_data_dict,
which must be a dict of {role_index : path-list} OR {role_index : DatasetInfo-list}
As shown above, you may pass either filepaths OR preconfigured DatasetInfo objects.
The latter is useful if you are batch processing data that already exists in memory as a numpy array.
(See DatasetInfo.preloaded_array for how to provide a numpy array instead of a filepath.)
For each dataset:
1. Append a lane to the workflow
2. Configure the new lane's DataSelection inputs with the new file (or files, if there is more than one
role).
3. Export the results from the new lane
4. Remove the lane from the workflow.
By appending/removing the batch lane for EACH dataset we process, we trigger the workflow's usual
prepareForNewLane() and connectLane() logic, which ensures that we get a fresh new lane that's
ready to process data.
After each lane is processed, the given post-processing callback will be executed.
signature: lane_postprocessing_callback(batch_lane_index)
export_to_array: If True do NOT export to disk as usual.
Instead, export the results to a list of arrays, which is returned.
If False, return a list of the filenames we produced to.
"""
results = []
self.progressSignal(0)
try:
assert isinstance(role_data_dict, OrderedDict)
template_infos = self._get_template_dataset_infos(
input_axes, sequence_axis)
# Invert dict from [role][batch_index] -> path to a list-of-tuples, indexed by batch_index:
# [ (role-1-path, role-2-path, ...),
# (role-1-path, role-2-path,...) ]
datas_by_batch_index = list(zip(*list(role_data_dict.values())))
# Call customization hook
self.dataExportApplet.prepare_for_entire_export()
batch_lane_index = len(self.dataSelectionApplet.topLevelOperator)
for batch_dataset_index, role_input_datas in enumerate(
datas_by_batch_index):
# Add a lane to the end of the workflow for batch processing
# (Expanding OpDataSelection by one has the effect of expanding the whole workflow.)
self.dataSelectionApplet.topLevelOperator.addLane(
batch_lane_index)
try:
# The above setup can take a long time for a big workflow.
# If the user has ALREADY cancelled, quit now instead of waiting for the first request to begin.
Request.raise_if_cancelled()
def emit_progress(dataset_percent):
overall_progress = (batch_dataset_index +
dataset_percent /
100.0) / len(datas_by_batch_index)
self.progressSignal(100 * overall_progress)
# Now use the new lane to export the batch results for the current file.
result = self._run_export_with_empty_batch_lane(
role_input_datas,
batch_lane_index,
template_infos,
emit_progress,
export_to_array=export_to_array)
if export_to_array:
assert isinstance(result, numpy.ndarray)
else:
assert isinstance(result, str)
results.append(result)
finally:
# Remove the batch lane. See docstring above for explanation.
try:
self.dataSelectionApplet.topLevelOperator.removeLane(
batch_lane_index, batch_lane_index)
except Request.CancellationException:
log_exception(logger)
# If you see this, something went wrong in a graph setup operation.
raise RuntimeError(
"Encountered an unexpected CancellationException while removing the batch lane."
)
assert len(self.dataSelectionApplet.topLevelOperator.
DatasetGroup) == batch_lane_index
# Call customization hook
self.dataExportApplet.post_process_entire_export()
return results
finally:
self.progressSignal(100)
def run_export(self, role_data_dict, input_axes=None, export_to_array=False ):
"""
Run the export for each dataset listed in role_data_dict,
which must be a dict of {role_index : path-list} OR {role_index : DatasetInfo-list}
As shown above, you may pass either filepaths OR preconfigured DatasetInfo objects.
The latter is useful if you are batch processing data that already exists in memory as a numpy array.
(See DatasetInfo.preloaded_array for how to provide a numpy array instead of a filepath.)
For each dataset:
1. Append a lane to the workflow
2. Configure the new lane's DataSelection inputs with the new file (or files, if there is more than one role).
3. Export the results from the new lane
4. Remove the lane from the workflow.
By appending/removing the batch lane for EACH dataset we process, we trigger the workflow's usual
prepareForNewLane() and connectLane() logic, which ensures that we get a fresh new lane that's
ready to process data.
After each lane is processed, the given post-processing callback will be executed.
signature: lane_postprocessing_callback(batch_lane_index)
export_to_array: If True do NOT export to disk as usual.
Instead, export the results to a list of arrays, which is returned.
If False, return a list of the filenames we produced to.
"""
results = []
self.progressSignal.emit(0)
try:
assert isinstance(role_data_dict, OrderedDict)
template_infos = self._get_template_dataset_infos(input_axes)
# Invert dict from [role][batch_index] -> path to a list-of-tuples, indexed by batch_index:
# [ (role-1-path, role-2-path, ...),
# (role-1-path, role-2-path,...) ]
datas_by_batch_index = zip( *role_data_dict.values() )
# Call customization hook
self.dataExportApplet.prepare_for_entire_export()
batch_lane_index = len(self.dataSelectionApplet.topLevelOperator)
for batch_dataset_index, role_input_datas in enumerate(datas_by_batch_index):
# Add a lane to the end of the workflow for batch processing
# (Expanding OpDataSelection by one has the effect of expanding the whole workflow.)
self.dataSelectionApplet.topLevelOperator.addLane( batch_lane_index )
try:
# The above setup can take a long time for a big workflow.
# If the user has ALREADY cancelled, quit now instead of waiting for the first request to begin.
Request.raise_if_cancelled()
def emit_progress(dataset_percent):
overall_progress = (batch_dataset_index + dataset_percent/100.0)/len(datas_by_batch_index)
self.progressSignal.emit(100*overall_progress)
# Now use the new lane to export the batch results for the current file.
result = self._run_export_with_empty_batch_lane( role_input_datas,
batch_lane_index,
template_infos,
emit_progress,
export_to_array=export_to_array )
if export_to_array:
assert isinstance(result, numpy.ndarray)
else:
assert isinstance(result, str)
results.append( result )
finally:
# Remove the batch lane. See docstring above for explanation.
try:
self.dataSelectionApplet.topLevelOperator.removeLane( batch_lane_index, batch_lane_index )
except Request.CancellationException:
log_exception(logger)
# If you see this, something went wrong in a graph setup operation.
raise RuntimeError("Encountered an unexpected CancellationException while removing the batch lane.")
assert len(self.dataSelectionApplet.topLevelOperator.DatasetGroup) == batch_lane_index
# Call customization hook
self.dataExportApplet.post_process_entire_export()
return results
finally:
self.progressSignal.emit(100)
