def __init__(self, outputSlot, roiIterator, totalVolume=None, batchSize=2, allowParallelResults=False):
"""
Constructor.
:param outputSlot: The slot to request data from.
:param roiIterator: An iterator providing new rois.
:param totalVolume: The total volume to be processed.
Used to provide the progress reporting signal.
If not provided, then no intermediate progress will be signaled.
:param batchSize: The maximum number of requests to launch in parallel.
:param allowParallelResults: If False, The resultSignal will not be called in parallel.
In that case, your handler function has no need for locks.
"""
self._resultSignal = OrderedSignal()
self._progressSignal = OrderedSignal()
assert isinstance(
outputSlot.stype, lazyflow.stype.ArrayLike
), "Only Array-like slots supported." # Because progress reporting depends on the roi shape
self._outputSlot = outputSlot
self._roiIter = roiIterator
self._batchSize = batchSize
self._allowParallelResults = allowParallelResults
self._condition = SimpleRequestCondition()
self._activated_count = 0
self._completed_count = 0
self._failure_excinfo = None
# Progress bookkeeping
self._totalVolume = totalVolume
self._processedVolume = 0
def __init__(self, *args, **kwargs):
super(OpConcatenateFeatureMatrices, self).__init__(*args, **kwargs)
self._dirty_slots = set()
self.progressSignal = OrderedSignal()
self._num_feature_channels = 0 # Not including the labels...
self._channel_names = []
# Normally, lane removal does not trigger a dirty notification.
# But in this case, if the lane contained any label data whatsoever,
# the classifier needs to be marked dirty.
# We know which slots contain (or contained) label data because they have
# been 'touched' at some point (they became dirty at some point).
self._touched_slots = set()
def handle_new_lane( multislot, index, newlength ):
def handle_dirty_lane( slot, roi ):
self._touched_slots.add(slot)
multislot[index].notifyDirty( handle_dirty_lane )
self.FeatureMatrices.notifyInserted( handle_new_lane )
def handle_remove_lane( multislot, index, newlength ):
# If the lane we're removing contained
# label data, then mark the downstream dirty
if multislot[index] in self._touched_slots:
self.ConcatenatedOutput.setDirty()
self._touched_slots.remove(multislot[index])
self.FeatureMatrices.notifyRemove( handle_remove_lane )
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
# Set up the impl function lookup dict
export_impls = {}
export_impls["hdf5"] = ("h5", self._export_h5n5)
export_impls["compressed hdf5"] = ("h5",
partial(self._export_h5n5, True))
export_impls["n5"] = ("n5", self._export_h5n5)
export_impls["compressed n5"] = ("n5", partial(self._export_h5n5,
True))
export_impls["numpy"] = ("npy", self._export_npy)
export_impls["dvid"] = ("", self._export_dvid)
export_impls["blockwise hdf5"] = ("json", self._export_blockwise_hdf5)
for fmt in self._2d_formats:
export_impls[fmt.name] = (fmt.extension,
partial(self._export_2d, fmt.extension))
for fmt in self._3d_sequence_formats:
export_impls[fmt.name] = (fmt.extension,
partial(self._export_3d_sequence,
fmt.extension))
export_impls["multipage tiff"] = ("tiff", self._export_multipage_tiff)
export_impls["multipage tiff sequence"] = (
"tiff", self._export_multipage_tiff_sequence)
self._export_impls = export_impls
self.Input.notifyMetaChanged(self._updateFormatSelectionErrorMsg)
def __init__(self, *args, **kwargs):
super(OpExportSlot, self).__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
# Set up the impl function lookup dict
export_impls = {}
export_impls['hdf5'] = ('h5', self._export_hdf5)
export_impls['npy'] = ('npy', self._export_npy)
export_impls['dvid'] = ('', self._export_dvid)
export_impls['blockwise hdf5'] = ('json', self._export_blockwise_hdf5)
for fmt in self._2d_formats:
export_impls[fmt.name] = (fmt.extension,
partial(self._export_2d, fmt.extension))
for fmt in self._3d_sequence_formats:
export_impls[fmt.name] = (fmt.extension,
partial(self._export_3d_sequence,
fmt.extension))
export_impls['multipage tiff'] = ('tiff', self._export_multipage_tiff)
export_impls['multipage tiff sequence'] = (
'tiff', self._export_multipage_tiff_sequence)
self._export_impls = export_impls
self.Input.notifyMetaChanged(self._updateFormatSelectionErrorMsg)
def __init__(self, *args, **kwargs):
super(OpWsdt, self).__init__(*args, **kwargs)
self.debug_results = None
self.watershed_completed = OrderedSignal()
self._opSelectedInput = OpSumChannels(parent=self)
self._opSelectedInput.ChannelSelections.connect(self.ChannelSelections)
self._opSelectedInput.Input.connect(self.Input)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
self._opReorderAxes = OpReorderAxes(parent=self)
self._opReorderAxes.Input.connect(self.Input)
self._opReorderAxes.AxisOrder.setValue(self._EXPORT_AXES)
def __init__(self, *args, **kwargs):
super(OpFeatureMatrixCache, self).__init__(*args, **kwargs)
self._lock = RequestLock()
self.progressSignal = OrderedSignal()
self._progress_lock = RequestLock()
self._blockshape = None
self._dirty_blocks = set()
self._blockwise_feature_matrices = {}
self._block_locks = {} # One lock per stored block
self._init_blocks(None, None)
def __init__(self, *args, **kwargs):
super(OpFeatureMatrixCache, self).__init__(*args, **kwargs)
self._blockshape = None
self._lock = RequestLock()
self.progressSignal = OrderedSignal()
self._progress_lock = RequestLock()
# In these set/dict members, the block id (dict key)
# is simply the block's start coordinate (as a tuple)
self._blockwise_feature_matrices = {}
self._dirty_blocks = set()
self._block_locks = {} # One lock per stored block
class ExportFile(object):
ExportProgress = OrderedSignal()
InsertionProgress = OrderedSignal()
def __init__(self, file_name):
self.file_name = file_name
self.table_dict = {}
self.meta_dict = {}
def add_columns(self, table_name, col_data, mode, extra=None):
"""
Adds new columns to the table ( creates the table if neccessary )
:param table_name: the table name
:type table_name: str
:param col_data: the actual data to be added
:type col_data: list, dict, numpy.array, whatever is supported
:param mode: the type of the table data
:type mode: exportFile.Mode
:param extra: extra information for the given mode
:type extra: dict
"""
if extra is None:
extra = {}
if mode == Mode.IlastikTrackingTable:
if not "counts" in extra or not "max" in extra:
raise AttributeError("Tracking need 'counts', 'max' extra")
columns = flatten_tracking_table(col_data, extra["extra ids"],
extra["counts"], extra["max"],
extra["range"])
elif mode == Mode.List:
if not "names" in extra:
raise AttributeError(
"[Tuple]List needs a tuple for the column name (extra 'names')"
)
dtypes = extra["dtypes"] if "dtypes" in extra else None
columns = prepare_list(col_data, extra["names"], dtypes)
elif mode == Mode.IlastikFeatureTable:
if "selection" not in extra:
raise AttributeError(
"IlastikFeatureTable needs a feature selection (extra 'selection')"
)
columns = flatten_ilastik_feature_table(col_data,
extra["selection"],
self.InsertionProgress)
elif mode == Mode.NumpyStructArray:
columns = col_data
else:
raise AttributeError("Invalid Mode")
self._add_columns(table_name, columns)
def add_rois(self,
table_path,
image_slot,
feature_table_name,
margin,
type_="image"):
"""
Adds the rois as images to the table
:param table_path: the new name for the table
:type table_path: str
:param image_slot: the slot to read the data from
:type image_slot: lazyflow.slot.Slot
:param feature_table_name: the already added feature table to read the coords from
:type feature_table_name: str
:param margin: the margin to be added around the images
:type margin: int
:param type_: "image" for normal images, "labeling" for labeling images
:type type_: str
"""
assert type_ in ("labeling",
"image"), "Type must be 'labeling' or 'image'"
slicings = create_slicing(image_slot.meta.axistags,
image_slot.meta.shape, margin,
self.table_dict[feature_table_name])
self.InsertionProgress(0)
if type_ == "labeling":
vec = self._normalize
else:
vec = lambda _: lambda y: y
for i, (slicing, oid) in enumerate(slicings):
roi = image_slot(slicing).wait()
roi = vec(oid)(roi)
roi_path = table_path.format(i)
self.meta_dict[roi_path] = {
"type":
type_,
"axistags":
actual_axistags(image_slot.meta.axistags, roi.shape).toJSON()
}
self.table_dict[roi_path] = roi.squeeze()
self.InsertionProgress(
100 * i / self.table_dict[feature_table_name].shape[0])
self.InsertionProgress(100)
@staticmethod
def _normalize(oid):
def f(pixel_value):
return 1 if pixel_value == oid else 0
return np.vectorize(f)
def add_image(self, table, image_slot):
"""
Adds an image as a table
:param table: the name for the image
:type table: str
:param image_slot: the slot to read the image from
:type image_slot: lazyflow.slot.Slot
"""
self.table_dict[table] = image_slot([]).wait().squeeze()
self.meta_dict[table] = {
"type":
"image",
"axistags":
actual_axistags(image_slot.meta.axistags,
image_slot.meta.shape).toJSON()
}
def update_meta(self, table, meta):
"""
Adds meta information to the table
:param table: the table to add meta to
:type table: str
:param meta: the meta information to add
:type meta: dict
"""
self.meta_dict.setdefault(table, {})
self.meta_dict[table].update(meta)
def write_all(self, mode, compression=None):
"""
Writes all tables to the file
:param mode: "h[d[f]]5" or "csv" at the moment
:type mode: str
:param compression: the compression settings
:type compression: dict
"""
count = 0
self.ExportProgress(0)
if mode in ("h5", "hd5", "hdf5"):
with h5py.File(self.file_name, "w") as fout:
for table_name, table in self.table_dict.iteritems():
self._make_h5_dataset(
fout, table_name, table,
self.meta_dict.get(table_name, {}),
compression if compression is not None else {})
count += 1
self.ExportProgress(count * 100 / len(self.table_dict))
elif mode == "csv":
f_name = self.file_name.rsplit(".", 1)
if len(f_name) == 1:
base, ext = f_name, ""
else:
base, ext = f_name
file_names = []
for table_name, table in self.table_dict.iteritems():
file_names.append("{name}_{table}.{ext}".format(
name=base, table=table_name, ext=ext))
with open(file_names[-1], "w") as fout:
self._make_csv_table(fout, table)
count += 1
self.ExportProgress(count * 100 / len(self.table_dict))
if False:
with ZipFile("{name}.zip".format(name=base), "w") as zip_file:
for file_name in file_names:
zip_file.write(file_name)
self.ExportProgress(100)
logger.info("exported %i tables" % count)
def _add_columns(self, table_name, columns):
if table_name in self.table_dict.iterkeys():
old = self.table_dict[table_name]
columns = nlr.merge_arrays((old, columns), flatten=True)
self.table_dict[table_name] = columns
@staticmethod
def _make_h5_dataset(fout, table_name, table, meta, compression):
try:
dset = fout.create_dataset(table_name,
table.shape,
data=table,
**compression)
except TypeError:
dset = fout.create_dataset(table_name, table.shape, data=table)
for k, v in meta.iteritems():
dset.attrs[k] = v
@staticmethod
def _make_csv_table(fout, table):
line = ",".join(table.dtype.names)
fout.write(line)
fout.write("\n")
for row in table:
line = ",".join(map(str, row))
fout.write(line)
fout.write("\n")
def __init__(self, *args, **kwargs):
super(OpNpyWriter, self).__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
def __enter__(self):
if self._graph:
with self._graph._lock:
if self._graph._setup_depth == 0:
self._graph._sig_setup_complete = OrderedSignal()
self._graph._setup_depth += 1
def __init__(self, transpose_axes, *args, **kwargs):
super(OpExportDvidVolume, self).__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
self._transpose_axes = transpose_axes
class Slot(object):
"""
Base class for InputSlot, OutputSlot
"""
loggerName = __name__ + '.Slot'
logger = logging.getLogger(loggerName)
traceLogger = logging.getLogger('TRACE.' + loggerName)
# Allow slots to be sorted by their order of creation for debug
# output and diagramming purposes.
_global_counter = itertools.count()
class SlotNotReadyError(Exception):
pass
@property
def graph(self):
return self.operator.graph
def __init__(self, name="", operator=None, stype=ArrayLike,
rtype=rtype.SubRegion, value=None, optional=False,
level=0, nonlane=False):
"""Constructor of the Slot class.
:param name: user readable name of the slot, is normally
assigned automatically by the Operator
:param operator: the parent operator of a slot
:param stype: the slot type (see stype.py)
:param rtype: the region of interest type (see rtype.py)
:param value: the default value of the slot
:param optional: if True this means the slot needs a value or
connection for its parent operator to be functional
:param level: defines the dimensionality of the slot. 0 for
single element (e.g. single numpy.ndarray), 1 for list of
elements (e.g. list of strings), 2 for list of list of
elements.
:param nonlane: For multislot, this flag protects it from
being considered lane-indexed
"""
# This assertion is here for a reason: default values do NOT work on OutputSlots.
# (We should probably change that at some point...)
assert value is None or isinstance(self, InputSlot), "Only InputSlots can have default values. OutputSlots cannot."
if not hasattr(self, "_type"):
self._type = None
if type(stype) == str:
stype = ArrayLike
self.partners = []
self.name = name
self._optional = optional
self.operator = operator
self._real_operator = None # Memoized in getRealOperator()
# in the case of an InputSlot this is the slot to which it is
# connected
self.partner = None
self.level = level
# in the case of an InputSlot one can directly assign a value
# to a slot instead of connecting it to a partner, this
# attribute holds the value
self._value = None
self._defaultValue = value
# Causes calls to setValue to be propagated backwards to the
# partner slot. Used by the OperatorWrapper.
self._backpropagate_values = False
self.rtype = rtype
# the MetaDict that holds the slots meta information
self.meta = MetaDict()
# if level > 0, this holds the sub-Input/Output slots
self._subSlots = []
self._stypeType = stype
# the slot type instance
self.stype = stype(self)
self.nonlane = nonlane
self._sig_changed = OrderedSignal()
self._sig_value_changed = OrderedSignal()
self._sig_ready = OrderedSignal()
self._sig_unready = OrderedSignal()
self._sig_dirty = OrderedSignal()
self._sig_connect = OrderedSignal()
self._sig_disconnect = OrderedSignal()
self._sig_resize = OrderedSignal()
self._sig_resized = OrderedSignal()
self._sig_remove = OrderedSignal()
self._sig_removed = OrderedSignal()
self._sig_preinsertion = OrderedSignal()
self._sig_inserted = OrderedSignal()
self._resizing = False
self._executionCount = 0
self._settingUp = False
self._condition = threading.Condition()
# Allow slots to be sorted by their order of creation for
# debug output and diagramming purposes.
self._global_slot_id = Slot._global_counter.next()
###########################
# A p i M e t h o d s #
###########################
def notifyDirty(self, function, **kwargs):
"""
calls the corresponding function when the slot gets dirty
first argument of the function is the slot, second argument the roi
the keyword arguments follow
"""
self._sig_dirty.subscribe(function, **kwargs)
def notifyMetaChanged(self, function, **kwargs):
"""calls the corresponding function when the slot meta
information is changed
first argument of the function is the slot
the keyword arguments follow
"""
self._sig_changed.subscribe(function, **kwargs)
def notifyValueChanged(self, function, **kwargs):
"""Used by slots with cached values to notify when the cache
has changed, even if the output is not dirty.
"""
self._sig_value_changed.subscribe(function, **kwargs)
def notifyReady(self, function, **kwargs):
"""Calls the corresponding function when the slot is "ready",
meaning it is connected and will produce data when called.
This is implemented by manipulating and monitoring a flag in
the slot metadata.
first argument of the function is the slot
the keyword arguments follow
"""
self._sig_ready.subscribe(function, **kwargs)
def notifyUnready(self, function, **kwargs):
"""
Subscribe to "unready" callbacks. See notifyReady for details.
"""
self._sig_unready.subscribe(function, **kwargs)
def _notifyConnect(self, function, **kwargs):
"""
calls the corresponding function when the slot is connected
first argument of the function is the slot
the keyword arguments follow
"""
self._sig_connect.subscribe(function, **kwargs)
def notifyDisconnect(self, function, **kwargs):
"""
calls the corresponding function when the slot is disconnected
first argument of the function is the slot
the keyword arguments follow
"""
self._sig_disconnect.subscribe(function, **kwargs)
def notifyResize(self, function, **kwargs):
"""
calls the corresponding function before the slot is resized
first argument of the function is the slot
second argument is the old size and the third
argument is the new size
the keyword arguments follow
"""
self._sig_resize.subscribe(function, **kwargs)
def notifyResized(self, function, **kwargs):
"""
calls the corresponding function after the slot is resized
first argument of the function is the slot
second argument is the old size and the third
argument is the new size
the keyword arguments follow
"""
self._sig_resized.subscribe(function, **kwargs)
def notifyRemove(self, function, **kwargs):
"""
calls the corresponding function BEFORE a slot is removed
first argument of the function is the slot
second argument is the old size and the third
argument is the new size
the keyword arguments follow
"""
self._sig_remove.subscribe(function, **kwargs)
def notifyRemoved(self, function, **kwargs):
"""
calls the corresponding function AFTER a slot is removed
first argument of the function is the slot
second argument is the old size and the third
argument is the new size
the keyword arguments follow
"""
self._sig_removed.subscribe(function, **kwargs)
def notifyPreInsertion(self, function, **kwargs):
"""
Called immediately before a slot is going to be inserted into a multi-slot.
Same signature as the notifyInserted signal.
"""
self._sig_preinsertion.subscribe(function, **kwargs)
def notifyInserted(self, function, **kwargs):
"""
calls the corresponding function after a slot has been added
first argument of the function is the slot
second argument is the old size and the third
argument is the new size
the keyword arguments follow
"""
self._sig_inserted.subscribe(function, **kwargs)
def unregisterDirty(self, function):
"""
unregister a dirty callback
"""
self._sig_dirty.unsubscribe(function)
def _unregisterConnect(self, function):
"""
unregister a connect callback
"""
self._sig_connect.unsubscribe(function)
def unregisterDisconnect(self, function):
"""
unregister a disconnect callback
"""
self._sig_disconnect.unsubscribe(function)
def unregisterMetaChanged(self, function):
"""
unregister a changed callback
"""
self._sig_changed.unsubscribe(function)
def unregisterValueChanged(self, function):
"""
unregister a value changed callback
"""
self._sig_value_changed.unsubscribe(function)
def unregisterReady(self, function):
"""
unregister a ready callback
"""
self._sig_ready.unsubscribe(function)
def unregisterUnready(self, function):
"""
unregister an unready callback
"""
self._sig_unready.unsubscribe(function)
def unregisterResize(self, function):
"""
unregister a resize callback
"""
self._sig_resize.unsubscribe(function)
def unregisterResized(self, function):
"""
unregister a resized callback
"""
self._sig_resized.unsubscribe(function)
def unregisterRemove(self, function):
"""
unregister a remove callback
"""
self._sig_remove.unsubscribe(function)
def unregisterRemoved(self, function):
"""
unregister a removed callback
"""
self._sig_removed.unsubscribe(function)
def unregisterPreInsertion(self, function):
"""
unregister a inserted callback
"""
self._sig_preinsertion.unsubscribe(function)
def unregisterInserted(self, function):
"""
unregister a inserted callback
"""
self._sig_inserted.unsubscribe(function)
def _handleUpstreamUnready(self, slot):
"""
This handler ensures that UNready status propagates quickly
through the graph (before the normal _changed path)
"""
if self.meta._ready:
self.meta._ready = False
self._sig_unready(self)
def connect(self, partner, notify=True):
"""
Connect a slot to another slot
Arguments:
partner : the slot to which this slot is conencted
"""
try:
if partner is None:
self.disconnect()
return
assert isinstance(partner, Slot), ("Slot.connect() can only be used to"
" connect other Slots. Did you mean"
" to use Slot.setValue()?")
my_op = self.getRealOperator()
partner_op = partner.getRealOperator()
if not( partner_op.parent == my_op.parent or \
(self._type == "output" and partner_op.parent == my_op) or \
(self._type == "input" and my_op.parent == partner_op) or \
my_op == partner_op):
msg = "It is forbidden to connect slots of operators that are not siblings "\
"or not directly related as parent and child."
if partner_op.parent is None or my_op.parent is None:
msg += "\n(For one of your operators, parent=None. Was it already cleaned up?"
raise Exception(msg)
if self.partner == partner and partner.level == self.level:
return
if self.level == 0:
self.disconnect()
if partner is not None:
partner._sig_unready.subscribe( self._handleUpstreamUnready )
self._value = None
if partner.level == self.level:
assert isinstance(partner.stype, type(self.stype)), \
"Can't connect slots of non-matching stypes!" \
" Attempting to connect '{}' (stype: {}) to '{}' (stype: {})".format(self.name, self.stype, partner.name, partner.stype)
self.partner = partner
notifyReady = (self.partner.meta._ready and
not self.meta._ready)
self.meta = self.partner.meta.copy()
# the slot with more sub-slots determines
# the number of subslots
if len(self) < len(partner):
self.resize(len(partner))
elif len(self) > len(partner):
partner.resize(len(self))
partner.partners.append(self)
for i in range(len(self.partner)):
p = self.partner[i]
self[i].connect(p)
# call slot type connect function
self.stype.connect(partner)
if self.level > 0 or self.stype.isConfigured():
self._changed()
# call connect callbacks
self._sig_connect(self)
# Notify readiness after partner is updated
if notifyReady:
self._sig_ready(self)
elif partner.level < self.level:
self.partner = partner
notifyReady = (self.partner.meta._ready and not
self.meta._ready)
self.meta = self.partner.meta.copy()
for i, slot in enumerate(self._subSlots):
slot.connect(partner)
if notifyReady:
self._sig_ready(self)
self._changed()
# call connect callbacks
self._sig_connect(self)
elif partner.level > self.level:
msg = str("Can't connect slots:"
" {}.{}.level={}, but"
" {}.{}.level={}"
" (Implicit OpearatorWrapper creation"
" is no longer supported.)").format(
self.getRealOperator().name,
self.name, self.level,
partner.getRealOperator().name,
partner.name, partner.level)
raise RuntimeError(msg)
# propagate value changed signals from inner to outer
# operators.
if self._type == partner._type == "output":
partner.notifyValueChanged(self._sig_value_changed)
except:
# If anything went wrong, we revert to the disconnected state.
try:
exc_info = sys.exc_info()
self.disconnect()
except:
# Well, this is bad. We caused an exception while handling an exception.
# We're more interested in the FIRST excpetion, so print this one out and
# continue unwinding the stack with the first one.
self.logger.error("Error: Caught a secondary exception while handling a different exception.")
import traceback
traceback.print_exc()
raise exc_info[0], exc_info[1], exc_info[2]
raise
def disconnect(self):
"""
Disconnect a InputSlot from its partner
"""
if self.backpropagate_values:
if self.partner is not None:
self.partner.disconnect()
return
for slot in self._subSlots:
slot.disconnect()
had_partner = False
if self.partner is not None:
had_partner = True
try:
self.partner.partners.remove(self)
except ValueError:
pass
self.partner = None
had_value = self._value is not None
self._value = None
oldReady = self.meta._ready
self.meta = MetaDict()
if len(self._subSlots) > 0:
self.resize(0)
# call callbacks
if had_partner or had_value:
self._sig_disconnect(self)
# Notify our partners that we changed.
self._changed()
# If we were ready before, signal that we aren't any more
if oldReady:
self._sig_unready(self)
def resize(self, size):
"""
Resizes a slot to the desired length
Arguments:
size : the desired number of subslots
"""
assert numpy.issubdtype(type(size), numpy.integer), \
"Bug: 'size' must be int, not {}".format( type(size) )
if self._resizing:
return
if self.level == 0:
raise RuntimeError("Can't resize a level-0 slot!")
oldsize = len(self)
if size == oldsize:
return
self._resizing = True
if self.operator is not None:
self.logger.debug("Resizing slot {} of operator {} to size {}".format(
self.name, self.operator.name, size))
# call before resize callbacks
self._sig_resize(self, oldsize, size)
new_subslots = []
while size > len(self):
self.insertSlot(len(self), len(self)+1, propagate=False)
new_subslots.append( len(self) - 1 )
while size < len(self):
self.removeSlot(len(self)-1, len(self)-1, propagate=False)
# propagate size change downward
for c in self.partners:
if c.level == self.level:
c.resize(size)
# propagate size change upward
if (self.partner and len(self.partner) < size and self.partner.level == self.level):
self.partner.resize(size)
# connect newly added slots
# We must connect these subslots here, AFTER all resizes have propagated up and down through the graph.
# Otherwise, our new subslots may lose downstream partners (happens in "diamond" shaped graphs.)
for i in new_subslots:
self._connectSubSlot(i)
# call after resize callbacks
self._sig_resized(self, oldsize, size)
self._resizing = False
def insertSlot(self, position, finalsize, propagate=True):
"""
Insert a new slot at the specififed position
finalsize indicates the final destination size
"""
if len(self) >= finalsize:
return self[position]
# pre-insert callbacks
self._sig_preinsertion(self, position, finalsize)
slot = self._insertNew(position)
self.logger.debug("Inserting slot {} into slot {} of operator {} to size {}".format(
position, self.name, self.operator.name, finalsize))
if propagate:
if self.partner is not None and self.partner.level == self.level:
self.partner.insertSlot(position, finalsize)
for p in self.partners:
if p.level == self.level:
p.insertSlot(position, finalsize)
self._connectSubSlot(position)
# call after insert callbacks
self._sig_inserted(self, position, finalsize)
return slot
def removeSlot(self, position, finalsize, propagate=True):
"""
Remove the slot at position
finalsize indicates the final size of all subslots
"""
if len(self) <= finalsize:
return None
assert position < len(self)
if self.operator is not None:
self.logger.debug("Removing slot {} into slot {} of operator {} to size {}".format(
position, self.name, self.operator.name, finalsize))
# call before-remove callbacks
self._sig_remove(self, position, finalsize)
slot = self._subSlots.pop(position)
slot.operator = None
slot.disconnect()
if propagate:
if self.partner is not None and self.partner.level == self.level:
self.partner.removeSlot(position, finalsize)
for p in self.partners:
if p.level == self.level:
p.removeSlot(position, finalsize)
# call after-remove callbacks
self._sig_removed(self, position, finalsize)
def get(self, roi):
"""This method is used to retrieve the actual content of a Slot.
:param roi: the region of interest, e.g. a subregion in the
case of an ArrayLike stype
:param destination: this may define a destination area for the
request, for example a ndarray into which the results should
be written in the case of an ArrayLike stype
Returns:
a request.Request object.
"""
if self._value is not None:
# this handles the case of an inputslot
# having a ._value
# --> construct cheaper request object for this case
result = self.stype.writeIntoDestination(None, self._value, roi)
return ValueRequest(result)
elif self.partner is not None:
# this handles the case of an inputslot
# --> just relay the request
return self.partner.get(roi)
else:
if not self.ready():
# Something is wrong. Are we cancelled?
Request.raise_if_cancelled()
msg = "Can't get data from slot {}.{} yet."\
" It isn't ready."\
"First upstream problem slot is: {}"
msg = msg.format( self.getRealOperator().__class__, self.name, Slot._findUpstreamProblemSlot(self) )
raise Slot.SlotNotReadyError(msg)
# If someone is asking for data from an inputslot that has
# no value and no partner, then something is wrong.
if self._type == "input":
# Something is wrong. Are we cancelled?
Request.raise_if_cancelled()
assert self._type != "input", "This inputSlot has no value and no partner. You can't ask for its data yet!"
# normal (outputslot) case
# --> construct heavy request object..
execWrapper = Slot.RequestExecutionWrapper(self, roi)
request = Request(execWrapper)
# We must decrement the execution count even if the
# request is cancelled
request.notify_cancelled(execWrapper.handleCancel)
return request
@staticmethod
def _findUpstreamProblemSlot(slot):
if slot.partner is not None:
return Slot._findUpstreamProblemSlot( slot.partner )
for inputSlot in slot.getRealOperator().inputs.values():
if not inputSlot._optional and not inputSlot.ready():
return inputSlot
return "Couldn't find an upstream problem slot."
class RequestExecutionWrapper(object):
def __init__(self, slot, roi):
self.started = False
self.finished = False
self.slot = slot
self.operator = slot.operator
self.lock = threading.Lock()
self.roi = roi
def __call__(self, destination=None):
# store whether the user wants the results in a given
# destination area
destination_given = destination is not None
if destination is None:
destination = self.slot.stype.allocateDestination(self.roi)
else:
if self.slot.meta.dtype is not None and hasattr(destination, 'dtype'):
assert self.slot.meta.dtype == destination.dtype, \
"Can't provide a destination array of the wrong dtype. "\
"Slot generates {}, but you gave {}".format( self.slot.meta.dtype, destination.dtype )
# We are executing the operator. Incremement the execution
# count to protect against simultaneous setupOutputs()
# calls.
self._incrementOperatorExecutionCount()
try:
# Execute the workload, which might not ever return
# (if we get cancelled).
result_op = self.operator.execute(self.slot, (), self.roi, destination)
# copy data from result_op to destination, if
# destination was actually given by the user, and the
# returned result_op is different from destination.
# (but don't copy if result_op is None, this means
# legacy op which wrote into destination anyway)
if destination_given and result_op is not None and id(result_op) != id(destination):
# check that the returned value is compatible with the requested roi
self.slot.stype.check_result_valid(self.roi, result_op)
self.slot.stype.copy_data(dst=destination, src = result_op)
elif result_op is not None:
# FIXME: this should be moved to a isCompatible
# check in stypes.py
if hasattr(result_op, "shape"):
assert result_op.shape == destination.shape, \
("ERROR: Operator {} has failed to provide a"
" result of correct shape. result shape is"
" {} vs {}. roi was {}".format(
self.operator, result_op.shape,
destination.shape, str(self.roi)))
destination = result_op
# check that the returned value is compatible with the requested roi
self.slot.stype.check_result_valid(self.roi, destination)
# Decrement the execution count
self._decrementOperatorExecutionCount()
return destination
except: # except Request.CancellationException
# Decrement the execution count
self._decrementOperatorExecutionCount()
raise
def _incrementOperatorExecutionCount(self):
self.started = True
assert self.operator._executionCount >= 0, \
"BUG: How did the execution count get negative?"
# We can't execute while the operator is in the middle of
# setupOutputs
with self.operator._condition:
while self.operator._settingUp:
self.operator._condition.wait()
self.operator._executionCount += 1
def handleCancel(self, *args):
# The new request api does clean up by handling an
# exception, not in this callback. Only clean up if we are
# using the old request api
using_old_api = len(args) > 0 and not hasattr(args[0], 'notify_cancelled')
if using_old_api:
self._decrementOperatorExecutionCount()
def _decrementOperatorExecutionCount(self):
# Must lock here because cancel callbacks are
# asynchronous. (Perhaps it would be better if they were
# called from the worker thread instead...)
with self.lock:
# Only do this once per execution. If we were cancelled
# after we finished working, don't do anything
if self.started and not self.finished:
assert self.operator._executionCount > 0, \
"BUG: Can't decrement the execution count below zero!"
self.finished = True
with self.operator._condition:
self.operator._executionCount -= 1
self.operator._condition.notifyAll()
def setDirty(self, *args, **kwargs):
"""This method is called by a partnering OutputSlot when its
content changes.
The 'key' parameter identifies the changed region
of an numpy.ndarray
"""
assert self.operator is not None, ("Slot '{}' cannot be set dirty,"
" slot not belonging to any"
" actual operator instance".format(self.name))
if self.stype.isConfigured():
if len(args) == 0 or not isinstance(args[0], rtype.Roi):
roi = self.rtype(self, *args, **kwargs)
else:
roi = args[0]
for c in self.partners:
c.setDirty(roi)
# call callbacks
self._sig_dirty(self, roi)
if self._type == "input" and self.operator.configured():
self.operator.propagateDirty(self, (), roi)
def __iter__(self):
assert self.level >= 1
return self._subSlots.__iter__()
def __getitem__(self, key):
"""If level=0, emulate __call__ but with a slicing instead of
a roi.
If level>0, return the subslot corresponding to the key, which
may be a tuple
"""
if self.level > 0:
if isinstance(key, tuple):
assert len(key) > 0
assert len(key) <= self.level
if len(key) == 1:
return self._subSlots[key[0]]
else:
return self._subSlots[key[0]][key[1:]]
return self._subSlots[key]
else:
if self.meta.shape is None:
# Something is wrong. Are we cancelled?
Request.raise_if_cancelled()
if not self.ready():
msg = "This slot ({}.{}) isn't ready yet, which means " \
"you can't ask for its data. Is it connected?".format(self.getRealOperator().name, self.name)
self.logger.error(msg)
slotInfoMsg = "Can't get data from slot {}.{} yet."\
" It isn't ready."\
"First upstream problem slot is: {}"\
"".format( self.getRealOperator().__class__, self.name, Slot._findUpstreamProblemSlot(self) )
self.logger.error(slotInfoMsg)
raise Slot.SlotNotReadyError("Slot isn't ready. See error log.")
assert self.meta.shape is not None, \
("Can't ask for slices of this slot yet:"
" self.meta.shape is None!"
" (operator {} [self={}] slot: {}, key={}".format(
self.operator.name, self.operator, self.name, key))
return self(pslice=key)
def __setitem__(self, key, value):
"""This method provied access to the subslots of a
MultiSlot.
"""
assert not isinstance(value, Slot), \
"Can't use setitem to connect slots. Use connect()"
assert self.level == 0, \
("setitem can only be used with slots of level 0."
" Did you forget to append a key?")
assert self.operator is not None, \
"cannot do __setitem__ on Slot '{}' -> no operator !!"
assert slicingtools.is_bounded(key), \
"Can't use Slot.__setitem__ with keys that include : or ..."
roi = self.rtype(self, pslice=key)
if self._value is not None:
self._value[key] = value
# only propagate the dirty key at the very beginning of
# the chain
self.setDirty(roi)
if self._type == "input":
self.operator.setInSlot(self, (), roi, value)
# Forward to partners
for p in self.partners:
p[key] = value
def index(self, slot):
return self._subSlots.index(slot)
def setInSlot(self, slot, subindex, roi, value):
"""For now, Slots of level > 0 pretend to be operators (as far
as their subslots are concerned). That's why they have to have
this setInSlot() method.
"""
# Determine which subslot this is and prepend it to the totalIndex
totalIndex = (self._subSlots.index(slot),) + subindex
# Forward the call to our operator
self.operator.setInSlot(self, totalIndex, roi, value)
def __len__(self):
"""In the case of a MultiSlot this returns the number of
subslots, i.e. the length of the list
"""
return len(self._subSlots)
@property
def value(self):
"""This method directly returns the full content of a slot.
Is mainly used when region of interest specification make no
sense, e.g. in the case of slots which hold a single integer
or float value
"""
if self.partner is not None:
# outputslot-inputsslot, inputslot-inputslot and outputslot-outputslot case
temp = self[:].wait()
elif self._value is None:
# outputslot case
temp = self[:].wait()
else:
# _value case
return self._value
if isinstance(temp, numpy.ndarray) and temp.shape != (1,):
return temp
else:
try:
return temp[0]
except IndexError:
self.logger.warn("FIXME: Slot.value for slot {} is {},"
" which should be wrapped in an ndarray.".format(
self.name, temp))
return temp
def setValue(self, value, notify=True, check_changed=True):
"""This method can be used to directly assign a value to an
InputSlot.
Usually a slot is either connected to another slot from which
it retrieves the content when it is queried, or it directly
holds a value itself. This method can be used to set such a
value.
If check_changed is True, the new value is compared to the
current one and updates are only triggered if the new value differs
from the old one according to the __eq__ operator.
The check can be turned off with the check_changed flag.
"""
try:
assert isinstance(notify, bool)
assert isinstance(check_changed, bool)
# This assertion is here to prevent accidental use of setValue
# when connect should be used. If your use case requires
# passing slots as values, then this assertion can be refined.
assert not isinstance(value, Slot), \
"When using setValue, value cannot be a slot. Use connect instead."
if not self.backpropagate_values:
assert self.partner is None, \
("Cannot call setValue on this slot."
" It is already connected to a partner."
" Call disconnect first if that's what you really wanted.")
elif self.partner is not None:
self.partner.setValue(value, notify, check_changed)
return
changed = True
# We use == here instead of 'is' to avoid subtle bugs that
# can occur if you supplied an equivalent value that 'is not' the original.
# For example: x=numpy.uint8(3); y=numpy.int64(3); assert x == y; assert x is not y
if check_changed:
if isinstance(value, vigra.VigraArray) or isinstance(self._value, vigra.VigraArray):
if type(value) != type(self._value) or value.axistags != self._value.axistags:
changed = True
else:
same = (value is self._value)
if not same:
try:
same = ( value == self._value )
except ValueError:
# Some values can't be compared with __eq__,
# in which case we assume the values are different
same = False
if isinstance(same, (numpy.ndarray, TinyVector)):
same = same.all()
changed = not same
if changed:
# call disconnect callbacks
self._sig_disconnect(self)
self._value = value
self.stype.setupMetaForValue(value)
self.meta._dirty = True
for s in self._subSlots:
s.setValue(self._value)
notify = (self.meta._ready == False)
# a slot with a value is always ready
self.meta._ready = True
if notify:
self._sig_ready(self)
# call connect callbacks
self._sig_connect(self)
self._changed()
# Propagate dirtyness
if self.rtype == rtype.List:
self.setDirty(())
else:
self.setDirty(slice(None))
except:
try:
exc_info = sys.exc_info()
self.disconnect()
except:
# Well, this is bad. We caused an exception while handling an exception.
# We're more interested in the FIRST excpetion, so print this one out and
# continue unwinding the stack with the first one.
self.logger.error("Error: Caught a secondary exception while handling a different exception.")
import traceback
traceback.print_exc()
raise exc_info[0], exc_info[1], exc_info[2]
raise
def setValues(self, values):
"""Set values of subslots with arraylike object. Resizes the
multinputslot with the length of the values array
"""
try:
# call disconnect callbacks
self._sig_disconnect(self)
self.resize(len(values))
for i, s in enumerate(self._subSlots):
s.setValue(values[i])
# call connect callbacks
self._changed()
self._sig_connect(self)
except:
try:
exc_info = sys.exc_info()
self.disconnect()
except:
# Well, this is bad. We caused an exception while handling an exception.
# We're more interested in the FIRST excpetion, so print this one out and
# continue unwinding the stack with the first one.
self.logger.error("Error: Caught a secondary exception while handling a different exception.")
import traceback
traceback.print_exc()
raise exc_info[0], exc_info[1], exc_info[2]
raise
@property
def backpropagate_values(self):
return self._backpropagate_values
@backpropagate_values.setter
def backpropagate_values(self, backprop):
self._backpropagate_values = backprop
for slot in self._subSlots:
slot.backpropagate_values = backprop
def connected(self):
"""Returns True if the slot is conencted to a partner slot or
has a _value assigned as input
"""
answer = True
if self._value is None and self.partner is None:
answer = False
if answer is False and len(self._subSlots) > 0:
answer = True
for s in self._subSlots:
if s.connected() is False:
answer = False
break
return answer
def configured(self):
"""Slots of level >= 1 must implement parts of the operator
interface, including this function. This "operator" is
considered "configured" if it is ready.
"""
return self._optional or self.ready()
def ready(self):
if self.level == 0:
# If this slot is non-multi, then just check our own
# status
ready = self.meta._ready
else:
# If this slot is multi, check all of our subslots. (If we
# have no subslots, then we are NOT ready). Operators that
# can properly handle an empty multi-input slot should
# mark the input as optional.
ready = len(self._subSlots) > 0 and all(p.ready() for p in self._subSlots)
return ready
def _setReady(self):
wasReady = self.ready()
for p in self._subSlots:
p._setReady()
self.meta._ready = (self.level == 0) or (len(self._subSlots) > 0)
# If we just became ready...
if not wasReady and self.meta._ready:
# Notify partners of changed readystatus
self._changed()
self._sig_ready(self)
def __call__(self, *args, **kwargs):
"""The slot relays all arguments to the __init__ method of the
Roi type. This allows lazyflow to support different types of
rois without knowing anything about them.
"""
roi = self.rtype(self, *args, **kwargs)
return self.get(roi)
def getRealOperator(self):
"""If a slot is owned by a higher-level slot, self.operator is
a slot. This function keeps going up the hierarchy until it
finds the actual operator this slot belongs to.
"""
if self._real_operator is not None:
# use memoized
return self._real_operator
if isinstance(self.operator, Slot):
self._real_operator = self.operator.getRealOperator()
else:
self._real_operator = self.operator
return self._real_operator
#####################
# Private Methods #
#####################
def _getInstance(self, operator, level=None):
"""This method constructs a copy of the slot.
This method is used when creating an Instance of an Operator.
All defined Input and Output slots of the Class are cloned and
inserted into the instance of the Operator.
"""
if level is None:
level = self.level
if self._type == "input":
s = InputSlot(self.name, operator, stype=self._stypeType,
rtype=self.rtype, value=self._defaultValue,
optional=self._optional, level=level,
nonlane=self.nonlane)
elif self._type == "output":
s = OutputSlot(self.name, operator, stype=self._stypeType,
rtype=self.rtype, value=self._defaultValue,
level=level,
nonlane=self.nonlane)
return s
def _changed(self):
oldMeta = self.meta
if self.partner is not None and self.meta != self.partner.meta:
self.meta = self.partner.meta.copy()
if self._type == "output":
for o in self._subSlots:
o._changed()
# Notify readiness after subslots are updated
if self.meta._ready != oldMeta._ready:
if self.meta._ready:
self._sig_ready(self)
else:
self._sig_unready(self)
wasdirty = self.meta._dirty
if self.meta._dirty:
for c in self.partners:
c._changed()
self.meta._dirty = False
if self._type != "output":
op = self.getRealOperator()
if op is not None and not op._cleaningUp:
self._configureOperator(self)
if wasdirty:
# call changed callbacks
self._sig_changed(self)
def _configureOperator(self, slot, oldSize=0, newSize=0, notify=True):
"""Call setupOutputs of Operator if all slots of the operator
are connected and configured.
"""
if self.operator is not None:
# check whether all slots are connected and notify operator
if self.operator.configured():
self.operator._setupOutputs()
def _requiredLength(self):
"""
Returns the required number of subslots
"""
if self.partner is not None:
if self.partner.level == self.level:
return len(self.partner)
elif self.partner.level < self.level:
return 1
elif self._value is not None:
return 1
else:
return 0
def _setupOutputs(self):
"""
"""
self._changed()
def _connectSubSlot(self, slot, notify=True):
"""Connect a subslot either to the partner, or set the correct
value in case of self._value != None
"""
if type(slot) == int:
index = slot
slot = self._subSlots[slot]
else:
index = self._subSlots.index(slot)
if self.partner is not None:
if self.partner.level == self.level:
if len(self.partner) > index:
slot.connect(self.partner[index])
else:
slot.connect(self.partner)
if self._value is not None:
slot.setValue(self._value, notify=notify)
def _insertNew(self, position):
"""Construct a new subSlot of correct type and level and
insert it to the list of subslots
"""
assert position >= 0 and position <= len(self._subSlots)
slot = self._getInstance(self, level=self.level - 1)
self._subSlots.insert(position, slot)
slot.name = self.name
if self._value is not None:
slot.setValue(self._value)
return slot
def pop(self, index=-1, event=None):
if index < 0:
index = len(self) + index
self._subSlots.pop(index)
def propagateDirty(self, slot, subindex, roi):
"""Slots with level > 0 must implement part of the operator
interface so they look like an operator as far as their
subslots are concerned. That's why this function is here.
"""
totalIndex = (self._subSlots.index(slot),) + subindex
self.operator.propagateDirty(self, totalIndex, roi)
######################################
# methods aimed to enhance usability #
######################################
def setShapeAtAxisTo(self, axis, size):
tmpshape = list(self.meta.shape)
tmpshape[self.meta.axistags.index(axis)] = size
self.meta.shape = tuple(tmpshape)
def __str__(self):
mslot_info = ""
if self.level > 0 or isinstance(self.operator, Slot):
mslot_info += "["
if isinstance(self.operator, Slot):
mslot_info += " index={}".format( self.operator.index(self) )
if self.level > 0:
mslot_info += " len={}".format( len(self) )
if self.level > 1:
mslot_info += " level={}".format( self.level )
mslot_info += " ] "
return '"{}" {}: \t{}\n'.format( self.name, mslot_info, self.meta )
def __repr__(self):
return self.__str__()
def __init__(self, *args, **kwargs):
super(OpExportMultipageTiff, self).__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
def __init__(self, name="", operator=None, stype=ArrayLike,
rtype=rtype.SubRegion, value=None, optional=False,
level=0, nonlane=False):
"""Constructor of the Slot class.
:param name: user readable name of the slot, is normally
assigned automatically by the Operator
:param operator: the parent operator of a slot
:param stype: the slot type (see stype.py)
:param rtype: the region of interest type (see rtype.py)
:param value: the default value of the slot
:param optional: if True this means the slot needs a value or
connection for its parent operator to be functional
:param level: defines the dimensionality of the slot. 0 for
single element (e.g. single numpy.ndarray), 1 for list of
elements (e.g. list of strings), 2 for list of list of
elements.
:param nonlane: For multislot, this flag protects it from
being considered lane-indexed
"""
# This assertion is here for a reason: default values do NOT work on OutputSlots.
# (We should probably change that at some point...)
assert value is None or isinstance(self, InputSlot), "Only InputSlots can have default values. OutputSlots cannot."
if not hasattr(self, "_type"):
self._type = None
if type(stype) == str:
stype = ArrayLike
self.partners = []
self.name = name
self._optional = optional
self.operator = operator
self._real_operator = None # Memoized in getRealOperator()
# in the case of an InputSlot this is the slot to which it is
# connected
self.partner = None
self.level = level
# in the case of an InputSlot one can directly assign a value
# to a slot instead of connecting it to a partner, this
# attribute holds the value
self._value = None
self._defaultValue = value
# Causes calls to setValue to be propagated backwards to the
# partner slot. Used by the OperatorWrapper.
self._backpropagate_values = False
self.rtype = rtype
# the MetaDict that holds the slots meta information
self.meta = MetaDict()
# if level > 0, this holds the sub-Input/Output slots
self._subSlots = []
self._stypeType = stype
# the slot type instance
self.stype = stype(self)
self.nonlane = nonlane
self._sig_changed = OrderedSignal()
self._sig_value_changed = OrderedSignal()
self._sig_ready = OrderedSignal()
self._sig_unready = OrderedSignal()
self._sig_dirty = OrderedSignal()
self._sig_connect = OrderedSignal()
self._sig_disconnect = OrderedSignal()
self._sig_resize = OrderedSignal()
self._sig_resized = OrderedSignal()
self._sig_remove = OrderedSignal()
self._sig_removed = OrderedSignal()
self._sig_preinsertion = OrderedSignal()
self._sig_inserted = OrderedSignal()
self._resizing = False
self._executionCount = 0
self._settingUp = False
self._condition = threading.Condition()
# Allow slots to be sorted by their order of creation for
# debug output and diagramming purposes.
self._global_slot_id = Slot._global_counter.next()
def __init__(self, *args, **kwargs):
super(OpWsdt, self).__init__(*args, **kwargs)
self.debug_results = None
self.watershed_completed = OrderedSignal()
def __init__(self, *args, **kwargs):
super(OpResize5D, self).__init__(*args, **kwargs)
self._input_to_output_scales = None
self.progressSignal = OrderedSignal()
class ArrayCacheMemoryMgr(threading.Thread):
totalCacheMemory = OrderedSignal()
loggingName = __name__ + ".ArrayCacheMemoryMgr"
logger = logging.getLogger(loggingName)
traceLogger = logging.getLogger("TRACE." + loggingName)
def __init__(self):
threading.Thread.__init__(self)
self.daemon = True
self.caches = self._new_list()
self.namedCaches = []
self._max_usage = 85
self._target_usage = 70
self._lock = threading.Lock()
self._last_usage = memoryUsagePercentage()
def _new_list(self):
def getPrio(array_cache):
return array_cache._cache_priority
return blist.sortedlist((), getPrio)
def addNamedCache(self, array_cache):
"""add a cache to a special list of named caches
The list of named caches should contain only top-level caches.
This way, when showing memory usage, we can provide a tree-view, where the
named caches are the top-level items and the user can then drill down into the caches
that are children of the top-level caches.
"""
self.namedCaches.append(array_cache)
def add(self, array_cache):
with self._lock:
self.caches.add(array_cache)
def remove(self, array_cache):
with self._lock:
try:
self.caches.remove(array_cache)
except ValueError:
pass
def run(self):
while True:
mem_usage = memoryUsagePercentage()
mem_usage_gb = memoryUsageGB()
delta = abs(self._last_usage - mem_usage)
if delta > 10 or self.logger.level == logging.DEBUG:
cpu_usages = psutil.cpu_percent(interval=1, percpu=True)
avg = sum(cpu_usages) / len(cpu_usages)
self.logger.info(
"RAM: {:1.3f} GB ({:02.0f}%), CPU: Avg={:02.0f}%, {}".
format(mem_usage_gb, mem_usage, avg, cpu_usages))
if delta > 10:
self._last_usage = mem_usage
#calculate total memory usage and send as signal
tot = 0.0
for c in self.namedCaches:
if c.usedMemory() is None:
continue
else:
tot += c.usedMemory()
self.totalCacheMemory(tot)
time.sleep(10)
if mem_usage > self._max_usage:
self.logger.info("freeing memory...")
with self._lock:
count = 0
not_freed = []
old_length = len(self.caches)
new_caches = self._new_list()
self.traceLogger.debug("Updating {} caches".format(
len(self.caches)))
for c in iter(self.caches):
c._updatePriority(c._last_access)
new_caches.add(c)
self.caches = new_caches
gc.collect()
self.traceLogger.debug("Target mem usage: {}".format(
self._target_usage))
while mem_usage > self._target_usage and len(
self.caches) > 0:
self.traceLogger.debug(
"Mem usage: {}".format(mem_usage))
last_cache = self.caches.pop(-1)
freed = last_cache._freeMemory(refcheck=True)
self.traceLogger.debug("Freed: {}".format(freed))
mem_usage = memoryUsagePercentage()
count += 1
if freed == 0:
# store the caches which could not be freed
not_freed.append(last_cache)
gc.collect()
self.logger.info("freed %d/%d blocks, new usage = %f%%" %
(count, old_length, mem_usage))
for c in not_freed:
# add the caches which could not be freed
self.add(c)
def __init__(self, *args, **kwargs):
super(OpExportMultipageTiff, self).__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
self._opReorderAxes = OpReorderAxes(parent=self)
self._opReorderAxes.Input.connect(self.Input)
class CacheMemoryManager(threading.Thread):
"""
class for the management of cache memory
The cache memory manager is a background thread that observes caches
in use and cleans them up when the total memory consumption by the
process exceeds the limit defined by the `lazyflow.utility.Memory`
class. See the definition of the cache interfaces (opCache.py) to
get an overview over the possible caches.
Usage:
This manager is a singleton - just call its constructor somewhere
and you will get a reference to the *only* running memory management
thread.
Interface:
The manager provides a signal you can subscribe to
>>> def writeFunction(x): print("total mem: {}".format(x))
>>> mgr = CacheMemoryManager()
>>> mgr.totalCacheMemory.subscribe(writeFunction)
which emits the size of all observable caches, combined, in regular
intervals.
The update interval (for the signal and for automated cache release)
can be set with a call to a class method
>>> CacheMemoryManager().setRefreshInterval(5)
the interval is measured in seconds. Each change of refresh interval
triggers cleanup.
"""
__metaclass__ = Singleton
totalCacheMemory = OrderedSignal()
def __init__(self):
threading.Thread.__init__(self)
self.daemon = True
self._caches = weakref.WeakSet()
self._first_class_caches = weakref.WeakSet()
self._observable_caches = weakref.WeakSet()
self._managed_caches = weakref.WeakSet()
self._managed_blocked_caches = weakref.WeakSet()
self._condition = threading.Condition()
self._disable_lock = threading.Condition()
self._disabled = False
self._refresh_interval = default_refresh_interval
self._first_class_caches_lock = threading.Lock()
# maximum fraction of *allowed memory* used
self._max_usage = 1.0
# target usage fraction
self._target_usage = .90
self._stopped = False
self.start()
atexit.register(self.stop)
def addFirstClassCache(self, cache):
"""
add a first class cache (root cache) to the manager
First class caches are handled differently so we are able to
show a tree view of the caches (e.g. in ilastik). This method
calls addCache() automatically.
"""
# late import to prevent import loop
from lazyflow.operators.opCache import Cache
if isinstance(cache, Cache):
with self._first_class_caches_lock:
self._first_class_caches.add(cache)
self.addCache(cache)
def getFirstClassCaches(self):
"""
get a list of first class caches
"""
with self._first_class_caches_lock:
return list(self._first_class_caches)
def getCaches(self):
"""
get a list of all caches (including first class caches)
"""
return list(self._caches)
def addCache(self, cache):
"""
add a cache to be managed
Caches are kept with weak references, so there is no need to
remove them from the manager.
"""
# late import to prevent import loop
from lazyflow.operators.opCache import Cache
from lazyflow.operators.opCache import ObservableCache
from lazyflow.operators.opCache import ManagedCache
from lazyflow.operators.opCache import ManagedBlockedCache
assert isinstance(cache, Cache),\
"Only Cache instances can be managed by CacheMemoryManager"
self._caches.add(cache)
if isinstance(cache, ObservableCache):
self._observable_caches.add(cache)
if isinstance(cache, ManagedBlockedCache):
self._managed_blocked_caches.add(cache)
elif isinstance(cache, ManagedCache):
self._managed_caches.add(cache)
def run(self):
"""
main loop
"""
while not self._stopped:
self._wait()
# acquire lock so that we don't get disabled during cleanup
with self._disable_lock:
if self._disabled or self._stopped:
continue
self._cleanup()
def _cleanup(self):
"""
clean up once
"""
from lazyflow.operators.opCache import ObservableCache
try:
# notify subscribed functions about current cache memory
total = 0
# Avoid "RuntimeError: Set changed size during iteration"
with self._first_class_caches_lock:
first_class_caches = self._first_class_caches.copy()
for cache in first_class_caches:
if isinstance(cache, ObservableCache):
total += cache.usedMemory()
self.totalCacheMemory(total)
cache = None
# check current memory state
cache_memory = Memory.getAvailableRamCaches()
if total <= self._max_usage * cache_memory:
return
# === we need a cache cleanup ===
# queue holds time stamps and cleanup functions
q = PriorityQueue()
caches = list(self._managed_caches)
for c in caches:
q.push((c.lastAccessTime(), c.name, c.freeMemory))
caches = list(self._managed_blocked_caches)
for c in caches:
for k, t in c.getBlockAccessTimes():
cleanupFun = functools.partial(c.freeBlock, k)
info = "{}: {}".format(c.name, k)
q.push((t, info, cleanupFun))
c = None
caches = None
msg = "Caches are using {} memory".format(
Memory.format(total))
if cache_memory > 0:
msg += " ({:.1f}% of allowed)".format(
total*100.0/cache_memory)
logger.debug(msg)
while (total > self._target_usage * cache_memory
and len(q) > 0):
t, info, cleanupFun = q.pop()
mem = cleanupFun()
logger.debug("Cleaned up {} ({})".format(
info, Memory.format(mem)))
total -= mem
gc.collect()
# don't keep a reference until next loop iteration
cleanupFun = None
q = None
msg = ("Done cleaning up, cache memory usage is now at "
"{}".format(Memory.format(total)))
if cache_memory > 0:
msg += " ({:.1f}% of allowed)".format(
total*100.0/cache_memory)
logger.debug(msg)
except:
log_exception(logger)
def _wait(self):
"""
sleep for _refresh_interval seconds or until woken up
"""
with self._condition:
self._condition.wait(self._refresh_interval)
def stop(self):
"""
Stop the memory manager thread in preparation for app exit.
"""
self._stopped = True
with self._condition:
self._condition.notify()
self.join()
def setRefreshInterval(self, t):
"""
set the clean up period and wake up the cleaning thread
"""
with self._condition:
self._refresh_interval = t
self._condition.notifyAll()
def disable(self):
"""
disable all memory management
This method blocks until current memory management tasks are finished.
"""
with self._disable_lock:
self._disabled = True
def enable(self):
"""
enable cache management and wake the thread
"""
with self._disable_lock:
self._disabled = False
with self._condition:
self._condition.notifyAll()
class CacheMemoryManager(with_metaclass(Singleton, threading.Thread)):
"""
class for the management of cache memory
The cache memory manager is a background thread that observes caches
in use and cleans them up when the total memory consumption by the
process exceeds the limit defined by the `lazyflow.utility.Memory`
class. See the definition of the cache interfaces (opCache.py) to
get an overview over the possible caches.
Usage:
This manager is a singleton - just call its constructor somewhere
and you will get a reference to the *only* running memory management
thread.
Interface:
The manager provides a signal you can subscribe to::
def writeFunction(x): print("total mem: {}".format(x))
mgr = CacheMemoryManager()
mgr.totalCacheMemory.subscribe(writeFunction)
which emits the size of all observable caches, combined, in regular
intervals.
The update interval (for the signal and for automated cache release)
can be set with a call to a class method::
CacheMemoryManager().setRefreshInterval(5)
the interval is measured in seconds. Each change of refresh interval
triggers cleanup.
"""
totalCacheMemory = OrderedSignal()
def __init__(self):
threading.Thread.__init__(self)
self.daemon = True
self._caches = weakref.WeakSet()
self._first_class_caches = weakref.WeakSet()
self._observable_caches = weakref.WeakSet()
self._managed_caches = weakref.WeakSet()
self._managed_blocked_caches = weakref.WeakSet()
self._condition = threading.Condition()
self._disable_lock = threading.Condition()
self._disabled = False
self._refresh_interval = default_refresh_interval
self._first_class_caches_lock = threading.Lock()
# maximum fraction of *allowed memory* used
self._max_usage = 1.0
# target usage fraction
self._target_usage = 0.90
self._stopped = False
self.start()
atexit.register(self.stop)
def addFirstClassCache(self, cache):
"""
add a first class cache (root cache) to the manager
First class caches are handled differently so we are able to
show a tree view of the caches (e.g. in ilastik). This method
calls addCache() automatically.
"""
# late import to prevent import loop
from lazyflow.operators.opCache import Cache
if isinstance(cache, Cache):
with self._first_class_caches_lock:
self._first_class_caches.add(cache)
self.addCache(cache)
def getFirstClassCaches(self):
"""
get a list of first class caches
"""
with self._first_class_caches_lock:
return list(self._first_class_caches)
def getCaches(self):
"""
get a list of all caches (including first class caches)
"""
return list(self._caches)
def addCache(self, cache):
"""
add a cache to be managed
Caches are kept with weak references, so there is no need to
remove them from the manager.
"""
# late import to prevent import loop
from lazyflow.operators.opCache import Cache
from lazyflow.operators.opCache import ObservableCache
from lazyflow.operators.opCache import ManagedCache
from lazyflow.operators.opCache import ManagedBlockedCache
assert isinstance(cache, Cache), "Only Cache instances can be managed by CacheMemoryManager"
self._caches.add(cache)
if isinstance(cache, ObservableCache):
self._observable_caches.add(cache)
if isinstance(cache, ManagedBlockedCache):
self._managed_blocked_caches.add(cache)
elif isinstance(cache, ManagedCache):
self._managed_caches.add(cache)
def run(self):
"""
main loop
"""
while not self._stopped:
self._wait()
# acquire lock so that we don't get disabled during cleanup
with self._disable_lock:
if self._disabled or self._stopped:
continue
self._cleanup()
def _cleanup(self):
"""
clean up once
"""
from lazyflow.operators.opCache import ObservableCache
try:
# notify subscribed functions about current cache memory
total = 0
# Avoid "RuntimeError: Set changed size during iteration"
with self._first_class_caches_lock:
first_class_caches = self._first_class_caches.copy()
for cache in first_class_caches:
if isinstance(cache, ObservableCache):
total += cache.usedMemory()
self.totalCacheMemory(total)
cache = None
# check current memory state
cache_memory = Memory.getAvailableRamCaches()
cache_pct = 0.0
if cache_memory:
cache_pct = total * 100.0 / cache_memory
logger.debug(
"Process memory usage is {:0.2f} GB out of {:0.2f} (caches are {}, {:.1f}% of allowed)".format(
Memory.getMemoryUsage() / 2.0 ** 30,
Memory.getAvailableRam() / 2.0 ** 30,
Memory.format(total),
cache_pct,
)
)
if total <= self._max_usage * cache_memory:
return
cache_entries = []
cache_entries += [
(cache.lastAccessTime(), cache.name, cache.freeMemory) for cache in list(self._managed_caches)
]
cache_entries += [
(lastAccessTime, f"{cache.name}: {blockKey}", functools.partial(cache.freeBlock, blockKey))
for cache in list(self._managed_blocked_caches)
for blockKey, lastAccessTime in cache.getBlockAccessTimes()
]
cache_entries.sort(key=lambda entry: entry[0])
for lastAccessTime, info, cleanupFun in cache_entries:
if total <= self._target_usage * cache_memory:
break
mem = cleanupFun()
logger.debug(f"Cleaned up {info} ({Memory.format(mem)})")
total -= mem
# Remove references to cache entries before triggering garbage collection.
cleanupFun = None
cache_entries = None
gc.collect()
msg = "Done cleaning up, cache memory usage is now at {}".format(Memory.format(total))
if cache_memory > 0:
msg += " ({:.1f}% of allowed)".format(total * 100.0 / cache_memory)
logger.debug(msg)
except:
log_exception(logger)
def _wait(self):
"""
sleep for _refresh_interval seconds or until woken up
"""
with self._condition:
self._condition.wait(self._refresh_interval)
def stop(self):
"""
Stop the memory manager thread in preparation for app exit.
"""
self._stopped = True
with self._condition:
self._condition.notify()
self.join()
def setRefreshInterval(self, t):
"""
set the clean up period and wake up the cleaning thread
"""
with self._condition:
self._refresh_interval = t
self._condition.notifyAll()
def disable(self):
"""
disable all memory management
This method blocks until current memory management tasks are finished.
"""
with self._disable_lock:
self._disabled = True
def enable(self):
"""
enable cache management and wake the thread
"""
with self._disable_lock:
self._disabled = False
with self._condition:
self._condition.notifyAll()
def __init__(self, *args, **kwargs):
super(OpConcatenateFeatureMatrices, self).__init__(*args, **kwargs)
self._dirty_slots = set()
self.progressSignal = OrderedSignal()
self._num_feature_channels = 0 # Not including the labels...
def __init__(self, *args, **kwargs):
super(OpExportToArray, self).__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
def __init__(self, *args, **kwargs):
super(OpConcatenateFeatureMatrices, self).__init__(*args, **kwargs)
self._dirty_slots = set()
self.progressSignal = OrderedSignal()
