def __init__(self, outputSlot, roiIterator, totalVolume=None, batchSize=2, allowParallelResults=False): """ Constructor. :param outputSlot: The slot to request data from. :param roiIterator: An iterator providing new rois. :param totalVolume: The total volume to be processed. Used to provide the progress reporting signal. If not provided, then no intermediate progress will be signaled. :param batchSize: The maximum number of requests to launch in parallel. :param allowParallelResults: If False, The resultSignal will not be called in parallel. In that case, your handler function has no need for locks. """ self._resultSignal = OrderedSignal() self._progressSignal = OrderedSignal() assert isinstance( outputSlot.stype, lazyflow.stype.ArrayLike ), "Only Array-like slots supported." # Because progress reporting depends on the roi shape self._outputSlot = outputSlot self._roiIter = roiIterator self._batchSize = batchSize self._allowParallelResults = allowParallelResults self._condition = SimpleRequestCondition() self._activated_count = 0 self._completed_count = 0 self._failure_excinfo = None # Progress bookkeeping self._totalVolume = totalVolume self._processedVolume = 0
def __init__(self, *args, **kwargs): super(OpConcatenateFeatureMatrices, self).__init__(*args, **kwargs) self._dirty_slots = set() self.progressSignal = OrderedSignal() self._num_feature_channels = 0 # Not including the labels... self._channel_names = [] # Normally, lane removal does not trigger a dirty notification. # But in this case, if the lane contained any label data whatsoever, # the classifier needs to be marked dirty. # We know which slots contain (or contained) label data because they have # been 'touched' at some point (they became dirty at some point). self._touched_slots = set() def handle_new_lane( multislot, index, newlength ): def handle_dirty_lane( slot, roi ): self._touched_slots.add(slot) multislot[index].notifyDirty( handle_dirty_lane ) self.FeatureMatrices.notifyInserted( handle_new_lane ) def handle_remove_lane( multislot, index, newlength ): # If the lane we're removing contained # label data, then mark the downstream dirty if multislot[index] in self._touched_slots: self.ConcatenatedOutput.setDirty() self._touched_slots.remove(multislot[index]) self.FeatureMatrices.notifyRemove( handle_remove_lane )
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.progressSignal = OrderedSignal() # Set up the impl function lookup dict export_impls = {} export_impls["hdf5"] = ("h5", self._export_h5n5) export_impls["compressed hdf5"] = ("h5", partial(self._export_h5n5, True)) export_impls["n5"] = ("n5", self._export_h5n5) export_impls["compressed n5"] = ("n5", partial(self._export_h5n5, True)) export_impls["numpy"] = ("npy", self._export_npy) export_impls["dvid"] = ("", self._export_dvid) export_impls["blockwise hdf5"] = ("json", self._export_blockwise_hdf5) for fmt in self._2d_formats: export_impls[fmt.name] = (fmt.extension, partial(self._export_2d, fmt.extension)) for fmt in self._3d_sequence_formats: export_impls[fmt.name] = (fmt.extension, partial(self._export_3d_sequence, fmt.extension)) export_impls["multipage tiff"] = ("tiff", self._export_multipage_tiff) export_impls["multipage tiff sequence"] = ( "tiff", self._export_multipage_tiff_sequence) self._export_impls = export_impls self.Input.notifyMetaChanged(self._updateFormatSelectionErrorMsg)
def __init__(self, *args, **kwargs): super(OpExportSlot, self).__init__(*args, **kwargs) self.progressSignal = OrderedSignal() # Set up the impl function lookup dict export_impls = {} export_impls['hdf5'] = ('h5', self._export_hdf5) export_impls['npy'] = ('npy', self._export_npy) export_impls['dvid'] = ('', self._export_dvid) export_impls['blockwise hdf5'] = ('json', self._export_blockwise_hdf5) for fmt in self._2d_formats: export_impls[fmt.name] = (fmt.extension, partial(self._export_2d, fmt.extension)) for fmt in self._3d_sequence_formats: export_impls[fmt.name] = (fmt.extension, partial(self._export_3d_sequence, fmt.extension)) export_impls['multipage tiff'] = ('tiff', self._export_multipage_tiff) export_impls['multipage tiff sequence'] = ( 'tiff', self._export_multipage_tiff_sequence) self._export_impls = export_impls self.Input.notifyMetaChanged(self._updateFormatSelectionErrorMsg)
def __init__(self, *args, **kwargs): super(OpWsdt, self).__init__(*args, **kwargs) self.debug_results = None self.watershed_completed = OrderedSignal() self._opSelectedInput = OpSumChannels(parent=self) self._opSelectedInput.ChannelSelections.connect(self.ChannelSelections) self._opSelectedInput.Input.connect(self.Input)
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.progressSignal = OrderedSignal() self._opReorderAxes = OpReorderAxes(parent=self) self._opReorderAxes.Input.connect(self.Input) self._opReorderAxes.AxisOrder.setValue(self._EXPORT_AXES)
def __init__(self, *args, **kwargs): super(OpFeatureMatrixCache, self).__init__(*args, **kwargs) self._lock = RequestLock() self.progressSignal = OrderedSignal() self._progress_lock = RequestLock() self._blockshape = None self._dirty_blocks = set() self._blockwise_feature_matrices = {} self._block_locks = {} # One lock per stored block self._init_blocks(None, None)
def __init__(self, *args, **kwargs): super(OpFeatureMatrixCache, self).__init__(*args, **kwargs) self._blockshape = None self._lock = RequestLock() self.progressSignal = OrderedSignal() self._progress_lock = RequestLock() # In these set/dict members, the block id (dict key) # is simply the block's start coordinate (as a tuple) self._blockwise_feature_matrices = {} self._dirty_blocks = set() self._block_locks = {} # One lock per stored block
class ExportFile(object): ExportProgress = OrderedSignal() InsertionProgress = OrderedSignal() def __init__(self, file_name): self.file_name = file_name self.table_dict = {} self.meta_dict = {} def add_columns(self, table_name, col_data, mode, extra=None): """ Adds new columns to the table ( creates the table if neccessary ) :param table_name: the table name :type table_name: str :param col_data: the actual data to be added :type col_data: list, dict, numpy.array, whatever is supported :param mode: the type of the table data :type mode: exportFile.Mode :param extra: extra information for the given mode :type extra: dict """ if extra is None: extra = {} if mode == Mode.IlastikTrackingTable: if not "counts" in extra or not "max" in extra: raise AttributeError("Tracking need 'counts', 'max' extra") columns = flatten_tracking_table(col_data, extra["extra ids"], extra["counts"], extra["max"], extra["range"]) elif mode == Mode.List: if not "names" in extra: raise AttributeError( "[Tuple]List needs a tuple for the column name (extra 'names')" ) dtypes = extra["dtypes"] if "dtypes" in extra else None columns = prepare_list(col_data, extra["names"], dtypes) elif mode == Mode.IlastikFeatureTable: if "selection" not in extra: raise AttributeError( "IlastikFeatureTable needs a feature selection (extra 'selection')" ) columns = flatten_ilastik_feature_table(col_data, extra["selection"], self.InsertionProgress) elif mode == Mode.NumpyStructArray: columns = col_data else: raise AttributeError("Invalid Mode") self._add_columns(table_name, columns) def add_rois(self, table_path, image_slot, feature_table_name, margin, type_="image"): """ Adds the rois as images to the table :param table_path: the new name for the table :type table_path: str :param image_slot: the slot to read the data from :type image_slot: lazyflow.slot.Slot :param feature_table_name: the already added feature table to read the coords from :type feature_table_name: str :param margin: the margin to be added around the images :type margin: int :param type_: "image" for normal images, "labeling" for labeling images :type type_: str """ assert type_ in ("labeling", "image"), "Type must be 'labeling' or 'image'" slicings = create_slicing(image_slot.meta.axistags, image_slot.meta.shape, margin, self.table_dict[feature_table_name]) self.InsertionProgress(0) if type_ == "labeling": vec = self._normalize else: vec = lambda _: lambda y: y for i, (slicing, oid) in enumerate(slicings): roi = image_slot(slicing).wait() roi = vec(oid)(roi) roi_path = table_path.format(i) self.meta_dict[roi_path] = { "type": type_, "axistags": actual_axistags(image_slot.meta.axistags, roi.shape).toJSON() } self.table_dict[roi_path] = roi.squeeze() self.InsertionProgress( 100 * i / self.table_dict[feature_table_name].shape[0]) self.InsertionProgress(100) @staticmethod def _normalize(oid): def f(pixel_value): return 1 if pixel_value == oid else 0 return np.vectorize(f) def add_image(self, table, image_slot): """ Adds an image as a table :param table: the name for the image :type table: str :param image_slot: the slot to read the image from :type image_slot: lazyflow.slot.Slot """ self.table_dict[table] = image_slot([]).wait().squeeze() self.meta_dict[table] = { "type": "image", "axistags": actual_axistags(image_slot.meta.axistags, image_slot.meta.shape).toJSON() } def update_meta(self, table, meta): """ Adds meta information to the table :param table: the table to add meta to :type table: str :param meta: the meta information to add :type meta: dict """ self.meta_dict.setdefault(table, {}) self.meta_dict[table].update(meta) def write_all(self, mode, compression=None): """ Writes all tables to the file :param mode: "h[d[f]]5" or "csv" at the moment :type mode: str :param compression: the compression settings :type compression: dict """ count = 0 self.ExportProgress(0) if mode in ("h5", "hd5", "hdf5"): with h5py.File(self.file_name, "w") as fout: for table_name, table in self.table_dict.iteritems(): self._make_h5_dataset( fout, table_name, table, self.meta_dict.get(table_name, {}), compression if compression is not None else {}) count += 1 self.ExportProgress(count * 100 / len(self.table_dict)) elif mode == "csv": f_name = self.file_name.rsplit(".", 1) if len(f_name) == 1: base, ext = f_name, "" else: base, ext = f_name file_names = [] for table_name, table in self.table_dict.iteritems(): file_names.append("{name}_{table}.{ext}".format( name=base, table=table_name, ext=ext)) with open(file_names[-1], "w") as fout: self._make_csv_table(fout, table) count += 1 self.ExportProgress(count * 100 / len(self.table_dict)) if False: with ZipFile("{name}.zip".format(name=base), "w") as zip_file: for file_name in file_names: zip_file.write(file_name) self.ExportProgress(100) logger.info("exported %i tables" % count) def _add_columns(self, table_name, columns): if table_name in self.table_dict.iterkeys(): old = self.table_dict[table_name] columns = nlr.merge_arrays((old, columns), flatten=True) self.table_dict[table_name] = columns @staticmethod def _make_h5_dataset(fout, table_name, table, meta, compression): try: dset = fout.create_dataset(table_name, table.shape, data=table, **compression) except TypeError: dset = fout.create_dataset(table_name, table.shape, data=table) for k, v in meta.iteritems(): dset.attrs[k] = v @staticmethod def _make_csv_table(fout, table): line = ",".join(table.dtype.names) fout.write(line) fout.write("\n") for row in table: line = ",".join(map(str, row)) fout.write(line) fout.write("\n")
def __init__(self, *args, **kwargs): super(OpNpyWriter, self).__init__(*args, **kwargs) self.progressSignal = OrderedSignal()
def __enter__(self): if self._graph: with self._graph._lock: if self._graph._setup_depth == 0: self._graph._sig_setup_complete = OrderedSignal() self._graph._setup_depth += 1
def __init__(self, transpose_axes, *args, **kwargs): super(OpExportDvidVolume, self).__init__(*args, **kwargs) self.progressSignal = OrderedSignal() self._transpose_axes = transpose_axes
class Slot(object): """ Base class for InputSlot, OutputSlot """ loggerName = __name__ + '.Slot' logger = logging.getLogger(loggerName) traceLogger = logging.getLogger('TRACE.' + loggerName) # Allow slots to be sorted by their order of creation for debug # output and diagramming purposes. _global_counter = itertools.count() class SlotNotReadyError(Exception): pass @property def graph(self): return self.operator.graph def __init__(self, name="", operator=None, stype=ArrayLike, rtype=rtype.SubRegion, value=None, optional=False, level=0, nonlane=False): """Constructor of the Slot class. :param name: user readable name of the slot, is normally assigned automatically by the Operator :param operator: the parent operator of a slot :param stype: the slot type (see stype.py) :param rtype: the region of interest type (see rtype.py) :param value: the default value of the slot :param optional: if True this means the slot needs a value or connection for its parent operator to be functional :param level: defines the dimensionality of the slot. 0 for single element (e.g. single numpy.ndarray), 1 for list of elements (e.g. list of strings), 2 for list of list of elements. :param nonlane: For multislot, this flag protects it from being considered lane-indexed """ # This assertion is here for a reason: default values do NOT work on OutputSlots. # (We should probably change that at some point...) assert value is None or isinstance(self, InputSlot), "Only InputSlots can have default values. OutputSlots cannot." if not hasattr(self, "_type"): self._type = None if type(stype) == str: stype = ArrayLike self.partners = [] self.name = name self._optional = optional self.operator = operator self._real_operator = None # Memoized in getRealOperator() # in the case of an InputSlot this is the slot to which it is # connected self.partner = None self.level = level # in the case of an InputSlot one can directly assign a value # to a slot instead of connecting it to a partner, this # attribute holds the value self._value = None self._defaultValue = value # Causes calls to setValue to be propagated backwards to the # partner slot. Used by the OperatorWrapper. self._backpropagate_values = False self.rtype = rtype # the MetaDict that holds the slots meta information self.meta = MetaDict() # if level > 0, this holds the sub-Input/Output slots self._subSlots = [] self._stypeType = stype # the slot type instance self.stype = stype(self) self.nonlane = nonlane self._sig_changed = OrderedSignal() self._sig_value_changed = OrderedSignal() self._sig_ready = OrderedSignal() self._sig_unready = OrderedSignal() self._sig_dirty = OrderedSignal() self._sig_connect = OrderedSignal() self._sig_disconnect = OrderedSignal() self._sig_resize = OrderedSignal() self._sig_resized = OrderedSignal() self._sig_remove = OrderedSignal() self._sig_removed = OrderedSignal() self._sig_preinsertion = OrderedSignal() self._sig_inserted = OrderedSignal() self._resizing = False self._executionCount = 0 self._settingUp = False self._condition = threading.Condition() # Allow slots to be sorted by their order of creation for # debug output and diagramming purposes. self._global_slot_id = Slot._global_counter.next() ########################### # A p i M e t h o d s # ########################### def notifyDirty(self, function, **kwargs): """ calls the corresponding function when the slot gets dirty first argument of the function is the slot, second argument the roi the keyword arguments follow """ self._sig_dirty.subscribe(function, **kwargs) def notifyMetaChanged(self, function, **kwargs): """calls the corresponding function when the slot meta information is changed first argument of the function is the slot the keyword arguments follow """ self._sig_changed.subscribe(function, **kwargs) def notifyValueChanged(self, function, **kwargs): """Used by slots with cached values to notify when the cache has changed, even if the output is not dirty. """ self._sig_value_changed.subscribe(function, **kwargs) def notifyReady(self, function, **kwargs): """Calls the corresponding function when the slot is "ready", meaning it is connected and will produce data when called. This is implemented by manipulating and monitoring a flag in the slot metadata. first argument of the function is the slot the keyword arguments follow """ self._sig_ready.subscribe(function, **kwargs) def notifyUnready(self, function, **kwargs): """ Subscribe to "unready" callbacks. See notifyReady for details. """ self._sig_unready.subscribe(function, **kwargs) def _notifyConnect(self, function, **kwargs): """ calls the corresponding function when the slot is connected first argument of the function is the slot the keyword arguments follow """ self._sig_connect.subscribe(function, **kwargs) def notifyDisconnect(self, function, **kwargs): """ calls the corresponding function when the slot is disconnected first argument of the function is the slot the keyword arguments follow """ self._sig_disconnect.subscribe(function, **kwargs) def notifyResize(self, function, **kwargs): """ calls the corresponding function before the slot is resized first argument of the function is the slot second argument is the old size and the third argument is the new size the keyword arguments follow """ self._sig_resize.subscribe(function, **kwargs) def notifyResized(self, function, **kwargs): """ calls the corresponding function after the slot is resized first argument of the function is the slot second argument is the old size and the third argument is the new size the keyword arguments follow """ self._sig_resized.subscribe(function, **kwargs) def notifyRemove(self, function, **kwargs): """ calls the corresponding function BEFORE a slot is removed first argument of the function is the slot second argument is the old size and the third argument is the new size the keyword arguments follow """ self._sig_remove.subscribe(function, **kwargs) def notifyRemoved(self, function, **kwargs): """ calls the corresponding function AFTER a slot is removed first argument of the function is the slot second argument is the old size and the third argument is the new size the keyword arguments follow """ self._sig_removed.subscribe(function, **kwargs) def notifyPreInsertion(self, function, **kwargs): """ Called immediately before a slot is going to be inserted into a multi-slot. Same signature as the notifyInserted signal. """ self._sig_preinsertion.subscribe(function, **kwargs) def notifyInserted(self, function, **kwargs): """ calls the corresponding function after a slot has been added first argument of the function is the slot second argument is the old size and the third argument is the new size the keyword arguments follow """ self._sig_inserted.subscribe(function, **kwargs) def unregisterDirty(self, function): """ unregister a dirty callback """ self._sig_dirty.unsubscribe(function) def _unregisterConnect(self, function): """ unregister a connect callback """ self._sig_connect.unsubscribe(function) def unregisterDisconnect(self, function): """ unregister a disconnect callback """ self._sig_disconnect.unsubscribe(function) def unregisterMetaChanged(self, function): """ unregister a changed callback """ self._sig_changed.unsubscribe(function) def unregisterValueChanged(self, function): """ unregister a value changed callback """ self._sig_value_changed.unsubscribe(function) def unregisterReady(self, function): """ unregister a ready callback """ self._sig_ready.unsubscribe(function) def unregisterUnready(self, function): """ unregister an unready callback """ self._sig_unready.unsubscribe(function) def unregisterResize(self, function): """ unregister a resize callback """ self._sig_resize.unsubscribe(function) def unregisterResized(self, function): """ unregister a resized callback """ self._sig_resized.unsubscribe(function) def unregisterRemove(self, function): """ unregister a remove callback """ self._sig_remove.unsubscribe(function) def unregisterRemoved(self, function): """ unregister a removed callback """ self._sig_removed.unsubscribe(function) def unregisterPreInsertion(self, function): """ unregister a inserted callback """ self._sig_preinsertion.unsubscribe(function) def unregisterInserted(self, function): """ unregister a inserted callback """ self._sig_inserted.unsubscribe(function) def _handleUpstreamUnready(self, slot): """ This handler ensures that UNready status propagates quickly through the graph (before the normal _changed path) """ if self.meta._ready: self.meta._ready = False self._sig_unready(self) def connect(self, partner, notify=True): """ Connect a slot to another slot Arguments: partner : the slot to which this slot is conencted """ try: if partner is None: self.disconnect() return assert isinstance(partner, Slot), ("Slot.connect() can only be used to" " connect other Slots. Did you mean" " to use Slot.setValue()?") my_op = self.getRealOperator() partner_op = partner.getRealOperator() if not( partner_op.parent == my_op.parent or \ (self._type == "output" and partner_op.parent == my_op) or \ (self._type == "input" and my_op.parent == partner_op) or \ my_op == partner_op): msg = "It is forbidden to connect slots of operators that are not siblings "\ "or not directly related as parent and child." if partner_op.parent is None or my_op.parent is None: msg += "\n(For one of your operators, parent=None. Was it already cleaned up?" raise Exception(msg) if self.partner == partner and partner.level == self.level: return if self.level == 0: self.disconnect() if partner is not None: partner._sig_unready.subscribe( self._handleUpstreamUnready ) self._value = None if partner.level == self.level: assert isinstance(partner.stype, type(self.stype)), \ "Can't connect slots of non-matching stypes!" \ " Attempting to connect '{}' (stype: {}) to '{}' (stype: {})".format(self.name, self.stype, partner.name, partner.stype) self.partner = partner notifyReady = (self.partner.meta._ready and not self.meta._ready) self.meta = self.partner.meta.copy() # the slot with more sub-slots determines # the number of subslots if len(self) < len(partner): self.resize(len(partner)) elif len(self) > len(partner): partner.resize(len(self)) partner.partners.append(self) for i in range(len(self.partner)): p = self.partner[i] self[i].connect(p) # call slot type connect function self.stype.connect(partner) if self.level > 0 or self.stype.isConfigured(): self._changed() # call connect callbacks self._sig_connect(self) # Notify readiness after partner is updated if notifyReady: self._sig_ready(self) elif partner.level < self.level: self.partner = partner notifyReady = (self.partner.meta._ready and not self.meta._ready) self.meta = self.partner.meta.copy() for i, slot in enumerate(self._subSlots): slot.connect(partner) if notifyReady: self._sig_ready(self) self._changed() # call connect callbacks self._sig_connect(self) elif partner.level > self.level: msg = str("Can't connect slots:" " {}.{}.level={}, but" " {}.{}.level={}" " (Implicit OpearatorWrapper creation" " is no longer supported.)").format( self.getRealOperator().name, self.name, self.level, partner.getRealOperator().name, partner.name, partner.level) raise RuntimeError(msg) # propagate value changed signals from inner to outer # operators. if self._type == partner._type == "output": partner.notifyValueChanged(self._sig_value_changed) except: # If anything went wrong, we revert to the disconnected state. try: exc_info = sys.exc_info() self.disconnect() except: # Well, this is bad. We caused an exception while handling an exception. # We're more interested in the FIRST excpetion, so print this one out and # continue unwinding the stack with the first one. self.logger.error("Error: Caught a secondary exception while handling a different exception.") import traceback traceback.print_exc() raise exc_info[0], exc_info[1], exc_info[2] raise def disconnect(self): """ Disconnect a InputSlot from its partner """ if self.backpropagate_values: if self.partner is not None: self.partner.disconnect() return for slot in self._subSlots: slot.disconnect() had_partner = False if self.partner is not None: had_partner = True try: self.partner.partners.remove(self) except ValueError: pass self.partner = None had_value = self._value is not None self._value = None oldReady = self.meta._ready self.meta = MetaDict() if len(self._subSlots) > 0: self.resize(0) # call callbacks if had_partner or had_value: self._sig_disconnect(self) # Notify our partners that we changed. self._changed() # If we were ready before, signal that we aren't any more if oldReady: self._sig_unready(self) def resize(self, size): """ Resizes a slot to the desired length Arguments: size : the desired number of subslots """ assert numpy.issubdtype(type(size), numpy.integer), \ "Bug: 'size' must be int, not {}".format( type(size) ) if self._resizing: return if self.level == 0: raise RuntimeError("Can't resize a level-0 slot!") oldsize = len(self) if size == oldsize: return self._resizing = True if self.operator is not None: self.logger.debug("Resizing slot {} of operator {} to size {}".format( self.name, self.operator.name, size)) # call before resize callbacks self._sig_resize(self, oldsize, size) new_subslots = [] while size > len(self): self.insertSlot(len(self), len(self)+1, propagate=False) new_subslots.append( len(self) - 1 ) while size < len(self): self.removeSlot(len(self)-1, len(self)-1, propagate=False) # propagate size change downward for c in self.partners: if c.level == self.level: c.resize(size) # propagate size change upward if (self.partner and len(self.partner) < size and self.partner.level == self.level): self.partner.resize(size) # connect newly added slots # We must connect these subslots here, AFTER all resizes have propagated up and down through the graph. # Otherwise, our new subslots may lose downstream partners (happens in "diamond" shaped graphs.) for i in new_subslots: self._connectSubSlot(i) # call after resize callbacks self._sig_resized(self, oldsize, size) self._resizing = False def insertSlot(self, position, finalsize, propagate=True): """ Insert a new slot at the specififed position finalsize indicates the final destination size """ if len(self) >= finalsize: return self[position] # pre-insert callbacks self._sig_preinsertion(self, position, finalsize) slot = self._insertNew(position) self.logger.debug("Inserting slot {} into slot {} of operator {} to size {}".format( position, self.name, self.operator.name, finalsize)) if propagate: if self.partner is not None and self.partner.level == self.level: self.partner.insertSlot(position, finalsize) for p in self.partners: if p.level == self.level: p.insertSlot(position, finalsize) self._connectSubSlot(position) # call after insert callbacks self._sig_inserted(self, position, finalsize) return slot def removeSlot(self, position, finalsize, propagate=True): """ Remove the slot at position finalsize indicates the final size of all subslots """ if len(self) <= finalsize: return None assert position < len(self) if self.operator is not None: self.logger.debug("Removing slot {} into slot {} of operator {} to size {}".format( position, self.name, self.operator.name, finalsize)) # call before-remove callbacks self._sig_remove(self, position, finalsize) slot = self._subSlots.pop(position) slot.operator = None slot.disconnect() if propagate: if self.partner is not None and self.partner.level == self.level: self.partner.removeSlot(position, finalsize) for p in self.partners: if p.level == self.level: p.removeSlot(position, finalsize) # call after-remove callbacks self._sig_removed(self, position, finalsize) def get(self, roi): """This method is used to retrieve the actual content of a Slot. :param roi: the region of interest, e.g. a subregion in the case of an ArrayLike stype :param destination: this may define a destination area for the request, for example a ndarray into which the results should be written in the case of an ArrayLike stype Returns: a request.Request object. """ if self._value is not None: # this handles the case of an inputslot # having a ._value # --> construct cheaper request object for this case result = self.stype.writeIntoDestination(None, self._value, roi) return ValueRequest(result) elif self.partner is not None: # this handles the case of an inputslot # --> just relay the request return self.partner.get(roi) else: if not self.ready(): # Something is wrong. Are we cancelled? Request.raise_if_cancelled() msg = "Can't get data from slot {}.{} yet."\ " It isn't ready."\ "First upstream problem slot is: {}" msg = msg.format( self.getRealOperator().__class__, self.name, Slot._findUpstreamProblemSlot(self) ) raise Slot.SlotNotReadyError(msg) # If someone is asking for data from an inputslot that has # no value and no partner, then something is wrong. if self._type == "input": # Something is wrong. Are we cancelled? Request.raise_if_cancelled() assert self._type != "input", "This inputSlot has no value and no partner. You can't ask for its data yet!" # normal (outputslot) case # --> construct heavy request object.. execWrapper = Slot.RequestExecutionWrapper(self, roi) request = Request(execWrapper) # We must decrement the execution count even if the # request is cancelled request.notify_cancelled(execWrapper.handleCancel) return request @staticmethod def _findUpstreamProblemSlot(slot): if slot.partner is not None: return Slot._findUpstreamProblemSlot( slot.partner ) for inputSlot in slot.getRealOperator().inputs.values(): if not inputSlot._optional and not inputSlot.ready(): return inputSlot return "Couldn't find an upstream problem slot." class RequestExecutionWrapper(object): def __init__(self, slot, roi): self.started = False self.finished = False self.slot = slot self.operator = slot.operator self.lock = threading.Lock() self.roi = roi def __call__(self, destination=None): # store whether the user wants the results in a given # destination area destination_given = destination is not None if destination is None: destination = self.slot.stype.allocateDestination(self.roi) else: if self.slot.meta.dtype is not None and hasattr(destination, 'dtype'): assert self.slot.meta.dtype == destination.dtype, \ "Can't provide a destination array of the wrong dtype. "\ "Slot generates {}, but you gave {}".format( self.slot.meta.dtype, destination.dtype ) # We are executing the operator. Incremement the execution # count to protect against simultaneous setupOutputs() # calls. self._incrementOperatorExecutionCount() try: # Execute the workload, which might not ever return # (if we get cancelled). result_op = self.operator.execute(self.slot, (), self.roi, destination) # copy data from result_op to destination, if # destination was actually given by the user, and the # returned result_op is different from destination. # (but don't copy if result_op is None, this means # legacy op which wrote into destination anyway) if destination_given and result_op is not None and id(result_op) != id(destination): # check that the returned value is compatible with the requested roi self.slot.stype.check_result_valid(self.roi, result_op) self.slot.stype.copy_data(dst=destination, src = result_op) elif result_op is not None: # FIXME: this should be moved to a isCompatible # check in stypes.py if hasattr(result_op, "shape"): assert result_op.shape == destination.shape, \ ("ERROR: Operator {} has failed to provide a" " result of correct shape. result shape is" " {} vs {}. roi was {}".format( self.operator, result_op.shape, destination.shape, str(self.roi))) destination = result_op # check that the returned value is compatible with the requested roi self.slot.stype.check_result_valid(self.roi, destination) # Decrement the execution count self._decrementOperatorExecutionCount() return destination except: # except Request.CancellationException # Decrement the execution count self._decrementOperatorExecutionCount() raise def _incrementOperatorExecutionCount(self): self.started = True assert self.operator._executionCount >= 0, \ "BUG: How did the execution count get negative?" # We can't execute while the operator is in the middle of # setupOutputs with self.operator._condition: while self.operator._settingUp: self.operator._condition.wait() self.operator._executionCount += 1 def handleCancel(self, *args): # The new request api does clean up by handling an # exception, not in this callback. Only clean up if we are # using the old request api using_old_api = len(args) > 0 and not hasattr(args[0], 'notify_cancelled') if using_old_api: self._decrementOperatorExecutionCount() def _decrementOperatorExecutionCount(self): # Must lock here because cancel callbacks are # asynchronous. (Perhaps it would be better if they were # called from the worker thread instead...) with self.lock: # Only do this once per execution. If we were cancelled # after we finished working, don't do anything if self.started and not self.finished: assert self.operator._executionCount > 0, \ "BUG: Can't decrement the execution count below zero!" self.finished = True with self.operator._condition: self.operator._executionCount -= 1 self.operator._condition.notifyAll() def setDirty(self, *args, **kwargs): """This method is called by a partnering OutputSlot when its content changes. The 'key' parameter identifies the changed region of an numpy.ndarray """ assert self.operator is not None, ("Slot '{}' cannot be set dirty," " slot not belonging to any" " actual operator instance".format(self.name)) if self.stype.isConfigured(): if len(args) == 0 or not isinstance(args[0], rtype.Roi): roi = self.rtype(self, *args, **kwargs) else: roi = args[0] for c in self.partners: c.setDirty(roi) # call callbacks self._sig_dirty(self, roi) if self._type == "input" and self.operator.configured(): self.operator.propagateDirty(self, (), roi) def __iter__(self): assert self.level >= 1 return self._subSlots.__iter__() def __getitem__(self, key): """If level=0, emulate __call__ but with a slicing instead of a roi. If level>0, return the subslot corresponding to the key, which may be a tuple """ if self.level > 0: if isinstance(key, tuple): assert len(key) > 0 assert len(key) <= self.level if len(key) == 1: return self._subSlots[key[0]] else: return self._subSlots[key[0]][key[1:]] return self._subSlots[key] else: if self.meta.shape is None: # Something is wrong. Are we cancelled? Request.raise_if_cancelled() if not self.ready(): msg = "This slot ({}.{}) isn't ready yet, which means " \ "you can't ask for its data. Is it connected?".format(self.getRealOperator().name, self.name) self.logger.error(msg) slotInfoMsg = "Can't get data from slot {}.{} yet."\ " It isn't ready."\ "First upstream problem slot is: {}"\ "".format( self.getRealOperator().__class__, self.name, Slot._findUpstreamProblemSlot(self) ) self.logger.error(slotInfoMsg) raise Slot.SlotNotReadyError("Slot isn't ready. See error log.") assert self.meta.shape is not None, \ ("Can't ask for slices of this slot yet:" " self.meta.shape is None!" " (operator {} [self={}] slot: {}, key={}".format( self.operator.name, self.operator, self.name, key)) return self(pslice=key) def __setitem__(self, key, value): """This method provied access to the subslots of a MultiSlot. """ assert not isinstance(value, Slot), \ "Can't use setitem to connect slots. Use connect()" assert self.level == 0, \ ("setitem can only be used with slots of level 0." " Did you forget to append a key?") assert self.operator is not None, \ "cannot do __setitem__ on Slot '{}' -> no operator !!" assert slicingtools.is_bounded(key), \ "Can't use Slot.__setitem__ with keys that include : or ..." roi = self.rtype(self, pslice=key) if self._value is not None: self._value[key] = value # only propagate the dirty key at the very beginning of # the chain self.setDirty(roi) if self._type == "input": self.operator.setInSlot(self, (), roi, value) # Forward to partners for p in self.partners: p[key] = value def index(self, slot): return self._subSlots.index(slot) def setInSlot(self, slot, subindex, roi, value): """For now, Slots of level > 0 pretend to be operators (as far as their subslots are concerned). That's why they have to have this setInSlot() method. """ # Determine which subslot this is and prepend it to the totalIndex totalIndex = (self._subSlots.index(slot),) + subindex # Forward the call to our operator self.operator.setInSlot(self, totalIndex, roi, value) def __len__(self): """In the case of a MultiSlot this returns the number of subslots, i.e. the length of the list """ return len(self._subSlots) @property def value(self): """This method directly returns the full content of a slot. Is mainly used when region of interest specification make no sense, e.g. in the case of slots which hold a single integer or float value """ if self.partner is not None: # outputslot-inputsslot, inputslot-inputslot and outputslot-outputslot case temp = self[:].wait() elif self._value is None: # outputslot case temp = self[:].wait() else: # _value case return self._value if isinstance(temp, numpy.ndarray) and temp.shape != (1,): return temp else: try: return temp[0] except IndexError: self.logger.warn("FIXME: Slot.value for slot {} is {}," " which should be wrapped in an ndarray.".format( self.name, temp)) return temp def setValue(self, value, notify=True, check_changed=True): """This method can be used to directly assign a value to an InputSlot. Usually a slot is either connected to another slot from which it retrieves the content when it is queried, or it directly holds a value itself. This method can be used to set such a value. If check_changed is True, the new value is compared to the current one and updates are only triggered if the new value differs from the old one according to the __eq__ operator. The check can be turned off with the check_changed flag. """ try: assert isinstance(notify, bool) assert isinstance(check_changed, bool) # This assertion is here to prevent accidental use of setValue # when connect should be used. If your use case requires # passing slots as values, then this assertion can be refined. assert not isinstance(value, Slot), \ "When using setValue, value cannot be a slot. Use connect instead." if not self.backpropagate_values: assert self.partner is None, \ ("Cannot call setValue on this slot." " It is already connected to a partner." " Call disconnect first if that's what you really wanted.") elif self.partner is not None: self.partner.setValue(value, notify, check_changed) return changed = True # We use == here instead of 'is' to avoid subtle bugs that # can occur if you supplied an equivalent value that 'is not' the original. # For example: x=numpy.uint8(3); y=numpy.int64(3); assert x == y; assert x is not y if check_changed: if isinstance(value, vigra.VigraArray) or isinstance(self._value, vigra.VigraArray): if type(value) != type(self._value) or value.axistags != self._value.axistags: changed = True else: same = (value is self._value) if not same: try: same = ( value == self._value ) except ValueError: # Some values can't be compared with __eq__, # in which case we assume the values are different same = False if isinstance(same, (numpy.ndarray, TinyVector)): same = same.all() changed = not same if changed: # call disconnect callbacks self._sig_disconnect(self) self._value = value self.stype.setupMetaForValue(value) self.meta._dirty = True for s in self._subSlots: s.setValue(self._value) notify = (self.meta._ready == False) # a slot with a value is always ready self.meta._ready = True if notify: self._sig_ready(self) # call connect callbacks self._sig_connect(self) self._changed() # Propagate dirtyness if self.rtype == rtype.List: self.setDirty(()) else: self.setDirty(slice(None)) except: try: exc_info = sys.exc_info() self.disconnect() except: # Well, this is bad. We caused an exception while handling an exception. # We're more interested in the FIRST excpetion, so print this one out and # continue unwinding the stack with the first one. self.logger.error("Error: Caught a secondary exception while handling a different exception.") import traceback traceback.print_exc() raise exc_info[0], exc_info[1], exc_info[2] raise def setValues(self, values): """Set values of subslots with arraylike object. Resizes the multinputslot with the length of the values array """ try: # call disconnect callbacks self._sig_disconnect(self) self.resize(len(values)) for i, s in enumerate(self._subSlots): s.setValue(values[i]) # call connect callbacks self._changed() self._sig_connect(self) except: try: exc_info = sys.exc_info() self.disconnect() except: # Well, this is bad. We caused an exception while handling an exception. # We're more interested in the FIRST excpetion, so print this one out and # continue unwinding the stack with the first one. self.logger.error("Error: Caught a secondary exception while handling a different exception.") import traceback traceback.print_exc() raise exc_info[0], exc_info[1], exc_info[2] raise @property def backpropagate_values(self): return self._backpropagate_values @backpropagate_values.setter def backpropagate_values(self, backprop): self._backpropagate_values = backprop for slot in self._subSlots: slot.backpropagate_values = backprop def connected(self): """Returns True if the slot is conencted to a partner slot or has a _value assigned as input """ answer = True if self._value is None and self.partner is None: answer = False if answer is False and len(self._subSlots) > 0: answer = True for s in self._subSlots: if s.connected() is False: answer = False break return answer def configured(self): """Slots of level >= 1 must implement parts of the operator interface, including this function. This "operator" is considered "configured" if it is ready. """ return self._optional or self.ready() def ready(self): if self.level == 0: # If this slot is non-multi, then just check our own # status ready = self.meta._ready else: # If this slot is multi, check all of our subslots. (If we # have no subslots, then we are NOT ready). Operators that # can properly handle an empty multi-input slot should # mark the input as optional. ready = len(self._subSlots) > 0 and all(p.ready() for p in self._subSlots) return ready def _setReady(self): wasReady = self.ready() for p in self._subSlots: p._setReady() self.meta._ready = (self.level == 0) or (len(self._subSlots) > 0) # If we just became ready... if not wasReady and self.meta._ready: # Notify partners of changed readystatus self._changed() self._sig_ready(self) def __call__(self, *args, **kwargs): """The slot relays all arguments to the __init__ method of the Roi type. This allows lazyflow to support different types of rois without knowing anything about them. """ roi = self.rtype(self, *args, **kwargs) return self.get(roi) def getRealOperator(self): """If a slot is owned by a higher-level slot, self.operator is a slot. This function keeps going up the hierarchy until it finds the actual operator this slot belongs to. """ if self._real_operator is not None: # use memoized return self._real_operator if isinstance(self.operator, Slot): self._real_operator = self.operator.getRealOperator() else: self._real_operator = self.operator return self._real_operator ##################### # Private Methods # ##################### def _getInstance(self, operator, level=None): """This method constructs a copy of the slot. This method is used when creating an Instance of an Operator. All defined Input and Output slots of the Class are cloned and inserted into the instance of the Operator. """ if level is None: level = self.level if self._type == "input": s = InputSlot(self.name, operator, stype=self._stypeType, rtype=self.rtype, value=self._defaultValue, optional=self._optional, level=level, nonlane=self.nonlane) elif self._type == "output": s = OutputSlot(self.name, operator, stype=self._stypeType, rtype=self.rtype, value=self._defaultValue, level=level, nonlane=self.nonlane) return s def _changed(self): oldMeta = self.meta if self.partner is not None and self.meta != self.partner.meta: self.meta = self.partner.meta.copy() if self._type == "output": for o in self._subSlots: o._changed() # Notify readiness after subslots are updated if self.meta._ready != oldMeta._ready: if self.meta._ready: self._sig_ready(self) else: self._sig_unready(self) wasdirty = self.meta._dirty if self.meta._dirty: for c in self.partners: c._changed() self.meta._dirty = False if self._type != "output": op = self.getRealOperator() if op is not None and not op._cleaningUp: self._configureOperator(self) if wasdirty: # call changed callbacks self._sig_changed(self) def _configureOperator(self, slot, oldSize=0, newSize=0, notify=True): """Call setupOutputs of Operator if all slots of the operator are connected and configured. """ if self.operator is not None: # check whether all slots are connected and notify operator if self.operator.configured(): self.operator._setupOutputs() def _requiredLength(self): """ Returns the required number of subslots """ if self.partner is not None: if self.partner.level == self.level: return len(self.partner) elif self.partner.level < self.level: return 1 elif self._value is not None: return 1 else: return 0 def _setupOutputs(self): """ """ self._changed() def _connectSubSlot(self, slot, notify=True): """Connect a subslot either to the partner, or set the correct value in case of self._value != None """ if type(slot) == int: index = slot slot = self._subSlots[slot] else: index = self._subSlots.index(slot) if self.partner is not None: if self.partner.level == self.level: if len(self.partner) > index: slot.connect(self.partner[index]) else: slot.connect(self.partner) if self._value is not None: slot.setValue(self._value, notify=notify) def _insertNew(self, position): """Construct a new subSlot of correct type and level and insert it to the list of subslots """ assert position >= 0 and position <= len(self._subSlots) slot = self._getInstance(self, level=self.level - 1) self._subSlots.insert(position, slot) slot.name = self.name if self._value is not None: slot.setValue(self._value) return slot def pop(self, index=-1, event=None): if index < 0: index = len(self) + index self._subSlots.pop(index) def propagateDirty(self, slot, subindex, roi): """Slots with level > 0 must implement part of the operator interface so they look like an operator as far as their subslots are concerned. That's why this function is here. """ totalIndex = (self._subSlots.index(slot),) + subindex self.operator.propagateDirty(self, totalIndex, roi) ###################################### # methods aimed to enhance usability # ###################################### def setShapeAtAxisTo(self, axis, size): tmpshape = list(self.meta.shape) tmpshape[self.meta.axistags.index(axis)] = size self.meta.shape = tuple(tmpshape) def __str__(self): mslot_info = "" if self.level > 0 or isinstance(self.operator, Slot): mslot_info += "[" if isinstance(self.operator, Slot): mslot_info += " index={}".format( self.operator.index(self) ) if self.level > 0: mslot_info += " len={}".format( len(self) ) if self.level > 1: mslot_info += " level={}".format( self.level ) mslot_info += " ] " return '"{}" {}: \t{}\n'.format( self.name, mslot_info, self.meta ) def __repr__(self): return self.__str__()
def __init__(self, *args, **kwargs): super(OpExportMultipageTiff, self).__init__(*args, **kwargs) self.progressSignal = OrderedSignal()
def __init__(self, name="", operator=None, stype=ArrayLike, rtype=rtype.SubRegion, value=None, optional=False, level=0, nonlane=False): """Constructor of the Slot class. :param name: user readable name of the slot, is normally assigned automatically by the Operator :param operator: the parent operator of a slot :param stype: the slot type (see stype.py) :param rtype: the region of interest type (see rtype.py) :param value: the default value of the slot :param optional: if True this means the slot needs a value or connection for its parent operator to be functional :param level: defines the dimensionality of the slot. 0 for single element (e.g. single numpy.ndarray), 1 for list of elements (e.g. list of strings), 2 for list of list of elements. :param nonlane: For multislot, this flag protects it from being considered lane-indexed """ # This assertion is here for a reason: default values do NOT work on OutputSlots. # (We should probably change that at some point...) assert value is None or isinstance(self, InputSlot), "Only InputSlots can have default values. OutputSlots cannot." if not hasattr(self, "_type"): self._type = None if type(stype) == str: stype = ArrayLike self.partners = [] self.name = name self._optional = optional self.operator = operator self._real_operator = None # Memoized in getRealOperator() # in the case of an InputSlot this is the slot to which it is # connected self.partner = None self.level = level # in the case of an InputSlot one can directly assign a value # to a slot instead of connecting it to a partner, this # attribute holds the value self._value = None self._defaultValue = value # Causes calls to setValue to be propagated backwards to the # partner slot. Used by the OperatorWrapper. self._backpropagate_values = False self.rtype = rtype # the MetaDict that holds the slots meta information self.meta = MetaDict() # if level > 0, this holds the sub-Input/Output slots self._subSlots = [] self._stypeType = stype # the slot type instance self.stype = stype(self) self.nonlane = nonlane self._sig_changed = OrderedSignal() self._sig_value_changed = OrderedSignal() self._sig_ready = OrderedSignal() self._sig_unready = OrderedSignal() self._sig_dirty = OrderedSignal() self._sig_connect = OrderedSignal() self._sig_disconnect = OrderedSignal() self._sig_resize = OrderedSignal() self._sig_resized = OrderedSignal() self._sig_remove = OrderedSignal() self._sig_removed = OrderedSignal() self._sig_preinsertion = OrderedSignal() self._sig_inserted = OrderedSignal() self._resizing = False self._executionCount = 0 self._settingUp = False self._condition = threading.Condition() # Allow slots to be sorted by their order of creation for # debug output and diagramming purposes. self._global_slot_id = Slot._global_counter.next()
def __init__(self, *args, **kwargs): super(OpWsdt, self).__init__(*args, **kwargs) self.debug_results = None self.watershed_completed = OrderedSignal()
def __init__(self, *args, **kwargs): super(OpResize5D, self).__init__(*args, **kwargs) self._input_to_output_scales = None self.progressSignal = OrderedSignal()
class ArrayCacheMemoryMgr(threading.Thread): totalCacheMemory = OrderedSignal() loggingName = __name__ + ".ArrayCacheMemoryMgr" logger = logging.getLogger(loggingName) traceLogger = logging.getLogger("TRACE." + loggingName) def __init__(self): threading.Thread.__init__(self) self.daemon = True self.caches = self._new_list() self.namedCaches = [] self._max_usage = 85 self._target_usage = 70 self._lock = threading.Lock() self._last_usage = memoryUsagePercentage() def _new_list(self): def getPrio(array_cache): return array_cache._cache_priority return blist.sortedlist((), getPrio) def addNamedCache(self, array_cache): """add a cache to a special list of named caches The list of named caches should contain only top-level caches. This way, when showing memory usage, we can provide a tree-view, where the named caches are the top-level items and the user can then drill down into the caches that are children of the top-level caches. """ self.namedCaches.append(array_cache) def add(self, array_cache): with self._lock: self.caches.add(array_cache) def remove(self, array_cache): with self._lock: try: self.caches.remove(array_cache) except ValueError: pass def run(self): while True: mem_usage = memoryUsagePercentage() mem_usage_gb = memoryUsageGB() delta = abs(self._last_usage - mem_usage) if delta > 10 or self.logger.level == logging.DEBUG: cpu_usages = psutil.cpu_percent(interval=1, percpu=True) avg = sum(cpu_usages) / len(cpu_usages) self.logger.info( "RAM: {:1.3f} GB ({:02.0f}%), CPU: Avg={:02.0f}%, {}". format(mem_usage_gb, mem_usage, avg, cpu_usages)) if delta > 10: self._last_usage = mem_usage #calculate total memory usage and send as signal tot = 0.0 for c in self.namedCaches: if c.usedMemory() is None: continue else: tot += c.usedMemory() self.totalCacheMemory(tot) time.sleep(10) if mem_usage > self._max_usage: self.logger.info("freeing memory...") with self._lock: count = 0 not_freed = [] old_length = len(self.caches) new_caches = self._new_list() self.traceLogger.debug("Updating {} caches".format( len(self.caches))) for c in iter(self.caches): c._updatePriority(c._last_access) new_caches.add(c) self.caches = new_caches gc.collect() self.traceLogger.debug("Target mem usage: {}".format( self._target_usage)) while mem_usage > self._target_usage and len( self.caches) > 0: self.traceLogger.debug( "Mem usage: {}".format(mem_usage)) last_cache = self.caches.pop(-1) freed = last_cache._freeMemory(refcheck=True) self.traceLogger.debug("Freed: {}".format(freed)) mem_usage = memoryUsagePercentage() count += 1 if freed == 0: # store the caches which could not be freed not_freed.append(last_cache) gc.collect() self.logger.info("freed %d/%d blocks, new usage = %f%%" % (count, old_length, mem_usage)) for c in not_freed: # add the caches which could not be freed self.add(c)
def __init__(self, *args, **kwargs): super(OpExportMultipageTiff, self).__init__(*args, **kwargs) self.progressSignal = OrderedSignal() self._opReorderAxes = OpReorderAxes(parent=self) self._opReorderAxes.Input.connect(self.Input)
class CacheMemoryManager(threading.Thread): """ class for the management of cache memory The cache memory manager is a background thread that observes caches in use and cleans them up when the total memory consumption by the process exceeds the limit defined by the `lazyflow.utility.Memory` class. See the definition of the cache interfaces (opCache.py) to get an overview over the possible caches. Usage: This manager is a singleton - just call its constructor somewhere and you will get a reference to the *only* running memory management thread. Interface: The manager provides a signal you can subscribe to >>> def writeFunction(x): print("total mem: {}".format(x)) >>> mgr = CacheMemoryManager() >>> mgr.totalCacheMemory.subscribe(writeFunction) which emits the size of all observable caches, combined, in regular intervals. The update interval (for the signal and for automated cache release) can be set with a call to a class method >>> CacheMemoryManager().setRefreshInterval(5) the interval is measured in seconds. Each change of refresh interval triggers cleanup. """ __metaclass__ = Singleton totalCacheMemory = OrderedSignal() def __init__(self): threading.Thread.__init__(self) self.daemon = True self._caches = weakref.WeakSet() self._first_class_caches = weakref.WeakSet() self._observable_caches = weakref.WeakSet() self._managed_caches = weakref.WeakSet() self._managed_blocked_caches = weakref.WeakSet() self._condition = threading.Condition() self._disable_lock = threading.Condition() self._disabled = False self._refresh_interval = default_refresh_interval self._first_class_caches_lock = threading.Lock() # maximum fraction of *allowed memory* used self._max_usage = 1.0 # target usage fraction self._target_usage = .90 self._stopped = False self.start() atexit.register(self.stop) def addFirstClassCache(self, cache): """ add a first class cache (root cache) to the manager First class caches are handled differently so we are able to show a tree view of the caches (e.g. in ilastik). This method calls addCache() automatically. """ # late import to prevent import loop from lazyflow.operators.opCache import Cache if isinstance(cache, Cache): with self._first_class_caches_lock: self._first_class_caches.add(cache) self.addCache(cache) def getFirstClassCaches(self): """ get a list of first class caches """ with self._first_class_caches_lock: return list(self._first_class_caches) def getCaches(self): """ get a list of all caches (including first class caches) """ return list(self._caches) def addCache(self, cache): """ add a cache to be managed Caches are kept with weak references, so there is no need to remove them from the manager. """ # late import to prevent import loop from lazyflow.operators.opCache import Cache from lazyflow.operators.opCache import ObservableCache from lazyflow.operators.opCache import ManagedCache from lazyflow.operators.opCache import ManagedBlockedCache assert isinstance(cache, Cache),\ "Only Cache instances can be managed by CacheMemoryManager" self._caches.add(cache) if isinstance(cache, ObservableCache): self._observable_caches.add(cache) if isinstance(cache, ManagedBlockedCache): self._managed_blocked_caches.add(cache) elif isinstance(cache, ManagedCache): self._managed_caches.add(cache) def run(self): """ main loop """ while not self._stopped: self._wait() # acquire lock so that we don't get disabled during cleanup with self._disable_lock: if self._disabled or self._stopped: continue self._cleanup() def _cleanup(self): """ clean up once """ from lazyflow.operators.opCache import ObservableCache try: # notify subscribed functions about current cache memory total = 0 # Avoid "RuntimeError: Set changed size during iteration" with self._first_class_caches_lock: first_class_caches = self._first_class_caches.copy() for cache in first_class_caches: if isinstance(cache, ObservableCache): total += cache.usedMemory() self.totalCacheMemory(total) cache = None # check current memory state cache_memory = Memory.getAvailableRamCaches() if total <= self._max_usage * cache_memory: return # === we need a cache cleanup === # queue holds time stamps and cleanup functions q = PriorityQueue() caches = list(self._managed_caches) for c in caches: q.push((c.lastAccessTime(), c.name, c.freeMemory)) caches = list(self._managed_blocked_caches) for c in caches: for k, t in c.getBlockAccessTimes(): cleanupFun = functools.partial(c.freeBlock, k) info = "{}: {}".format(c.name, k) q.push((t, info, cleanupFun)) c = None caches = None msg = "Caches are using {} memory".format( Memory.format(total)) if cache_memory > 0: msg += " ({:.1f}% of allowed)".format( total*100.0/cache_memory) logger.debug(msg) while (total > self._target_usage * cache_memory and len(q) > 0): t, info, cleanupFun = q.pop() mem = cleanupFun() logger.debug("Cleaned up {} ({})".format( info, Memory.format(mem))) total -= mem gc.collect() # don't keep a reference until next loop iteration cleanupFun = None q = None msg = ("Done cleaning up, cache memory usage is now at " "{}".format(Memory.format(total))) if cache_memory > 0: msg += " ({:.1f}% of allowed)".format( total*100.0/cache_memory) logger.debug(msg) except: log_exception(logger) def _wait(self): """ sleep for _refresh_interval seconds or until woken up """ with self._condition: self._condition.wait(self._refresh_interval) def stop(self): """ Stop the memory manager thread in preparation for app exit. """ self._stopped = True with self._condition: self._condition.notify() self.join() def setRefreshInterval(self, t): """ set the clean up period and wake up the cleaning thread """ with self._condition: self._refresh_interval = t self._condition.notifyAll() def disable(self): """ disable all memory management This method blocks until current memory management tasks are finished. """ with self._disable_lock: self._disabled = True def enable(self): """ enable cache management and wake the thread """ with self._disable_lock: self._disabled = False with self._condition: self._condition.notifyAll()
class CacheMemoryManager(with_metaclass(Singleton, threading.Thread)): """ class for the management of cache memory The cache memory manager is a background thread that observes caches in use and cleans them up when the total memory consumption by the process exceeds the limit defined by the `lazyflow.utility.Memory` class. See the definition of the cache interfaces (opCache.py) to get an overview over the possible caches. Usage: This manager is a singleton - just call its constructor somewhere and you will get a reference to the *only* running memory management thread. Interface: The manager provides a signal you can subscribe to:: def writeFunction(x): print("total mem: {}".format(x)) mgr = CacheMemoryManager() mgr.totalCacheMemory.subscribe(writeFunction) which emits the size of all observable caches, combined, in regular intervals. The update interval (for the signal and for automated cache release) can be set with a call to a class method:: CacheMemoryManager().setRefreshInterval(5) the interval is measured in seconds. Each change of refresh interval triggers cleanup. """ totalCacheMemory = OrderedSignal() def __init__(self): threading.Thread.__init__(self) self.daemon = True self._caches = weakref.WeakSet() self._first_class_caches = weakref.WeakSet() self._observable_caches = weakref.WeakSet() self._managed_caches = weakref.WeakSet() self._managed_blocked_caches = weakref.WeakSet() self._condition = threading.Condition() self._disable_lock = threading.Condition() self._disabled = False self._refresh_interval = default_refresh_interval self._first_class_caches_lock = threading.Lock() # maximum fraction of *allowed memory* used self._max_usage = 1.0 # target usage fraction self._target_usage = 0.90 self._stopped = False self.start() atexit.register(self.stop) def addFirstClassCache(self, cache): """ add a first class cache (root cache) to the manager First class caches are handled differently so we are able to show a tree view of the caches (e.g. in ilastik). This method calls addCache() automatically. """ # late import to prevent import loop from lazyflow.operators.opCache import Cache if isinstance(cache, Cache): with self._first_class_caches_lock: self._first_class_caches.add(cache) self.addCache(cache) def getFirstClassCaches(self): """ get a list of first class caches """ with self._first_class_caches_lock: return list(self._first_class_caches) def getCaches(self): """ get a list of all caches (including first class caches) """ return list(self._caches) def addCache(self, cache): """ add a cache to be managed Caches are kept with weak references, so there is no need to remove them from the manager. """ # late import to prevent import loop from lazyflow.operators.opCache import Cache from lazyflow.operators.opCache import ObservableCache from lazyflow.operators.opCache import ManagedCache from lazyflow.operators.opCache import ManagedBlockedCache assert isinstance(cache, Cache), "Only Cache instances can be managed by CacheMemoryManager" self._caches.add(cache) if isinstance(cache, ObservableCache): self._observable_caches.add(cache) if isinstance(cache, ManagedBlockedCache): self._managed_blocked_caches.add(cache) elif isinstance(cache, ManagedCache): self._managed_caches.add(cache) def run(self): """ main loop """ while not self._stopped: self._wait() # acquire lock so that we don't get disabled during cleanup with self._disable_lock: if self._disabled or self._stopped: continue self._cleanup() def _cleanup(self): """ clean up once """ from lazyflow.operators.opCache import ObservableCache try: # notify subscribed functions about current cache memory total = 0 # Avoid "RuntimeError: Set changed size during iteration" with self._first_class_caches_lock: first_class_caches = self._first_class_caches.copy() for cache in first_class_caches: if isinstance(cache, ObservableCache): total += cache.usedMemory() self.totalCacheMemory(total) cache = None # check current memory state cache_memory = Memory.getAvailableRamCaches() cache_pct = 0.0 if cache_memory: cache_pct = total * 100.0 / cache_memory logger.debug( "Process memory usage is {:0.2f} GB out of {:0.2f} (caches are {}, {:.1f}% of allowed)".format( Memory.getMemoryUsage() / 2.0 ** 30, Memory.getAvailableRam() / 2.0 ** 30, Memory.format(total), cache_pct, ) ) if total <= self._max_usage * cache_memory: return cache_entries = [] cache_entries += [ (cache.lastAccessTime(), cache.name, cache.freeMemory) for cache in list(self._managed_caches) ] cache_entries += [ (lastAccessTime, f"{cache.name}: {blockKey}", functools.partial(cache.freeBlock, blockKey)) for cache in list(self._managed_blocked_caches) for blockKey, lastAccessTime in cache.getBlockAccessTimes() ] cache_entries.sort(key=lambda entry: entry[0]) for lastAccessTime, info, cleanupFun in cache_entries: if total <= self._target_usage * cache_memory: break mem = cleanupFun() logger.debug(f"Cleaned up {info} ({Memory.format(mem)})") total -= mem # Remove references to cache entries before triggering garbage collection. cleanupFun = None cache_entries = None gc.collect() msg = "Done cleaning up, cache memory usage is now at {}".format(Memory.format(total)) if cache_memory > 0: msg += " ({:.1f}% of allowed)".format(total * 100.0 / cache_memory) logger.debug(msg) except: log_exception(logger) def _wait(self): """ sleep for _refresh_interval seconds or until woken up """ with self._condition: self._condition.wait(self._refresh_interval) def stop(self): """ Stop the memory manager thread in preparation for app exit. """ self._stopped = True with self._condition: self._condition.notify() self.join() def setRefreshInterval(self, t): """ set the clean up period and wake up the cleaning thread """ with self._condition: self._refresh_interval = t self._condition.notifyAll() def disable(self): """ disable all memory management This method blocks until current memory management tasks are finished. """ with self._disable_lock: self._disabled = True def enable(self): """ enable cache management and wake the thread """ with self._disable_lock: self._disabled = False with self._condition: self._condition.notifyAll()
def __init__(self, *args, **kwargs): super(OpConcatenateFeatureMatrices, self).__init__(*args, **kwargs) self._dirty_slots = set() self.progressSignal = OrderedSignal() self._num_feature_channels = 0 # Not including the labels...
def __init__(self, *args, **kwargs): super(OpExportToArray, self).__init__(*args, **kwargs) self.progressSignal = OrderedSignal()
def __init__(self, *args, **kwargs): super(OpConcatenateFeatureMatrices, self).__init__(*args, **kwargs) self._dirty_slots = set() self.progressSignal = OrderedSignal()