def __init__(self, outputSlot, roiIterator, totalVolume=None, batchSize=2, allowParallelResults=False):
"""
Constructor.
:param outputSlot: The slot to request data from.
:param roiIterator: An iterator providing new rois.
:param totalVolume: The total volume to be processed.
Used to provide the progress reporting signal.
If not provided, then no intermediate progress will be signaled.
:param batchSize: The maximum number of requests to launch in parallel.
:param allowParallelResults: If False, The resultSignal will not be called in parallel.
In that case, your handler function has no need for locks.
"""
self._resultSignal = OrderedSignal()
self._progressSignal = OrderedSignal()
assert isinstance(
outputSlot.stype, lazyflow.stype.ArrayLike
), "Only Array-like slots supported." # Because progress reporting depends on the roi shape
self._outputSlot = outputSlot
self._roiIter = roiIterator
self._batchSize = batchSize
self._allowParallelResults = allowParallelResults
self._condition = SimpleRequestCondition()
self._activated_count = 0
self._completed_count = 0
self._failure_excinfo = None
# Progress bookkeeping
self._totalVolume = totalVolume
self._processedVolume = 0
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
# Set up the impl function lookup dict
export_impls = {}
export_impls["hdf5"] = ("h5", self._export_h5n5)
export_impls["compressed hdf5"] = ("h5",
partial(self._export_h5n5, True))
export_impls["n5"] = ("n5", self._export_h5n5)
export_impls["compressed n5"] = ("n5", partial(self._export_h5n5,
True))
export_impls["numpy"] = ("npy", self._export_npy)
export_impls["dvid"] = ("", self._export_dvid)
export_impls["blockwise hdf5"] = ("json", self._export_blockwise_hdf5)
for fmt in self._2d_formats:
export_impls[fmt.name] = (fmt.extension,
partial(self._export_2d, fmt.extension))
for fmt in self._3d_sequence_formats:
export_impls[fmt.name] = (fmt.extension,
partial(self._export_3d_sequence,
fmt.extension))
export_impls["multipage tiff"] = ("tiff", self._export_multipage_tiff)
export_impls["multipage tiff sequence"] = (
"tiff", self._export_multipage_tiff_sequence)
self._export_impls = export_impls
self.Input.notifyMetaChanged(self._updateFormatSelectionErrorMsg)
def __init__(self, *args, **kwargs):
super(OpConcatenateFeatureMatrices, self).__init__(*args, **kwargs)
self._dirty_slots = set()
self.progressSignal = OrderedSignal()
self._num_feature_channels = 0 # Not including the labels...
self._channel_names = []
# Normally, lane removal does not trigger a dirty notification.
# But in this case, if the lane contained any label data whatsoever,
# the classifier needs to be marked dirty.
# We know which slots contain (or contained) label data because they have
# been 'touched' at some point (they became dirty at some point).
self._touched_slots = set()
def handle_new_lane( multislot, index, newlength ):
def handle_dirty_lane( slot, roi ):
self._touched_slots.add(slot)
multislot[index].notifyDirty( handle_dirty_lane )
self.FeatureMatrices.notifyInserted( handle_new_lane )
def handle_remove_lane( multislot, index, newlength ):
# If the lane we're removing contained
# label data, then mark the downstream dirty
if multislot[index] in self._touched_slots:
self.ConcatenatedOutput.setDirty()
self._touched_slots.remove(multislot[index])
self.FeatureMatrices.notifyRemove( handle_remove_lane )
def __init__(self, *args, **kwargs):
super(OpExportSlot, self).__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
# Set up the impl function lookup dict
export_impls = {}
export_impls['hdf5'] = ('h5', self._export_hdf5)
export_impls['npy'] = ('npy', self._export_npy)
export_impls['dvid'] = ('', self._export_dvid)
export_impls['blockwise hdf5'] = ('json', self._export_blockwise_hdf5)
for fmt in self._2d_formats:
export_impls[fmt.name] = (fmt.extension,
partial(self._export_2d, fmt.extension))
for fmt in self._3d_sequence_formats:
export_impls[fmt.name] = (fmt.extension,
partial(self._export_3d_sequence,
fmt.extension))
export_impls['multipage tiff'] = ('tiff', self._export_multipage_tiff)
export_impls['multipage tiff sequence'] = (
'tiff', self._export_multipage_tiff_sequence)
self._export_impls = export_impls
self.Input.notifyMetaChanged(self._updateFormatSelectionErrorMsg)
def __init__(self, *args, **kwargs):
super(OpWsdt, self).__init__(*args, **kwargs)
self.debug_results = None
self.watershed_completed = OrderedSignal()
self._opSelectedInput = OpSumChannels(parent=self)
self._opSelectedInput.ChannelSelections.connect(self.ChannelSelections)
self._opSelectedInput.Input.connect(self.Input)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
self._opReorderAxes = OpReorderAxes(parent=self)
self._opReorderAxes.Input.connect(self.Input)
self._opReorderAxes.AxisOrder.setValue(self._EXPORT_AXES)
def __init__(self, *args, **kwargs):
super(OpFeatureMatrixCache, self).__init__(*args, **kwargs)
self._lock = RequestLock()
self.progressSignal = OrderedSignal()
self._progress_lock = RequestLock()
self._blockshape = None
self._dirty_blocks = set()
self._blockwise_feature_matrices = {}
self._block_locks = {} # One lock per stored block
self._init_blocks(None, None)
def __init__(self, *args, **kwargs):
super(OpFeatureMatrixCache, self).__init__(*args, **kwargs)
self._blockshape = None
self._lock = RequestLock()
self.progressSignal = OrderedSignal()
self._progress_lock = RequestLock()
# In these set/dict members, the block id (dict key)
# is simply the block's start coordinate (as a tuple)
self._blockwise_feature_matrices = {}
self._dirty_blocks = set()
self._block_locks = {} # One lock per stored block
def __enter__(self):
if self._graph:
with self._graph._lock:
if self._graph._setup_depth == 0:
self._graph._sig_setup_complete = OrderedSignal()
self._graph._setup_depth += 1
def __init__(self, transpose_axes, *args, **kwargs):
super(OpExportDvidVolume, self).__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
self._transpose_axes = transpose_axes
def __init__(self, *args, **kwargs):
super(OpNpyWriter, self).__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
def __init__(self, *args, **kwargs):
super(OpExportMultipageTiff, self).__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
def __init__(self, *args, **kwargs):
super(OpConcatenateFeatureMatrices, self).__init__(*args, **kwargs)
self._dirty_slots = set()
self.progressSignal = OrderedSignal()
self._num_feature_channels = 0 # Not including the labels...
def __init__(self, *args, **kwargs):
super(OpWsdt, self).__init__(*args, **kwargs)
self.debug_results = None
self.watershed_completed = OrderedSignal()
class ArrayCacheMemoryMgr(threading.Thread):
totalCacheMemory = OrderedSignal()
loggingName = __name__ + ".ArrayCacheMemoryMgr"
logger = logging.getLogger(loggingName)
traceLogger = logging.getLogger("TRACE." + loggingName)
def __init__(self):
threading.Thread.__init__(self)
self.daemon = True
self.caches = self._new_list()
self.namedCaches = []
self._max_usage = 85
self._target_usage = 70
self._lock = threading.Lock()
self._last_usage = memoryUsagePercentage()
def _new_list(self):
def getPrio(array_cache):
return array_cache._cache_priority
return blist.sortedlist((), getPrio)
def addNamedCache(self, array_cache):
"""add a cache to a special list of named caches
The list of named caches should contain only top-level caches.
This way, when showing memory usage, we can provide a tree-view, where the
named caches are the top-level items and the user can then drill down into the caches
that are children of the top-level caches.
"""
self.namedCaches.append(array_cache)
def add(self, array_cache):
with self._lock:
self.caches.add(array_cache)
def remove(self, array_cache):
with self._lock:
try:
self.caches.remove(array_cache)
except ValueError:
pass
def run(self):
while True:
mem_usage = memoryUsagePercentage()
mem_usage_gb = memoryUsageGB()
delta = abs(self._last_usage - mem_usage)
if delta > 10 or self.logger.level == logging.DEBUG:
cpu_usages = psutil.cpu_percent(interval=1, percpu=True)
avg = sum(cpu_usages) / len(cpu_usages)
self.logger.info(
"RAM: {:1.3f} GB ({:02.0f}%), CPU: Avg={:02.0f}%, {}".
format(mem_usage_gb, mem_usage, avg, cpu_usages))
if delta > 10:
self._last_usage = mem_usage
#calculate total memory usage and send as signal
tot = 0.0
for c in self.namedCaches:
if c.usedMemory() is None:
continue
else:
tot += c.usedMemory()
self.totalCacheMemory(tot)
time.sleep(10)
if mem_usage > self._max_usage:
self.logger.info("freeing memory...")
with self._lock:
count = 0
not_freed = []
old_length = len(self.caches)
new_caches = self._new_list()
self.traceLogger.debug("Updating {} caches".format(
len(self.caches)))
for c in iter(self.caches):
c._updatePriority(c._last_access)
new_caches.add(c)
self.caches = new_caches
gc.collect()
self.traceLogger.debug("Target mem usage: {}".format(
self._target_usage))
while mem_usage > self._target_usage and len(
self.caches) > 0:
self.traceLogger.debug(
"Mem usage: {}".format(mem_usage))
last_cache = self.caches.pop(-1)
freed = last_cache._freeMemory(refcheck=True)
self.traceLogger.debug("Freed: {}".format(freed))
mem_usage = memoryUsagePercentage()
count += 1
if freed == 0:
# store the caches which could not be freed
not_freed.append(last_cache)
gc.collect()
self.logger.info("freed %d/%d blocks, new usage = %f%%" %
(count, old_length, mem_usage))
for c in not_freed:
# add the caches which could not be freed
self.add(c)
class CacheMemoryManager(with_metaclass(Singleton, threading.Thread)):
"""
class for the management of cache memory
The cache memory manager is a background thread that observes caches
in use and cleans them up when the total memory consumption by the
process exceeds the limit defined by the `lazyflow.utility.Memory`
class. See the definition of the cache interfaces (opCache.py) to
get an overview over the possible caches.
Usage:
This manager is a singleton - just call its constructor somewhere
and you will get a reference to the *only* running memory management
thread.
Interface:
The manager provides a signal you can subscribe to::
def writeFunction(x): print("total mem: {}".format(x))
mgr = CacheMemoryManager()
mgr.totalCacheMemory.subscribe(writeFunction)
which emits the size of all observable caches, combined, in regular
intervals.
The update interval (for the signal and for automated cache release)
can be set with a call to a class method::
CacheMemoryManager().setRefreshInterval(5)
the interval is measured in seconds. Each change of refresh interval
triggers cleanup.
"""
totalCacheMemory = OrderedSignal()
def __init__(self):
threading.Thread.__init__(self)
self.daemon = True
self._caches = weakref.WeakSet()
self._first_class_caches = weakref.WeakSet()
self._observable_caches = weakref.WeakSet()
self._managed_caches = weakref.WeakSet()
self._managed_blocked_caches = weakref.WeakSet()
self._condition = threading.Condition()
self._disable_lock = threading.Condition()
self._disabled = False
self._refresh_interval = default_refresh_interval
self._first_class_caches_lock = threading.Lock()
# maximum fraction of *allowed memory* used
self._max_usage = 1.0
# target usage fraction
self._target_usage = 0.90
self._stopped = False
self.start()
atexit.register(self.stop)
def addFirstClassCache(self, cache):
"""
add a first class cache (root cache) to the manager
First class caches are handled differently so we are able to
show a tree view of the caches (e.g. in ilastik). This method
calls addCache() automatically.
"""
# late import to prevent import loop
from lazyflow.operators.opCache import Cache
if isinstance(cache, Cache):
with self._first_class_caches_lock:
self._first_class_caches.add(cache)
self.addCache(cache)
def getFirstClassCaches(self):
"""
get a list of first class caches
"""
with self._first_class_caches_lock:
return list(self._first_class_caches)
def getCaches(self):
"""
get a list of all caches (including first class caches)
"""
return list(self._caches)
def addCache(self, cache):
"""
add a cache to be managed
Caches are kept with weak references, so there is no need to
remove them from the manager.
"""
# late import to prevent import loop
from lazyflow.operators.opCache import Cache
from lazyflow.operators.opCache import ObservableCache
from lazyflow.operators.opCache import ManagedCache
from lazyflow.operators.opCache import ManagedBlockedCache
assert isinstance(cache, Cache), "Only Cache instances can be managed by CacheMemoryManager"
self._caches.add(cache)
if isinstance(cache, ObservableCache):
self._observable_caches.add(cache)
if isinstance(cache, ManagedBlockedCache):
self._managed_blocked_caches.add(cache)
elif isinstance(cache, ManagedCache):
self._managed_caches.add(cache)
def run(self):
"""
main loop
"""
while not self._stopped:
self._wait()
# acquire lock so that we don't get disabled during cleanup
with self._disable_lock:
if self._disabled or self._stopped:
continue
self._cleanup()
def _cleanup(self):
"""
clean up once
"""
from lazyflow.operators.opCache import ObservableCache
try:
# notify subscribed functions about current cache memory
total = 0
# Avoid "RuntimeError: Set changed size during iteration"
with self._first_class_caches_lock:
first_class_caches = self._first_class_caches.copy()
for cache in first_class_caches:
if isinstance(cache, ObservableCache):
total += cache.usedMemory()
self.totalCacheMemory(total)
cache = None
# check current memory state
cache_memory = Memory.getAvailableRamCaches()
cache_pct = 0.0
if cache_memory:
cache_pct = total * 100.0 / cache_memory
logger.debug(
"Process memory usage is {:0.2f} GB out of {:0.2f} (caches are {}, {:.1f}% of allowed)".format(
Memory.getMemoryUsage() / 2.0 ** 30,
Memory.getAvailableRam() / 2.0 ** 30,
Memory.format(total),
cache_pct,
)
)
if total <= self._max_usage * cache_memory:
return
cache_entries = []
cache_entries += [
(cache.lastAccessTime(), cache.name, cache.freeMemory) for cache in list(self._managed_caches)
]
cache_entries += [
(lastAccessTime, f"{cache.name}: {blockKey}", functools.partial(cache.freeBlock, blockKey))
for cache in list(self._managed_blocked_caches)
for blockKey, lastAccessTime in cache.getBlockAccessTimes()
]
cache_entries.sort(key=lambda entry: entry[0])
for lastAccessTime, info, cleanupFun in cache_entries:
if total <= self._target_usage * cache_memory:
break
mem = cleanupFun()
logger.debug(f"Cleaned up {info} ({Memory.format(mem)})")
total -= mem
# Remove references to cache entries before triggering garbage collection.
cleanupFun = None
cache_entries = None
gc.collect()
msg = "Done cleaning up, cache memory usage is now at {}".format(Memory.format(total))
if cache_memory > 0:
msg += " ({:.1f}% of allowed)".format(total * 100.0 / cache_memory)
logger.debug(msg)
except:
log_exception(logger)
def _wait(self):
"""
sleep for _refresh_interval seconds or until woken up
"""
with self._condition:
self._condition.wait(self._refresh_interval)
def stop(self):
"""
Stop the memory manager thread in preparation for app exit.
"""
self._stopped = True
with self._condition:
self._condition.notify()
self.join()
def setRefreshInterval(self, t):
"""
set the clean up period and wake up the cleaning thread
"""
with self._condition:
self._refresh_interval = t
self._condition.notifyAll()
def disable(self):
"""
disable all memory management
This method blocks until current memory management tasks are finished.
"""
with self._disable_lock:
self._disabled = True
def enable(self):
"""
enable cache management and wake the thread
"""
with self._disable_lock:
self._disabled = False
with self._condition:
self._condition.notifyAll()
def __init__(self, *args, **kwargs):
super(OpExportMultipageTiff, self).__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
self._opReorderAxes = OpReorderAxes(parent=self)
self._opReorderAxes.Input.connect(self.Input)
class CacheMemoryManager(threading.Thread):
"""
class for the management of cache memory
The cache memory manager is a background thread that observes caches
in use and cleans them up when the total memory consumption by the
process exceeds the limit defined by the `lazyflow.utility.Memory`
class. See the definition of the cache interfaces (opCache.py) to
get an overview over the possible caches.
Usage:
This manager is a singleton - just call its constructor somewhere
and you will get a reference to the *only* running memory management
thread.
Interface:
The manager provides a signal you can subscribe to
>>> def writeFunction(x): print("total mem: {}".format(x))
>>> mgr = CacheMemoryManager()
>>> mgr.totalCacheMemory.subscribe(writeFunction)
which emits the size of all observable caches, combined, in regular
intervals.
The update interval (for the signal and for automated cache release)
can be set with a call to a class method
>>> CacheMemoryManager().setRefreshInterval(5)
the interval is measured in seconds. Each change of refresh interval
triggers cleanup.
"""
__metaclass__ = Singleton
totalCacheMemory = OrderedSignal()
def __init__(self):
threading.Thread.__init__(self)
self.daemon = True
self._caches = weakref.WeakSet()
self._first_class_caches = weakref.WeakSet()
self._observable_caches = weakref.WeakSet()
self._managed_caches = weakref.WeakSet()
self._managed_blocked_caches = weakref.WeakSet()
self._condition = threading.Condition()
self._disable_lock = threading.Condition()
self._disabled = False
self._refresh_interval = default_refresh_interval
self._first_class_caches_lock = threading.Lock()
# maximum fraction of *allowed memory* used
self._max_usage = 1.0
# target usage fraction
self._target_usage = .90
self._stopped = False
self.start()
atexit.register(self.stop)
def addFirstClassCache(self, cache):
"""
add a first class cache (root cache) to the manager
First class caches are handled differently so we are able to
show a tree view of the caches (e.g. in ilastik). This method
calls addCache() automatically.
"""
# late import to prevent import loop
from lazyflow.operators.opCache import Cache
if isinstance(cache, Cache):
with self._first_class_caches_lock:
self._first_class_caches.add(cache)
self.addCache(cache)
def getFirstClassCaches(self):
"""
get a list of first class caches
"""
with self._first_class_caches_lock:
return list(self._first_class_caches)
def getCaches(self):
"""
get a list of all caches (including first class caches)
"""
return list(self._caches)
def addCache(self, cache):
"""
add a cache to be managed
Caches are kept with weak references, so there is no need to
remove them from the manager.
"""
# late import to prevent import loop
from lazyflow.operators.opCache import Cache
from lazyflow.operators.opCache import ObservableCache
from lazyflow.operators.opCache import ManagedCache
from lazyflow.operators.opCache import ManagedBlockedCache
assert isinstance(cache, Cache),\
"Only Cache instances can be managed by CacheMemoryManager"
self._caches.add(cache)
if isinstance(cache, ObservableCache):
self._observable_caches.add(cache)
if isinstance(cache, ManagedBlockedCache):
self._managed_blocked_caches.add(cache)
elif isinstance(cache, ManagedCache):
self._managed_caches.add(cache)
def run(self):
"""
main loop
"""
while not self._stopped:
self._wait()
# acquire lock so that we don't get disabled during cleanup
with self._disable_lock:
if self._disabled or self._stopped:
continue
self._cleanup()
def _cleanup(self):
"""
clean up once
"""
from lazyflow.operators.opCache import ObservableCache
try:
# notify subscribed functions about current cache memory
total = 0
# Avoid "RuntimeError: Set changed size during iteration"
with self._first_class_caches_lock:
first_class_caches = self._first_class_caches.copy()
for cache in first_class_caches:
if isinstance(cache, ObservableCache):
total += cache.usedMemory()
self.totalCacheMemory(total)
cache = None
# check current memory state
cache_memory = Memory.getAvailableRamCaches()
if total <= self._max_usage * cache_memory:
return
# === we need a cache cleanup ===
# queue holds time stamps and cleanup functions
q = PriorityQueue()
caches = list(self._managed_caches)
for c in caches:
q.push((c.lastAccessTime(), c.name, c.freeMemory))
caches = list(self._managed_blocked_caches)
for c in caches:
for k, t in c.getBlockAccessTimes():
cleanupFun = functools.partial(c.freeBlock, k)
info = "{}: {}".format(c.name, k)
q.push((t, info, cleanupFun))
c = None
caches = None
msg = "Caches are using {} memory".format(
Memory.format(total))
if cache_memory > 0:
msg += " ({:.1f}% of allowed)".format(
total*100.0/cache_memory)
logger.debug(msg)
while (total > self._target_usage * cache_memory
and len(q) > 0):
t, info, cleanupFun = q.pop()
mem = cleanupFun()
logger.debug("Cleaned up {} ({})".format(
info, Memory.format(mem)))
total -= mem
gc.collect()
# don't keep a reference until next loop iteration
cleanupFun = None
q = None
msg = ("Done cleaning up, cache memory usage is now at "
"{}".format(Memory.format(total)))
if cache_memory > 0:
msg += " ({:.1f}% of allowed)".format(
total*100.0/cache_memory)
logger.debug(msg)
except:
log_exception(logger)
def _wait(self):
"""
sleep for _refresh_interval seconds or until woken up
"""
with self._condition:
self._condition.wait(self._refresh_interval)
def stop(self):
"""
Stop the memory manager thread in preparation for app exit.
"""
self._stopped = True
with self._condition:
self._condition.notify()
self.join()
def setRefreshInterval(self, t):
"""
set the clean up period and wake up the cleaning thread
"""
with self._condition:
self._refresh_interval = t
self._condition.notifyAll()
def disable(self):
"""
disable all memory management
This method blocks until current memory management tasks are finished.
"""
with self._disable_lock:
self._disabled = True
def enable(self):
"""
enable cache management and wake the thread
"""
with self._disable_lock:
self._disabled = False
with self._condition:
self._condition.notifyAll()
class ExportFile(object):
ExportProgress = OrderedSignal()
InsertionProgress = OrderedSignal()
def __init__(self, file_name):
self.file_name = file_name
self.table_dict = {}
self.meta_dict = {}
def add_columns(self, table_name, col_data, mode, extra=None):
"""
Adds new columns to the table ( creates the table if neccessary )
:param table_name: the table name
:type table_name: str
:param col_data: the actual data to be added
:type col_data: list, dict, numpy.array, whatever is supported
:param mode: the type of the table data
:type mode: exportFile.Mode
:param extra: extra information for the given mode
:type extra: dict
"""
if extra is None:
extra = {}
if mode == Mode.IlastikTrackingTable:
if not "counts" in extra or not "max" in extra:
raise AttributeError("Tracking need 'counts', 'max' extra")
columns = flatten_tracking_table(col_data, extra["extra ids"],
extra["counts"], extra["max"],
extra["range"])
elif mode == Mode.List:
if not "names" in extra:
raise AttributeError(
"[Tuple]List needs a tuple for the column name (extra 'names')"
)
dtypes = extra["dtypes"] if "dtypes" in extra else None
columns = prepare_list(col_data, extra["names"], dtypes)
elif mode == Mode.IlastikFeatureTable:
if "selection" not in extra:
raise AttributeError(
"IlastikFeatureTable needs a feature selection (extra 'selection')"
)
columns = flatten_ilastik_feature_table(col_data,
extra["selection"],
self.InsertionProgress)
elif mode == Mode.NumpyStructArray:
columns = col_data
else:
raise AttributeError("Invalid Mode")
self._add_columns(table_name, columns)
def add_rois(self,
table_path,
image_slot,
feature_table_name,
margin,
type_="image"):
"""
Adds the rois as images to the table
:param table_path: the new name for the table
:type table_path: str
:param image_slot: the slot to read the data from
:type image_slot: lazyflow.slot.Slot
:param feature_table_name: the already added feature table to read the coords from
:type feature_table_name: str
:param margin: the margin to be added around the images
:type margin: int
:param type_: "image" for normal images, "labeling" for labeling images
:type type_: str
"""
assert type_ in ("labeling",
"image"), "Type must be 'labeling' or 'image'"
slicings = create_slicing(image_slot.meta.axistags,
image_slot.meta.shape, margin,
self.table_dict[feature_table_name])
self.InsertionProgress(0)
if type_ == "labeling":
vec = self._normalize
else:
vec = lambda _: lambda y: y
for i, (slicing, oid) in enumerate(slicings):
roi = image_slot(slicing).wait()
roi = vec(oid)(roi)
roi_path = table_path.format(i)
self.meta_dict[roi_path] = {
"type":
type_,
"axistags":
actual_axistags(image_slot.meta.axistags, roi.shape).toJSON()
}
self.table_dict[roi_path] = roi.squeeze()
self.InsertionProgress(
100 * i / self.table_dict[feature_table_name].shape[0])
self.InsertionProgress(100)
@staticmethod
def _normalize(oid):
def f(pixel_value):
return 1 if pixel_value == oid else 0
return np.vectorize(f)
def add_image(self, table, image_slot):
"""
Adds an image as a table
:param table: the name for the image
:type table: str
:param image_slot: the slot to read the image from
:type image_slot: lazyflow.slot.Slot
"""
self.table_dict[table] = image_slot([]).wait().squeeze()
self.meta_dict[table] = {
"type":
"image",
"axistags":
actual_axistags(image_slot.meta.axistags,
image_slot.meta.shape).toJSON()
}
def update_meta(self, table, meta):
"""
Adds meta information to the table
:param table: the table to add meta to
:type table: str
:param meta: the meta information to add
:type meta: dict
"""
self.meta_dict.setdefault(table, {})
self.meta_dict[table].update(meta)
def write_all(self, mode, compression=None):
"""
Writes all tables to the file
:param mode: "h[d[f]]5" or "csv" at the moment
:type mode: str
:param compression: the compression settings
:type compression: dict
"""
count = 0
self.ExportProgress(0)
if mode in ("h5", "hd5", "hdf5"):
with h5py.File(self.file_name, "w") as fout:
for table_name, table in self.table_dict.iteritems():
self._make_h5_dataset(
fout, table_name, table,
self.meta_dict.get(table_name, {}),
compression if compression is not None else {})
count += 1
self.ExportProgress(count * 100 / len(self.table_dict))
elif mode == "csv":
f_name = self.file_name.rsplit(".", 1)
if len(f_name) == 1:
base, ext = f_name, ""
else:
base, ext = f_name
file_names = []
for table_name, table in self.table_dict.iteritems():
file_names.append("{name}_{table}.{ext}".format(
name=base, table=table_name, ext=ext))
with open(file_names[-1], "w") as fout:
self._make_csv_table(fout, table)
count += 1
self.ExportProgress(count * 100 / len(self.table_dict))
if False:
with ZipFile("{name}.zip".format(name=base), "w") as zip_file:
for file_name in file_names:
zip_file.write(file_name)
self.ExportProgress(100)
logger.info("exported %i tables" % count)
def _add_columns(self, table_name, columns):
if table_name in self.table_dict.iterkeys():
old = self.table_dict[table_name]
columns = nlr.merge_arrays((old, columns), flatten=True)
self.table_dict[table_name] = columns
@staticmethod
def _make_h5_dataset(fout, table_name, table, meta, compression):
try:
dset = fout.create_dataset(table_name,
table.shape,
data=table,
**compression)
except TypeError:
dset = fout.create_dataset(table_name, table.shape, data=table)
for k, v in meta.iteritems():
dset.attrs[k] = v
@staticmethod
def _make_csv_table(fout, table):
line = ",".join(table.dtype.names)
fout.write(line)
fout.write("\n")
for row in table:
line = ",".join(map(str, row))
fout.write(line)
fout.write("\n")
def __init__(self, *args, **kwargs):
super(OpResize5D, self).__init__(*args, **kwargs)
self._input_to_output_scales = None
self.progressSignal = OrderedSignal()
def __init__(self, *args, **kwargs):
super(OpExportToArray, self).__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
def __init__(self, *args, **kwargs):
super(OpConcatenateFeatureMatrices, self).__init__(*args, **kwargs)
self._dirty_slots = set()
self.progressSignal = OrderedSignal()
