def __init__(self, outputSlot, roiIterator, totalVolume=None, batchSize=2, allowParallelResults=False): """ Constructor. :param outputSlot: The slot to request data from. :param roiIterator: An iterator providing new rois. :param totalVolume: The total volume to be processed. Used to provide the progress reporting signal. If not provided, then no intermediate progress will be signaled. :param batchSize: The maximum number of requests to launch in parallel. :param allowParallelResults: If False, The resultSignal will not be called in parallel. In that case, your handler function has no need for locks. """ self._resultSignal = OrderedSignal() self._progressSignal = OrderedSignal() assert isinstance( outputSlot.stype, lazyflow.stype.ArrayLike ), "Only Array-like slots supported." # Because progress reporting depends on the roi shape self._outputSlot = outputSlot self._roiIter = roiIterator self._batchSize = batchSize self._allowParallelResults = allowParallelResults self._condition = SimpleRequestCondition() self._activated_count = 0 self._completed_count = 0 self._failure_excinfo = None # Progress bookkeeping self._totalVolume = totalVolume self._processedVolume = 0
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.progressSignal = OrderedSignal() # Set up the impl function lookup dict export_impls = {} export_impls["hdf5"] = ("h5", self._export_h5n5) export_impls["compressed hdf5"] = ("h5", partial(self._export_h5n5, True)) export_impls["n5"] = ("n5", self._export_h5n5) export_impls["compressed n5"] = ("n5", partial(self._export_h5n5, True)) export_impls["numpy"] = ("npy", self._export_npy) export_impls["dvid"] = ("", self._export_dvid) export_impls["blockwise hdf5"] = ("json", self._export_blockwise_hdf5) for fmt in self._2d_formats: export_impls[fmt.name] = (fmt.extension, partial(self._export_2d, fmt.extension)) for fmt in self._3d_sequence_formats: export_impls[fmt.name] = (fmt.extension, partial(self._export_3d_sequence, fmt.extension)) export_impls["multipage tiff"] = ("tiff", self._export_multipage_tiff) export_impls["multipage tiff sequence"] = ( "tiff", self._export_multipage_tiff_sequence) self._export_impls = export_impls self.Input.notifyMetaChanged(self._updateFormatSelectionErrorMsg)
def __init__(self, *args, **kwargs): super(OpConcatenateFeatureMatrices, self).__init__(*args, **kwargs) self._dirty_slots = set() self.progressSignal = OrderedSignal() self._num_feature_channels = 0 # Not including the labels... self._channel_names = [] # Normally, lane removal does not trigger a dirty notification. # But in this case, if the lane contained any label data whatsoever, # the classifier needs to be marked dirty. # We know which slots contain (or contained) label data because they have # been 'touched' at some point (they became dirty at some point). self._touched_slots = set() def handle_new_lane( multislot, index, newlength ): def handle_dirty_lane( slot, roi ): self._touched_slots.add(slot) multislot[index].notifyDirty( handle_dirty_lane ) self.FeatureMatrices.notifyInserted( handle_new_lane ) def handle_remove_lane( multislot, index, newlength ): # If the lane we're removing contained # label data, then mark the downstream dirty if multislot[index] in self._touched_slots: self.ConcatenatedOutput.setDirty() self._touched_slots.remove(multislot[index]) self.FeatureMatrices.notifyRemove( handle_remove_lane )
def __init__(self, *args, **kwargs): super(OpExportSlot, self).__init__(*args, **kwargs) self.progressSignal = OrderedSignal() # Set up the impl function lookup dict export_impls = {} export_impls['hdf5'] = ('h5', self._export_hdf5) export_impls['npy'] = ('npy', self._export_npy) export_impls['dvid'] = ('', self._export_dvid) export_impls['blockwise hdf5'] = ('json', self._export_blockwise_hdf5) for fmt in self._2d_formats: export_impls[fmt.name] = (fmt.extension, partial(self._export_2d, fmt.extension)) for fmt in self._3d_sequence_formats: export_impls[fmt.name] = (fmt.extension, partial(self._export_3d_sequence, fmt.extension)) export_impls['multipage tiff'] = ('tiff', self._export_multipage_tiff) export_impls['multipage tiff sequence'] = ( 'tiff', self._export_multipage_tiff_sequence) self._export_impls = export_impls self.Input.notifyMetaChanged(self._updateFormatSelectionErrorMsg)
def __init__(self, *args, **kwargs): super(OpWsdt, self).__init__(*args, **kwargs) self.debug_results = None self.watershed_completed = OrderedSignal() self._opSelectedInput = OpSumChannels(parent=self) self._opSelectedInput.ChannelSelections.connect(self.ChannelSelections) self._opSelectedInput.Input.connect(self.Input)
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.progressSignal = OrderedSignal() self._opReorderAxes = OpReorderAxes(parent=self) self._opReorderAxes.Input.connect(self.Input) self._opReorderAxes.AxisOrder.setValue(self._EXPORT_AXES)
def __init__(self, *args, **kwargs): super(OpFeatureMatrixCache, self).__init__(*args, **kwargs) self._lock = RequestLock() self.progressSignal = OrderedSignal() self._progress_lock = RequestLock() self._blockshape = None self._dirty_blocks = set() self._blockwise_feature_matrices = {} self._block_locks = {} # One lock per stored block self._init_blocks(None, None)
def __init__(self, *args, **kwargs): super(OpFeatureMatrixCache, self).__init__(*args, **kwargs) self._blockshape = None self._lock = RequestLock() self.progressSignal = OrderedSignal() self._progress_lock = RequestLock() # In these set/dict members, the block id (dict key) # is simply the block's start coordinate (as a tuple) self._blockwise_feature_matrices = {} self._dirty_blocks = set() self._block_locks = {} # One lock per stored block
def __enter__(self): if self._graph: with self._graph._lock: if self._graph._setup_depth == 0: self._graph._sig_setup_complete = OrderedSignal() self._graph._setup_depth += 1
def __init__(self, transpose_axes, *args, **kwargs): super(OpExportDvidVolume, self).__init__(*args, **kwargs) self.progressSignal = OrderedSignal() self._transpose_axes = transpose_axes
def __init__(self, *args, **kwargs): super(OpNpyWriter, self).__init__(*args, **kwargs) self.progressSignal = OrderedSignal()
def __init__(self, *args, **kwargs): super(OpExportMultipageTiff, self).__init__(*args, **kwargs) self.progressSignal = OrderedSignal()
def __init__(self, *args, **kwargs): super(OpConcatenateFeatureMatrices, self).__init__(*args, **kwargs) self._dirty_slots = set() self.progressSignal = OrderedSignal() self._num_feature_channels = 0 # Not including the labels...
def __init__(self, *args, **kwargs): super(OpWsdt, self).__init__(*args, **kwargs) self.debug_results = None self.watershed_completed = OrderedSignal()
class ArrayCacheMemoryMgr(threading.Thread): totalCacheMemory = OrderedSignal() loggingName = __name__ + ".ArrayCacheMemoryMgr" logger = logging.getLogger(loggingName) traceLogger = logging.getLogger("TRACE." + loggingName) def __init__(self): threading.Thread.__init__(self) self.daemon = True self.caches = self._new_list() self.namedCaches = [] self._max_usage = 85 self._target_usage = 70 self._lock = threading.Lock() self._last_usage = memoryUsagePercentage() def _new_list(self): def getPrio(array_cache): return array_cache._cache_priority return blist.sortedlist((), getPrio) def addNamedCache(self, array_cache): """add a cache to a special list of named caches The list of named caches should contain only top-level caches. This way, when showing memory usage, we can provide a tree-view, where the named caches are the top-level items and the user can then drill down into the caches that are children of the top-level caches. """ self.namedCaches.append(array_cache) def add(self, array_cache): with self._lock: self.caches.add(array_cache) def remove(self, array_cache): with self._lock: try: self.caches.remove(array_cache) except ValueError: pass def run(self): while True: mem_usage = memoryUsagePercentage() mem_usage_gb = memoryUsageGB() delta = abs(self._last_usage - mem_usage) if delta > 10 or self.logger.level == logging.DEBUG: cpu_usages = psutil.cpu_percent(interval=1, percpu=True) avg = sum(cpu_usages) / len(cpu_usages) self.logger.info( "RAM: {:1.3f} GB ({:02.0f}%), CPU: Avg={:02.0f}%, {}". format(mem_usage_gb, mem_usage, avg, cpu_usages)) if delta > 10: self._last_usage = mem_usage #calculate total memory usage and send as signal tot = 0.0 for c in self.namedCaches: if c.usedMemory() is None: continue else: tot += c.usedMemory() self.totalCacheMemory(tot) time.sleep(10) if mem_usage > self._max_usage: self.logger.info("freeing memory...") with self._lock: count = 0 not_freed = [] old_length = len(self.caches) new_caches = self._new_list() self.traceLogger.debug("Updating {} caches".format( len(self.caches))) for c in iter(self.caches): c._updatePriority(c._last_access) new_caches.add(c) self.caches = new_caches gc.collect() self.traceLogger.debug("Target mem usage: {}".format( self._target_usage)) while mem_usage > self._target_usage and len( self.caches) > 0: self.traceLogger.debug( "Mem usage: {}".format(mem_usage)) last_cache = self.caches.pop(-1) freed = last_cache._freeMemory(refcheck=True) self.traceLogger.debug("Freed: {}".format(freed)) mem_usage = memoryUsagePercentage() count += 1 if freed == 0: # store the caches which could not be freed not_freed.append(last_cache) gc.collect() self.logger.info("freed %d/%d blocks, new usage = %f%%" % (count, old_length, mem_usage)) for c in not_freed: # add the caches which could not be freed self.add(c)
class CacheMemoryManager(with_metaclass(Singleton, threading.Thread)): """ class for the management of cache memory The cache memory manager is a background thread that observes caches in use and cleans them up when the total memory consumption by the process exceeds the limit defined by the `lazyflow.utility.Memory` class. See the definition of the cache interfaces (opCache.py) to get an overview over the possible caches. Usage: This manager is a singleton - just call its constructor somewhere and you will get a reference to the *only* running memory management thread. Interface: The manager provides a signal you can subscribe to:: def writeFunction(x): print("total mem: {}".format(x)) mgr = CacheMemoryManager() mgr.totalCacheMemory.subscribe(writeFunction) which emits the size of all observable caches, combined, in regular intervals. The update interval (for the signal and for automated cache release) can be set with a call to a class method:: CacheMemoryManager().setRefreshInterval(5) the interval is measured in seconds. Each change of refresh interval triggers cleanup. """ totalCacheMemory = OrderedSignal() def __init__(self): threading.Thread.__init__(self) self.daemon = True self._caches = weakref.WeakSet() self._first_class_caches = weakref.WeakSet() self._observable_caches = weakref.WeakSet() self._managed_caches = weakref.WeakSet() self._managed_blocked_caches = weakref.WeakSet() self._condition = threading.Condition() self._disable_lock = threading.Condition() self._disabled = False self._refresh_interval = default_refresh_interval self._first_class_caches_lock = threading.Lock() # maximum fraction of *allowed memory* used self._max_usage = 1.0 # target usage fraction self._target_usage = 0.90 self._stopped = False self.start() atexit.register(self.stop) def addFirstClassCache(self, cache): """ add a first class cache (root cache) to the manager First class caches are handled differently so we are able to show a tree view of the caches (e.g. in ilastik). This method calls addCache() automatically. """ # late import to prevent import loop from lazyflow.operators.opCache import Cache if isinstance(cache, Cache): with self._first_class_caches_lock: self._first_class_caches.add(cache) self.addCache(cache) def getFirstClassCaches(self): """ get a list of first class caches """ with self._first_class_caches_lock: return list(self._first_class_caches) def getCaches(self): """ get a list of all caches (including first class caches) """ return list(self._caches) def addCache(self, cache): """ add a cache to be managed Caches are kept with weak references, so there is no need to remove them from the manager. """ # late import to prevent import loop from lazyflow.operators.opCache import Cache from lazyflow.operators.opCache import ObservableCache from lazyflow.operators.opCache import ManagedCache from lazyflow.operators.opCache import ManagedBlockedCache assert isinstance(cache, Cache), "Only Cache instances can be managed by CacheMemoryManager" self._caches.add(cache) if isinstance(cache, ObservableCache): self._observable_caches.add(cache) if isinstance(cache, ManagedBlockedCache): self._managed_blocked_caches.add(cache) elif isinstance(cache, ManagedCache): self._managed_caches.add(cache) def run(self): """ main loop """ while not self._stopped: self._wait() # acquire lock so that we don't get disabled during cleanup with self._disable_lock: if self._disabled or self._stopped: continue self._cleanup() def _cleanup(self): """ clean up once """ from lazyflow.operators.opCache import ObservableCache try: # notify subscribed functions about current cache memory total = 0 # Avoid "RuntimeError: Set changed size during iteration" with self._first_class_caches_lock: first_class_caches = self._first_class_caches.copy() for cache in first_class_caches: if isinstance(cache, ObservableCache): total += cache.usedMemory() self.totalCacheMemory(total) cache = None # check current memory state cache_memory = Memory.getAvailableRamCaches() cache_pct = 0.0 if cache_memory: cache_pct = total * 100.0 / cache_memory logger.debug( "Process memory usage is {:0.2f} GB out of {:0.2f} (caches are {}, {:.1f}% of allowed)".format( Memory.getMemoryUsage() / 2.0 ** 30, Memory.getAvailableRam() / 2.0 ** 30, Memory.format(total), cache_pct, ) ) if total <= self._max_usage * cache_memory: return cache_entries = [] cache_entries += [ (cache.lastAccessTime(), cache.name, cache.freeMemory) for cache in list(self._managed_caches) ] cache_entries += [ (lastAccessTime, f"{cache.name}: {blockKey}", functools.partial(cache.freeBlock, blockKey)) for cache in list(self._managed_blocked_caches) for blockKey, lastAccessTime in cache.getBlockAccessTimes() ] cache_entries.sort(key=lambda entry: entry[0]) for lastAccessTime, info, cleanupFun in cache_entries: if total <= self._target_usage * cache_memory: break mem = cleanupFun() logger.debug(f"Cleaned up {info} ({Memory.format(mem)})") total -= mem # Remove references to cache entries before triggering garbage collection. cleanupFun = None cache_entries = None gc.collect() msg = "Done cleaning up, cache memory usage is now at {}".format(Memory.format(total)) if cache_memory > 0: msg += " ({:.1f}% of allowed)".format(total * 100.0 / cache_memory) logger.debug(msg) except: log_exception(logger) def _wait(self): """ sleep for _refresh_interval seconds or until woken up """ with self._condition: self._condition.wait(self._refresh_interval) def stop(self): """ Stop the memory manager thread in preparation for app exit. """ self._stopped = True with self._condition: self._condition.notify() self.join() def setRefreshInterval(self, t): """ set the clean up period and wake up the cleaning thread """ with self._condition: self._refresh_interval = t self._condition.notifyAll() def disable(self): """ disable all memory management This method blocks until current memory management tasks are finished. """ with self._disable_lock: self._disabled = True def enable(self): """ enable cache management and wake the thread """ with self._disable_lock: self._disabled = False with self._condition: self._condition.notifyAll()
def __init__(self, *args, **kwargs): super(OpExportMultipageTiff, self).__init__(*args, **kwargs) self.progressSignal = OrderedSignal() self._opReorderAxes = OpReorderAxes(parent=self) self._opReorderAxes.Input.connect(self.Input)
class CacheMemoryManager(threading.Thread): """ class for the management of cache memory The cache memory manager is a background thread that observes caches in use and cleans them up when the total memory consumption by the process exceeds the limit defined by the `lazyflow.utility.Memory` class. See the definition of the cache interfaces (opCache.py) to get an overview over the possible caches. Usage: This manager is a singleton - just call its constructor somewhere and you will get a reference to the *only* running memory management thread. Interface: The manager provides a signal you can subscribe to >>> def writeFunction(x): print("total mem: {}".format(x)) >>> mgr = CacheMemoryManager() >>> mgr.totalCacheMemory.subscribe(writeFunction) which emits the size of all observable caches, combined, in regular intervals. The update interval (for the signal and for automated cache release) can be set with a call to a class method >>> CacheMemoryManager().setRefreshInterval(5) the interval is measured in seconds. Each change of refresh interval triggers cleanup. """ __metaclass__ = Singleton totalCacheMemory = OrderedSignal() def __init__(self): threading.Thread.__init__(self) self.daemon = True self._caches = weakref.WeakSet() self._first_class_caches = weakref.WeakSet() self._observable_caches = weakref.WeakSet() self._managed_caches = weakref.WeakSet() self._managed_blocked_caches = weakref.WeakSet() self._condition = threading.Condition() self._disable_lock = threading.Condition() self._disabled = False self._refresh_interval = default_refresh_interval self._first_class_caches_lock = threading.Lock() # maximum fraction of *allowed memory* used self._max_usage = 1.0 # target usage fraction self._target_usage = .90 self._stopped = False self.start() atexit.register(self.stop) def addFirstClassCache(self, cache): """ add a first class cache (root cache) to the manager First class caches are handled differently so we are able to show a tree view of the caches (e.g. in ilastik). This method calls addCache() automatically. """ # late import to prevent import loop from lazyflow.operators.opCache import Cache if isinstance(cache, Cache): with self._first_class_caches_lock: self._first_class_caches.add(cache) self.addCache(cache) def getFirstClassCaches(self): """ get a list of first class caches """ with self._first_class_caches_lock: return list(self._first_class_caches) def getCaches(self): """ get a list of all caches (including first class caches) """ return list(self._caches) def addCache(self, cache): """ add a cache to be managed Caches are kept with weak references, so there is no need to remove them from the manager. """ # late import to prevent import loop from lazyflow.operators.opCache import Cache from lazyflow.operators.opCache import ObservableCache from lazyflow.operators.opCache import ManagedCache from lazyflow.operators.opCache import ManagedBlockedCache assert isinstance(cache, Cache),\ "Only Cache instances can be managed by CacheMemoryManager" self._caches.add(cache) if isinstance(cache, ObservableCache): self._observable_caches.add(cache) if isinstance(cache, ManagedBlockedCache): self._managed_blocked_caches.add(cache) elif isinstance(cache, ManagedCache): self._managed_caches.add(cache) def run(self): """ main loop """ while not self._stopped: self._wait() # acquire lock so that we don't get disabled during cleanup with self._disable_lock: if self._disabled or self._stopped: continue self._cleanup() def _cleanup(self): """ clean up once """ from lazyflow.operators.opCache import ObservableCache try: # notify subscribed functions about current cache memory total = 0 # Avoid "RuntimeError: Set changed size during iteration" with self._first_class_caches_lock: first_class_caches = self._first_class_caches.copy() for cache in first_class_caches: if isinstance(cache, ObservableCache): total += cache.usedMemory() self.totalCacheMemory(total) cache = None # check current memory state cache_memory = Memory.getAvailableRamCaches() if total <= self._max_usage * cache_memory: return # === we need a cache cleanup === # queue holds time stamps and cleanup functions q = PriorityQueue() caches = list(self._managed_caches) for c in caches: q.push((c.lastAccessTime(), c.name, c.freeMemory)) caches = list(self._managed_blocked_caches) for c in caches: for k, t in c.getBlockAccessTimes(): cleanupFun = functools.partial(c.freeBlock, k) info = "{}: {}".format(c.name, k) q.push((t, info, cleanupFun)) c = None caches = None msg = "Caches are using {} memory".format( Memory.format(total)) if cache_memory > 0: msg += " ({:.1f}% of allowed)".format( total*100.0/cache_memory) logger.debug(msg) while (total > self._target_usage * cache_memory and len(q) > 0): t, info, cleanupFun = q.pop() mem = cleanupFun() logger.debug("Cleaned up {} ({})".format( info, Memory.format(mem))) total -= mem gc.collect() # don't keep a reference until next loop iteration cleanupFun = None q = None msg = ("Done cleaning up, cache memory usage is now at " "{}".format(Memory.format(total))) if cache_memory > 0: msg += " ({:.1f}% of allowed)".format( total*100.0/cache_memory) logger.debug(msg) except: log_exception(logger) def _wait(self): """ sleep for _refresh_interval seconds or until woken up """ with self._condition: self._condition.wait(self._refresh_interval) def stop(self): """ Stop the memory manager thread in preparation for app exit. """ self._stopped = True with self._condition: self._condition.notify() self.join() def setRefreshInterval(self, t): """ set the clean up period and wake up the cleaning thread """ with self._condition: self._refresh_interval = t self._condition.notifyAll() def disable(self): """ disable all memory management This method blocks until current memory management tasks are finished. """ with self._disable_lock: self._disabled = True def enable(self): """ enable cache management and wake the thread """ with self._disable_lock: self._disabled = False with self._condition: self._condition.notifyAll()
class ExportFile(object): ExportProgress = OrderedSignal() InsertionProgress = OrderedSignal() def __init__(self, file_name): self.file_name = file_name self.table_dict = {} self.meta_dict = {} def add_columns(self, table_name, col_data, mode, extra=None): """ Adds new columns to the table ( creates the table if neccessary ) :param table_name: the table name :type table_name: str :param col_data: the actual data to be added :type col_data: list, dict, numpy.array, whatever is supported :param mode: the type of the table data :type mode: exportFile.Mode :param extra: extra information for the given mode :type extra: dict """ if extra is None: extra = {} if mode == Mode.IlastikTrackingTable: if not "counts" in extra or not "max" in extra: raise AttributeError("Tracking need 'counts', 'max' extra") columns = flatten_tracking_table(col_data, extra["extra ids"], extra["counts"], extra["max"], extra["range"]) elif mode == Mode.List: if not "names" in extra: raise AttributeError( "[Tuple]List needs a tuple for the column name (extra 'names')" ) dtypes = extra["dtypes"] if "dtypes" in extra else None columns = prepare_list(col_data, extra["names"], dtypes) elif mode == Mode.IlastikFeatureTable: if "selection" not in extra: raise AttributeError( "IlastikFeatureTable needs a feature selection (extra 'selection')" ) columns = flatten_ilastik_feature_table(col_data, extra["selection"], self.InsertionProgress) elif mode == Mode.NumpyStructArray: columns = col_data else: raise AttributeError("Invalid Mode") self._add_columns(table_name, columns) def add_rois(self, table_path, image_slot, feature_table_name, margin, type_="image"): """ Adds the rois as images to the table :param table_path: the new name for the table :type table_path: str :param image_slot: the slot to read the data from :type image_slot: lazyflow.slot.Slot :param feature_table_name: the already added feature table to read the coords from :type feature_table_name: str :param margin: the margin to be added around the images :type margin: int :param type_: "image" for normal images, "labeling" for labeling images :type type_: str """ assert type_ in ("labeling", "image"), "Type must be 'labeling' or 'image'" slicings = create_slicing(image_slot.meta.axistags, image_slot.meta.shape, margin, self.table_dict[feature_table_name]) self.InsertionProgress(0) if type_ == "labeling": vec = self._normalize else: vec = lambda _: lambda y: y for i, (slicing, oid) in enumerate(slicings): roi = image_slot(slicing).wait() roi = vec(oid)(roi) roi_path = table_path.format(i) self.meta_dict[roi_path] = { "type": type_, "axistags": actual_axistags(image_slot.meta.axistags, roi.shape).toJSON() } self.table_dict[roi_path] = roi.squeeze() self.InsertionProgress( 100 * i / self.table_dict[feature_table_name].shape[0]) self.InsertionProgress(100) @staticmethod def _normalize(oid): def f(pixel_value): return 1 if pixel_value == oid else 0 return np.vectorize(f) def add_image(self, table, image_slot): """ Adds an image as a table :param table: the name for the image :type table: str :param image_slot: the slot to read the image from :type image_slot: lazyflow.slot.Slot """ self.table_dict[table] = image_slot([]).wait().squeeze() self.meta_dict[table] = { "type": "image", "axistags": actual_axistags(image_slot.meta.axistags, image_slot.meta.shape).toJSON() } def update_meta(self, table, meta): """ Adds meta information to the table :param table: the table to add meta to :type table: str :param meta: the meta information to add :type meta: dict """ self.meta_dict.setdefault(table, {}) self.meta_dict[table].update(meta) def write_all(self, mode, compression=None): """ Writes all tables to the file :param mode: "h[d[f]]5" or "csv" at the moment :type mode: str :param compression: the compression settings :type compression: dict """ count = 0 self.ExportProgress(0) if mode in ("h5", "hd5", "hdf5"): with h5py.File(self.file_name, "w") as fout: for table_name, table in self.table_dict.iteritems(): self._make_h5_dataset( fout, table_name, table, self.meta_dict.get(table_name, {}), compression if compression is not None else {}) count += 1 self.ExportProgress(count * 100 / len(self.table_dict)) elif mode == "csv": f_name = self.file_name.rsplit(".", 1) if len(f_name) == 1: base, ext = f_name, "" else: base, ext = f_name file_names = [] for table_name, table in self.table_dict.iteritems(): file_names.append("{name}_{table}.{ext}".format( name=base, table=table_name, ext=ext)) with open(file_names[-1], "w") as fout: self._make_csv_table(fout, table) count += 1 self.ExportProgress(count * 100 / len(self.table_dict)) if False: with ZipFile("{name}.zip".format(name=base), "w") as zip_file: for file_name in file_names: zip_file.write(file_name) self.ExportProgress(100) logger.info("exported %i tables" % count) def _add_columns(self, table_name, columns): if table_name in self.table_dict.iterkeys(): old = self.table_dict[table_name] columns = nlr.merge_arrays((old, columns), flatten=True) self.table_dict[table_name] = columns @staticmethod def _make_h5_dataset(fout, table_name, table, meta, compression): try: dset = fout.create_dataset(table_name, table.shape, data=table, **compression) except TypeError: dset = fout.create_dataset(table_name, table.shape, data=table) for k, v in meta.iteritems(): dset.attrs[k] = v @staticmethod def _make_csv_table(fout, table): line = ",".join(table.dtype.names) fout.write(line) fout.write("\n") for row in table: line = ",".join(map(str, row)) fout.write(line) fout.write("\n")
def __init__(self, *args, **kwargs): super(OpResize5D, self).__init__(*args, **kwargs) self._input_to_output_scales = None self.progressSignal = OrderedSignal()
def __init__(self, *args, **kwargs): super(OpExportToArray, self).__init__(*args, **kwargs) self.progressSignal = OrderedSignal()
def __init__(self, *args, **kwargs): super(OpConcatenateFeatureMatrices, self).__init__(*args, **kwargs) self._dirty_slots = set() self.progressSignal = OrderedSignal()