def _getOpenHdf5Blockfile(self, blockFilePath):
"""
Return a handle to the open hdf5File at the given path.
If we haven't opened the file yet, open it first.
"""
# Try once without locking
if blockFilePath in list(self._openBlockFiles.keys()):
return self._openBlockFiles[blockFilePath]
# Obtain the lock and try again
with self._lock:
if blockFilePath not in list(self._openBlockFiles.keys()):
try:
writeLock = FileLock(blockFilePath, timeout=10)
if self.mode == "a":
acquired = writeLock.acquire(blocking=False)
assert acquired, "Couldn't obtain an exclusive lock for writing to file: {}".format(
blockFilePath
)
self._fileLocks[blockFilePath] = writeLock
elif self.mode == "r":
assert writeLock.available(), "Can't read from a file that is being written to elsewhere."
else:
assert False, "Unsupported mode"
self._openBlockFiles[blockFilePath] = h5py.File(blockFilePath, self.mode)
except:
log_exception(logger, "Couldn't open {}".format(blockFilePath))
raise
return self._openBlockFiles[blockFilePath]
def _downloadFromQueue(self, num_blocks, blockQueue, failedBlockQueue):
"""
Helper function for downloadAllBlocks(), above.
"""
try:
while not blockQueue.empty():
block_start = blockQueue.get(block=False)
entire_block_roi = self.getEntireBlockRoi(block_start) # Roi of this whole block within the whole dataset
blockFilePathComponents = self.getDatasetPathComponents( block_start )
# Obtain lock
fileLock = FileLock( blockFilePathComponents.externalPath )
if not fileLock.acquire(False):
failedBlockQueue.put( block_start )
else:
try:
# Download the block
# (This function releases the lock for us.)
self._downloadBlock(fileLock, entire_block_roi, blockFilePathComponents)
logger.debug( "Finished downloading {}/{}".format( num_blocks-blockQueue.qsize(), num_blocks ) )
except:
if fileLock.locked():
fileLock.release()
self.setBlockStatus(entire_block_roi[0], BlockwiseFileset.BLOCK_NOT_AVAILABLE)
failedBlockQueue.put( block_start )
raise
except Queue.Empty:
return
def _waitForBlocks(self, block_starts):
"""
Initiate downloads for those blocks that need it.
(Some blocks in the list may already be downloading.)
Then wait for all necessary downloads to complete (including the ones that we didn't initiate).
"""
# Only wait for those that are missing.
missing_blocks = []
for block_start in block_starts:
if self.getBlockStatus(
block_start) == BlockwiseFileset.BLOCK_NOT_AVAILABLE:
missing_blocks.append(block_start)
# Start by creating all necessary directories.
self._ensureDirectoriesExist(missing_blocks)
# Attempt to lock each path we need to create.
# Locks we fail to obtain are already being fetched by other processes, which is okay.
acquired_locks = []
unobtained_locks = []
for block_start in missing_blocks:
entire_block_roi = self.getEntireBlockRoi(
block_start
) # Roi of this whole block within the whole dataset
blockFilePathComponents = self.getDatasetPathComponents(
block_start)
fileLock = FileLock(blockFilePathComponents.externalPath)
if fileLock.acquire(False):
acquired_locks.append((entire_block_roi, fileLock))
else:
unobtained_locks.append((entire_block_roi, fileLock))
# We are now responsible for downloading the data for the file paths we were able to lock.
# Start a separate thread for each.
downloadThreads = []
for block_roi, fileLock in acquired_locks:
blockFilePathComponents = self.getDatasetPathComponents(
block_roi[0])
th = threading.Thread(
target=functools.partial(self._downloadBlock, fileLock,
block_roi, blockFilePathComponents))
downloadThreads.append(th)
# Start all the threads
for th in downloadThreads:
th.start()
# Wait for them all to complete
for th in downloadThreads:
th.join()
# Finally, wait for the blocks that we COULDN'T lock (they must be downloading in other processes somewhere...)
for block_roi, fileLock in unobtained_locks:
while self.getBlockStatus(
block_roi[0]) == BlockwiseFileset.BLOCK_NOT_AVAILABLE:
time.sleep(5)
def isBlockLocked(self, blockstart):
"""
Return True if the block is locked for writing.
Note that both 'available' and 'not available' blocks might be locked.
"""
datasetPathComponents = self.getDatasetPathComponents(blockstart)
hdf5FilePath = datasetPathComponents.externalPath
testLock = FileLock(hdf5FilePath)
return not testLock.available()
def _waitForBlocks(self, block_starts):
"""
Initiate downloads for those blocks that need it.
(Some blocks in the list may already be downloading.)
Then wait for all necessary downloads to complete (including the ones that we didn't initiate).
"""
# Only wait for those that are missing.
missing_blocks = []
for block_start in block_starts:
if self.getBlockStatus(block_start) == BlockwiseFileset.BLOCK_NOT_AVAILABLE:
missing_blocks.append(block_start)
# Start by creating all necessary directories.
self._ensureDirectoriesExist(missing_blocks)
# Attempt to lock each path we need to create.
# Locks we fail to obtain are already being fetched by other processes, which is okay.
acquired_locks = []
unobtained_locks = []
for block_start in missing_blocks:
entire_block_roi = self.getEntireBlockRoi(block_start) # Roi of this whole block within the whole dataset
blockFilePathComponents = self.getDatasetPathComponents(block_start)
fileLock = FileLock(blockFilePathComponents.externalPath)
if fileLock.acquire(False):
acquired_locks.append((entire_block_roi, fileLock))
else:
unobtained_locks.append((entire_block_roi, fileLock))
# We are now responsible for downloading the data for the file paths we were able to lock.
# Start a separate thread for each.
downloadThreads = []
for block_roi, fileLock in acquired_locks:
blockFilePathComponents = self.getDatasetPathComponents(block_roi[0])
th = threading.Thread(
target=functools.partial(self._downloadBlock, fileLock, block_roi, blockFilePathComponents)
)
downloadThreads.append(th)
# Start all the threads
for th in downloadThreads:
th.start()
# Wait for them all to complete
for th in downloadThreads:
th.join()
# Finally, wait for the blocks that we COULDN'T lock (they must be downloading in other processes somewhere...)
for block_roi, fileLock in unobtained_locks:
while self.getBlockStatus(block_roi[0]) == BlockwiseFileset.BLOCK_NOT_AVAILABLE:
time.sleep(5)
def purgeAllLocks(self):
"""
Clears all .lock files from the local blockwise fileset.
This may be necessary if previous processes crashed or were killed while some blocks were downloading.
You must ensure that this is NOT called while more than one process (or thread) has access to the fileset.
For example, in a master/worker situation, call this only from the master, before the workers have been started.
"""
found_lock = False
view_shape = self.description.view_shape
view_roi = ([0] * len(view_shape), view_shape)
block_starts = list(getIntersectingBlocks(self.description.block_shape, view_roi))
for block_start in block_starts:
blockFilePathComponents = self.getDatasetPathComponents(block_start)
fileLock = FileLock(blockFilePathComponents.externalPath)
found_lock |= fileLock.purge()
if found_lock:
logger.warning("Purged lock for block: {}".format(tuple(block_start)))
return found_lock
def _downloadFromQueue(self, num_blocks, blockQueue, failedBlockQueue):
"""
Helper function for downloadAllBlocks(), above.
"""
try:
while not blockQueue.empty():
block_start = blockQueue.get(block=False)
entire_block_roi = self.getEntireBlockRoi(
block_start
) # Roi of this whole block within the whole dataset
blockFilePathComponents = self.getDatasetPathComponents(
block_start)
# Obtain lock
fileLock = FileLock(blockFilePathComponents.externalPath)
if not fileLock.acquire(False):
failedBlockQueue.put(block_start)
else:
try:
# Download the block
# (This function releases the lock for us.)
self._downloadBlock(fileLock, entire_block_roi,
blockFilePathComponents)
logger.debug("Finished downloading {}/{}".format(
num_blocks - blockQueue.qsize(), num_blocks))
except:
if fileLock.locked():
fileLock.release()
self.setBlockStatus(
entire_block_roi[0],
BlockwiseFileset.BLOCK_NOT_AVAILABLE)
failedBlockQueue.put(block_start)
raise
except Queue.Empty:
return
class OpColorizeLabels(Operator):
name = "OpColorizeLabels"
category = "display adaptor"
Input = InputSlot()
OverrideColors = InputSlot(stype='object', value={0 : (0,0,0,0)} ) # dict of { label : (R,G,B,A) }
# By default, label 0 is black and transparent
Output = OutputSlot() # 4 channels: RGBA
colortable = None
_lock = threading.Lock()
def __init__(self, *args, **kwargs):
super(OpColorizeLabels, self).__init__(*args, **kwargs)
self.overrideColors = {}
with OpColorizeLabels._lock:
if OpColorizeLabels.colortable is None:
OpColorizeLabels.colortable = OpColorizeLabels.generateColortable(2**22)
# Pre-generate the table of data
self.colortable = copy.copy(OpColorizeLabels.colortable)
def setupOutputs(self):
inputTags = self.Input.meta.axistags
inputShape = self.Input.meta.shape
self.channelIndex = inputTags.index('c')
assert inputShape[self.channelIndex] == 1, "Input must be a single-channel label image"
self.Output.meta.assignFrom(self.Input.meta)
applyToChannel = partial(applyToElement, inputTags, 'c')
self.Output.meta.shape = applyToChannel(inputShape, 4) # RGBA
self.Output.meta.dtype = numpy.uint8
self.Output.meta.drange = (0, 255)
self.Output.meta.axistags["c"].description = "rgba"
newOverrideColors = self.OverrideColors.value
if newOverrideColors != self.overrideColors:
# Add new overrides
for label, color in newOverrideColors.items():
if label not in self.overrideColors:
self.colortable[label] = color
# Replace removed overrides with their original random values
for label, color in self.overrideColors.items():
if label not in newOverrideColors:
self.colortable[label] = OpColorizeLabels.colortable[label]
self.overrideColors = newOverrideColors
def setStartToZero(self,start,stop):
start = [0]*len(start)
stop = [end-begin for begin,end in zip(start,stop)]
start = TinyVector(start)
stop = TinyVector(stop)
return start,stop
def execute(self, slot, subindex, roi, result):
fullKey = roi.toSlice()
roiCopy = copy.copy(roi)
roiCopy.start, roiCopy.stop = self.setStartToZero(roi.start, roi.stop)
resultKey = roiCopy.toSlice()
# Input has only one channel
thinKey = applyToElement(self.Input.meta.axistags, 'c', fullKey, slice(0,1))
inputData = self.Input[thinKey].wait()
dropChannelKey = applyToElement(self.Input.meta.axistags, 'c', resultKey, 0)
channellessInput = inputData[dropChannelKey]
# Advanced indexing with colortable applies the relabeling from labels to colors.
# If we get an error here, we may need to expand the colortable (currently supports only 2**22 labels.)
channelSlice = getElement(self.Input.meta.axistags, 'c', fullKey)
# channellessInput % self.colortable.shape[0]
channellessInput &= self.colortable.shape[0]-1 # Cheaper than mod for 2**X
result[...] = self.colortable[:, channelSlice][channellessInput]
return result
@staticmethod
def generateColortable(size):
lg = math.log(size, 2)
# The execute function makes this assumption, so check it.
assert lg == math.floor(lg), "Colortable size must be a power of 2."
lazyflowSettingsDir = os.path.expanduser('~/.lazyflow')
try:
if not os.path.exists( lazyflowSettingsDir ):
os.makedirs( lazyflowSettingsDir )
except Exception, ex:
import warnings
warnings.warn("Not able to create dir: ~/.lazyflow. Writing random_color_table.npy to /tmp instead.")
# Write to a temporary directory.
lazyflowSettingsDir = tempfile.mkdtemp()
cachedColortablePath = os.path.join(lazyflowSettingsDir, 'random_color_table.npy')
with FileLock(cachedColortablePath):
# If possible, load the table from disk
loadedTable = False
if os.path.exists( cachedColortablePath ):
try:
table = numpy.load(cachedColortablePath)
if table.shape[0] == size:
loadedTable = True
else:
table = None
except:
table = None
if not loadedTable:
randState = numpy.random.mtrand.RandomState(0)
table = numpy.zeros((size,4), dtype=numpy.uint8)
table[...] = randState.random_integers( 0, 255, table.shape )
table[...,3] = 255 # Alpha is 255 by default.
# Save it for next session
saved = False
ex = None
try:
if not os.path.exists( lazyflowSettingsDir ):
os.makedirs( lazyflowSettingsDir )
except Exception, ex:
pass
else:
try:
numpy.save(cachedColortablePath, table)
saved = True
except Exception, ex:
pass
if not saved:
# It's not worth crashing if the table can't be cached.
logger.warn( "Wasn't able to create cache file: " + cachedColortablePath )
logger.warn( "Caught exception: " + str(ex) )
