def _downloadFromQueue(self, num_blocks, blockQueue, failedBlockQueue):
"""
Helper function for downloadAllBlocks(), above.
"""
try:
while not blockQueue.empty():
block_start = blockQueue.get(block=False)
entire_block_roi = self.getEntireBlockRoi(
block_start
) # Roi of this whole block within the whole dataset
blockFilePathComponents = self.getDatasetPathComponents(
block_start)
# Obtain lock
fileLock = FileLock(blockFilePathComponents.externalPath)
if not fileLock.acquire(False):
failedBlockQueue.put(block_start)
else:
try:
# Download the block
# (This function releases the lock for us.)
self._downloadBlock(fileLock, entire_block_roi,
blockFilePathComponents)
logger.debug("Finished downloading {}/{}".format(
num_blocks - blockQueue.qsize(), num_blocks))
except:
if fileLock.locked():
fileLock.release()
self.setBlockStatus(
entire_block_roi[0],
BlockwiseFileset.BLOCK_NOT_AVAILABLE)
failedBlockQueue.put(block_start)
raise
except Queue.Empty:
return
def _downloadFromQueue(self, num_blocks, blockQueue, failedBlockQueue):
"""
Helper function for downloadAllBlocks(), above.
"""
try:
while not blockQueue.empty():
block_start = blockQueue.get(block=False)
entire_block_roi = self.getEntireBlockRoi(block_start) # Roi of this whole block within the whole dataset
blockFilePathComponents = self.getDatasetPathComponents( block_start )
# Obtain lock
fileLock = FileLock( blockFilePathComponents.externalPath )
if not fileLock.acquire(False):
failedBlockQueue.put( block_start )
else:
try:
# Download the block
# (This function releases the lock for us.)
self._downloadBlock(fileLock, entire_block_roi, blockFilePathComponents)
logger.debug( "Finished downloading {}/{}".format( num_blocks-blockQueue.qsize(), num_blocks ) )
except:
if fileLock.locked():
fileLock.release()
self.setBlockStatus(entire_block_roi[0], BlockwiseFileset.BLOCK_NOT_AVAILABLE)
failedBlockQueue.put( block_start )
raise
except Queue.Empty:
return
