def _downloadFromQueue(self, num_blocks, blockQueue, failedBlockQueue): """ Helper function for downloadAllBlocks(), above. """ try: while not blockQueue.empty(): block_start = blockQueue.get(block=False) entire_block_roi = self.getEntireBlockRoi( block_start ) # Roi of this whole block within the whole dataset blockFilePathComponents = self.getDatasetPathComponents( block_start) # Obtain lock fileLock = FileLock(blockFilePathComponents.externalPath) if not fileLock.acquire(False): failedBlockQueue.put(block_start) else: try: # Download the block # (This function releases the lock for us.) self._downloadBlock(fileLock, entire_block_roi, blockFilePathComponents) logger.debug("Finished downloading {}/{}".format( num_blocks - blockQueue.qsize(), num_blocks)) except: if fileLock.locked(): fileLock.release() self.setBlockStatus( entire_block_roi[0], BlockwiseFileset.BLOCK_NOT_AVAILABLE) failedBlockQueue.put(block_start) raise except Queue.Empty: return
def _downloadFromQueue(self, num_blocks, blockQueue, failedBlockQueue): """ Helper function for downloadAllBlocks(), above. """ try: while not blockQueue.empty(): block_start = blockQueue.get(block=False) entire_block_roi = self.getEntireBlockRoi(block_start) # Roi of this whole block within the whole dataset blockFilePathComponents = self.getDatasetPathComponents( block_start ) # Obtain lock fileLock = FileLock( blockFilePathComponents.externalPath ) if not fileLock.acquire(False): failedBlockQueue.put( block_start ) else: try: # Download the block # (This function releases the lock for us.) self._downloadBlock(fileLock, entire_block_roi, blockFilePathComponents) logger.debug( "Finished downloading {}/{}".format( num_blocks-blockQueue.qsize(), num_blocks ) ) except: if fileLock.locked(): fileLock.release() self.setBlockStatus(entire_block_roi[0], BlockwiseFileset.BLOCK_NOT_AVAILABLE) failedBlockQueue.put( block_start ) raise except Queue.Empty: return