def _determine_blockshape(self, outputSlot): """ Choose a blockshape using the slot metadata (if available) or an arbitrary guess otherwise. """ input_shape = outputSlot.meta.shape ideal_blockshape = outputSlot.meta.ideal_blockshape ram_usage_per_requested_pixel = outputSlot.meta.ram_usage_per_requested_pixel num_channels = 1 tagged_shape = outputSlot.meta.getTaggedShape() # Generally, we don't want to split requests across channels. if 'c' in tagged_shape.keys(): num_channels = tagged_shape['c'] channel_index = tagged_shape.keys().index('c') input_shape = input_shape[:channel_index] + input_shape[channel_index+1:] if ideal_blockshape: # Never enlarge 'ideal' in the channel dimension. num_channels = ideal_blockshape[channel_index] ideal_blockshape = ideal_blockshape[:channel_index] + ideal_blockshape[channel_index+1:] max_blockshape = input_shape available_ram = Memory.getAvailableRamComputation() if ram_usage_per_requested_pixel is None: # Make a conservative guess: 2*(bytes for dtype) * (num channels) + (fudge factor=4) ram_usage_per_requested_pixel = 2*outputSlot.meta.dtype().nbytes*num_channels + 4 warnings.warn( "Unknown per-pixel RAM requirement. Making a guess." ) # Safety factor (fudge factor): Double the estimated RAM usage per pixel safety_factor = 2.0 logger.info("Estimated RAM usage per pixel is {} * safety factor ({})" .format( Memory.format(ram_usage_per_requested_pixel), safety_factor ) ) ram_usage_per_requested_pixel *= safety_factor if ideal_blockshape is None: blockshape = determineBlockShape( input_shape, available_ram/(self._num_threads*ram_usage_per_requested_pixel) ) if 'c' in outputSlot.meta.getAxisKeys(): blockshape = blockshape[:channel_index] + (num_channels,) + blockshape[channel_index:] warnings.warn( "Chose an arbitrary request blockshape {}".format( blockshape ) ) else: logger.info("determining blockshape assuming available_ram is {}" ", split between {} threads" .format(Memory.format(available_ram), self._num_threads)) # By convention, ram_usage_per_requested_pixel refers to the ram used when requesting ALL channels of a 'pixel' # Therefore, we do not include the channel dimension in the blockshapes here. blockshape = determine_optimal_request_blockshape( max_blockshape, ideal_blockshape, ram_usage_per_requested_pixel, self._num_threads, available_ram ) if 'c' in outputSlot.meta.getAxisKeys(): blockshape = blockshape[:channel_index] + (num_channels,) + blockshape[channel_index:] logger.info( "Chose blockshape: {}".format( blockshape ) ) fmt = Memory.format(ram_usage_per_requested_pixel * numpy.prod(blockshape[:-1])) logger.info("Estimated RAM usage per block is {}".format(fmt)) return blockshape
def _determine_blockshape(self, outputSlot): """ Choose a blockshape using the slot metadata (if available) or an arbitrary guess otherwise. """ input_shape = outputSlot.meta.shape max_blockshape = input_shape ideal_blockshape = outputSlot.meta.ideal_blockshape ram_usage_per_requested_pixel = outputSlot.meta.ram_usage_per_requested_pixel num_threads = Request.global_thread_pool.num_workers if lazyflow.AVAILABLE_RAM_MB != 0: available_ram = lazyflow.AVAILABLE_RAM_MB * 1e6 else: available_ram = psutil.virtual_memory().available if ram_usage_per_requested_pixel is None: # Make a conservative guess: 2*(bytes for dtype) * (num channels) + (fudge factor=4) ram_usage_per_requested_pixel = 2*outputSlot.meta.dtype().nbytes*outputSlot.meta.shape[-1] + 4 logger.warn( "Unknown per-pixel RAM requirement. Making a guess." ) # Safety factor (fudge factor): Double the estimated RAM usage per pixel safety_factor = 2.0 logger.info( "Estimated RAM usage per pixel is {} bytes * safety factor ({})" .format( ram_usage_per_requested_pixel, safety_factor ) ) ram_usage_per_requested_pixel *= safety_factor if ideal_blockshape is None: blockshape = determineBlockShape( input_shape, available_ram/(num_threads*ram_usage_per_requested_pixel) ) logger.warn( "Chose an arbitrary request blockshape {}".format( blockshape ) ) else: logger.info( "determining blockshape assuming available_ram is {} GB, split between {} threads" .format( available_ram/1e9, num_threads ) ) # By convention, ram_usage_per_requested_pixel refers to the ram used when requesting ALL channels of a 'pixel' # Therefore, we do not include the channel dimension in the blockshapes here. blockshape = determine_optimal_request_blockshape( max_blockshape[:-1], ideal_blockshape[:-1], ram_usage_per_requested_pixel, num_threads, available_ram ) blockshape += (outputSlot.meta.shape[-1],) logger.info( "Chose blockshape: {}".format( blockshape ) ) logger.info( "Estimated RAM usage per block is {} GB" .format( ram_usage_per_requested_pixel * numpy.prod( blockshape[:-1] ) / 1e9 ) ) return blockshape
def _determine_blockshape(self, outputSlot): """ Choose a blockshape using the slot metadata (if available) or an arbitrary guess otherwise. """ input_shape = outputSlot.meta.shape max_blockshape = input_shape ideal_blockshape = outputSlot.meta.ideal_blockshape ram_usage_per_requested_pixel = outputSlot.meta.ram_usage_per_requested_pixel num_threads = Request.global_thread_pool.num_workers available_ram = psutil.virtual_memory().available # Fudge factor: Reduce RAM usage by a bit available_ram *= 0.5 if ram_usage_per_requested_pixel is None: # Make a conservative guess: (bytes for dtype) * (num channels) + (fudge factor=4) ram_usage_per_requested_pixel = 4 + 2*outputSlot.meta.dtype().nbytes*outputSlot.meta.shape[-1] logger.warn( "Unknown RAM usage. Making a guess." ) else: logger.info( "Estimated RAM usage per pixel is {} bytes" .format( ram_usage_per_requested_pixel ) ) if ideal_blockshape is None: blockshape = determineBlockShape( input_shape, available_ram/(num_threads*ram_usage_per_requested_pixel) ) logger.warn( "Chose an arbitrary request blockshape {}".format( blockshape ) ) else: logger.info( "determining blockshape assuming available_ram is {} GB, split between {} threads" .format( available_ram/1e9, num_threads ) ) # By convention, ram_usage_per_requested_pixel refers to the ram used when requesting ALL channels of a 'pixel' # Therefore, we do not include the channel dimension in the blockshapes here. blockshape = determine_optimal_request_blockshape( max_blockshape[:-1], ideal_blockshape[:-1], ram_usage_per_requested_pixel, num_threads, available_ram ) blockshape += (outputSlot.meta.shape[-1],) logger.info( "Chose blockshape: {}".format( blockshape ) ) logger.info( "Estimated RAM usage per block is {} GB" .format( ram_usage_per_requested_pixel * numpy.prod( blockshape[:-1] ) / 1e9 ) ) return blockshape
def test_determine_optimal_request_blockshape_exceeds_memory(): assert (64, 64, 1) == determine_optimal_request_blockshape( (1000, 1000, 100), (0, 0, 1), 100000, 10, 1000)
def _determine_blockshape(self, outputSlot): """ Choose a blockshape using the slot metadata (if available) or an arbitrary guess otherwise. """ input_shape = outputSlot.meta.shape ideal_blockshape = outputSlot.meta.ideal_blockshape ram_usage_per_requested_pixel = outputSlot.meta.ram_usage_per_requested_pixel num_channels = 1 tagged_shape = outputSlot.meta.getTaggedShape() # Generally, we don't want to split requests across channels. if 'c' in tagged_shape.keys(): num_channels = tagged_shape['c'] channel_index = tagged_shape.keys().index('c') input_shape = input_shape[:channel_index] + input_shape[channel_index+1:] if ideal_blockshape: # Never enlarge 'ideal' in the channel dimension. num_channels = ideal_blockshape[channel_index] ideal_blockshape = ideal_blockshape[:channel_index] + ideal_blockshape[channel_index+1:] max_blockshape = input_shape num_threads = max(1, Request.global_thread_pool.num_workers) available_ram = Memory.getAvailableRamComputation() if ram_usage_per_requested_pixel is None: # Make a conservative guess: 2*(bytes for dtype) * (num channels) + (fudge factor=4) ram_usage_per_requested_pixel = 2*outputSlot.meta.dtype().nbytes*num_channels + 4 warnings.warn( "Unknown per-pixel RAM requirement. Making a guess." ) # Safety factor (fudge factor): Double the estimated RAM usage per pixel safety_factor = 2.0 logger.info("Estimated RAM usage per pixel is {} * safety factor ({})" .format( Memory.format(ram_usage_per_requested_pixel), safety_factor ) ) ram_usage_per_requested_pixel *= safety_factor if ideal_blockshape is None: blockshape = determineBlockShape( input_shape, available_ram/(num_threads*ram_usage_per_requested_pixel) ) if 'c' in outputSlot.meta.getAxisKeys(): blockshape = blockshape[:channel_index] + (num_channels,) + blockshape[channel_index:] warnings.warn( "Chose an arbitrary request blockshape {}".format( blockshape ) ) else: logger.info("determining blockshape assuming available_ram is {}" ", split between {} threads" .format(Memory.format(available_ram), num_threads)) # By convention, ram_usage_per_requested_pixel refers to the ram used when requesting ALL channels of a 'pixel' # Therefore, we do not include the channel dimension in the blockshapes here. blockshape = determine_optimal_request_blockshape( max_blockshape, ideal_blockshape, ram_usage_per_requested_pixel, num_threads, available_ram ) if 'c' in outputSlot.meta.getAxisKeys(): blockshape = blockshape[:channel_index] + (num_channels,) + blockshape[channel_index:] logger.info( "Chose blockshape: {}".format( blockshape ) ) fmt = Memory.format(ram_usage_per_requested_pixel * numpy.prod(blockshape[:-1])) logger.info("Estimated RAM usage per block is {}".format(fmt)) return blockshape
def _determine_blockshape(self, outputSlot): """ Choose a blockshape using the slot metadata (if available) or an arbitrary guess otherwise. """ input_shape = outputSlot.meta.shape ideal_blockshape = outputSlot.meta.ideal_blockshape ram_usage_per_requested_pixel = outputSlot.meta.ram_usage_per_requested_pixel max_blockshape = outputSlot.meta.max_blockshape or input_shape num_channels = 1 tagged_shape = outputSlot.meta.getTaggedShape() available_ram = Memory.getAvailableRamComputation() # Generally, we don't want to split requests across channels. if "c" in list(tagged_shape.keys()): num_channels = tagged_shape["c"] channel_index = list(tagged_shape.keys()).index("c") input_shape = input_shape[:channel_index] + input_shape[ channel_index + 1:] max_blockshape = max_blockshape[:channel_index] + max_blockshape[ channel_index + 1:] if ideal_blockshape: # Never enlarge 'ideal' in the channel dimension. num_channels = ideal_blockshape[channel_index] ideal_blockshape = ideal_blockshape[: channel_index] + ideal_blockshape[ channel_index + 1:] del tagged_shape["c"] # Generally, we don't want to join time slices if "t" in tagged_shape.keys(): blockshape_time_steps = 1 time_index = list(tagged_shape.keys()).index("t") input_shape = input_shape[:time_index] + input_shape[time_index + 1:] max_blockshape = max_blockshape[:time_index] + max_blockshape[ time_index + 1:] if ideal_blockshape: # Never enlarge 'ideal' in the time dimension. blockshape_time_steps = ideal_blockshape[time_index] ideal_blockshape = ideal_blockshape[: time_index] + ideal_blockshape[ time_index + 1:] available_ram /= blockshape_time_steps del tagged_shape["t"] if ram_usage_per_requested_pixel is None: # Make a conservative guess: 2*(bytes for dtype) * (num channels) + (fudge factor=4) ram_usage_per_requested_pixel = 2 * outputSlot.meta.dtype( ).nbytes * num_channels + 4 warnings.warn( "Unknown per-pixel RAM requirement. Making a guess.") # Safety factor (fudge factor): Double the estimated RAM usage per pixel safety_factor = 2.0 logger.info( "Estimated RAM usage per pixel is {} * safety factor ({})".format( Memory.format(ram_usage_per_requested_pixel), safety_factor)) ram_usage_per_requested_pixel *= safety_factor if ideal_blockshape is None: blockshape = determineBlockShape( input_shape, (available_ram // (self._num_threads * ram_usage_per_requested_pixel))) blockshape = tuple(numpy.minimum(max_blockshape, blockshape)) warnings.warn("Chose an arbitrary request blockshape") else: logger.info("determining blockshape assuming available_ram is {}" ", split between {} threads".format( Memory.format(available_ram), self._num_threads)) # By convention, ram_usage_per_requested_pixel refers to the ram used when requesting ALL channels of a 'pixel' # Therefore, we do not include the channel dimension in the blockshapes here. # # Also, it rarely makes sense to request more than one time slice, so we omit that, too. (See above.) blockshape = determine_optimal_request_blockshape( max_blockshape, ideal_blockshape, ram_usage_per_requested_pixel, self._num_threads, available_ram) # compute the RAM size of the block before adding back t anc c dimensions fmt = Memory.format(ram_usage_per_requested_pixel * numpy.prod(blockshape)) # If we removed time and channel from consideration, add them back now before returning if "t" in outputSlot.meta.getAxisKeys(): blockshape = blockshape[:time_index] + ( blockshape_time_steps, ) + blockshape[time_index:] if "c" in outputSlot.meta.getAxisKeys(): blockshape = blockshape[:channel_index] + ( num_channels, ) + blockshape[channel_index:] logger.info("Chose blockshape: {}".format(blockshape)) logger.info("Estimated RAM usage per block is {}".format(fmt)) return blockshape
def _determine_blockshape(self, outputSlot): """ Choose a blockshape using the slot metadata (if available) or an arbitrary guess otherwise. """ input_shape = outputSlot.meta.shape ideal_blockshape = outputSlot.meta.ideal_blockshape ram_usage_per_requested_pixel = outputSlot.meta.ram_usage_per_requested_pixel max_blockshape = outputSlot.meta.max_blockshape or input_shape num_channels = 1 tagged_shape = outputSlot.meta.getTaggedShape() available_ram = Memory.getAvailableRamComputation() # Generally, we don't want to split requests across channels. if "c" in list(tagged_shape.keys()): num_channels = tagged_shape["c"] channel_index = list(tagged_shape.keys()).index("c") input_shape = input_shape[:channel_index] + input_shape[channel_index + 1 :] max_blockshape = max_blockshape[:channel_index] + max_blockshape[channel_index + 1 :] if ideal_blockshape: # Never enlarge 'ideal' in the channel dimension. num_channels = ideal_blockshape[channel_index] ideal_blockshape = ideal_blockshape[:channel_index] + ideal_blockshape[channel_index + 1 :] del tagged_shape["c"] # Generally, we don't want to join time slices if "t" in tagged_shape.keys(): blockshape_time_steps = 1 time_index = list(tagged_shape.keys()).index("t") input_shape = input_shape[:time_index] + input_shape[time_index + 1 :] max_blockshape = max_blockshape[:time_index] + max_blockshape[time_index + 1 :] if ideal_blockshape: # Never enlarge 'ideal' in the time dimension. blockshape_time_steps = ideal_blockshape[time_index] ideal_blockshape = ideal_blockshape[:time_index] + ideal_blockshape[time_index + 1 :] available_ram /= blockshape_time_steps del tagged_shape["t"] if ram_usage_per_requested_pixel is None: # Make a conservative guess: 2*(bytes for dtype) * (num channels) + (fudge factor=4) ram_usage_per_requested_pixel = 2 * outputSlot.meta.dtype().nbytes * num_channels + 4 warnings.warn("Unknown per-pixel RAM requirement. Making a guess.") # Safety factor (fudge factor): Double the estimated RAM usage per pixel safety_factor = 2.0 logger.info( "Estimated RAM usage per pixel is {} * safety factor ({})".format( Memory.format(ram_usage_per_requested_pixel), safety_factor ) ) ram_usage_per_requested_pixel *= safety_factor if ideal_blockshape is None: blockshape = determineBlockShape( input_shape, (available_ram // (self._num_threads * ram_usage_per_requested_pixel)) ) blockshape = tuple(numpy.minimum(max_blockshape, blockshape)) warnings.warn("Chose an arbitrary request blockshape") else: logger.info( "determining blockshape assuming available_ram is {}" ", split between {} threads".format(Memory.format(available_ram), self._num_threads) ) # By convention, ram_usage_per_requested_pixel refers to the ram used when requesting ALL channels of a 'pixel' # Therefore, we do not include the channel dimension in the blockshapes here. # # Also, it rarely makes sense to request more than one time slice, so we omit that, too. (See above.) blockshape = determine_optimal_request_blockshape( max_blockshape, ideal_blockshape, ram_usage_per_requested_pixel, self._num_threads, available_ram ) # If we removed time and channel from consideration, add them back now before returning if "t" in outputSlot.meta.getAxisKeys(): blockshape = blockshape[:time_index] + (blockshape_time_steps,) + blockshape[time_index:] if "c" in outputSlot.meta.getAxisKeys(): blockshape = blockshape[:channel_index] + (num_channels,) + blockshape[channel_index:] logger.info("Chose blockshape: {}".format(blockshape)) fmt = Memory.format(ram_usage_per_requested_pixel * numpy.prod(blockshape[:-1])) logger.info("Estimated RAM usage per block is {}".format(fmt)) return blockshape