def initialize(self): """ Initialise the processing """ # Get the start time start_time = time() # Ensure the reflections contain bounding boxes assert "bbox" in self.reflections, "Reflections have no bbox" if self.params.mp.nproc is libtbx.Auto: self.params.mp.nproc = available_cores() logger.info("Setting nproc={}".format(self.params.mp.nproc)) # Compute the block size and processors self.compute_jobs() self.split_reflections() self.compute_processors() # Create the reflection manager self.manager = ReflectionManager(self.jobs, self.reflections) # Parallel reading of HDF5 from the same handle is not allowed. Python # multiprocessing is a bit messed up and used fork on linux so need to # close and reopen file. for exp in self.experiments: if exp.imageset.reader().is_single_file_reader(): exp.imageset.reader().nullify_format_instance() # Set the initialization time self.time.initialize = time() - start_time
def initialize(self): """ Initialise the processing """ # Get the start time start_time = time() # Ensure the reflections contain bounding boxes assert "bbox" in self.reflections, "Reflections have no bbox" if self.params.mp.nproc is libtbx.Auto: self.params.mp.nproc = available_cores() logger.info(f"Setting nproc={self.params.mp.nproc}") # Compute the block size and processors self.compute_jobs() self.split_reflections() self.compute_processors() # Create the reflection manager self.manager = ReflectionManager(self.jobs, self.reflections) # Set the initialization time self.time.initialize = time() - start_time
def _find_spots(self, imageset): """ Find the spots in the imageset :param imageset: The imageset to process :return: The list of spot shoeboxes """ # Change the number of processors if necessary mp_nproc = self.mp_nproc mp_njobs = self.mp_njobs if mp_nproc is libtbx.Auto: mp_nproc = available_cores() logger.info(f"Setting nproc={mp_nproc}") if mp_nproc * mp_njobs > len(imageset): mp_nproc = min(mp_nproc, len(imageset)) mp_njobs = int(math.ceil(len(imageset) / mp_nproc)) mp_method = self.mp_method mp_chunksize = self.mp_chunksize if mp_chunksize is libtbx.Auto: mp_chunksize = self._compute_chunksize(len(imageset), mp_njobs * mp_nproc, self.min_chunksize) logger.info("Setting chunksize=%i", mp_chunksize) len_by_nproc = int(math.floor(len(imageset) / (mp_njobs * mp_nproc))) if mp_chunksize > len_by_nproc: mp_chunksize = len_by_nproc if mp_chunksize == 0: mp_chunksize = 1 assert mp_nproc > 0, "Invalid number of processors" assert mp_njobs > 0, "Invalid number of jobs" assert mp_njobs == 1 or mp_method is not None, "Invalid cluster method" assert mp_chunksize > 0, "Invalid chunk size" # The extract pixels function function = ExtractPixelsFromImage( imageset=imageset, threshold_function=self.threshold_function, mask=self.mask, max_strong_pixel_fraction=self.max_strong_pixel_fraction, compute_mean_background=self.compute_mean_background, region_of_interest=self.region_of_interest, ) # The indices to iterate over indices = list(range(len(imageset))) # Initialise the pixel labeller num_panels = len(imageset.get_detector()) pixel_labeller = [PixelListLabeller() for p in range(num_panels)] # Do the processing logger.info("Extracting strong pixels from images") if mp_njobs > 1: logger.info( " Using %s with %d parallel job(s) and %d processes per node\n", mp_method, mp_njobs, mp_nproc, ) else: logger.info(" Using multiprocessing with %d parallel job(s)\n", mp_nproc) if mp_nproc > 1 or mp_njobs > 1: def process_output(result): rehandle_cached_records(result[1]) assert len(pixel_labeller) == len( result[0]), "Inconsistent size" for plabeller, plist in zip(pixel_labeller, result[0]): plabeller.add(plist) batch_multi_node_parallel_map( func=ExtractSpotsParallelTask(function), iterable=indices, nproc=mp_nproc, njobs=mp_njobs, cluster_method=mp_method, chunksize=mp_chunksize, callback=process_output, ) else: for task in indices: result = function(task) assert len(pixel_labeller) == len(result), "Inconsistent size" for plabeller, plist in zip(pixel_labeller, result): plabeller.add(plist) result.clear() # Create shoeboxes from pixel list return pixel_list_to_reflection_table( imageset, pixel_labeller, filter_spots=self.filter_spots, min_spot_size=self.min_spot_size, max_spot_size=self.max_spot_size, write_hot_pixel_mask=self.write_hot_pixel_mask, )