def tst_count_mask_values(self): from dials.model.data import Shoebox from random import randint, sample from dials.array_family import flex shoebox = flex.shoebox(10) num = flex.int(10) value = (1 << 2) for i in range(10): x0 = randint(0, 90) y0 = randint(0, 90) z0 = randint(0, 90) x1 = randint(1, 10) + x0 y1 = randint(1, 10) + y0 z1 = randint(1, 10) + z0 shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1)) shoebox[i].allocate() maxnum = len(shoebox[i].mask) num[i] = randint(1, maxnum) indices = sample(list(range(maxnum)), num[i]) for j in indices: shoebox[i].mask[j] = value assert(shoebox.count_mask_values(value) == num) # Test passed print 'OK'
def test_Simple6ProfileModel_compute_mask(simple6_profile_model, test_experiment): experiments = [test_experiment] # Create the index generator index_generator = IndexGenerator( experiments[0].crystal.get_unit_cell(), experiments[0].crystal.get_space_group().type(), 2.0, ) # Get an array of miller indices miller_indices = index_generator.to_array() reflections = simple6_profile_model.predict_reflections(experiments, miller_indices, probability=0.9973) s2 = reflections["s2"] s0 = matrix.col(experiments[0].beam.get_s0()) quantile = chisq_quantile(3, 0.9973) sigma_inv = matrix.sqr(flumpy.from_numpy( simple6_profile_model.sigma())).inverse() for s2 in map(matrix.col, reflections["s2"]): x = s2.normalize() * s0.length() - s2 d = (x.transpose() * sigma_inv * x)[0] assert d < quantile simple6_profile_model.compute_bbox(experiments, reflections) reflections["shoebox"] = flex.shoebox(reflections["panel"], reflections["bbox"], allocate=True) simple6_profile_model.compute_mask(experiments, reflections)
def test_does_bbox_contain_bad_pixels(): from dials.array_family import flex from dials.model.data import Shoebox mask = flex.bool(flex.grid(100, 100), True) for j in range(100): for i in range(40, 60): mask[j, i] = False mask[i, j] = False shoebox = flex.shoebox(1000) res = flex.bool(1000) for i in range(1000): x0 = random.randint(0, 90) y0 = random.randint(0, 90) z0 = random.randint(0, 90) x1 = random.randint(1, 10) + x0 y1 = random.randint(1, 10) + y0 z1 = random.randint(1, 10) + z0 shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1)) res2 = False if x0 >= 40 and x0 < 60: res2 = True if x1 > 40 and x1 <= 60: res2 = True if y0 >= 40 and y0 < 60: res2 = True if y1 > 40 and y1 <= 60: res2 = True res[i] = res2 assert shoebox.does_bbox_contain_bad_pixels(mask) == res
def test_consistent(): from dials.array_family import flex from dials.model.data import Shoebox shoebox = flex.shoebox(10) for i in range(10): x0 = random.randint(0, 1000) y0 = random.randint(0, 1000) z0 = random.randint(0, 1000) x1 = random.randint(1, 10) + x0 y1 = random.randint(1, 10) + y0 z1 = random.randint(1, 10) + z0 shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1)) assert shoebox.is_consistent() == flex.bool(10, False) for i in range(10): shoebox[i].allocate() assert shoebox.is_consistent() == flex.bool(10, True) for i in [0, 2, 4, 6, 8]: shoebox[i].data.resize(flex.grid(10, 10, 10)) assert shoebox.is_consistent() == flex.bool([False, True] * 5) for i in range(10): shoebox[i].deallocate() assert shoebox.is_consistent() == flex.bool(10, False)
def tst_count_mask_values(self): from dials.model.data import Shoebox from random import randint, sample from dials.array_family import flex shoebox = flex.shoebox(10) num = flex.int(10) value = (1 << 2) for i in range(10): x0 = randint(0, 90) y0 = randint(0, 90) z0 = randint(0, 90) x1 = randint(1, 10) + x0 y1 = randint(1, 10) + y0 z1 = randint(1, 10) + z0 shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1)) shoebox[i].allocate() maxnum = len(shoebox[i].mask) num[i] = randint(1, maxnum) indices = sample(list(range(maxnum)), num[i]) for j in indices: shoebox[i].mask[j] = value assert (shoebox.count_mask_values(value) == num) # Test passed print 'OK'
def tst_is_bbox_within_image_volume(self): from dials.model.data import Shoebox from dials.array_family import flex isize = (1000, 1000) srange = (0, 100) shoebox = flex.shoebox(7) shoebox[0] = Shoebox((10, 20, 10, 20, 10, 20)) shoebox[1] = Shoebox((-10, 20, 10, 20, 10, 20)) shoebox[2] = Shoebox((10, 20, -10, 20, 10, 20)) shoebox[3] = Shoebox((10, 20, 10, 20, -10, 20)) shoebox[4] = Shoebox((10, 1020, 10, 20, 10, 20)) shoebox[5] = Shoebox((10, 20, 10, 1020, 10, 20)) shoebox[6] = Shoebox((10, 20, 10, 20, 10, 1020)) assert (shoebox.is_bbox_within_image_volume(isize, srange) == flex.bool([ True, False, False, False, False, False, False ])) # Test passed print 'OK'
def _allocate_shoebox(self): """ Allocate the shoebox """ self.reflections["shoebox"] = flex.shoebox(self.reflections["panel"], self.reflections["bbox"], allocate=True)
def test(dials_regression, tmpdir): tmpdir.chdir() from dxtbx.serialize import load from dials.algorithms import shoebox from dials.array_family import flex # Load the sweep and crystal sweep_filename = os.path.join(dials_regression, 'centroid_test_data', 'sweep.json') crystal_filename = os.path.join(dials_regression, 'centroid_test_data', 'crystal.json') sweep = load.imageset(sweep_filename) crystal = load.crystal(crystal_filename) # Get models from the sweep beam = sweep.get_beam() detector = sweep.get_detector() gonio = sweep.get_goniometer() scan = sweep.get_scan() # Get the reflections and overlaps reflections, adjacency_list = predict_reflections(sweep, crystal) reflections['shoebox'] = flex.shoebox( reflections['panel'], reflections['bbox']) reflections['shoebox'].allocate_with_value(shoebox.MaskCode.Valid) # If the adjacency list is given, then create the reflection mask assert(len(detector) == 1) image_size = detector[0].get_image_size() shoeboxes = reflections['shoebox'] coords = reflections['xyzcal.px'] shoebox_masker = shoebox.MaskOverlapping() shoebox_masker(shoeboxes, coords, adjacency_list) # Loop through all edges overlapping = [] for e in adjacency_list.edges(): v1, v2 = adjacency_list.source(e), adjacency_list.target(e) overlapping.append(v1) overlapping.append(v2) # Ensure elements are unique overlapping = set(overlapping) # Ensure we have some overlaps assert len(overlapping) > 0 # Get all non-overlapping reflections all_r = set(range(len(reflections))) non_overlapping = all_r.difference(overlapping) # Run the tests tst_non_overlapping(reflections, non_overlapping, detector[0].get_image_size()) tst_overlapping(reflections, overlapping, adjacency_list, image_size)
def __init__(self, data, jobs, npanels, callback): from dials.array_family import flex # Allocate shoeboxes data["shoebox"] = flex.shoebox(data["panel"], data["bbox"]) data["shoebox"].allocate() # Set the callback self._callback = callback # Initialise the base class super(IntegrationTask3DMultiExecutor, self).__init__(data, jobs, npanels)
def test_RefinerData(testdata): experiment = testdata.experiment reflections = testdata.reflections panel = experiment.detector[0] s0_length = matrix.col(experiment.beam.get_s0()).length() reflections["bbox"] = flex.int6(len(reflections)) reflections["xyzobs.px.value"] = flex.vec3_double(len(reflections)) reflections["s2"] = reflections["s1"].each_normalize() * s0_length reflections["sp"] = flex.vec3_double(len(reflections)) for i, (x, y, z) in enumerate(reflections["xyzcal.px"]): x0 = int(x) - 5 x1 = int(x) + 5 + 1 y0 = int(y) - 5 y1 = int(y) + 5 + 1 z0 = int(z) z1 = z0 + 1 reflections["bbox"][i] = x0, x1, y0, y1, z0, z1 reflections["xyzobs.px.value"][i] = (int(x) + 0.5, int(y) + 0.5, int(z) + 0.5) reflections["sp"][i] = (matrix.col( panel.get_pixel_lab_coord( reflections["xyzobs.px.value"][i][0:2])).normalize() * s0_length) reflections["shoebox"] = flex.shoebox(reflections["panel"], reflections["bbox"], allocate=True) shoebox_data = flex.float(flex.grid(1, 11, 11)) shoebox_mask = flex.int(flex.grid(1, 11, 11)) for j in range(11): for i in range(11): shoebox_data[0, j, i] = (100 * exp(-0.5 * (j - 5)**2 / 1**2) * exp(-0.5 * (i - 5)**2 / 1**2)) shoebox_mask[0, j, i] = 5 for sbox in reflections["shoebox"]: sbox.data = shoebox_data sbox.mask = shoebox_mask data = RefinerData.from_reflections(experiment, reflections) assert tuple(data.s0) == pytest.approx(experiment.beam.get_s0()) assert data.h_list == reflections["miller_index"] for i, sp in enumerate(reflections["sp"]): assert data.sp_list[:, i] == pytest.approx(sp) assert data.ctot_list[0] == sum(shoebox_data) mobs1 = np.abs(data.mobs_list[0, :]) mobs2 = np.abs(data.mobs_list[1, :]) assert np.max(mobs1) < 1e-6 assert np.max(mobs2) < 1e-6
def final_integrator( experiments, reflection_table, sigma_d, use_crude_shoebox_mask=False, shoebox_probability=0.9973, ): """Performs an initial integration of all predicted spots""" experiment = experiments[0] logger.info("\n" + "=" * 80 + "\nIntegrating reflections") # first compute the bbox if use_crude_shoebox_mask: reflection_table["bbox"] = _compute_bbox(experiment, reflection_table, sigma_d, "s1") else: # compute bbox from model profile = experiment.crystal.mosaicity profile.parameterisation.compute_bbox(experiments, reflection_table, shoebox_probability) # Select reflections within detector x0, x1, y0, y1, _, _ = reflection_table["bbox"].parts() xsize, ysize = experiment.detector[0].get_image_size() selection = (x1 > 0) & (y1 > 0) & (x0 < xsize) & (y0 < ysize) reflection_table = reflection_table.select(selection) reflection_table["shoebox"] = flex.shoebox(reflection_table["panel"], reflection_table["bbox"], allocate=True) reflection_table.extract_shoeboxes(experiment.imageset) if use_crude_shoebox_mask: _compute_mask(experiment, reflection_table, sigma_d, "s1") else: # compute the mask from the model. profile = experiment.crystal.mosaicity profile.parameterisation.compute_mask(experiments, reflection_table, shoebox_probability) logger.info( f"Computing background, intensity, corrections for {len(reflection_table)} reflections" ) reflection_table.compute_background(experiments) reflection_table.compute_summed_intensity() reflection_table.compute_corrections(experiments) reflection_table.compute_centroid(experiments) profile = experiment.crystal.mosaicity profile.parameterisation.compute_partiality(experiments, reflection_table) return reflection_table
def __call__(self, imageset, pixel_labeller): """ Convert the pixel list to shoeboxes """ from dxtbx.imageset import ImageSequence # Extract the pixel lists into a list of reflections shoeboxes = flex.shoebox() spotsizes = flex.size_t() hotpixels = tuple(flex.size_t() for i in range(len(imageset.get_detector()))) if isinstance(imageset, ImageSequence): scan = imageset.get_scan() if scan.is_still(): twod = True else: twod = False else: twod = True for i, (p, hp) in enumerate(zip(pixel_labeller, hotpixels)): if p.num_pixels() > 0: creator = flex.PixelListShoeboxCreator( p, i, # panel 0, # zrange twod, # twod self.min_spot_size, # min_pixels self.max_spot_size, # max_pixels self.write_hot_pixel_mask, ) shoeboxes.extend(creator.result()) spotsizes.extend(creator.spot_size()) hp.extend(creator.hot_pixels()) logger.info("") logger.info("Extracted {} spots".format(len(shoeboxes))) # Get the unallocated spots and print some info selection = shoeboxes.is_allocated() shoeboxes = shoeboxes.select(selection) ntoosmall = (spotsizes < self.min_spot_size).count(True) ntoolarge = (spotsizes > self.max_spot_size).count(True) assert ntoosmall + ntoolarge == selection.count(False) logger.info( "Removed %d spots with size < %d pixels" % (ntoosmall, self.min_spot_size) ) logger.info( "Removed %d spots with size > %d pixels" % (ntoolarge, self.max_spot_size) ) # Return the shoeboxes return shoeboxes, hotpixels
def test(dials_data): from dxtbx.serialize import load from dials.algorithms import shoebox from dials.array_family import flex # Load the sequence and crystal sequence = load.imageset( dials_data("centroid_test_data").join("sweep.json").strpath ) crystal = load.crystal( dials_data("centroid_test_data").join("crystal.json").strpath ) # Get models from the sequence detector = sequence.get_detector() # Get the reflections and overlaps reflections, adjacency_list = predict_reflections(sequence, crystal) reflections["shoebox"] = flex.shoebox(reflections["panel"], reflections["bbox"]) reflections["shoebox"].allocate_with_value(shoebox.MaskCode.Valid) # If the adjacency list is given, then create the reflection mask assert len(detector) == 1 image_size = detector[0].get_image_size() shoeboxes = reflections["shoebox"] coords = reflections["xyzcal.px"] shoebox_masker = shoebox.MaskOverlapping() shoebox_masker(shoeboxes, coords, adjacency_list) # Loop through all edges overlapping = [] for e in adjacency_list.edges(): v1, v2 = adjacency_list.source(e), adjacency_list.target(e) overlapping.append(v1) overlapping.append(v2) # Ensure elements are unique overlapping = set(overlapping) # Ensure we have some overlaps assert len(overlapping) > 0 # Get all non-overlapping reflections all_r = set(range(len(reflections))) non_overlapping = all_r.difference(overlapping) # Run the tests tst_non_overlapping(reflections, non_overlapping, detector[0].get_image_size()) tst_overlapping(reflections, overlapping, adjacency_list, image_size)
def integrate(experiments, table, params): _params = Parameters.from_phil(params.integration) experiments[0].scan = None BackgroundAlgorithm = dials.extensions.Background.load( params.integration.background.algorithm ) flex.reflection_table.background_algorithm = functools.partial( BackgroundAlgorithm, params ) CentroidAlgorithm = dials.extensions.Centroid.load( params.integration.centroid.algorithm ) flex.reflection_table.centroid_algorithm = functools.partial( CentroidAlgorithm, params ) _initialize_stills(experiments, _params, table) table["shoebox"] = flex.shoebox(table["panel"], table["bbox"], allocate=True) table.extract_shoeboxes(experiments[0].imageset) # From integratorexecutor table.is_overloaded(experiments) table.compute_mask(experiments) # Check for invalid pixels in foreground/background table.contains_invalid_pixels() sbox = table["shoebox"] nvalfg = sbox.count_mask_values(MaskCode.Valid | MaskCode.Foreground) nforeg = sbox.count_mask_values(MaskCode.Foreground) fraction_valid = nvalfg.as_double() / nforeg.as_double() selection = ( fraction_valid < params.integration.profile.valid_foreground_threshold ) table.set_flags(selection, table.flags.dont_integrate) # Process the data table.compute_background(experiments) table.compute_centroid(experiments) table.compute_summed_intensity() table["num_pixels.valid"] = sbox.count_mask_values(MaskCode.Valid) table["num_pixels.background"] = sbox.count_mask_values( MaskCode.Valid | MaskCode.Background ) table["num_pixels.background_used"] = sbox.count_mask_values( MaskCode.Valid | MaskCode.Background | MaskCode.BackgroundUsed ) table["num_pixels.foreground"] = nvalfg table, experiments = _finalize_stills(table, experiments, _params) return table, experiments
def run(self): from dials.model.serialize import load from dials.algorithms import shoebox from dxtbx.serialize.load import crystal as load_crystal from dials.array_family import flex # Load the sweep and crystal self.sweep = load.sweep(self.sweep_filename) self.crystal = load_crystal(self.crystal_filename) # Get the reflections and overlaps reflections, adjacency_list = self.predict_reflections() reflections['shoebox'] = flex.shoebox( reflections['panel'], reflections['bbox']) reflections['shoebox'].allocate_with_value(shoebox.MaskCode.Valid) # If the adjacency list is given, then create the reflection mask assert(len(self.detector) == 1) image_size = self.detector[0].get_image_size() shoeboxes = reflections['shoebox'] coords = reflections['xyzcal.px'] shoebox_masker = shoebox.MaskOverlapping() shoebox_masker(shoeboxes, coords, adjacency_list) # Loop through all edges overlapping = [] for e in adjacency_list.edges(): v1, v2 = adjacency_list.source(e), adjacency_list.target(e) overlapping.append(v1) overlapping.append(v2) # Ensure elements are unique overlapping = set(overlapping) # Ensure we have some overlaps assert(len(overlapping) > 0) # Get all non-overlapping reflections all_r = set(range(len(reflections))) non_overlapping = all_r.difference(overlapping) # Run the tests self.tst_non_overlapping(reflections, non_overlapping, self.detector[0].get_image_size()) self.tst_overlapping(reflections, overlapping, adjacency_list, image_size)
def __call__(self, imageset, pixel_labeller): ''' Convert the pixel list to shoeboxes ''' from dxtbx.imageset import ImageSweep from dials.array_family import flex # Extract the pixel lists into a list of reflections shoeboxes = flex.shoebox() spotsizes = flex.size_t() hotpixels = tuple(flex.size_t() for i in range(len(imageset.get_detector()))) if isinstance(imageset, ImageSweep): twod = False else: twod = True for i, (p, hp) in enumerate(zip(pixel_labeller, hotpixels)): if p.num_pixels() > 0: creator = flex.PixelListShoeboxCreator( p, i, # panel 0, # zrange twod, # twod self.min_spot_size, # min_pixels self.max_spot_size, # max_pixels self.write_hot_pixel_mask) shoeboxes.extend(creator.result()) spotsizes.extend(creator.spot_size()) hp.extend(creator.hot_pixels()) logger.info('') logger.info('Extracted {0} spots'.format(len(shoeboxes))) # Get the unallocated spots and print some info selection = shoeboxes.is_allocated() shoeboxes = shoeboxes.select(selection) ntoosmall = (spotsizes < self.min_spot_size).count(True) ntoolarge = (spotsizes > self.max_spot_size).count(True) assert ntoosmall + ntoolarge == selection.count(False) logger.info('Removed %d spots with size < %d pixels' % ( ntoosmall, self.min_spot_size)) logger.info('Removed %d spots with size > %d pixels' % ( ntoolarge, self.max_spot_size)) # Return the shoeboxes return shoeboxes, hotpixels
def pixel_list_to_shoeboxes( imageset: ImageSet, pixel_labeller: Iterable[PixelListLabeller], min_spot_size: int, max_spot_size: int, write_hot_pixel_mask: bool, ) -> Tuple[flex.shoebox, Tuple[flex.size_t, ...]]: """Convert a pixel list to shoeboxes""" # Extract the pixel lists into a list of reflections shoeboxes = flex.shoebox() spotsizes = flex.size_t() hotpixels = tuple(flex.size_t() for i in range(len(imageset.get_detector()))) if isinstance(imageset, ImageSequence): twod = imageset.get_scan().is_still() else: twod = True for i, (p, hp) in enumerate(zip(pixel_labeller, hotpixels)): if p.num_pixels() > 0: creator = flex.PixelListShoeboxCreator( p, i, # panel 0, # zrange twod, # twod min_spot_size, # min_pixels max_spot_size, # max_pixels write_hot_pixel_mask, ) shoeboxes.extend(creator.result()) spotsizes.extend(creator.spot_size()) hp.extend(creator.hot_pixels()) logger.info("\nExtracted %d spots", len(shoeboxes)) # Get the unallocated spots and print some info selection = shoeboxes.is_allocated() shoeboxes = shoeboxes.select(selection) ntoosmall = (spotsizes < min_spot_size).count(True) ntoolarge = (spotsizes > max_spot_size).count(True) assert ntoosmall + ntoolarge == selection.count(False) logger.info("Removed %d spots with size < %d pixels", ntoosmall, min_spot_size) logger.info("Removed %d spots with size > %d pixels", ntoolarge, max_spot_size) # Return the shoeboxes return shoeboxes, hotpixels
def run(self): from dials.model.serialize import load from dials.algorithms import shoebox from dxtbx.serialize.load import crystal as load_crystal from dials.array_family import flex # Load the sweep and crystal self.sweep = load.sweep(self.sweep_filename) self.crystal = load_crystal(self.crystal_filename) # Get the reflections and overlaps reflections, adjacency_list = self.predict_reflections() reflections['shoebox'] = flex.shoebox(reflections['panel'], reflections['bbox']) reflections['shoebox'].allocate_with_value(shoebox.MaskCode.Valid) # If the adjacency list is given, then create the reflection mask assert (len(self.detector) == 1) image_size = self.detector[0].get_image_size() shoeboxes = reflections['shoebox'] coords = reflections['xyzcal.px'] shoebox_masker = shoebox.MaskOverlapping() shoebox_masker(shoeboxes, coords, adjacency_list) # Loop through all edges overlapping = [] for e in adjacency_list.edges(): v1, v2 = adjacency_list.source(e), adjacency_list.target(e) overlapping.append(v1) overlapping.append(v2) # Ensure elements are unique overlapping = set(overlapping) # Ensure we have some overlaps assert (len(overlapping) > 0) # Get all non-overlapping reflections all_r = set(range(len(reflections))) non_overlapping = all_r.difference(overlapping) # Run the tests self.tst_non_overlapping(reflections, non_overlapping, self.detector[0].get_image_size()) self.tst_overlapping(reflections, overlapping, adjacency_list, image_size)
def initial_integrator(experiments, reflection_table): """Performs an initial integration of strong spots""" # some functions require an experimentlist, others just the experiment experiment = experiments[0] sel = reflection_table.get_flags(reflection_table.flags.strong) strong_refls = reflection_table.select(sel) strong_shoeboxes = copy.deepcopy( strong_refls["shoebox"]) # Save the strong shoeboxes # Compute and initial spot size estimate and beam vector sigma_d = ComputeEsdBeamDivergence(experiment.detector, strong_refls).sigma() sigma_degrees = sigma_d * 180 / pi logger.info( f"Initial sigma d estimate for {len(strong_refls)} reflections\n" + f"Sigma D: {sigma_degrees:.5f} degrees\n", ) strong_refls["s1_obs"] = _compute_beam_vector(experiment, strong_refls) strong_refls["bbox"] = _compute_bbox(experiment, strong_refls, sigma_d, "s1_obs") # allocate and extract shoebox strong_refls["shoebox"] = flex.shoebox(strong_refls["panel"], strong_refls["bbox"], allocate=True) strong_refls.extract_shoeboxes(experiment.imageset) _compute_mask(experiment, strong_refls, sigma_d, "s1_obs", strong_shoeboxes) logger.info( f"Computing background, intensity and centroids for {len(strong_refls)} reflections" ) strong_refls.compute_background(experiments) strong_refls.compute_summed_intensity() n_sum = strong_refls.get_flags( strong_refls.flags.integrated_sum).count(True) logger.info(f"{n_sum} reflections integrated") strong_refls.compute_centroid(experiments) strong_refls.compute_d(experiments) return strong_refls, sigma_d
def test_bounding_boxes(): from dials.array_family import flex from dials.model.data import Shoebox shoebox = flex.shoebox(10) bbox = flex.int6(10) for i in range(10): x0 = random.randint(0, 90) y0 = random.randint(0, 90) z0 = random.randint(0, 90) x1 = random.randint(1, 10) + x0 y1 = random.randint(1, 10) + y0 z1 = random.randint(1, 10) + z0 bbox[i] = (x0, x1, y0, y1, z0, z1) shoebox[i] = Shoebox(bbox[i]) bbox2 = shoebox.bounding_boxes() for i in range(10): assert bbox2[i] == bbox[i]
def refinerdata_testdata(testdata): experiment = testdata.experiment reflections = testdata.reflections panel = experiment.detector[0] s0_length = matrix.col(experiment.beam.get_s0()).length() reflections["bbox"] = flex.int6(len(reflections)) reflections["xyzobs.px.value"] = flex.vec3_double(len(reflections)) reflections["s2"] = reflections["s1"].each_normalize() * s0_length reflections["sp"] = flex.vec3_double(len(reflections)) for i, (x, y, z) in enumerate(reflections["xyzcal.px"]): x0 = int(x) - 5 x1 = int(x) + 5 + 1 y0 = int(y) - 5 y1 = int(y) + 5 + 1 z0 = int(z) z1 = z0 + 1 reflections["bbox"][i] = x0, x1, y0, y1, z0, z1 reflections["xyzobs.px.value"][i] = (int(x) + 0.5, int(y) + 0.5, int(z) + 0.5) reflections["sp"][i] = (matrix.col( panel.get_pixel_lab_coord( reflections["xyzobs.px.value"][i][0:2])).normalize() * s0_length) reflections["shoebox"] = flex.shoebox(reflections["panel"], reflections["bbox"], allocate=True) shoebox_data = flex.float(flex.grid(1, 11, 11)) shoebox_mask = flex.int(flex.grid(1, 11, 11)) for j in range(11): for i in range(11): shoebox_data[0, j, i] = (100 * exp(-0.5 * (j - 5)**2 / 1**2) * exp(-0.5 * (i - 5)**2 / 1**2)) shoebox_mask[0, j, i] = 5 for sbox in reflections["shoebox"]: sbox.data = shoebox_data sbox.mask = shoebox_mask return RefinerData.from_reflections(experiment, reflections)
def generate_reflections(detector, beam, scan, experiment, num): seed(0) assert len(detector) == 1 beam_vector = flex.vec3_double(num) xyzcal_px = flex.vec3_double(num) xyzcal_mm = flex.vec3_double(num) panel = flex.size_t(num) s0_length = matrix.col(beam.get_s0()).length() for i in range(num): x = randint(0, 2000) y = randint(0, 2000) z = randint(0, 8) s1 = detector[0].get_pixel_lab_coord((x, y)) s1 = matrix.col(s1).normalize() * s0_length phi = scan.get_angle_from_array_index(z, deg=False) beam_vector[i] = s1 xyzcal_px[i] = (x, y, z) (x, y) = detector[0].pixel_to_millimeter((x, y)) xyzcal_mm[i] = (x, y, phi) panel[i] = 0 rlist = flex.reflection_table() rlist["id"] = flex.int(len(beam_vector), 0) rlist["s1"] = beam_vector rlist["panel"] = panel rlist["xyzcal.px"] = xyzcal_px rlist["xyzcal.mm"] = xyzcal_mm rlist["bbox"] = rlist.compute_bbox(experiment) index = [] image_size = experiment[0].detector[0].get_image_size() array_range = experiment[0].scan.get_array_range() bbox = rlist["bbox"] for i in range(len(rlist)): x0, x1, y0, y1, z0, z1 = bbox[i] if (x0 < 0 or x1 > image_size[0] or y0 < 0 or y1 > image_size[1] or z0 < array_range[0] or z1 > array_range[1]): index.append(i) rlist.del_selected(flex.size_t(index)) rlist["shoebox"] = flex.shoebox(rlist["panel"], rlist["bbox"]) rlist["shoebox"].allocate_with_value(MaskCode.Valid) return rlist
def tst_is_bbox_within_image_volume(self): from dials.model.data import Shoebox from dials.array_family import flex isize = (1000, 1000) srange = (0, 100) shoebox = flex.shoebox(7) shoebox[0] = Shoebox((10, 20, 10, 20, 10, 20)) shoebox[1] = Shoebox((-10, 20, 10, 20, 10, 20)) shoebox[2] = Shoebox((10, 20, -10, 20, 10, 20)) shoebox[3] = Shoebox((10, 20, 10, 20, -10, 20)) shoebox[4] = Shoebox((10, 1020, 10, 20, 10, 20)) shoebox[5] = Shoebox((10, 20, 10, 1020, 10, 20)) shoebox[6] = Shoebox((10, 20, 10, 20, 10, 1020)) assert(shoebox.is_bbox_within_image_volume(isize, srange) == flex.bool([True, False, False, False, False, False, False])) # Test passed print 'OK'
def tst_does_bbox_contain_bad_pixels(self): from dials.array_family import flex from dials.model.data import Shoebox from random import randint mask = flex.bool(flex.grid(100, 100), True) for j in range(100): for i in range(40, 60): mask[j,i] = False mask[i,j] = False shoebox = flex.shoebox(1000) res = flex.bool(1000) for i in range(1000): x0 = randint(0, 90) y0 = randint(0, 90) z0 = randint(0, 90) x1 = randint(1, 10) + x0 y1 = randint(1, 10) + y0 z1 = randint(1, 10) + z0 shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1)) res2 = False if x0 >= 40 and x0 < 60: res2 = True if x1 > 40 and x1 <= 60: res2 = True if y0 >= 40 and y0 < 60: res2 = True if y1 > 40 and y1 <= 60: res2 = True res[i] = res2 assert(shoebox.does_bbox_contain_bad_pixels(mask) == res) # Test passed print 'OK'
def __init__(self): from dials.algorithms.integration.profile import GridSampler2D from dials.array_family import flex from random import randint, uniform # Number of reflections nrefl = 1000 # Size of the images width = 1000 height = 1000 # Create the grid self.grid = GridSampler2D((width, height), (5, 5)) # Create the list of xyz and bboxes self.xyz = flex.vec3_double(nrefl) self.bbox = flex.int6(nrefl) self.panel = flex.size_t(nrefl, 0) for i in range(nrefl): x0 = randint(0, width - 10) x1 = x0 + randint(3, 10) y0 = randint(0, height - 10) y1 = y0 + randint(3, 10) z0 = randint(0, 10) z1 = z0 + randint(1, 10) b = x0, x1, y0, y1, z0, z1 c = (x1 + x0) / 2, (y1 + y0) / 2, (z1 + z0) / 2 self.xyz[i] = c self.bbox[i] = b # Create the array of shoeboxes self.sbox = flex.shoebox(self.panel, self.bbox) self.sbox.allocate() for i in range(len(self.sbox)): data = self.sbox[i].data for j in range(len(data)): data[j] = uniform(0, 100)
def __init__(self): from dials.algorithms.integration.profile import GridSampler2D from dials.array_family import flex from random import randint, uniform # Number of reflections nrefl = 1000 # Size of the images width = 1000 height = 1000 # Create the grid self.grid = GridSampler2D((width, height), (5,5)) # Create the list of xyz and bboxes self.xyz = flex.vec3_double(nrefl) self.bbox = flex.int6(nrefl) self.panel = flex.size_t(nrefl, 0) for i in range(nrefl): x0 = randint(0,width-10) x1 = x0 + randint(3,10) y0 = randint(0,height-10) y1 = y0 + randint(3,10) z0 = randint(0,10) z1 = z0 + randint(1,10) b = x0, x1, y0, y1, z0, z1 c = (x1 + x0) / 2, (y1 + y0) / 2, (z1 + z0) / 2 self.xyz[i] = c self.bbox[i] = b # Create the array of shoeboxes self.sbox = flex.shoebox(self.panel, self.bbox) self.sbox.allocate() for i in range(len(self.sbox)): data = self.sbox[i].data for j in range(len(data)): data[j] = uniform(0, 100)
def tst_bounding_boxes(self): from dials.model.data import Shoebox from random import randint from dials.array_family import flex shoebox = flex.shoebox(10) bbox = flex.int6(10) for i in range(10): x0 = randint(0, 90) y0 = randint(0, 90) z0 = randint(0, 90) x1 = randint(1, 10) + x0 y1 = randint(1, 10) + y0 z1 = randint(1, 10) + z0 bbox[i] = (x0, x1, y0, y1, z0, z1) shoebox[i] = Shoebox(bbox[i]) bbox2 = shoebox.bounding_boxes() for i in range(10): assert(bbox2[i] == bbox[i]) # Test passed print 'OK'
def tst_consistent(self): from random import randint from dials.model.data import Shoebox from dials.array_family import flex shoebox = flex.shoebox(10) for i in range(10): x0 = randint(0, 1000) y0 = randint(0, 1000) z0 = randint(0, 1000) x1 = randint(1, 10) + x0 y1 = randint(1, 10) + y0 z1 = randint(1, 10) + z0 shoebox[i] = Shoebox((x0, x1, y0, y1, z0, z1)) assert(shoebox.is_consistent() == flex.bool(10, False)) for i in range(10): shoebox[i].allocate() assert(shoebox.is_consistent() == flex.bool(10, True)) for i in [0, 2, 4, 6, 8]: shoebox[i].data.resize(flex.grid(10, 10, 10)) assert(shoebox.is_consistent() == flex.bool([False, True] * 5)) for i in range(10): shoebox[i].deallocate() assert(shoebox.is_consistent() == flex.bool(10, False)) # Test passed print 'OK'
def test_Simple1ProfileModel_compute_mask(simple1_profile_model, test_experiment): experiments = [test_experiment] # Create the index generator index_generator = IndexGenerator( experiments[0].crystal.get_unit_cell(), experiments[0].crystal.get_space_group().type(), 2.0, ) # Get an array of miller indices miller_indices = index_generator.to_array() reflections = simple1_profile_model.predict_reflections(experiments, miller_indices, probability=0.9973) simple1_profile_model.compute_bbox(experiments, reflections) reflections["shoebox"] = flex.shoebox(reflections["panel"], reflections["bbox"], allocate=True) simple1_profile_model.compute_mask(experiments, reflections)
def run(i, imp): from random import randint from dials.array_family import flex #building a reflection table num_ref = 5 ref_table = flex.reflection_table() shoebox = flex.shoebox(num_ref) ref_table['shoebox'] = shoebox intensity = flex.double(num_ref) ref_table['intensity.sum.value'] = intensity intensity_var = flex.double(num_ref) ref_table['intensity.sum.variance'] = intensity_var iterate = ref_table['shoebox'] i_to_compare = [] # bulding the shoebox with a desired content # which is a reflection with noise included n = 0 for arr in iterate: img = flex.double(flex.grid(3, 3, 3)) bkg = flex.double(flex.grid(3, 3, 3)) msk = flex.int(flex.grid(3, 3, 3)) for row in range(3): for col in range(3): for fra in range(3): img[row, col, fra] = row + col + fra + n * 9 + randint(0, i) bkg[row, col, fra] = 0.0 msk[row, col, fra] = 3 n += 1 msk[1, 1, 1] = 5 tmp_i = n * n * n * 3 i_to_compare.append(tmp_i) img[1, 1, 1] += tmp_i arr.data = img[:, :, :] arr.background = bkg[:, :, :] arr.mask = msk[:, :, :] # calling the functions that we need to test # first select the algorithm for background calculation if imp == "inclined": print "testing inclined_background_subtractor" from dials.algorithms.background.inclined_background_subtractor \ import layering_and_background_plane layering_and_background_plane(ref_table) elif imp == "flat": print "testing flat_background_subtractor" from dials.algorithms.background.flat_background_subtractor \ import layering_and_background_avg layering_and_background_avg(ref_table) elif imp == "curved": print "testing curved_background_subtractor" from dials.algorithms.background.curved_background_subtractor \ import layering_and_background_modl layering_and_background_modl(ref_table) # no matter which algorithm was used for background calculation # the integration summation must remain compatible from dials.algorithms.integration.summation2d \ import flex_2d_layering_n_integrating flex_2d_layering_n_integrating(ref_table) # comparing results result = "OK" resl_its = ref_table['intensity.sum.value'] resl_var = ref_table['intensity.sum.variance'] for n_its in range(len(resl_its)): if resl_its[n_its] <= i_to_compare[n_its] + i and \ resl_its[n_its] >= i_to_compare[n_its] - i and \ resl_var[n_its] > resl_its[n_its]: print "Ok ", n_its else: print "Wrong num", n_its print "i =", i print "resl_its[n_its] =", resl_its[n_its] print "i_to_compare[n_its] =", i_to_compare[n_its] print "resl_var[n_its] =", resl_var[n_its] result = "wrong" raise RuntimeError('wrong result') return result
def simple_gaussian_spots(params): from dials.array_family import flex from dials.algorithms import shoebox import random import math from scitbx import matrix r = params.rotation axis = matrix.col((r.axis.x, r.axis.y, r.axis.z)) if axis.length() > 0: rotation = axis.axis_and_angle_as_r3_rotation_matrix(r.angle, deg=True) else: rotation = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1)) # generate mask and peak values from dials.algorithms.shoebox import MaskCode mask_peak = MaskCode.Valid|MaskCode.Foreground mask_back = MaskCode.Valid|MaskCode.Background from dials.util.command_line import ProgressBar p = ProgressBar(title = 'Generating reflections') rlist = flex.reflection_table(params.nrefl) hkl = flex.miller_index(params.nrefl) s1 = flex.vec3_double(params.nrefl) xyzmm = flex.vec3_double(params.nrefl) xyzpx = flex.vec3_double(params.nrefl) panel = flex.size_t(params.nrefl) bbox = flex.int6(params.nrefl) for j in range(params.nrefl): p.update(j * 100.0 / params.nrefl) hkl[j] = (random.randint(0, 20), random.randint(0, 20), random.randint(0, 20)) phi = 2 * math.pi * random.random() s1[j] = (0, 0, 0) xyzpx[j] = (0, 0, 0) xyzmm[j] = (0, 0, phi) panel[j] = 0 bbox[j] = (0, params.shoebox_size.x, 0, params.shoebox_size.y, 0, params.shoebox_size.z) p.finished('Generating %d reflections' % params.nrefl) intensity = flex.double(params.nrefl) shoebox = flex.shoebox(panel, bbox) shoebox.allocate_with_value(MaskCode.Valid) p = ProgressBar(title = 'Generating shoeboxes') for i in range(len(rlist)): p.update(i * 100.0 / params.nrefl) mask = shoebox[i].mask if params.pixel_mask == 'precise': # flag everything as background: peak will me assigned later for j in range(len(mask)): mask[j] = mask_back elif params.pixel_mask == 'all': # flag we have no idea what anything is mask_none = MaskCode.Valid|MaskCode.Foreground|MaskCode.Background for j in range(len(mask)): mask[j] = mask_none elif params.pixel_mask == 'static': import itertools from scitbx.array_family import flex x0 = params.spot_offset.x + params.shoebox_size.x / 2 y0 = params.spot_offset.x + params.shoebox_size.y / 2 z0 = params.spot_offset.x + params.shoebox_size.z / 2 sx = params.mask_nsigma * params.spot_size.x sy = params.mask_nsigma * params.spot_size.y sz = params.mask_nsigma * params.spot_size.z # The x, y, z indices z, y, x = zip(*itertools.product(*(range(n) for n in mask.all()))) xyz = flex.vec3_double(flex.double(x), flex.double(y), flex.double(z)) # Calculate SUM(((xj - xj0) / sxj)**2) for each element xyz0 = (x0, y0, z0) isxyz = (1.0/sx, 1.0/sy, 1.0/sz) dxyz = sum([(x * isx)**2 for x, isx in zip(((xyz - xyz0) * rotation).parts(), isxyz)]) # Set the mask values index = dxyz <= 1.0 index.reshape(mask.accessor()) mask.set_selected(index, MaskCode.Valid | MaskCode.Foreground) mask.set_selected(index != True, MaskCode.Valid | MaskCode.Background) sbox = shoebox[i].data # reflection itself, including setting the peak region if we're doing that # FIXME use flex arrays to make the rotation bit more efficient as this is # now rather slow... counts_true = 0 for j in range(params.counts): _x = random.gauss(0, params.spot_size.x) _y = random.gauss(0, params.spot_size.y) _z = random.gauss(0, params.spot_size.z) Rxyz = rotation * matrix.col((_x, _y, _z)).elems x = int(Rxyz[0] + params.spot_offset.x + params.shoebox_size.x / 2) y = int(Rxyz[1] + params.spot_offset.y + params.shoebox_size.y / 2) z = int(Rxyz[2] + params.spot_offset.z + params.shoebox_size.z / 2) if x < 0 or x >= params.shoebox_size.x: continue if y < 0 or y >= params.shoebox_size.y: continue if z < 0 or z >= params.shoebox_size.z: continue sbox[z, y, x] += 1 counts_true += 1 if params.pixel_mask == 'precise': mask[z, y, x] = mask_peak intensity[i] = counts_true if params.background: # background:flat; for j in range(params.background * len(sbox)): x = random.randint(0, params.shoebox_size.x - 1) y = random.randint(0, params.shoebox_size.y - 1) z = random.randint(0, params.shoebox_size.z - 1) sbox[z, y, x] += 1 else: # or inclined random_background_plane(sbox, params.background_a, params.background_b, params.background_c, params.background_d) rlist['miller_index'] = hkl rlist['s1'] = s1 rlist['xyzcal.px'] = xyzpx rlist['xyzcal.mm'] = xyzmm rlist['bbox'] = bbox rlist['panel'] = panel rlist['shoebox'] = shoebox rlist['intensity.sum.value'] = intensity p.finished('Generating %d shoeboxes' % params.nrefl) return rlist
def tst_extract_shoeboxes(self): from dials.array_family import flex from random import randint, seed from dials.algorithms.shoebox import MaskCode import sys seed(0) reflections = flex.reflection_table() reflections['panel'] = flex.size_t() reflections['bbox'] = flex.int6() npanels = 2 width = 1000 height = 1000 nrefl = 10000 frame0 = 10 frame1 = 100 nrefl = 1000 for i in range(nrefl): xs = randint(5, 10) ys = randint(5, 10) x0 = randint(-xs+1, width-1) y0 = randint(-ys+1, height-1) z0 = randint(frame0, frame1-1) x1 = x0 + xs y1 = y0 + ys z1 = min([z0 + randint(1, 10), frame1]) assert(x1 > x0) assert(y1 > y0) assert(z1 > z0) assert(z0 >= frame0 and z1 <= frame1) bbox = (x0, x1, y0, y1, z0, z1) reflections.append({ "panel" : randint(0,1), "bbox" : bbox, }) reflections['shoebox'] = flex.shoebox( reflections['panel'], reflections['bbox']) reflections['shoebox'].allocate() class FakeImageSet(object): def __init__(self): from dials.array_family import flex self.data = flex.int(range(height*width)) self.data.reshape(flex.grid(height,width)) def get_array_range(self): return (frame0, frame1) def get_detector(self): class FakeDetector(object): def __len__(self): return npanels def __getitem__(self, index): class FakePanel(object): def get_trusted_range(self): return (-1, 1000000) return FakePanel() return FakeDetector() def __len__(self): return frame1 - frame0 def __getitem__(self, index): f = frame0+index return (self.data + f*1, self.data + f*2) def get_corrected_data(self, index): f = frame0+index return (self.data + f*1, self.data + f*2) def get_mask(self, index): image = self.get_corrected_data(index) return tuple(im >= 0 for im in image) imageset = FakeImageSet() stdout = sys.stdout class DevNull(object): def write(self, *args): pass def flush(self): pass sys.stdout = DevNull() reflections.extract_shoeboxes(imageset) sys.stdout = stdout for i in range(len(reflections)): sbox = reflections[i]["shoebox"] assert(sbox.is_consistent()) mask = sbox.mask data = sbox.data bbox = sbox.bbox panel = sbox.panel x0, x1, y0, y1, z0, z1 = bbox for z in range(z1 - z0): for y in range(y1 - y0): for x in range(x1 - x0): v1 = data[z,y,x] m1 = mask[z,y,x] if (x0 + x >= 0 and y0 + y >= 0 and x0 + x < width and y0 + y < height): v2 = imageset.data[y+y0,x+x0] + (z+z0)*(panel+1) m2 = MaskCode.Valid assert(v1 == v2) assert(m1 == m2) else: assert(v1 == 0) assert(m1 == 0) print 'OK'
def test_extract_shoeboxes(): from dials.algorithms.shoebox import MaskCode random.seed(0) reflections = flex.reflection_table() reflections["panel"] = flex.size_t() reflections["bbox"] = flex.int6() npanels = 2 width = 1000 height = 1000 frame0 = 10 frame1 = 100 nrefl = 1000 for i in range(nrefl): xs = random.randint(5, 10) ys = random.randint(5, 10) x0 = random.randint(-xs + 1, width - 1) y0 = random.randint(-ys + 1, height - 1) z0 = random.randint(frame0, frame1 - 1) x1 = x0 + xs y1 = y0 + ys z1 = min([z0 + random.randint(1, 10), frame1]) assert x1 > x0 assert y1 > y0 assert z1 > z0 assert z0 >= frame0 and z1 <= frame1 bbox = (x0, x1, y0, y1, z0, z1) reflections.append({"panel": random.randint(0, 1), "bbox": bbox}) reflections["shoebox"] = flex.shoebox(reflections["panel"], reflections["bbox"]) reflections["shoebox"].allocate() class FakeImageSet(object): def __init__(self): self.data = flex.int(range(height * width)) self.data.reshape(flex.grid(height, width)) def get_array_range(self): return (frame0, frame1) def get_detector(self): class FakeDetector(object): def __len__(self): return npanels def __getitem__(self, index): class FakePanel(object): def get_trusted_range(self): return (-1, 1000000) return FakePanel() return FakeDetector() def __len__(self): return frame1 - frame0 def __getitem__(self, index): f = frame0 + index return (self.data + f * 1, self.data + f * 2) def get_corrected_data(self, index): f = frame0 + index return (self.data + f * 1, self.data + f * 2) def get_mask(self, index): image = self.get_corrected_data(index) return tuple(im >= 0 for im in image) imageset = FakeImageSet() reflections.extract_shoeboxes(imageset) for i in range(len(reflections)): sbox = reflections[i]["shoebox"] assert sbox.is_consistent() mask = sbox.mask data = sbox.data bbox = sbox.bbox panel = sbox.panel x0, x1, y0, y1, z0, z1 = bbox for z in range(z1 - z0): for y in range(y1 - y0): for x in range(x1 - x0): v1 = data[z, y, x] m1 = mask[z, y, x] if (x0 + x >= 0 and y0 + y >= 0 and x0 + x < width and y0 + y < height): v2 = imageset.data[y + y0, x + x0] + (z + z0) * (panel + 1) m2 = MaskCode.Valid assert v1 == v2 assert m1 == m2 else: assert v1 == 0 assert m1 == 0
def set_reflections(self, reflections): '''Set the reflection data.''' self.set_data(reflections, ReflectionListEncoder()) def get_reflections(self): '''Get the reflection data.''' return self.get_data(ReflectionListDecoder()) if __name__ == '__main__': from dials.array_family import flex reflections = flex.reflection_table([('hkl', flex.miller_index(10)), ('s1', flex.vec3_double(10)), ('bbox', flex.int6(10)), ('id', flex.int(10)), ('shoebox', flex.shoebox(10))]) for i in range(10): reflections['shoebox'][i].data = flex.double(flex.grid(10, 10, 10)) reflections['shoebox'][i].mask = flex.int(flex.grid(10, 10, 10)) reflections['shoebox'][i].background = flex.double( flex.grid(10, 10, 10)) for i in range(10): print reflections['shoebox'][i].data.all() writer = NexusFile('temp.h5', 'w') writer.set_reflections(reflections) writer.close() reader = NexusFile('temp.h5', 'r')
def reconstruct_peabox(params): assert os.path.exists('xia2.json') from xia2.Schema.XProject import XProject xinfo = XProject.from_json(filename='xia2.json') from dxtbx.model.experiment_list import ExperimentListFactory import cPickle as pickle import dials # because WARNING:root:No profile class gaussian_rs registered from dials.array_family import flex crystals = xinfo.get_crystals() assert len(crystals) == 1 for xname in crystals: crystal = crystals[xname] scaler = crystal._get_scaler() epochs = scaler._sweep_handler.get_epochs() from xia2.command_line.rogues_gallery import munch_rogues from pprint import pprint batched_reflections = {} for epoch in epochs: si = scaler._sweep_handler.get_sweep_information(epoch) intgr = si.get_integrater() experiments = ExperimentListFactory.from_json_file( intgr.get_integrated_experiments()) reflections = pickle.load(open(intgr.get_integrated_reflections())) batched_reflections[si.get_batch_range()] = (experiments, reflections, si.get_sweep_name()) from dials.util import debug_console # debug_console() good = reflections.get_flags(reflections.flags.integrated) # bad = reflections.get_flags(reflections.flags.bad_spot) reflections = reflections.select(good) for r in reflections: flags = r['flags'] r['flags'] = [] for v, f in reflections.flags.values.iteritems(): if flags & f: r['flags'].append(str(f)) r['flags'] = ', '.join(r['flags']) # pprint(r) print "Collecting shoeboxes for %d reflections" % len(reflections) reflections["shoebox"] = flex.shoebox(reflections["panel"], reflections["bbox"], allocate=True) reflections.extract_shoeboxes(experiments.imagesets()[0], verbose=True) print "Consolidating..." sizes = {} for r in reflections: s = r['shoebox'] a = s.size() if a not in sizes: sizes[a] = { 'sum': flex.float([0] * a[0] * a[1] * a[2]), 'weights': flex.int([0] * a[0] * a[1] * a[2]), 'count': 0 } s.mask.set_selected(s.data < 0, 0) s.data.set_selected(s.mask == 0, 0) s.background.set_selected(s.mask == 0, 0) sizes[a]['sum'] += s.data - s.background sizes[a]['weights'] += s.mask sizes[a]['count'] += 1 print len(sizes), "shoebox sizes extracted" for s, c in sizes.iteritems(): print "%dx %s" % (c['count'], str(s)) sdat = iter(c['sum']) wdat = iter(c['weights']) for z in range(s[0]): for y in range(s[1]): count = [next(sdat) for x in range(s[2])] weight = [next(wdat) for x in range(s[2])] truecount = (0 if w == 0 else 10 * c / w for c, w in zip(count, weight)) visualize = ("X" if c < 0 else ("." if c < 10 else str(int(math.log10(c)))) for c in truecount) print "".join(visualize) print "" print ""
def simple_gaussian_spots(params): from scitbx import matrix from dials.array_family import flex r = params.rotation axis = matrix.col((r.axis.x, r.axis.y, r.axis.z)) if axis.length() > 0: rotation = axis.axis_and_angle_as_r3_rotation_matrix(r.angle, deg=True) else: rotation = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1)) # generate mask and peak values from dials.algorithms.shoebox import MaskCode mask_peak = MaskCode.Valid | MaskCode.Foreground mask_back = MaskCode.Valid | MaskCode.Background from dials.util.command_line import ProgressBar p = ProgressBar(title="Generating reflections") rlist = flex.reflection_table(params.nrefl) hkl = flex.miller_index(params.nrefl) s1 = flex.vec3_double(params.nrefl) xyzmm = flex.vec3_double(params.nrefl) xyzpx = flex.vec3_double(params.nrefl) panel = flex.size_t(params.nrefl) bbox = flex.int6(params.nrefl) for j in range(params.nrefl): p.update(j * 100.0 / params.nrefl) hkl[j] = (random.randint(0, 20), random.randint(0, 20), random.randint(0, 20)) phi = 2 * math.pi * random.random() s1[j] = (0, 0, 0) xyzpx[j] = (0, 0, 0) xyzmm[j] = (0, 0, phi) panel[j] = 0 bbox[j] = ( 0, params.shoebox_size.x, 0, params.shoebox_size.y, 0, params.shoebox_size.z, ) p.finished("Generating %d reflections" % params.nrefl) intensity = flex.double(params.nrefl) shoebox = flex.shoebox(panel, bbox) shoebox.allocate_with_value(MaskCode.Valid) p = ProgressBar(title="Generating shoeboxes") for i in range(len(rlist)): p.update(i * 100.0 / params.nrefl) mask = shoebox[i].mask if params.pixel_mask == "precise": # flag everything as background: peak will me assigned later for j in range(len(mask)): mask[j] = mask_back elif params.pixel_mask == "all": # flag we have no idea what anything is mask_none = MaskCode.Valid | MaskCode.Foreground | MaskCode.Background for j in range(len(mask)): mask[j] = mask_none elif params.pixel_mask == "static": from scitbx.array_family import flex x0 = params.spot_offset.x + params.shoebox_size.x / 2 y0 = params.spot_offset.x + params.shoebox_size.y / 2 z0 = params.spot_offset.x + params.shoebox_size.z / 2 sx = params.mask_nsigma * params.spot_size.x sy = params.mask_nsigma * params.spot_size.y sz = params.mask_nsigma * params.spot_size.z # The x, y, z indices z, y, x = zip(*itertools.product(*(range(n) for n in mask.all()))) xyz = flex.vec3_double(flex.double(x), flex.double(y), flex.double(z)) # Calculate SUM(((xj - xj0) / sxj)**2) for each element xyz0 = (x0, y0, z0) isxyz = (1.0 / sx, 1.0 / sy, 1.0 / sz) dxyz = sum( (x * isx) ** 2 for x, isx in zip(((xyz - xyz0) * rotation).parts(), isxyz) ) # Set the mask values index = dxyz <= 1.0 index.reshape(mask.accessor()) mask.set_selected(index, MaskCode.Valid | MaskCode.Foreground) mask.set_selected(not index, MaskCode.Valid | MaskCode.Background) sbox = shoebox[i].data # reflection itself, including setting the peak region if we're doing that # FIXME use flex arrays to make the rotation bit more efficient as this is # now rather slow... counts_true = 0 for j in range(params.counts): _x = random.gauss(0, params.spot_size.x) _y = random.gauss(0, params.spot_size.y) _z = random.gauss(0, params.spot_size.z) Rxyz = rotation * matrix.col((_x, _y, _z)).elems x = int(Rxyz[0] + params.spot_offset.x + params.shoebox_size.x / 2) y = int(Rxyz[1] + params.spot_offset.y + params.shoebox_size.y / 2) z = int(Rxyz[2] + params.spot_offset.z + params.shoebox_size.z / 2) if x < 0 or x >= params.shoebox_size.x: continue if y < 0 or y >= params.shoebox_size.y: continue if z < 0 or z >= params.shoebox_size.z: continue sbox[z, y, x] += 1 counts_true += 1 if params.pixel_mask == "precise": mask[z, y, x] = mask_peak intensity[i] = counts_true if params.background: # background:flat; for j in range(params.background * len(sbox)): x = random.randint(0, params.shoebox_size.x - 1) y = random.randint(0, params.shoebox_size.y - 1) z = random.randint(0, params.shoebox_size.z - 1) sbox[z, y, x] += 1 else: # or inclined random_background_plane( sbox, params.background_a, params.background_b, params.background_c, params.background_d, ) rlist["miller_index"] = hkl rlist["s1"] = s1 rlist["xyzcal.px"] = xyzpx rlist["xyzcal.mm"] = xyzmm rlist["bbox"] = bbox rlist["panel"] = panel rlist["shoebox"] = shoebox rlist["intensity.sum.value"] = intensity p.finished("Generating %d shoeboxes" % params.nrefl) return rlist
if __name__ == '__main__': from dials.array_family import flex from dxtbx.model.experiment.experiment_list import ExperimentListFactory import os.path path = "/home/upc86896/Projects/cctbx/sources/dials_regression/centroid_test_data" rlist_filename = os.path.join(path, "integrated.pickle") exlist_filename = os.path.join(path, "experiments.json") rlist = flex.reflection_table.from_pickle(rlist_filename) exlist = ExperimentListFactory.from_json_file(exlist_filename) panel = rlist['panel'] bbox = rlist['bbox'] rlist['shoebox'] = flex.shoebox(panel, bbox) rlist['shoebox'].allocate() rlist.fill_shoeboxes(exlist[0].imageset)
from dials.array_family import flex num_ref = 3 ref_table = flex.reflection_table() shoebox = flex.shoebox(num_ref) ref_table['shoebox'] = shoebox intensity = flex.double(num_ref) ref_table['intensity.sum.value'] = intensity intensity_var = flex.double(num_ref) ref_table['intensity.sum.variance'] = intensity_var iterate = ref_table['shoebox'] n = 0 for arr in iterate: img = flex.double(flex.grid(3, 3, 3)) bkg = flex.double(flex.grid(3, 3, 3)) msk = flex.int(flex.grid(3, 3, 3)) for row in range(3): for col in range(3): for fra in range(3): img[row, col, fra] = row + col + fra + n * 9 bkg[row, col, fra] = 0.0 msk[row, col, fra] = 3 n += 1 msk[1, 1, 1] = 5 tmp_i = n * n * n * 3 img[1, 1, 1] += tmp_i print "intensity must be =", tmp_i arr.data = img[:, :, :]
import numpy data2d = numpy.zeros((3, 3), dtype = numpy.float64) data2d[:, :] = 15 data2d[1:2, 1:2] = 50 for row in range(3): for col in range(3): data2d[row, col] += row * 2 data2d[row, col] += col * 2 ref_table = flex.reflection_table() ''' shoebox = flex.shoebox(5) #panel = flex.size_t(1) ref_table = flex.reflection_table() shoebox.data = flex.double(flex.grid(6, 6, 6)) ref_table['shoebox'] = shoebox #ref_table['panel'] = panel #efisient_way = ''' its = ref_table['shoebox'] for arr in its: print arr.data.as_numpy_array() #'''
dials_regression = libtbx.env.dist_path('dials_regression') except KeyError, e: print 'FAIL: dials_regression not configured' exit(0) path = join(dials_regression, "centroid_test_data") import sys assert(len(sys.argv) == 1) sys.argv.append(join(path, "experiments.json")) sys.argv.append(join(path, "profile.phil")) parser = OptionParser(phil=phil_scope) params, options, args = parser.parse_args() assert(len(args) == 1) exlist = ExperimentListFactory.from_json_file(args[0]) assert(len(exlist) == 1) profile_model = ProfileModelList.load(params) rlist = flex.reflection_table.from_predictions_multi(exlist) rlist.compute_bbox(exlist, profile_model) rlist['shoebox'] = flex.shoebox(rlist['panel'], rlist['bbox']) rlist['shoebox'].allocate() rlist.extract_shoeboxes(exlist[0].imageset) show_reflection(rlist[len(rlist)//2]) #show_reflection(rlist[len(rlist)//2], orient = "porTrait") #show_reflection(rlist[len(rlist)//2], orient = "lanDscape")
def __call__(self, imageset): """ Find the spots in the imageset :param imageset: The imageset to process :return: The list of spot shoeboxes """ from dials.util.command_line import Command from dials.array_family import flex from dxtbx.imageset import ImageSweep from dials.model.data import PixelListLabeller from logging import info, warn from math import floor import platform from logging import warn # Change the number of processors if necessary mp_nproc = self.nproc if platform.system() == "Windows" and mp_nproc > 1: warn("") warn("*" * 80) warn("Multiprocessing is not available on windows. Setting nproc = 1") warn("*" * 80) warn("") mp_nproc = 1 if mp_nproc > len(imageset): mp_nproc = len(imageset) mp_method = self.mp_method mp_chunksize = self.mp_chunksize len_by_nproc = int(floor(len(imageset) / mp_nproc)) if mp_chunksize > len_by_nproc: mp_chunksize = len_by_nproc assert mp_nproc > 0, "Invalid number of processors" assert mp_chunksize > 0, "Invalid chunk size" # The extract pixels function function = ExtractPixelsFromImage( imageset=imageset, threshold_function=self.threshold_function, mask=self.mask, max_strong_pixel_fraction=self.max_strong_pixel_fraction, region_of_interest=self.region_of_interest, ) # The indices to iterate over indices = list(range(len(imageset))) # Initialise the pixel labeller num_panels = len(imageset.get_detector()) pixel_labeller = [PixelListLabeller() for p in range(num_panels)] # Do the processing info("Extracting strong pixels from images") info(" Using %s with %d parallel job(s)\n" % (mp_method, mp_nproc)) if mp_nproc > 1: def process_output(result): import logging for message in result[1]: logging.log(message.levelno, message.msg) assert len(pixel_labeller) == len(result[0].pixel_list), "Inconsistent size" for plabeller, plist in zip(pixel_labeller, result[0].pixel_list): plabeller.add(plist) result[0].pixel_list = None # Parallel reading of HDF5 from the same handle is not allowed. Python # multiprocessing is a bit messed up and used fork on linux so need to # close and reopen file. from dxtbx.imageset import SingleFileReader if isinstance(imageset.reader(), SingleFileReader): imageset.reader().nullify_format_instance() batch_parallel_map( func=ExtractSpotsParallelTask(function), iterable=indices, processes=mp_nproc, callback=process_output, method=mp_method, chunksize=mp_chunksize, ) else: for task in indices: result = function(task) assert len(pixel_labeller) == len(result.pixel_list), "Inconsistent size" for plabeller, plist in zip(pixel_labeller, result.pixel_list): plabeller.add(plist) result.pixel_list = None # Extract the pixel lists into a list of reflections shoeboxes = flex.shoebox() if isinstance(imageset, ImageSweep): twod = False else: twod = True for i, p in enumerate(pixel_labeller): if p.num_pixels() > 0: shoeboxes.extend( flex.shoebox( p, i, 0, twod, self.min_spot_size, self.max_spot_size # panel # zrange # twod # min_pixels ) ) # max_pixels info("") info("Extracted {0} spots".format(len(shoeboxes))) # Get the unallocated spots and print some info selection = shoeboxes.is_allocated() shoeboxes = shoeboxes.select(selection) info( "Removed %d spots with size < %d or > %d pixels" % (selection.count(False), self.min_spot_size, self.max_spot_size) ) # Return the shoeboxes return shoeboxes
def run(self): ''' Extract the shoeboxes. ''' from dials.util.options import flatten_reflections from dials.util.options import flatten_experiments from dials.util.options import flatten_datablocks from dials.util import log from dials.array_family import flex from libtbx.utils import Sorry from logging import info # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) # Configure logging log.config() # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': info('The following parameters have been modified:\n') info(diff_phil) # Get the data reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) datablocks = flatten_datablocks(params.input.datablock) if len(experiments) == 0 and len(datablocks) == 0 and len(reflections) == 0: self.parser.print_help() exit(0) elif (len(experiments) != 0 and len(datablocks) != 0): raise Sorry('Both experiment list and datablocks set') elif len(experiments) > 1: raise Sorry('More than 1 experiment set') elif len(datablocks) > 1: raise Sorry('More than 1 datablock set') elif len(experiments) == 1: imageset = experiments[0].imageset elif len(datablocks) == 1: imagesets = datablocks[0].extract_imagesets() if len(imagesets) != 1: raise Sorry('Need 1 imageset, got %d' % len(imagesets)) imageset = imagesets[0] if len(reflections) != 1: raise Sorry('Need 1 reflection table, got %d' % len(reflections)) else: reflections = reflections[0] # Check the reflections contain the necessary stuff assert("bbox" in reflections) assert("panel" in reflections) # Get some models detector = imageset.get_detector() scan = imageset.get_scan() frame0, frame1 = scan.get_array_range() # Add some padding but limit to image volume if params.padding > 0: info('Adding %d pixels as padding' % params.padding) x0, x1, y0, y1, z0, z1 = reflections['bbox'].parts() x0 -= params.padding x1 += params.padding y0 -= params.padding y1 += params.padding z0 -= params.padding z1 += params.padding panel = reflections['panel'] for i in range(len(reflections)): width, height = detector[panel[i]].get_image_size() if z0[i] < frame0: z0[i] = frame0 if z1[i] > frame1: z1[i] = frame1 reflections['bbox'] = flex.int6(x0, x1, y0, y1, z0, z1) # Save the old shoeboxes if "shoebox" in reflections: old_shoebox = reflections['shoebox'] else: old_shoebox = None # Allocate the shoeboxes reflections["shoebox"] = flex.shoebox( reflections["panel"], reflections["bbox"], allocate=True) # Extract the shoeboxes reflections.extract_shoeboxes(imageset, verbose=True) # Preserve masking if old_shoebox is not None: info("Applying old shoebox mask") new_shoebox = reflections['shoebox'] for i in range(len(reflections)): bbox0 = old_shoebox[i].bbox bbox1 = new_shoebox[i].bbox mask0 = old_shoebox[i].mask mask1 = new_shoebox[i].mask mask2 = flex.int(mask1.accessor(), 0) x0 = bbox0[0] - bbox1[0] x1 = bbox0[1] - bbox0[0] + x0 y0 = bbox0[2] - bbox1[2] y1 = bbox0[3] - bbox0[2] + y0 z0 = bbox0[4] - bbox1[4] z1 = bbox0[5] - bbox0[4] + z0 mask2[z0:z1,y0:y1,x0:x1] = mask0 mask1 = mask1.as_1d() | mask2.as_1d() mask1.reshape(new_shoebox[i].mask.accessor()) new_shoebox[i].mask = mask1 # Saving the reflections to disk filename = params.output.reflections info('Saving %d reflections to %s' % (len(reflections), filename)) reflections.as_pickle(filename)
def reconstruct_rogues(params): assert os.path.exists('xia2.json') from xia2.Schema.XProject import XProject xinfo = XProject.from_json(filename='xia2.json') from dxtbx.model.experiment.experiment_list import ExperimentListFactory import cPickle as pickle import dials # because WARNING:root:No profile class gaussian_rs registered crystals = xinfo.get_crystals() assert len(crystals) == 1 for xname in crystals: crystal = crystals[xname] scaler = crystal._get_scaler() epochs = scaler._sweep_handler.get_epochs() rogues = os.path.join(scaler.get_working_directory(), xname, 'scale', 'ROGUES') rogue_reflections = munch_rogues(rogues) batched_reflections = { } for epoch in epochs: si = scaler._sweep_handler.get_sweep_information(epoch) intgr = si.get_integrater() experiments = ExperimentListFactory.from_json_file( intgr.get_integrated_experiments()) reflections = pickle.load(open(intgr.get_integrated_reflections())) batched_reflections[si.get_batch_range()] = (experiments, reflections, si.get_sweep_name()) # - look up reflection in reflection list, get bounding box # - pull pixels given from image set, flatten these, write out from dials.array_family import flex from annlib_ext import AnnAdaptor as ann_adaptor reflections_run = { } for run in batched_reflections: reflections_run[run] = [] for rogue in rogue_reflections: b = rogue[0] for run in batched_reflections: if b >= run[0] and b <= run[1]: reflections_run[run].append(rogue) break for run_no, run in enumerate(reflections_run): experiment = batched_reflections[run][0] reflections = batched_reflections[run][1] name = batched_reflections[run][2] rogues = reflections_run[run] reference = flex.double() scan = experiment.scans()[0] images = experiment.imagesets()[0] for xyz in reflections['xyzcal.px']: reference.append(xyz[0]) reference.append(xyz[1]) reference.append(xyz[2]) search = flex.double() for rogue in rogues: search.append(rogue[1]) search.append(rogue[2]) search.append(scan.get_array_index_from_angle(rogue[3])) ann = ann_adaptor(data=reference, dim=3, k=1) ann.query(search) keep = flex.bool(len(reflections), False) for j, rogue in enumerate(rogues): keep[ann.nn[j]] = True reflections = reflections.select(keep==True) if params.extract: reflections["shoebox"] = flex.shoebox( reflections["panel"], reflections["bbox"], allocate=True) reflections.extract_shoeboxes(images, verbose=False) if len(reflections_run) > 1: output = params.output.reflections.replace( '.pickle', '-%s.pickle' % name) print 'Extracted %d rogue reflections for %s to %s' % \ (len(reflections), name, output) reflections.as_pickle(output) else: output = params.output.reflections print 'Extracted %d rogue reflections to %s' % \ (len(reflections), output) reflections.as_pickle(output)
def run(self): ''' Extract the shoeboxes. ''' from dials.util.options import flatten_reflections from dials.util.options import flatten_experiments from dials.util.options import flatten_datablocks from dials.util import log from dials.array_family import flex from libtbx.utils import Sorry # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) # Configure logging log.config() # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) # Get the data reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) datablocks = flatten_datablocks(params.input.datablock) if not any([experiments, datablocks, reflections]): self.parser.print_help() exit(0) elif experiments and datablocks: raise Sorry('Both experiment list and datablocks set') elif len(experiments) > 1: raise Sorry('More than 1 experiment set') elif len(datablocks) > 1: raise Sorry('More than 1 datablock set') elif len(experiments) == 1: imageset = experiments[0].imageset elif len(datablocks) == 1: imagesets = datablocks[0].extract_imagesets() if len(imagesets) != 1: raise Sorry('Need 1 imageset, got %d' % len(imagesets)) imageset = imagesets[0] if len(reflections) != 1: raise Sorry('Need 1 reflection table, got %d' % len(reflections)) else: reflections = reflections[0] # Check the reflections contain the necessary stuff assert("bbox" in reflections) assert("panel" in reflections) # Get some models detector = imageset.get_detector() scan = imageset.get_scan() frame0, frame1 = scan.get_array_range() # Add some padding but limit to image volume if params.padding > 0: logger.info('Adding %d pixels as padding' % params.padding) x0, x1, y0, y1, z0, z1 = reflections['bbox'].parts() x0 -= params.padding x1 += params.padding y0 -= params.padding y1 += params.padding # z0 -= params.padding # z1 += params.padding panel = reflections['panel'] for i in range(len(reflections)): width, height = detector[panel[i]].get_image_size() if x0[i] < 0: x0[i] = 0 if x1[i] > width: x1[i] = width if y0[i] < 0: y0[i] = 0 if y1[i] > height: y1[i] = height if z0[i] < frame0: z0[i] = frame0 if z1[i] > frame1: z1[i] = frame1 reflections['bbox'] = flex.int6(x0, x1, y0, y1, z0, z1) # Save the old shoeboxes if "shoebox" in reflections: old_shoebox = reflections['shoebox'] else: old_shoebox = None # Allocate the shoeboxes reflections["shoebox"] = flex.shoebox( reflections["panel"], reflections["bbox"], allocate=True) # Extract the shoeboxes reflections.extract_shoeboxes(imageset, verbose=True) # Preserve masking if old_shoebox is not None: from dials.algorithms.shoebox import MaskCode logger.info("Applying old shoebox mask") new_shoebox = reflections['shoebox'] for i in range(len(reflections)): bbox0 = old_shoebox[i].bbox bbox1 = new_shoebox[i].bbox mask0 = old_shoebox[i].mask mask1 = new_shoebox[i].mask mask2 = flex.int(mask1.accessor(), 0) x0 = bbox0[0] - bbox1[0] x1 = bbox0[1] - bbox0[0] + x0 y0 = bbox0[2] - bbox1[2] y1 = bbox0[3] - bbox0[2] + y0 z0 = bbox0[4] - bbox1[4] z1 = bbox0[5] - bbox0[4] + z0 mask2[z0:z1,y0:y1,x0:x1] = mask0 mask1 = mask1.as_1d() | mask2.as_1d() if params.padding_is_background: selection = flex.size_t(range(len(mask1))).select(mask1 == MaskCode.Valid) values = flex.int(len(selection), MaskCode.Valid | MaskCode.Background) mask1.set_selected(selection, values) mask1.reshape(new_shoebox[i].mask.accessor()) new_shoebox[i].mask = mask1 # Saving the reflections to disk filename = params.output.reflections logger.info('Saving %d reflections to %s' % (len(reflections), filename)) reflections.as_pickle(filename)
def __call__(self): ''' Do the processing. :return: The processed data ''' from dials.array_family import flex from time import time from dials.model.data import make_image from libtbx.introspection import machine_memory_info # Get the start time start_time = time() # Set the global process ID job.index = self.index # Check all reflections have same imageset and get it exp_id = list(set(self.reflections['id'])) imageset = self.experiments[exp_id[0]].imageset for i in exp_id[1:]: assert self.experiments[i].imageset == imageset, "Task can only handle 1 imageset" # Get the sub imageset frame00, frame01 = self.job try: frame10, frame11 = imageset.get_array_range() except Exception: frame10, frame11 = (0, len(imageset)) try: assert frame00 < frame01 assert frame10 < frame11 assert frame00 >= frame10 assert frame01 <= frame11 index0 = frame00 - frame10 index1 = index0 + (frame01 - frame00) assert index0 < index1 assert index0 >= 0 assert index1 <= len(imageset) imageset = imageset[index0:index1] except Exception: raise RuntimeError('Programmer Error: bad array range') try: frame0, frame1 = imageset.get_array_range() except Exception: frame0, frame1 = (0, len(imageset)) # Initlize the executor self.executor.initialize(frame0, frame1, self.reflections) # Set the shoeboxes (dont't allocate) self.reflections['shoebox'] = flex.shoebox( self.reflections['panel'], self.reflections['bbox'], allocate=False, flatten=self.params.shoebox.flatten) # Create the processor processor = ShoeboxProcessor( self.reflections, len(imageset.get_detector()), frame0, frame1, self.params.debug.output) # Compute percentage of max available. The function is not portable to # windows so need to add a check if the function fails. On windows no # warning will be printed memory_info = machine_memory_info() total_memory = memory_info.memory_total() sbox_memory = processor.compute_max_memory_usage() if total_memory is not None: assert total_memory > 0, "Your system appears to have no memory!" assert self.params.block.max_memory_usage > 0.0, "maximum memory usage must be > 0" assert self.params.block.max_memory_usage <= 1.0, "maximum memory usage must be <= 1" limit_memory = total_memory * self.params.block.max_memory_usage if sbox_memory > limit_memory: raise RuntimeError(''' There was a problem allocating memory for shoeboxes. Possible solutions include increasing the percentage of memory allowed for shoeboxes or decreasing the block size. This could also be caused by a highly mosaic crystal model - is your crystal really this mosaic? Total system memory: %g GB Limit shoebox memory: %g GB Required shoebox memory: %g GB ''' % (total_memory/1e9, limit_memory/1e9, sbox_memory/1e9)) else: logger.info(' Memory usage:') logger.info(' Total system memory: %g GB' % (total_memory/1e9)) logger.info(' Limit shoebox memory: %g GB' % (limit_memory/1e9)) logger.info(' Required shoebox memory: %g GB' % (sbox_memory/1e9)) logger.info('') # Loop through the imageset, extract pixels and process reflections read_time = 0.0 for i in range(len(imageset)): st = time() image = imageset.get_corrected_data(i) mask = imageset.get_mask(i) if self.params.lookup.mask is not None: assert len(mask) == len(self.params.lookup.mask), \ "Mask/Image are incorrect size %d %d" % ( len(mask), len(self.params.lookup.mask)) mask = tuple(m1 & m2 for m1, m2 in zip(self.params.lookup.mask, mask)) read_time += time() - st processor.next(make_image(image, mask), self.executor) del image del mask assert processor.finished(), "Data processor is not finished" # Optionally save the shoeboxes if self.params.debug.output and self.params.debug.separate_files: output = self.reflections if self.params.debug.select is not None: output = output.select(self.params.debug.select(output)) if self.params.debug.split_experiments: output = output.split_by_experiment_id() for table in output: i = table['id'][0] table.as_pickle('shoeboxes_%d_%d.pickle' % (self.index, i)) else: output.as_pickle('shoeboxes_%d.pickle' % self.index) # Delete the shoeboxes if self.params.debug.separate_files or not self.params.debug.output: del self.reflections['shoebox'] # Finalize the executor self.executor.finalize() # Return the result result = Result(self.index, self.reflections, self.executor.data()) result.read_time = read_time result.extract_time = processor.extract_time() result.process_time = processor.process_time() result.total_time = time() - start_time return result
def __call__(self): """ Do the processing. :return: The processed data """ # Get the start time start_time = time() # Set the global process ID job.index = self.index # Check all reflections have same imageset and get it exp_id = list(set(self.reflections["id"])) imageset = self.experiments[exp_id[0]].imageset for i in exp_id[1:]: assert (self.experiments[i].imageset == imageset ), "Task can only handle 1 imageset" # Get the sub imageset frame0, frame1 = self.job try: allowed_range = imageset.get_array_range() except Exception: allowed_range = 0, len(imageset) try: # range increasing assert frame0 < frame1 # within an increasing range assert allowed_range[1] > allowed_range[0] # we are processing data which is within range assert frame0 >= allowed_range[0] assert frame1 <= allowed_range[1] # I am 99% sure this is implied by all the code above assert (frame1 - frame0) <= len(imageset) if len(imageset) > 1: imageset = imageset[frame0:frame1] except Exception as e: raise RuntimeError(f"Programmer Error: bad array range: {e}") try: frame0, frame1 = imageset.get_array_range() except Exception: frame0, frame1 = (0, len(imageset)) self.executor.initialize(frame0, frame1, self.reflections) # Set the shoeboxes (don't allocate) self.reflections["shoebox"] = flex.shoebox( self.reflections["panel"], self.reflections["bbox"], allocate=False, flatten=self.params.shoebox.flatten, ) # Create the processor processor = ShoeboxProcessor( self.reflections, len(imageset.get_detector()), frame0, frame1, self.params.debug.output, ) # Loop through the imageset, extract pixels and process reflections read_time = 0.0 for i in range(len(imageset)): st = time() image = imageset.get_corrected_data(i) if imageset.is_marked_for_rejection(i): mask = tuple(flex.bool(im.accessor(), False) for im in image) else: mask = imageset.get_mask(i) if self.params.lookup.mask is not None: assert len(mask) == len( self.params.lookup.mask ), "Mask/Image are incorrect size %d %d" % ( len(mask), len(self.params.lookup.mask), ) mask = tuple( m1 & m2 for m1, m2 in zip(self.params.lookup.mask, mask)) read_time += time() - st processor.next(make_image(image, mask), self.executor) del image del mask assert processor.finished(), "Data processor is not finished" # Optionally save the shoeboxes if self.params.debug.output and self.params.debug.separate_files: output = self.reflections if self.params.debug.select is not None: output = output.select(self.params.debug.select(output)) if self.params.debug.split_experiments: output = output.split_by_experiment_id() for table in output: i = table["id"][0] table.as_file("shoeboxes_%d_%d.refl" % (self.index, i)) else: output.as_file("shoeboxes_%d.refl" % self.index) # Delete the shoeboxes if self.params.debug.separate_files or not self.params.debug.output: del self.reflections["shoebox"] # Finalize the executor self.executor.finalize() # Return the result return dials.algorithms.integration.Result( index=self.index, reflections=self.reflections, data=self.executor.data(), read_time=read_time, extract_time=processor.extract_time(), process_time=processor.process_time(), total_time=time() - start_time, )
reflection_lookup[j].append(i) width, height = experiments[0].detector[0].get_image_size() sum_background = flex.double(flex.grid(height, width), 0) sum_sq_background = flex.double(flex.grid(height, width), 0) count = flex.int(flex.grid(height, width), 0) # Loop through all images print "START" for frame in range(len(imageset)): # Get the subset of reflections on this image and compute the mask subset = reflections.select(flex.size_t(reflection_lookup[frame])) subset['shoebox'] = flex.shoebox( subset['panel'], subset['bbox'], allocate=True) subset.compute_mask(experiments) # Get the mask and data mask = imageset.get_mask(frame)[0] data = imageset.get_raw_data(frame)[0] sbox_mask = subset['shoebox'].apply_background_mask(frame, 1, (height, width)) mask = mask & sbox_mask #from dials.algorithms.image.threshold import DispersionThreshold #threshold = DispersionThreshold( # data.all(),
def generate_predictions(self, N): """Generate some reflections.""" from dials.algorithms import filtering from dials.algorithms.profile_model.gaussian_rs import MaskCalculator3D from dials.algorithms.profile_model.gaussian_rs import Model as ProfileModel from dials.algorithms.shoebox import MaskCode from dials.util.command_line import Command # Set the profile model self.experiment.profile = ProfileModel(None, self.n_sigma, self.sigma_b, self.sigma_m) # Generate a list of reflections refl = flex.reflection_table.from_predictions(self.experiment) refl["id"] = flex.int(len(refl), 0) # Filter by zeta zeta = 0.05 Command.start(f"Filtering by zeta >= {zeta:f}") mask = filtering.by_zeta(self.experiment.goniometer, self.experiment.beam, refl["s1"], zeta) refl.del_selected(~mask) Command.end(f"Filtered {len(refl)} reflections by zeta >= {zeta:f}") # Compute the bounding box refl.compute_bbox([self.experiment]) index = [] image_size = self.experiment.detector[0].get_image_size() array_range = self.experiment.scan.get_array_range() bbox = refl["bbox"] for i in range(len(refl)): x0, x1, y0, y1, z0, z1 = bbox[i] if (x0 < 0 or x1 > image_size[0] or y0 < 0 or y1 > image_size[1] or z0 < array_range[0] or z1 > array_range[1]): index.append(i) refl.del_selected(flex.size_t(index)) # Sample if specified index = random.sample(range(len(refl)), N) refl = refl.select(flex.size_t(index)) # Compute the bounding box # Create a load of shoeboxes Command.start(f"Creating shoeboxes for {len(refl)} reflections") refl["shoebox"] = flex.shoebox(refl["panel"], refl["bbox"]) refl["shoebox"].allocate_with_value(MaskCode.Valid) Command.end(f"Created shoeboxes for {len(refl)} reflections") # Get the function object to mask the foreground Command.start(f"Masking Foreground for {len(refl)} reflections") mask_foreground = MaskCalculator3D( self.experiment.beam, self.experiment.detector, self.experiment.goniometer, self.experiment.scan, self.n_sigma * self.sigma_b, self.n_sigma * self.sigma_m, ) # Mask the foreground mask_foreground(refl["shoebox"], refl["s1"], refl["xyzcal.px"].parts()[2], refl["panel"]) Command.end(f"Masked foreground for {len(refl)} reflections") # Return the reflections return refl
imageset = experiments[0].imageset print list(reflections.keys()) reflections.unset_flags(flex.size_t(range(len(reflections))), reflections.flags.integrated_sum) reflections.unset_flags(flex.size_t(range(len(reflections))), reflections.flags.integrated_prf) del reflections['intensity.sum.value'] del reflections['intensity.sum.variance'] del reflections['background.mean'] del reflections['intensity.prf.value'] del reflections['intensity.prf.variance'] del reflections['profile.correlation'] del reflections['profile.rmsd'] # Create the reflection lookup reflections['shoebox'] = flex.shoebox( reflections['panel'], reflections['bbox']) bbox = reflections['bbox'] reflection_lookup = defaultdict(list) for i in range(len(bbox)): for j in range(bbox[i][4],bbox[i][5]): reflection_lookup[j].append(i) width, height = experiments[0].detector[0].get_image_size() # Loop through all images print "START" for frame in range(0, len(imageset)): # Get the subset of reflections on this image and compute the mask indices = flex.size_t(reflection_lookup[frame])
def test_to_from_msgpack(tmpdir): from dials.model.data import Shoebox def gen_shoebox(): shoebox = Shoebox(0, (0, 4, 0, 3, 0, 1)) shoebox.allocate() for k in range(1): for j in range(3): for i in range(4): shoebox.data[k, j, i] = i + j + k + 0.1 shoebox.mask[k, j, i] = i % 2 shoebox.background[k, j, i] = i * j + 0.2 return shoebox def compare(a, b): assert a.is_consistent() assert b.is_consistent() assert a.panel == b.panel assert a.bbox == b.bbox for aa, bb in zip(a.data, b.data): if abs(aa - bb) > 1e-9: return False for aa, bb in zip(a.background, b.background): if abs(aa - bb) > 1e-9: return False for aa, bb in zip(a.mask, b.mask): if aa != bb: return False return True # The columns as lists c1 = list(range(10)) c2 = list(range(10)) c3 = ["a", "b", "c", "d", "e", "f", "g", "i", "j", "k"] c4 = [True, False, True, False, True] * 2 c5 = list(range(10)) c6 = [(i + 1, i + 2) for i in range(10)] c7 = [(i + 1, i + 2, i + 3) for i in range(10)] c8 = [tuple(i + j for j in range(9)) for i in range(10)] c9 = [tuple(i + j for j in range(6)) for i in range(10)] c10 = [(i + 1, i + 2, i + 3) for i in range(10)] c11 = [gen_shoebox() for i in range(10)] # Create a table with some elements table = flex.reflection_table() table["col1"] = flex.int(c1) table["col2"] = flex.double(c2) table["col3"] = flex.std_string(c3) table["col4"] = flex.bool(c4) table["col5"] = flex.size_t(c5) table["col6"] = flex.vec2_double(c6) table["col7"] = flex.vec3_double(c7) table["col8"] = flex.mat3_double(c8) table["col9"] = flex.int6(c9) table["col10"] = flex.miller_index(c10) table["col11"] = flex.shoebox(c11) obj = table.as_msgpack() new_table = flex.reflection_table.from_msgpack(obj) assert new_table.is_consistent() assert new_table.nrows() == 10 assert new_table.ncols() == 11 assert all(tuple(a == b for a, b in zip(new_table["col1"], c1))) assert all(tuple(a == b for a, b in zip(new_table["col2"], c2))) assert all(tuple(a == b for a, b in zip(new_table["col3"], c3))) assert all(tuple(a == b for a, b in zip(new_table["col4"], c4))) assert all(tuple(a == b for a, b in zip(new_table["col5"], c5))) assert all(tuple(a == b for a, b in zip(new_table["col6"], c6))) assert all(tuple(a == b for a, b in zip(new_table["col7"], c7))) assert all(tuple(a == b for a, b in zip(new_table["col8"], c8))) assert all(tuple(a == b for a, b in zip(new_table["col9"], c9))) assert all(tuple(a == b for a, b in zip(new_table["col10"], c10))) assert all(tuple(compare(a, b) for a, b in zip(new_table["col11"], c11))) table.as_msgpack_file(tmpdir.join("reflections.mpack").strpath) new_table = flex.reflection_table.from_msgpack_file( tmpdir.join("reflections.mpack").strpath) assert new_table.is_consistent() assert new_table.nrows() == 10 assert new_table.ncols() == 11 assert all(tuple(a == b for a, b in zip(new_table["col1"], c1))) assert all(tuple(a == b for a, b in zip(new_table["col2"], c2))) assert all(tuple(a == b for a, b in zip(new_table["col3"], c3))) assert all(tuple(a == b for a, b in zip(new_table["col4"], c4))) assert all(tuple(a == b for a, b in zip(new_table["col5"], c5))) assert all(tuple(a == b for a, b in zip(new_table["col6"], c6))) assert all(tuple(a == b for a, b in zip(new_table["col7"], c7))) assert all(tuple(a == b for a, b in zip(new_table["col8"], c8))) assert all(tuple(a == b for a, b in zip(new_table["col9"], c9))) assert all(tuple(a == b for a, b in zip(new_table["col10"], c10))) assert all(tuple(compare(a, b) for a, b in zip(new_table["col11"], c11)))
def tst_split_partials_with_shoebox(self): from dials.array_family import flex from random import randint, uniform from dials.model.data import Shoebox r = flex.reflection_table() r['value1'] = flex.double() r['value2'] = flex.int() r['value3'] = flex.double() r['bbox'] = flex.int6() r['panel'] = flex.size_t() r['shoebox'] = flex.shoebox() expected = [] for i in range(100): x0 = randint(0, 100) x1 = x0 + randint(1, 10) y0 = randint(0, 100) y1 = y0 + randint(1, 10) z0 = randint(0, 100) z1 = z0 + randint(1, 10) v1 = uniform(0, 100) v2 = randint(0, 100) v3 = uniform(0, 100) sbox = Shoebox(0, (x0, x1, y0, y1, z0, z1)) sbox.allocate() assert(sbox.is_consistent()) w = x1 - x0 h = y1 - y0 for z in range(z0, z1): for y in range(y0, y1): for x in range(x0, x1): sbox.data[z-z0,y-y0,x-x0] = x+y*w+z*w*h r.append({ 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z0, z1), 'panel' : 0, 'shoebox' : sbox }) for z in range(z0, z1): sbox = Shoebox(0, (x0, x1, y0, y1, z, z+1)) sbox.allocate() assert(sbox.is_consistent()) w = x1 - x0 h = y1 - y0 for y in range(y0, y1): for x in range(x0, x1): sbox.data[0,y-y0,x-x0] = x+y*w+z*w*h expected.append({ 'value1' : v1, 'value2' : v2, 'value3' : v3, 'bbox' : (x0, x1, y0, y1, z, z+1), 'partial_id' : i, 'panel' : 0, 'shoebox' : sbox }) r.split_partials_with_shoebox() assert(len(r) == len(expected)) EPS = 1e-7 for r1, r2 in zip(r, expected): assert(abs(r1['value1'] - r2['value1']) < EPS) assert(r1['value2'] == r2['value2']) assert(abs(r1['value3'] - r2['value3']) < EPS) assert(r1['bbox'] == r2['bbox']) assert(r1['partial_id'] == r2['partial_id']) assert(r1['panel'] == r2['panel']) assert(r1['shoebox'].data.as_double().as_1d().all_approx_equal( r2['shoebox'].data.as_double().as_1d())) print 'OK'
def test_split_partials_with_shoebox(): from dials.model.data import Shoebox r = flex.reflection_table() r["value1"] = flex.double() r["value2"] = flex.int() r["value3"] = flex.double() r["bbox"] = flex.int6() r["panel"] = flex.size_t() r["shoebox"] = flex.shoebox() expected = [] for i in range(100): x0 = random.randint(0, 100) x1 = x0 + random.randint(1, 10) y0 = random.randint(0, 100) y1 = y0 + random.randint(1, 10) z0 = random.randint(0, 100) z1 = z0 + random.randint(1, 10) v1 = random.uniform(0, 100) v2 = random.randint(0, 100) v3 = random.uniform(0, 100) sbox = Shoebox(0, (x0, x1, y0, y1, z0, z1)) sbox.allocate() assert sbox.is_consistent() w = x1 - x0 h = y1 - y0 for z in range(z0, z1): for y in range(y0, y1): for x in range(x0, x1): sbox.data[z - z0, y - y0, x - x0] = x + y * w + z * w * h r.append({ "value1": v1, "value2": v2, "value3": v3, "bbox": (x0, x1, y0, y1, z0, z1), "panel": 0, "shoebox": sbox, }) for z in range(z0, z1): sbox = Shoebox(0, (x0, x1, y0, y1, z, z + 1)) sbox.allocate() assert sbox.is_consistent() w = x1 - x0 h = y1 - y0 for y in range(y0, y1): for x in range(x0, x1): sbox.data[0, y - y0, x - x0] = x + y * w + z * w * h expected.append({ "value1": v1, "value2": v2, "value3": v3, "bbox": (x0, x1, y0, y1, z, z + 1), "partial_id": i, "panel": 0, "shoebox": sbox, }) r.split_partials_with_shoebox() assert len(r) == len(expected) EPS = 1e-7 for r1, r2 in zip(r.rows(), expected): assert abs(r1["value1"] - r2["value1"]) < EPS assert r1["value2"] == r2["value2"] assert abs(r1["value3"] - r2["value3"]) < EPS assert r1["bbox"] == r2["bbox"] assert r1["partial_id"] == r2["partial_id"] assert r1["panel"] == r2["panel"] assert (r1["shoebox"].data.as_double().as_1d().all_approx_equal( r2["shoebox"].data.as_double().as_1d()))
'''Set the reflection data.''' self.set_data(reflections, ReflectionListEncoder()) def get_reflections(self): '''Get the reflection data.''' return self.get_data(ReflectionListDecoder()) if __name__ == '__main__': from dials.array_family import flex reflections = flex.reflection_table([ ('hkl', flex.miller_index(10)), ('s1', flex.vec3_double(10)), ('bbox', flex.int6(10)), ('id', flex.int(10)), ('shoebox', flex.shoebox(10)) ]) for i in range(10): reflections['shoebox'][i].data = flex.double(flex.grid(10,10,10)) reflections['shoebox'][i].mask = flex.int(flex.grid(10,10,10)) reflections['shoebox'][i].background = flex.double(flex.grid(10,10,10)) for i in range(10): print reflections['shoebox'][i].data.all() writer = NexusFile('temp.h5', 'w') writer.set_reflections(reflections) writer.close() reader = NexusFile('temp.h5', 'r')
def _find_in_imageset(self, imageset): """ Do the spot finding. :param imageset: The imageset to process :return: The observed spots """ from dials.util.masking import MaskGenerator from dials.array_family import flex from dials.util.command_line import Command from dxtbx.imageset import ImageSweep from logging import info # The input mask mask = self.mask_generator.generate(imageset) if self.mask is not None: mask = tuple(m1 & m2 for m1, m2 in zip(mask, self.mask)) # Set the spot finding algorithm extract_spots = ExtractSpots( threshold_function=self.threshold_function, mask=mask, region_of_interest=self.region_of_interest, max_strong_pixel_fraction=self.max_strong_pixel_fraction, mp_method=self.mp_method, nproc=self.nproc, mp_chunksize=self.mp_chunksize, min_spot_size=self.min_spot_size, max_spot_size=self.max_spot_size, ) # Get the max scan range if isinstance(imageset, ImageSweep): max_scan_range = imageset.get_array_range() else: max_scan_range = (0, len(imageset)) # Get list of scan ranges if not self.scan_range or self.scan_range[0] is None: scan_range = [(max_scan_range[0] + 1, max_scan_range[1])] else: scan_range = self.scan_range # Get spots from bits of scan spots_all = [] for scan in scan_range: j0, j1 = scan assert j1 >= j0 and j0 > max_scan_range[0] and j1 <= max_scan_range[1] info("\nFinding spots in image {0} to {1}...".format(j0, j1)) j0 -= 1 if isinstance(imageset, ImageSweep): j0 -= imageset.get_array_range()[0] j1 -= imageset.get_array_range()[0] spots_all.extend(extract_spots(imageset[j0:j1])) # Get the list of shoeboxes shoeboxes = flex.shoebox(spots_all) # Calculate the spot centroids centroid = shoeboxes.centroid_valid() info("Calculated {0} spot centroids".format(len(shoeboxes))) # Calculate the spot intensities intensity = shoeboxes.summed_intensity() info("Calculated {0} spot intensities".format(len(shoeboxes))) # Create the observations observed = flex.observation(shoeboxes.panels(), centroid, intensity) # Write the hot mask if self.write_hot_mask: # Find spots which cover the whole scan range bbox = flex.int6([sbox.bbox for sbox in shoeboxes]) z0, z1 = bbox.parts()[4:6] zr = z1 - z0 assert zr.all_gt(0) possible_hot_spots = zr == len(imageset) num_possible_hot_spots = possible_hot_spots.count(True) info("Found %d possible hot spots" % num_possible_hot_spots) # Create the hot pixel mask hot_mask = tuple(flex.bool(flex.grid(p.get_image_size()[::-1]), True) for p in imageset.get_detector()) if num_possible_hot_spots > 0: hot_shoeboxes = shoeboxes.select(possible_hot_spots) for sbox in hot_shoeboxes: x0, x1, y0, y1 = sbox.bbox[0:4] m = sbox.mask p = sbox.panel for y in range(m.all()[1]): for x in range(m.all()[2]): if m[:, y : y + 1, x : x + 1].all_ne(0): hot_mask[p][y0 + y, x0 + x] = False info("Found %d possible hot pixel(s)" % hot_mask.count(False)) else: hot_mask = None # Filter the reflections and select only the desired spots flags = self.filter_spots(None, sweep=imageset, observations=observed, shoeboxes=shoeboxes) observed = observed.select(flags) shoeboxes = shoeboxes.select(flags) # Return as a reflection list return flex.reflection_table(observed, shoeboxes), hot_mask
def generate_predictions(self, N): """ Generate some reflections. """ from dials.algorithms.profile_model.gaussian_rs import MaskCalculator3D from dials.array_family import flex from dials.util.command_line import Command from dials.algorithms import filtering from dials.algorithms.shoebox import MaskCode from dials.algorithms.profile_model.gaussian_rs import Model as ProfileModel import random # Set the profile model self.experiment.profile = ProfileModel(None, self.n_sigma, self.sigma_b, self.sigma_m) # Generate a list of reflections refl = flex.reflection_table.from_predictions(self.experiment) refl["id"] = flex.int(len(refl), 0) # Filter by zeta zeta = 0.05 Command.start("Filtering by zeta >= %f" % zeta) mask = filtering.by_zeta(self.experiment.goniometer, self.experiment.beam, refl["s1"], zeta) refl.del_selected(mask != True) Command.end("Filtered %d reflections by zeta >= %f" % (len(refl), zeta)) # Compute the bounding box refl.compute_bbox([self.experiment]) index = [] image_size = self.experiment.detector[0].get_image_size() array_range = self.experiment.scan.get_array_range() bbox = refl["bbox"] for i in range(len(refl)): x0, x1, y0, y1, z0, z1 = bbox[i] if ( x0 < 0 or x1 > image_size[0] or y0 < 0 or y1 > image_size[1] or z0 < array_range[0] or z1 > array_range[1] ): index.append(i) refl.del_selected(flex.size_t(index)) # Sample if specified index = random.sample(range(len(refl)), N) refl = refl.select(flex.size_t(index)) # Compute the bounding box # Create a load of shoeboxes Command.start("Creating shoeboxes for %d reflections" % len(refl)) refl["shoebox"] = flex.shoebox(refl["panel"], refl["bbox"]) refl["shoebox"].allocate_with_value(MaskCode.Valid) Command.end("Created shoeboxes for %d reflections" % len(refl)) # Get the function object to mask the foreground Command.start("Masking Foreground for %d reflections" % len(refl)) mask_foreground = MaskCalculator3D( self.experiment.beam, self.experiment.detector, self.experiment.goniometer, self.experiment.scan, self.n_sigma * self.sigma_b, self.n_sigma * self.sigma_m, ) # Mask the foreground mask_foreground(refl["shoebox"], refl["s1"], refl["xyzcal.px"].parts()[2], refl["panel"]) Command.end("Masked foreground for %d reflections" % len(refl)) # Return the reflections return refl