def signal(imageset, indx): from dials.array_family import flex from libtbx.phil import parse detectors = imageset.get_detector() assert len(detectors) == 1 detector = detectors[0] trusted = detector.get_trusted_range() data = imageset.get_raw_data(indx) assert len(data) == 1 data = data[0] negative = data < 0 hot = data > int(round(trusted[1])) bad = negative | hot from dials.algorithms.spot_finding.factory import SpotFinderFactory from dials.algorithms.spot_finding.factory import phil_scope data = data.as_double() spot_params = phil_scope.fetch(source=parse("min_spot_size=1")).extract() threshold_function = SpotFinderFactory.configure_threshold(spot_params) peak_pixels = threshold_function.compute_threshold(data, ~bad) return peak_pixels
def find_constant_signal_pixels(imageset, images): """Find pixels which are constantly reporting as signal through the images in imageset: on every image the pixel dispersion index is computed, and signal pixels identified using the default settings. A map is then calculated of the number of times a pixel is identified as signal: if this is >= 50% of the images (say) that pixel is untrustworthy.""" panels = imageset.get_detector() # only cope with monilithic detectors or the I23 Pilatus 12M assert len(panels) in (1, 24) # trusted range the same for all panels anyway detector = panels[0] trusted = detector.get_trusted_range() # construct an integer array same shape as image; accumulate number of # "signal" pixels in each pixel across data total = None for idx in images: pixels = imageset.get_raw_data(idx - 1) known_mask = imageset.get_mask(idx - 1) # apply known mask for _pixel, _panel, _mask in zip(pixels, panels, known_mask): _pixel.set_selected(~_mask, -1) for f0, s0, f1, s1 in _panel.get_mask(): blank = flex.int(flex.grid(s1 - s0, f1 - f0), 0) _pixel.matrix_paste_block_in_place(blank, s0, f0) if len(pixels) == 1: data = pixels[0] else: ny, nx = pixels[0].focus() data = flex.int(flex.grid(24 * ny + 23 * 17, nx), -1) for j in range(24): data.matrix_paste_block_in_place(pixels[j], j * (ny + 17), 0) negative = data < int(round(trusted[0])) hot = data > int(round(trusted[1])) bad = negative | hot data = data.as_double() spot_params = spot_phil.fetch( source=iotbx.phil.parse("min_spot_size=1") ).extract() threshold_function = SpotFinderFactory.configure_threshold(spot_params) peak_pixels = threshold_function.compute_threshold(data, ~bad) if total is None: total = peak_pixels.as_1d().as_int() else: total += peak_pixels.as_1d().as_int() return total
def extract_signal_mask(data): from dials.algorithms.spot_finding.factory import SpotFinderFactory from dials.algorithms.spot_finding.factory import phil_scope data = data.as_double() spot_params = phil_scope.fetch(source=iotbx.phil.parse( "spotfinder.threshold.dispersion.gain=1")).extract() threshold_function = SpotFinderFactory.configure_threshold(spot_params) negative = data < 0 signal = threshold_function.compute_threshold(data, ~negative) return signal
def extract_signal_mask(data): from dials.algorithms.spot_finding.factory import SpotFinderFactory from dials.algorithms.spot_finding.factory import phil_scope data = data.as_double() from dxtbx import datablock spot_params = phil_scope.fetch(source=iotbx.phil.parse( "spotfinder.threshold.xds.gain=1")).extract() threshold_function = SpotFinderFactory.configure_threshold( spot_params, None) negative = (data < 0) signal = threshold_function.compute_threshold(data, ~negative) return signal
def find_signal_pixels(mask, image): """Mask: 2D array of unsigned char: if 0 -> invalid, if 1 -> valid, image: stack of 32 modules, so 3D numpy array of unsigned short. Returns 3D array of unsigned char.""" shape = mask.shape assert mask.shape == image.shape spot_params = spot_phil.fetch(source=iotbx.phil.parse(PHIL_SETTINGS)).extract() signal = np.empty(shape, dtype=np.uint8) threshold_function = SpotFinderFactory.configure_threshold(spot_params) for module in range(shape[0]): _mask = flex.int(mask[module, :, :].astype(np.int32)) == 1 _image = flex.double(image[module, :, :].astype(np.float64)) _signal = threshold_function.compute_threshold(_image, _mask) signal[module, :, :] = _signal.as_numpy_array().astype(np.uint8) return signal
def background(imageset, indx, n_bins, mask_params=None): from dials.array_family import flex from libtbx.phil import parse from scitbx import matrix if mask_params is None: # Default mask params for trusted range mask_params = phil_scope.fetch(parse("")).extract().masking from dials.util.masking import MaskGenerator mask_generator = MaskGenerator(mask_params) mask = mask_generator.generate(imageset) detector = imageset.get_detector() beam = imageset.get_beam() # Only working with single panel detector for now assert len(detector) == 1 panel = detector[0] mask = mask[0] n = matrix.col(panel.get_normal()).normalize() b = matrix.col(beam.get_s0()).normalize() wavelength = beam.get_wavelength() if math.fabs(b.dot(n)) < 0.95: raise Sorry("Detector not perpendicular to beam") data = imageset.get_raw_data(indx) assert len(data) == 1 data = data[0] data = data.as_double() spot_params = spot_phil.fetch(source=parse("")).extract() threshold_function = SpotFinderFactory.configure_threshold(spot_params) peak_pixels = threshold_function.compute_threshold(data, mask) signal = data.select(peak_pixels.iselection()) background_pixels = mask & ~peak_pixels background = data.select(background_pixels.iselection()) # print some summary information print("Mean background: %.3f" % (flex.sum(background) / background.size())) if len(signal) > 0: print("Max/total signal pixels: %.0f / %.0f" % (flex.max(signal), flex.sum(signal))) else: print("No signal pixels on this image") print("Peak/background/masked pixels: %d / %d / %d" % (peak_pixels.count(True), background.size(), mask.count(False))) # compute histogram of two-theta values, then same weighted # by pixel values, finally divide latter by former to get # the radial profile out, need to set the number of bins # sensibly; inspired by method in PyFAI two_theta_array = panel.get_two_theta_array(beam.get_s0()) two_theta_array = two_theta_array.as_1d().select( background_pixels.iselection()) # Use flex.weighted_histogram h0 = flex.weighted_histogram(two_theta_array, n_slots=n_bins) h1 = flex.weighted_histogram(two_theta_array, background, n_slots=n_bins) h2 = flex.weighted_histogram(two_theta_array, background * background, n_slots=n_bins) d0 = h0.slots() d1 = h1.slots() d2 = h2.slots() I = d1 / d0 I2 = d2 / d0 sig = flex.sqrt(I2 - flex.pow2(I)) tt = h0.slot_centers() d_spacings = wavelength / (2.0 * flex.sin(0.5 * tt)) return d_spacings, I, sig
def run(args): from dials.util.options import OptionParser from dials.util.options import flatten_experiments from dials.util.command_line import ProgressBar usage = "%s [options] data_master.h5" % (libtbx.env.dispatcher_name) parser = OptionParser( usage=usage, phil=phil_scope, read_experiments=True, read_experiments_from_images=True, epilog=help_message, ) params, options = parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) if len(experiments) != 1: parser.print_help() print("Please pass an experiment list\n") return imagesets = experiments.imagesets() if len(imagesets) != 1: raise Sorry("Please pass an experiment list that contains one imageset") imageset = imagesets[0] first, last = imageset.get_scan().get_image_range() images = range(first, last + 1) if params.images is None and params.image_range is not None: start, end = params.image_range params.images = list(range(start, end + 1)) if params.images: if min(params.images) < first or max(params.images) > last: raise Sorry("image outside of scan range") images = params.images detectors = imageset.get_detector() assert len(detectors) == 1 detector = detectors[0] trusted = detector.get_trusted_range() # construct an integer array same shape as image; accumulate number of # "signal" pixels in each pixel across data total = None p = ProgressBar(title="Finding hot pixels") for idx in images: p.update(idx * 100.0 / len(images)) pixels = imageset.get_raw_data(idx - 1) assert len(pixels) == 1 data = pixels[0] negative = data < int(round(trusted[0])) hot = data > int(round(trusted[1])) bad = negative | hot data = data.as_double() spot_params = spot_phil.fetch( source=iotbx.phil.parse("min_spot_size=1") ).extract() threshold_function = SpotFinderFactory.configure_threshold(spot_params) peak_pixels = threshold_function.compute_threshold(data, ~bad) if total is None: total = peak_pixels.as_1d().as_int() else: total += peak_pixels.as_1d().as_int() p.finished("Finished finding hot pixels on %d images" % len(images)) hot_mask = total >= (len(images) // 2) hot_pixels = hot_mask.iselection() p = ProgressBar(title="Finding twinkies") twinkies = {} for h in hot_pixels: twinkies[h] = [] for idx in images: p.update(idx * 100.0 / len(images)) pixels = imageset.get_raw_data(idx - 1) data = pixels[0] for h in hot_pixels: twinkies[h].append(data[h]) p.finished("Finished hunting for twinkies on %d images" % len(images)) nslow, nfast = data.focus() ffff = 0 for h in hot_pixels: if total[h] == len(images) and data[h] >= trusted[1]: ffff += 1 continue print("Pixel %d at %d %d" % (total[h], h // nfast, h % nfast)) if len(set(twinkies[h])) >= len(twinkies[h]) // 2: print(" ... many possible values") continue values = set(twinkies[h]) result = [(twinkies[h].count(value), value) for value in values] for count, value in reversed(sorted(result)): print(" %08x %d" % (value, count)) print("Also found %d very hot pixels" % ffff) hot_mask.reshape(flex.grid(data.focus())) easy_pickle.dump(params.output.mask, (~hot_mask,))
def background(imageset, indx, n_bins): from dials.array_family import flex from libtbx.phil import parse from scitbx import matrix import math detector = imageset.get_detector() beam = imageset.get_beam() assert(len(detector) == 1) detector = detector[0] trusted = detector.get_trusted_range() n = matrix.col(detector.get_normal()).normalize() b = matrix.col(beam.get_s0()).normalize() wavelength = beam.get_wavelength() if math.fabs(b.dot(n)) < 0.95: from libtbx.utils import Sorry raise Sorry('Detector not perpendicular to beam') data = imageset.get_raw_data(indx) assert(len(data) == 1) data = data[0] negative = (data < 0) hot = (data > int(round(trusted[1]))) bad = negative | hot from dials.algorithms.spot_finding.factory import SpotFinderFactory from dials.algorithms.spot_finding.factory import phil_scope data = data.as_double() from dxtbx import datablock spot_params = phil_scope.fetch(source=parse("")).extract() threshold_function = SpotFinderFactory.configure_threshold( spot_params, datablock.DataBlock([imageset])) peak_pixels = threshold_function.compute_threshold(data, ~bad) signal = data.select(peak_pixels.iselection()) background = data.select((~bad & ~peak_pixels).iselection()) # print some summary information print 'Mean background: %.3f' % (flex.sum(background) / background.size()) print 'Max/total signal pixels: %.0f / %.0f' % (flex.max(signal), flex.sum(signal)) print 'Peak/background/hot pixels: %d / %d / %d' % (peak_pixels.count(True), background.size(), hot.count(True)) # compute histogram of two-theta values, then same weighted # by pixel values, finally divide latter by former to get # the radial profile out, need to set the number of bins # sensibly; flex.histogram does not allow weights so use # numpy.histogram to get the same effect... inspired by # method in PyFAI data = data.as_1d() two_theta_array = detector.get_two_theta_array(beam.get_s0()) two_theta_array.set_selected((bad | peak_pixels).iselection(), 0.0) data.set_selected((bad | peak_pixels).iselection(), 0.0) # new fangled flex.weighted_histogram :-) h0 = flex.weighted_histogram(two_theta_array, n_slots=n_bins) h1 = flex.weighted_histogram(two_theta_array, data, n_slots=n_bins) h2 = flex.weighted_histogram(two_theta_array, data * data, n_slots=n_bins) d0 = h0.slots() d1 = h1.slots() d2 = h2.slots() I = d1 / d0 I2 = d2 / d0 sig = flex.sqrt(I2 - flex.pow2(I)) tt = h0.slot_centers() d_spacings = wavelength / (2.0 * flex.sin(0.5 * tt)) return d_spacings, I, sig
def background(imageset, indx, n_bins, corrected=False, mask_params=None, show_summary=False): if mask_params is None: # Default mask params for trusted range mask_params = phil_scope.fetch(parse("")).extract().masking detector = imageset.get_detector() beam = imageset.get_beam() # Only working with single panel detector for now assert len(detector) == 1 panel = detector[0] imageset_mask = imageset.get_mask(indx)[0] mask = dials.util.masking.generate_mask(imageset, mask_params)[0] mask = imageset_mask & mask n = matrix.col(panel.get_normal()).normalize() b = matrix.col(beam.get_s0()).normalize() wavelength = beam.get_wavelength() if math.fabs(b.dot(n)) < 0.95: raise Sorry("Detector not perpendicular to beam") # Use corrected data to determine signal and background regions corrected_data = imageset.get_corrected_data(indx) assert len(corrected_data) == 1 corrected_data = corrected_data[0].as_double() # Use choice of raw or corrected data to evaluate the background values if corrected: data = corrected_data else: data = imageset.get_raw_data(indx)[0].as_double() spot_params = spot_phil.fetch(source=parse("")).extract() threshold_function = SpotFinderFactory.configure_threshold(spot_params) peak_pixels = threshold_function.compute_threshold(corrected_data, mask) signal = data.select(peak_pixels.iselection()) background_pixels = mask & ~peak_pixels background = data.select(background_pixels.iselection()) # print some summary information if show_summary: logger.info( f"Mean background: {flex.sum(background) / background.size():.3f}") if len(signal) > 0: logger.info( f"Max/total signal pixels: {flex.max(signal):.0f} / {flex.sum(signal):.0f}" ) else: logger.info("No signal pixels on this image") logger.info( "Peak/background/masked pixels: %d / %d / %d" % (peak_pixels.count(True), background.size(), mask.count(False))) # compute histogram of two-theta values, then same weighted # by pixel values, finally divide latter by former to get # the radial profile out, need to set the number of bins # sensibly; inspired by method in PyFAI two_theta_array = panel.get_two_theta_array(beam.get_s0()) two_theta_array = two_theta_array.as_1d().select( background_pixels.iselection()) # Use flex.weighted_histogram h0 = flex.weighted_histogram(two_theta_array, n_slots=n_bins) h1 = flex.weighted_histogram(two_theta_array, background, n_slots=n_bins) h2 = flex.weighted_histogram(two_theta_array, background * background, n_slots=n_bins) d0 = h0.slots() d1 = h1.slots() d2 = h2.slots() I = d1 / d0 I2 = d2 / d0 sig = flex.sqrt(I2 - flex.pow2(I)) tt = h0.slot_centers() d_spacings = wavelength / (2.0 * flex.sin(0.5 * tt)) return d_spacings, I, sig
def background(imageset, indx, n_bins): from dials.array_family import flex from libtbx.phil import parse from scitbx import matrix import math detector = imageset.get_detector() beam = imageset.get_beam() assert(len(detector) == 1) detector = detector[0] trusted = detector.get_trusted_range() n = matrix.col(detector.get_normal()).normalize() b = matrix.col(beam.get_s0()).normalize() wavelength = beam.get_wavelength() if math.fabs(b.dot(n)) < 0.95: from libtbx.utils import Sorry raise Sorry('Detector not perpendicular to beam') data = imageset.get_raw_data(indx) assert(len(data) == 1) data = data[0] negative = (data < 0) hot = (data > int(round(trusted[1]))) bad = negative | hot from dials.algorithms.spot_finding.factory import SpotFinderFactory from dials.algorithms.spot_finding.factory import phil_scope data = data.as_double() from dxtbx import datablock spot_params = phil_scope.fetch(source=parse("")).extract() threshold_function = SpotFinderFactory.configure_threshold( spot_params, datablock.DataBlock([imageset])) peak_pixels = threshold_function.compute_threshold(data, ~bad) signal = data.select(peak_pixels.iselection()) background = data.select((~bad & ~peak_pixels).iselection()) # print some summary information print('Mean background: %.3f' % (flex.sum(background) / background.size())) print('Max/total signal pixels: %.0f / %.0f' % (flex.max(signal), flex.sum(signal))) print('Peak/background/hot pixels: %d / %d / %d' % (peak_pixels.count(True), background.size(), hot.count(True))) # compute histogram of two-theta values, then same weighted # by pixel values, finally divide latter by former to get # the radial profile out, need to set the number of bins # sensibly; flex.histogram does not allow weights so use # numpy.histogram to get the same effect... inspired by # method in PyFAI data = data.as_1d() two_theta_array = detector.get_two_theta_array(beam.get_s0()) two_theta_array.set_selected((bad | peak_pixels).iselection(), 0.0) data.set_selected((bad | peak_pixels).iselection(), 0.0) # new fangled flex.weighted_histogram :-) h0 = flex.weighted_histogram(two_theta_array, n_slots=n_bins) h1 = flex.weighted_histogram(two_theta_array, data, n_slots=n_bins) h2 = flex.weighted_histogram(two_theta_array, data * data, n_slots=n_bins) d0 = h0.slots() d1 = h1.slots() d2 = h2.slots() I = d1 / d0 I2 = d2 / d0 sig = flex.sqrt(I2 - flex.pow2(I)) tt = h0.slot_centers() d_spacings = wavelength / (2.0 * flex.sin(0.5 * tt)) return d_spacings, I, sig
def stability_fft(imageset, params): from i19.util.time_analysis import fft scan = imageset.get_scan() detector = imageset.get_detector()[0] if params.exposure_time: exposure = params.exposure_time else: exposure = scan.get_exposure_times()[0] trusted = detector.get_trusted_range() indices = imageset.indices() if params.scan_range: start, end = params.scan_range indices = indices[start:end] else: start = 0 end = indices[-1] + 1 counts = flex.double(len(indices)) t0 = time.time() if params.remove_spots: from dials.algorithms.spot_finding.factory import SpotFinderFactory from dials.algorithms.spot_finding.factory import phil_scope from dxtbx import datablock spot_params = phil_scope.fetch(source=iotbx.phil.parse("")).extract() threshold_function = SpotFinderFactory.configure_threshold( spot_params, datablock.DataBlock([imageset])) else: threshold_function = None for i in indices: pixels = imageset.get_raw_data(i)[0].as_double() negative = (pixels < 0) hot = (pixels > int(round(trusted[1]))) bad = negative | hot if threshold_function: peak_pixels = threshold_function.compute_threshold(pixels, ~bad) good = pixels.select((~bad & ~peak_pixels).iselection()) else: good = pixels.select((~bad).iselection()) counts[i-start] = flex.sum(good) t1 = time.time() print 'Read data for %d images in %.1fs' % (len(indices), t1 - t0) # scale data to give sensible FFT values mean = flex.sum(counts) / counts.size() counts /= mean power = fft(counts) f_hz = 1.0 / exposure f_scale = f_hz / counts.size() print 'Sample frequency: %.2f Hz' % f_hz print 'Writing output to: %s' % params.output_file fout = open(params.output_file, 'w') for j in range(power.size()): fout.write('%f %f\n' % (j * f_scale, power[j])) fout.close()