def save_datablock(filename): from xia2.Schema import imageset_cache from dxtbx.datablock import DataBlock from dxtbx.serialize import dump datablock = DataBlock([]) for imagesets in imageset_cache.values(): for imageset in imagesets.values(): datablock.append(imageset) dump.datablock(datablock, filename, compact=True)
def __call__(self, datablock): ''' Transform the metadata ''' from dxtbx.datablock import DataBlock # Import the lookup data lookup = self.import_lookup_data(self.params) # Convert all to ImageGrid if self.params.input.as_grid_scan: datablock = self.convert_to_grid_scan(datablock, self.params) # Init imageset list imageset_list = [] # Loop through imagesets for imageset in datablock.extract_imagesets(): # Set the external lookups imageset = self.update_lookup(imageset, lookup) # Update the geometry for updater in self.update_geometry: imageset = updater(imageset) # Check beam and detector are present if imageset.get_beam() == None or imageset.get_detector() == None: raise Sorry(''' Imageset contains no beam or detector model. This means you will be unable to process your data. Possible causes of this error are: - A problem reading the images with one of the dxtbx format classes - A lack of header information in the file itself. You can override this by specifying the metadata as geometry parameters ''') # Check if dx and dy are set if [ imageset.external_lookup.dx.filename, imageset.external_lookup.dy.filename ].count(None) == 0: imageset.update_detector_px_mm_data() elif [ imageset.external_lookup.dx.filename, imageset.external_lookup.dy.filename ].count(None) == 1: raise Sorry(''' Only 1 offset map is set. Need to set both dx and d ''') # Append to new imageset list imageset_list.append(imageset) # Return the datablock return DataBlock(imageset_list)
def convert_to_grid_scan(self, datablock, params): ''' Convert the imagesets to grid scans ''' from dxtbx.datablock import DataBlock from dxtbx.imageset import ImageGrid if params.input.grid_size is None: raise Sorry("The input.grid_size parameter is required") sweeps = datablock.extract_sweeps() stills = datablock.extract_stills() imagesets = [] for iset in sweeps + stills: imagesets.append(ImageGrid.from_imageset(iset, params.input.grid_size)) return DataBlock(imagesets)
def __call__(self, datablock): ''' Transform the metadata ''' from dxtbx.datablock import DataBlock # Import the lookup data lookup = self.import_lookup_data(self.params) # Convert all to ImageGrid if self.params.input.as_grid_scan: datablock = self.convert_to_grid_scan(datablock, self.params) # Init imageset list imageset_list = [] # Loop through imagesets for imageset in datablock.extract_imagesets(): # Check beam and detector are present if imageset.get_beam() == None or imageset.get_detector() == None: raise Sorry(''' Imageset contains no beam or detector model. This means you will be unable to process your data. Possible causes of this error are: - A problem reading the images with one of the dxtbx format classes - A lack of header information in the file itself. ''') # Set the external lookups imageset = self.update_lookup(imageset, lookup) # Update the geometry imageset = self.update_geometry(imageset) # Append to new imageset list imageset_list.append(imageset) # Return the datablock return DataBlock(imageset_list)
def xia2_main(stop_after=None): '''Actually process something...''' Citations.cite('xia2') # print versions of related software Chatter.write(dials_version()) ccp4_version = get_ccp4_version() if ccp4_version is not None: Chatter.write('CCP4 %s' % ccp4_version) start_time = time.time() CommandLine = get_command_line() start_dir = Flags.get_starting_directory() # check that something useful has been assigned for processing... xtals = CommandLine.get_xinfo().get_crystals() no_images = True for name in xtals.keys(): xtal = xtals[name] if not xtal.get_all_image_names(): Chatter.write('-----------------------------------' + \ '-' * len(name)) Chatter.write('| No images assigned for crystal %s |' % name) Chatter.write('-----------------------------------' + '-' \ * len(name)) else: no_images = False args = [] from xia2.Handlers.Phil import PhilIndex params = PhilIndex.get_python_object() mp_params = params.xia2.settings.multiprocessing njob = mp_params.njob from libtbx import group_args xinfo = CommandLine.get_xinfo() if os.path.exists('xia2.json'): from xia2.Schema.XProject import XProject xinfo_new = xinfo xinfo = XProject.from_json(filename='xia2.json') crystals = xinfo.get_crystals() crystals_new = xinfo_new.get_crystals() for crystal_id in crystals_new.keys(): if crystal_id not in crystals: crystals[crystal_id] = crystals_new[crystal_id] continue crystals[crystal_id]._scaler = None # reset scaler for wavelength_id in crystals_new[crystal_id].get_wavelength_names( ): wavelength_new = crystals_new[crystal_id].get_xwavelength( wavelength_id) if wavelength_id not in crystals[ crystal_id].get_wavelength_names(): crystals[crystal_id].add_wavelength( crystals_new[crystal_id].get_xwavelength( wavelength_new)) continue wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) sweeps_new = wavelength_new.get_sweeps() sweeps = wavelength.get_sweeps() sweep_names = [s.get_name() for s in sweeps] sweep_keys = [(s.get_directory(), s.get_template(), s.get_image_range()) for s in sweeps] for sweep in sweeps_new: if ((sweep.get_directory(), sweep.get_template(), sweep.get_image_range()) not in sweep_keys): if sweep.get_name() in sweep_names: i = 1 while 'SWEEEP%i' % i in sweep_names: i += 1 sweep._name = 'SWEEP%i' % i break wavelength.add_sweep( name=sweep.get_name(), sample=sweep.get_xsample(), directory=sweep.get_directory(), image=sweep.get_image(), beam=sweep.get_beam_centre(), reversephi=sweep.get_reversephi(), distance=sweep.get_distance(), gain=sweep.get_gain(), dmin=sweep.get_resolution_high(), dmax=sweep.get_resolution_low(), polarization=sweep.get_polarization(), frames_to_process=sweep.get_frames_to_process(), user_lattice=sweep.get_user_lattice(), user_cell=sweep.get_user_cell(), epoch=sweep._epoch, ice=sweep._ice, excluded_regions=sweep._excluded_regions, ) sweep_names.append(sweep.get_name()) crystals = xinfo.get_crystals() failover = params.xia2.settings.failover if mp_params.mode == 'parallel' and njob > 1: driver_type = mp_params.type command_line_args = CommandLine.get_argv()[1:] for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) sweeps = wavelength.get_sweeps() for sweep in sweeps: sweep._get_indexer() sweep._get_refiner() sweep._get_integrater() args.append((group_args( driver_type=driver_type, stop_after=stop_after, failover=failover, command_line_args=command_line_args, nproc=mp_params.nproc, crystal_id=crystal_id, wavelength_id=wavelength_id, sweep_id=sweep.get_name(), ), )) from xia2.Driver.DriverFactory import DriverFactory default_driver_type = DriverFactory.get_driver_type() # run every nth job on the current computer (no need to submit to qsub) for i_job, arg in enumerate(args): if (i_job % njob) == 0: arg[0].driver_type = default_driver_type if mp_params.type == "qsub": method = "sge" else: method = "multiprocessing" nproc = mp_params.nproc qsub_command = mp_params.qsub_command if not qsub_command: qsub_command = 'qsub' qsub_command = '%s -V -cwd -pe smp %d' % (qsub_command, nproc) from libtbx import easy_mp results = easy_mp.parallel_map( process_one_sweep, args, processes=njob, #method=method, method="multiprocessing", qsub_command=qsub_command, preserve_order=True, preserve_exception_message=True) # Hack to update sweep with the serialized indexers/refiners/integraters i_sweep = 0 for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: success, output, xsweep_dict = results[i_sweep] if output is not None: Chatter.write(output) if not success: Chatter.write('Sweep failed: removing %s' % sweep.get_name()) remove_sweeps.append(sweep) else: assert xsweep_dict is not None Chatter.write('Loading sweep: %s' % sweep.get_name()) from xia2.Schema.XSweep import XSweep new_sweep = XSweep.from_dict(xsweep_dict) sweep._indexer = new_sweep._indexer sweep._refiner = new_sweep._refiner sweep._integrater = new_sweep._integrater i_sweep += 1 for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep) else: for crystal_id in crystals.keys(): for wavelength_id in crystals[crystal_id].get_wavelength_names(): wavelength = crystals[crystal_id].get_xwavelength( wavelength_id) remove_sweeps = [] sweeps = wavelength.get_sweeps() for sweep in sweeps: from dials.command_line.show import show_datablocks from dxtbx.datablock import DataBlock Debug.write(sweep.get_name()) Debug.write( show_datablocks([DataBlock([sweep.get_imageset()])])) try: if stop_after == 'index': sweep.get_indexer_cell() else: sweep.get_integrater_intensities() sweep.serialize() except Exception as e: if failover: Chatter.write('Processing sweep %s failed: %s' % \ (sweep.get_name(), str(e))) remove_sweeps.append(sweep) else: raise for sweep in remove_sweeps: wavelength.remove_sweep(sweep) sample = sweep.get_xsample() sample.remove_sweep(sweep) # save intermediate xia2.json file in case scaling step fails xinfo.as_json(filename='xia2.json') if stop_after not in ('index', 'integrate'): Chatter.write(xinfo.get_output(), strip=False) for crystal in crystals.values(): crystal.serialize() # save final xia2.json file in case report generation fails xinfo.as_json(filename='xia2.json') duration = time.time() - start_time # write out the time taken in a human readable way Chatter.write('Processing took %s' % \ time.strftime("%Hh %Mm %Ss", time.gmtime(duration))) if stop_after not in ('index', 'integrate'): # and the summary file with open('xia2-summary.dat', 'w') as fh: for record in xinfo.summarise(): fh.write('%s\n' % record) # looks like this import overwrites the initial command line # Phil overrides so... for https://github.com/xia2/xia2/issues/150 from xia2.command_line.html import generate_xia2_html if params.xia2.settings.small_molecule == True: params.xia2.settings.report.xtriage_analysis = False params.xia2.settings.report.include_radiation_damage = False generate_xia2_html(xinfo, filename='xia2.html', params=params.xia2.settings.report) write_citations() # delete all of the temporary mtz files... cleanup() Environment.cleanup()
def combine(datablock_list, reflections_list, params): ''' Combine the found spots. ''' from dxtbx.datablock import BeamComparison from dxtbx.datablock import DetectorComparison from dxtbx.datablock import GoniometerComparison from dxtbx.datablock import DataBlock from dxtbx.imageset import ImageSetFactory from dials.algorithms.spot_finding import StrongSpotCombiner from dials.array_family import flex assert len(datablock_list) == len(reflections_list) # Get a list of imagesets imageset_list = [] for db in datablock_list: iset = db.extract_imagesets() assert len(iset) == 1 imageset_list.append(iset[0]) compare_beam = BeamComparison( wavelength_tolerance=params.input.tolerance.beam.wavelength, direction_tolerance=params.input.tolerance.beam.direction, polarization_normal_tolerance=params.input.tolerance.beam.polarization_normal, polarization_fraction_tolerance=params.input.tolerance.beam.polarization_fraction) compare_detector = DetectorComparison( fast_axis_tolerance=params.input.tolerance.detector.fast_axis, slow_axis_tolerance=params.input.tolerance.detector.slow_axis, origin_tolerance=params.input.tolerance.detector.origin) compare_goniometer = GoniometerComparison( rotation_axis_tolerance=params.input.tolerance.goniometer.rotation_axis, fixed_rotation_tolerance=params.input.tolerance.goniometer.fixed_rotation, setting_rotation_tolerance=params.input.tolerance.goniometer.setting_rotation) scan_tolerance = params.input.tolerance.scan.oscillation # The initial models format_class = imageset_list[0].get_format_class() beam = imageset_list[0].get_beam() detector = imageset_list[0].get_detector() goniometer = imageset_list[0].get_goniometer() scan = imageset_list[0].get_scan() template = imageset_list[0].get_template() # Check all the models for imageset in imageset_list[1:]: b = imageset.get_beam() d = imageset.get_detector() g = imageset.get_goniometer() s = imageset.get_scan() if not imageset.get_format_class() == format_class: raise RuntimeError('Format classes do not match') if not imageset.get_template() == template: raise RuntimeError('Templates do not match') if not compare_beam(beam, b): raise RuntimeError('Beam models are too dissimilar') if not compare_detector(detector, d): raise RuntimeError('Detector models are too dissimilar') if not compare_goniometer(goniometer, g): raise RuntimeError('Goniometer models are too dissimilar') try: scan.append(s, scan_tolerance=scan_tolerance) except Exception: raise RuntimeError('Scans do not match') # Get the image range image_range = scan.get_image_range() image_range = (image_range[0], image_range[1]+1) # Create the sweep imageset = ImageSetFactory.make_sweep( template, range(*image_range), format_class, beam, detector, goniometer, scan) # Combine spots combiner = StrongSpotCombiner() for index, rlist in enumerate(reflections_list, start=1): assert rlist['id'].all_eq(0) logger.info("Combining %d reflections from reflection list %d" % ( len(rlist), index)) combiner.add(rlist['shoebox']) shoeboxes = combiner.shoeboxes() # Calculate the spot centroids and intensities logger.info('Combined into %d reflections' % len(shoeboxes)) centroid = shoeboxes.centroid_valid() logger.info('Calculated {0} spot centroids'.format(len(shoeboxes))) intensity = shoeboxes.summed_intensity() logger.info('Calculated {0} spot intensities'.format(len(shoeboxes))) # Construct the reflection table reflections = flex.reflection_table( flex.observation( shoeboxes.panels(), centroid, intensity), shoeboxes) reflections['id'] = flex.int(len(reflections), 0) reflections.set_flags( flex.size_t_range(len(reflections)), reflections.flags.strong) # Return the datablock and reflections return DataBlock([imageset]), reflections
def run(self): ''' Parse the options. ''' from dials.util import log import libtbx from uuid import uuid4 from dials.util.stream import ZMQStream, Decoder from os.path import join, exists import os import json from dxtbx.datablock import DataBlock # Parse the command line arguments in two passes to set up logging early params, options = self.parser.parse_args(show_diff_phil=False, quick_parse=True) # Configure logging log.config(params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Parse the command line arguments completely params, options = self.parser.parse_args(show_diff_phil=False) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) # Check a stream is given if params.input.host is None: raise Sorry("An input host needs to be given") # Check the directory if params.output.directory is None: raise Sorry("An output directory needs to be given") elif params.output.directory is libtbx.Auto: params.output.directory = "/dev/shm/dials-%s" % uuid4() # Make the output directory if exists(params.output.directory): raise Sorry('Directory "%s" already exists' % (params.output.directory)) # Make the directory os.mkdir(params.output.directory) # Initialise the stream stream = ZMQStream(params.input.host, params.input.port) decoder = Decoder(params.output.directory, params.output.image_template) imageset = None while True: # Get the frames from zmq frames = stream.receive() # Decode the frames obj = decoder.decode(frames) # Process the object if obj.is_header(): filename = join(params.output.directory, "metadata.json") with open(filename, "w") as outfile: json.dump(obj.header, outfile) imageset = obj.as_imageset(filename) datablocks = [DataBlock([imageset])] self.write_datablocks(datablocks, params) elif obj.is_image(): assert imageset is not None filename = join(params.output.directory, params.output.image_template % obj.count) with open(filename, "wb") as outfile: outfile.write(obj.data) filename = join( params.output.directory, "%s.info" % (params.output.image_template % obj.count)) with open(filename, "w") as outfile: json.dump(obj.info, outfile) elif obj.is_endofseries(): assert imageset is not None break else: raise RuntimeError("Unknown object") # Close the stream stream.close()
def _index_prepare(self): from xia2.Handlers.Citations import Citations Citations.cite('dials') #all_images = self.get_matching_images() #first = min(all_images) #last = max(all_images) spot_lists = [] datablocks = [] for imageset, xsweep in zip(self._indxr_imagesets, self._indxr_sweeps): Chatter.banner('Spotfinding %s' % xsweep.get_name()) first, last = imageset.get_scan().get_image_range() # at this stage, break out to run the DIALS code: this sets itself up # now cheat and pass in some information... save re-reading all of the # image headers # FIXME need to adjust this to allow (say) three chunks of images from dxtbx.serialize import dump from dxtbx.datablock import DataBlock sweep_filename = os.path.join( self.get_working_directory(), '%s_datablock.json' % xsweep.get_name()) dump.datablock(DataBlock([imageset]), sweep_filename) genmask = self.GenerateMask() genmask.set_input_datablock(sweep_filename) genmask.set_output_datablock( os.path.join( self.get_working_directory(), '%s_%s_datablock.json' % (genmask.get_xpid(), xsweep.get_name()))) genmask.set_params(PhilIndex.params.dials.masking) sweep_filename, mask_pickle = genmask.run() Debug.write('Generated mask for %s: %s' % (xsweep.get_name(), mask_pickle)) gain = PhilIndex.params.xia2.settings.input.gain if gain is libtbx.Auto: gain_estimater = self.EstimateGain() gain_estimater.set_sweep_filename(sweep_filename) gain_estimater.run() gain = gain_estimater.get_gain() Chatter.write('Estimated gain: %.2f' % gain) PhilIndex.params.xia2.settings.input.gain = gain # FIXME this should really use the assigned spot finding regions #offset = self.get_frame_offset() spotfinder = self.Spotfinder() if last - first > 10: spotfinder.set_write_hot_mask(True) spotfinder.set_input_sweep_filename(sweep_filename) spotfinder.set_output_sweep_filename( '%s_%s_datablock.json' % (spotfinder.get_xpid(), xsweep.get_name())) spotfinder.set_input_spot_filename( '%s_%s_strong.pickle' % (spotfinder.get_xpid(), xsweep.get_name())) if PhilIndex.params.dials.fast_mode: wedges = self._index_select_images_i(imageset) spotfinder.set_scan_ranges(wedges) else: spotfinder.set_scan_ranges([(first, last)]) if PhilIndex.params.dials.find_spots.phil_file is not None: spotfinder.set_phil_file( PhilIndex.params.dials.find_spots.phil_file) min_spot_size = PhilIndex.params.dials.find_spots.min_spot_size if min_spot_size is libtbx.Auto: if imageset.get_detector()[0].get_type() == 'SENSOR_PAD': min_spot_size = 3 else: min_spot_size = None if min_spot_size is not None: spotfinder.set_min_spot_size(min_spot_size) min_local = PhilIndex.params.dials.find_spots.min_local if min_local is not None: spotfinder.set_min_local(min_local) sigma_strong = PhilIndex.params.dials.find_spots.sigma_strong if sigma_strong: spotfinder.set_sigma_strong(sigma_strong) gain = PhilIndex.params.xia2.settings.input.gain if gain: spotfinder.set_gain(gain) filter_ice_rings = PhilIndex.params.dials.find_spots.filter_ice_rings if filter_ice_rings: spotfinder.set_filter_ice_rings(filter_ice_rings) kernel_size = PhilIndex.params.dials.find_spots.kernel_size if kernel_size: spotfinder.set_kernel_size(kernel_size) global_threshold = PhilIndex.params.dials.find_spots.global_threshold if global_threshold is not None: spotfinder.set_global_threshold(global_threshold) spotfinder.run() spot_filename = spotfinder.get_spot_filename() if not os.path.exists(spot_filename): raise RuntimeError("Spotfinding failed: %s does not exist." % os.path.basename(spot_filename)) spot_lists.append(spot_filename) datablocks.append(spotfinder.get_output_sweep_filename()) from libtbx import easy_pickle from dials.util.ascii_art import spot_counts_per_image_plot refl = easy_pickle.load(spot_filename) if not len(refl): raise RuntimeError('No spots found in sweep %s' % xsweep.get_name()) Chatter.write(spot_counts_per_image_plot(refl), strip=False) if not PhilIndex.params.dials.fast_mode: detectblanks = self.DetectBlanks() detectblanks.set_sweep_filename(datablocks[-1]) detectblanks.set_reflections_filename(spot_filename) detectblanks.run() json = detectblanks.get_results() offset = imageset.get_scan().get_image_range()[0] blank_regions = json['strong']['blank_regions'] if len(blank_regions): blank_regions = [(int(s), int(e)) for s, e in blank_regions] for blank_start, blank_end in blank_regions: Chatter.write( 'WARNING: Potential blank images: %i -> %i' % (blank_start + 1, blank_end)) if PhilIndex.params.xia2.settings.remove_blanks: non_blanks = [] start, end = imageset.get_array_range() last_blank_end = start for blank_start, blank_end in blank_regions: if blank_start > start: non_blanks.append( (last_blank_end, blank_start)) last_blank_end = blank_end if last_blank_end + 1 < end: non_blanks.append((last_blank_end, end)) xsweep = self.get_indexer_sweep() xwav = xsweep.get_wavelength() xsample = xsweep.get_xsample() sweep_name = xsweep.get_name() import string for i, (nb_start, nb_end) in enumerate(non_blanks): assert i < 26 if i == 0: sub_imageset = imageset[nb_start - start:nb_end - start] xsweep._frames_to_process = (nb_start + 1, nb_end + 1) self.set_indexer_prepare_done(done=False) self._indxr_imagesets[ self._indxr_imagesets.index( imageset)] = sub_imageset xsweep._integrater._setup_from_imageset( sub_imageset) else: new_name = '_'.join( (sweep_name, string.ascii_lowercase[i])) new_sweep = xwav.add_sweep( new_name, xsample, directory=os.path.join( os.path.basename( xsweep.get_directory()), new_name), image=imageset.get_path(nb_start - start), frames_to_process=(nb_start + 1, nb_end), ) Chatter.write( "Generating new sweep: %s (%s:%i:%i)" % (new_sweep.get_name(), new_sweep.get_image(), new_sweep.get_frames_to_process()[0], new_sweep.get_frames_to_process()[1])) return if not PhilIndex.params.xia2.settings.trust_beam_centre: discovery = self.DiscoverBetterExperimentalModel() discovery.set_sweep_filename(datablocks[-1]) discovery.set_spot_filename(spot_filename) #wedges = self._index_select_images_i(imageset) #discovery.set_scan_ranges(wedges) #discovery.set_scan_ranges([(first + offset, last + offset)]) try: discovery.run() except Exception as e: Debug.write('DIALS beam centre search failed: %s' % str(e)) else: # overwrite datablock.json in datablocks list datablocks[ -1] = discovery.get_optimized_datablock_filename() self.set_indexer_payload("spot_lists", spot_lists) self.set_indexer_payload("datablocks", datablocks) return
def combine_datablocks(self, datablocks): imageset_list = [] for d in datablocks: imageset_list.extend(d.extract_imagesets()) return DataBlock(imageset_list)
def run(self): '''Execute the script.''' from dials.util.options import flatten_reflections, flatten_experiments, \ flatten_datablocks import cPickle as pickle # Parse the command line params, options = self.parser.parse_args(show_diff_phil=True) reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) datablocks = flatten_datablocks(params.input.datablock) # Try to load the models and data slice_exps = len(experiments) > 0 slice_refs = len(reflections) > 0 slice_dbs = len(datablocks) > 0 # Catch case of nothing to do if not any([slice_exps, slice_refs, slice_dbs]): print "No suitable input provided" self.parser.print_help() return if reflections: if len(reflections) > 1: raise Sorry( "Only one reflections list can be imported at present") reflections = reflections[0] # calculate frame numbers if needed if experiments: reflections = calculate_frame_numbers(reflections, experiments) # if we still don't have the right column give up if 'xyzobs.px.value' not in reflections: raise Sorry( "These reflections do not have frame numbers set, and " "there are no experiments provided to calculate these.") # set trivial case where no scan range is provided at all if not params.scan_range: params.scan_range = [None] # check if slicing into blocks if params.block_size is not None: # in this case for simplicity, ensure that there is either an # an experiment list or datablocks, but not both. Ensure there is only # a single scan contained within. if [slice_exps, slice_dbs].count(True) != 1: raise Sorry( "For slicing into blocks please provide either datablocks" " or experiments, but not both.") if slice_exps: if len(experiments) > 1: raise Sorry( "For slicing into blocks please provide a single " "scan only") scan = experiments[0].scan if slice_dbs: scans = datablocks[0].unique_scans() if len(scans) > 1 or len(datablocks) > 1: raise Sorry( "For slicing into blocks please provide a single " "scan only") scan = scans[0] # Having extracted the scan, calculate the blocks params.scan_range = calculate_block_ranges(scan, params.block_size) # Do the slicing then recombine if slice_exps: sliced = [slice_experiments(experiments, [sr])[0] \ for sr in params.scan_range] sliced_experiments = ExperimentList() for exp in sliced: sliced_experiments.append(exp) if slice_dbs: sliced = [slice_datablocks(datablocks, [sr])[0] \ for sr in params.scan_range] imagesets = [db.extract_imagesets()[0] for db in sliced] sliced_datablocks = DataBlock(imagesets) # slice reflections if present if slice_refs: sliced = [slice_reflections(reflections, [sr]) \ for sr in params.scan_range] sliced_reflections = sliced[0] for i, rt in enumerate(sliced[1:]): rt['id'] += (i + 1) # set id sliced_reflections.extend(rt) else: # slice each dataset into the requested subset if slice_exps: sliced_experiments = slice_experiments(experiments, params.scan_range) if slice_refs: sliced_reflections = slice_reflections(reflections, params.scan_range) if slice_dbs: sliced_datablocks = slice_datablocks(datablocks, params.scan_range) # Save sliced experiments if slice_exps: output_experiments_filename = params.output.experiments_filename if output_experiments_filename is None: # take first filename as template bname = basename(params.input.experiments[0].filename) bname = splitext(bname)[0] if not bname: bname = "experiments" if len(params.scan_range ) == 1 and params.scan_range[0] is not None: ext = "_{0}_{1}.json".format(*params.scan_range[0]) else: ext = "_sliced.json" output_experiments_filename = bname + ext print 'Saving sliced experiments to {0}'.format( output_experiments_filename) from dxtbx.model.experiment.experiment_list import ExperimentListDumper dump = ExperimentListDumper(sliced_experiments) dump.as_json(output_experiments_filename) # Save sliced reflections if slice_refs: output_reflections_filename = params.output.reflections_filename if output_reflections_filename is None: # take first filename as template bname = basename(params.input.reflections[0].filename) bname = splitext(bname)[0] if not bname: bname = "reflections" if len(params.scan_range ) == 1 and params.scan_range[0] is not None: ext = "_{0}_{1}.pickle".format(*params.scan_range[0]) else: ext = "_sliced.pickle" output_reflections_filename = bname + ext print 'Saving sliced reflections to {0}'.format( output_reflections_filename) sliced_reflections.as_pickle(output_reflections_filename) # Save sliced datablocks if slice_dbs: output_datablocks_filename = params.output.datablocks_filename if output_datablocks_filename is None: # take first filename as template bname = basename(params.input.datablock[0].filename) bname = splitext(bname)[0] if not bname: bname = "datablock" if len(params.scan_range ) == 1 and params.scan_range[0] is not None: ext = "_{0}_{1}.json".format(*params.scan_range[0]) else: ext = "_sliced.json" output_datablocks_filename = bname + ext print 'Saving sliced datablocks to {0}'.format( output_datablocks_filename) from dxtbx.datablock import DataBlockDumper dump = DataBlockDumper(sliced_datablocks) dump.as_file(output_datablocks_filename) return
def _index_prepare(self): '''Prepare to do autoindexing - in XDS terms this will mean calling xycorr, init and colspot on the input images.''' # decide on images to work with Debug.write('XDS INDEX PREPARE:') Debug.write('Wavelength: %.6f' % self.get_wavelength()) Debug.write('Distance: %.2f' % self.get_distance()) if self._indxr_images == []: _select_images_function = getattr( self, '_index_select_images_%s' % (self._index_select_images)) wedges = _select_images_function() for wedge in wedges: self.add_indexer_image_wedge(wedge) self.set_indexer_prepare_done(True) all_images = self.get_matching_images() first = min(all_images) last = max(all_images) # next start to process these - first xycorr xycorr = self.Xycorr() xycorr.set_data_range(first, last) xycorr.set_background_range(self._indxr_images[0][0], self._indxr_images[0][1]) from dxtbx.serialize.xds import to_xds converter = to_xds(self.get_imageset()) xds_beam_centre = converter.detector_origin xycorr.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1]) for block in self._indxr_images: xycorr.add_spot_range(block[0], block[1]) # FIXME need to set the origin here xycorr.run() for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']: self._indxr_payload[file] = xycorr.get_output_data_file(file) # next start to process these - then init if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing: imageset = self._indxr_imagesets[0] masker = imageset.masker().format_class( imageset.paths()[0]).get_goniometer_shadow_masker() if masker is None: # disable dynamic_shadowing PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False if PhilIndex.params.xia2.settings.input.format.dynamic_shadowing: # find the region of the scan with the least predicted shadow # to use for background determination in XDS INIT step from dxtbx.serialize import dump from dxtbx.datablock import DataBlock imageset = self._indxr_imagesets[0] xsweep = self._indxr_sweeps[0] sweep_filename = os.path.join( self.get_working_directory(), '%s_datablock.json' % xsweep.get_name()) dump.datablock(DataBlock([imageset]), sweep_filename) from xia2.Wrappers.Dials.ShadowPlot import ShadowPlot shadow_plot = ShadowPlot() shadow_plot.set_working_directory(self.get_working_directory()) auto_logfiler(shadow_plot) shadow_plot.set_sweep_filename(sweep_filename) shadow_plot.set_json_filename( os.path.join(self.get_working_directory(), '%s_shadow_plot.json' % shadow_plot.get_xpid())) shadow_plot.run() results = shadow_plot.get_results() from scitbx.array_family import flex fraction_shadowed = flex.double(results['fraction_shadowed']) if flex.max(fraction_shadowed) == 0: PhilIndex.params.xia2.settings.input.format.dynamic_shadowing = False else: scan_points = flex.double(results['scan_points']) phi_width = self.get_phi_width() scan = imageset.get_scan() oscillation_range = scan.get_oscillation_range() oscillation = scan.get_oscillation() if self._background_images is not None: bg_images = self._background_images bg_range_deg = (scan.get_angle_from_image_index( bg_images[0]), scan.get_angle_from_image_index( bg_images[1])) bg_range_width = bg_range_deg[1] - bg_range_deg[0] min_shadow = 100 best_bg_range = bg_range_deg from libtbx.utils import frange for bg_range_start in frange(flex.min(scan_points), flex.max(scan_points) - bg_range_width, step=oscillation[1]): bg_range_deg = (bg_range_start, bg_range_start + bg_range_width) sel = (scan_points >= bg_range_deg[0]) & ( scan_points <= bg_range_deg[1]) mean_shadow = flex.mean(fraction_shadowed.select(sel)) if mean_shadow < min_shadow: min_shadow = mean_shadow best_bg_range = bg_range_deg self._background_images = (scan.get_image_index_from_angle( best_bg_range[0]), scan.get_image_index_from_angle( best_bg_range[1])) Debug.write('Setting background images: %s -> %s' % self._background_images) init = self.Init() for file in ['X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf']: init.set_input_data_file(file, self._indxr_payload[file]) init.set_data_range(first, last) if self._background_images: init.set_background_range(self._background_images[0], self._background_images[1]) else: init.set_background_range(self._indxr_images[0][0], self._indxr_images[0][1]) for block in self._indxr_images: init.add_spot_range(block[0], block[1]) init.run() # at this stage, need to (perhaps) modify the BKGINIT.cbf image # to mark out the back stop if PhilIndex.params.general.backstop_mask: Debug.write('Applying mask to BKGINIT.pck') # copy the original file cbf_old = os.path.join(init.get_working_directory(), 'BKGINIT.cbf') cbf_save = os.path.join(init.get_working_directory(), 'BKGINIT.sav') shutil.copyfile(cbf_old, cbf_save) # modify the file to give the new mask from xia2.Toolkit.BackstopMask import BackstopMask mask = BackstopMask(PhilIndex.params.general.backstop_mask) mask.apply_mask_xds(self.get_header(), cbf_save, cbf_old) init.reload() for file in ['BLANK.cbf', 'BKGINIT.cbf', 'GAIN.cbf']: self._indxr_payload[file] = init.get_output_data_file(file) if PhilIndex.params.xia2.settings.developmental.use_dials_spotfinder: spotfinder = self.DialsSpotfinder() for block in self._indxr_images: spotfinder.add_spot_range(block[0], block[1]) spotfinder.run() export = self.DialsExportSpotXDS() export.set_input_data_file( 'reflections.pickle', spotfinder.get_output_data_file('reflections.pickle')) export.run() for file in ['SPOT.XDS']: self._indxr_payload[file] = export.get_output_data_file(file) else: # next start to process these - then colspot colspot = self.Colspot() for file in [ 'X-CORRECTIONS.cbf', 'Y-CORRECTIONS.cbf', 'BLANK.cbf', 'BKGINIT.cbf', 'GAIN.cbf' ]: colspot.set_input_data_file(file, self._indxr_payload[file]) colspot.set_data_range(first, last) colspot.set_background_range(self._indxr_images[0][0], self._indxr_images[0][1]) for block in self._indxr_images: colspot.add_spot_range(block[0], block[1]) colspot.run() for file in ['SPOT.XDS']: self._indxr_payload[file] = colspot.get_output_data_file(file) # that should be everything prepared... all of the important # files should be loaded into memory to be able to cope with # integration happening somewhere else return