def run(args): assert len(args) == 2 expts = load.experiment_list(args[0], check_format=False) refl = flex.reflection_table.from_file(args[1]) params = phil_scope.fetch(source=phil.parse("")).extract() params.refinement.verbosity = 12 # no output by default print("Starting refinement #1") refiner = RefinerFactory.from_parameters_data_experiments( params, refl, expts) refiner.run() print("Finished refinement #1") # configure logging logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") # we should now get logging output from refinement print("Starting refinement #2") refiner = RefinerFactory.from_parameters_data_experiments( params, refl, expts) refiner.run() print("Finished refinement #2") # switch off logging for dials.algorithms.refinement logging.getLogger("dials.algorithms.refinement").setLevel(logging.ERROR) print("Starting refinement #3") refiner = RefinerFactory.from_parameters_data_experiments( params, refl, expts) refiner.run() print("Finished refinement #3")
def run(args): from dials.util import log import libtbx.load_env usage = "%s experiments.json indexed.pickle [options]" %libtbx.env.dispatcher_name from dials.util.options import OptionParser from dials.util.options import flatten_reflections from dials.util.options import flatten_experiments from dials.array_family import flex parser = OptionParser( usage=usage, phil=phil_scope, read_experiments=True, read_reflections=True, check_format=False, epilog=help_message) params, options = parser.parse_args(show_diff_phil=False) # Configure the logging #log.config(info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) assert(len(reflections) == 1) reflections = reflections[0] if len(experiments) == 0: parser.print_help() return #from dials.command_line import refine #params = refine.phil_scope.extract() indexed_reflections = reflections.select(reflections['id'] > -1) from dials.algorithms.refinement import RefinerFactory refiner = RefinerFactory.from_parameters_data_experiments( params, indexed_reflections, experiments) #refiner.run() rmsds = refiner.rmsds() import math xy_rmsds = math.sqrt(rmsds[0]**2 + rmsds[1]**2) print rmsds return
def __call__(self, experiments, reflections): self.working_phil.show() params = self.working_phil.extract() refiner = RefinerFactory.from_parameters_data_experiments( params, reflections, experiments, verbosity=2) refiner.run() experiments = refiner.get_experiments() return experiments
def __call__(self, experiments, reflections): self.working_phil.show() params = self.working_phil.extract() refiner = RefinerFactory.from_parameters_data_experiments( params, reflections, experiments, verbosity=2 ) refiner.run() experiments = refiner.get_experiments() return experiments
def refine(params, reflections, experiments, verbosity=0, debug_plots=False): detector = experiments.detectors()[0] from dials.algorithms.refinement import RefinerFactory refiner = RefinerFactory.from_parameters_data_experiments(params, reflections, experiments, verbosity=verbosity) outliers = None refined = refiner.run() if debug_plots: debug_plot_residuals(refiner) return refiner, refined, outliers
def refine(params, reflections, experiments, verbosity=0, debug_plots=False): detector = experiments.detectors()[0] from dials.algorithms.refinement import RefinerFactory refiner = RefinerFactory.from_parameters_data_experiments( params, reflections, experiments, verbosity=verbosity) outliers = None refined = refiner.run() if debug_plots: debug_plot_residuals(refiner) return refiner, refined, outliers
def get_rmsds_obs_pred(observations, experiment): from dials.algorithms.spot_prediction import ray_intersection from dials.algorithms.indexing.indexer import master_params from dials.algorithms.refinement import RefinerFactory from dxtbx.model.experiment.experiment_list import ExperimentList master_params.refinement.reflections.close_to_spindle_cutoff = 0.001 from dials.model.data import ReflectionList ref_list = ReflectionList.from_table(observations) ref_list = ray_intersection(experiment.detector, ref_list) ref_table = ref_list.to_table() import copy reflections = copy.deepcopy(observations) reflections['xyzcal.mm'] = ref_table['xyzcal.mm'] reflections['xyzcal.px'] = ref_table['xyzcal.px'] # XXX hack to make it work for a single lattice reflections['id'] = flex.int(len(reflections), 0) refine = RefinerFactory.from_parameters_data_experiments( master_params, reflections, ExperimentList([experiment]), verbosity=0) return refine.rmsds()
def _setup_perturbation(self): '''Obtain a PredictionParameterisation object, get the values of the parameters and their units''' self._refiner = RefinerFactory.from_parameters_data_experiments( self.params, self.dummy_reflections, self.original_experiments) pr = self._refiner.get_param_reporter() units = [p.param_type for p in pr.get_params()] self._pp = self._refiner._pred_param self._old_vals = self._pp.get_param_vals() # construct list of sigmas according to parameter units self._sigmas = [] for (u, val) in zip(units, self._old_vals): if '(mm)' in str(u): self._sigmas.append(self._sig_mm) elif '(mrad)' in str(u): self._sigmas.append(self._sig_mrad) else: # no recognised unit self._sigmas.append(self._frac_sig_unitless * val) return
def __call__(self, experiments, reflections): self.working_phil.show() params = self.working_phil.extract() for iexp, exp in enumerate(experiments): print "Refining crystal", iexp # reflection subset for a single experiment refs = reflections.select(reflections['id'] == iexp) refs['id'] = flex.size_t(len(refs),0) # experiment list for a single experiment exps=ExperimentList() exps.append(exp) refiner = RefinerFactory.from_parameters_data_experiments( params, refs, exps, verbosity=1) # do refinement refiner.run() refined_exps = refiner.get_experiments() # replace this experiment with the refined one experiments[iexp] = refined_exps[0] return experiments
def __call__(self, experiments, reflections): self.working_phil.show() params = self.working_phil.extract() for iexp, exp in enumerate(experiments): print("Refining crystal", iexp) # reflection subset for a single experiment refs = reflections.select(reflections["id"] == iexp) refs["id"] = flex.size_t(len(refs), 0) # experiment list for a single experiment exps = ExperimentList() exps.append(exp) refiner = RefinerFactory.from_parameters_data_experiments( params, refs, exps, verbosity=1 ) # do refinement refiner.run() refined_exps = refiner.get_experiments() # replace this experiment with the refined one experiments[iexp] = refined_exps[0] return experiments
def do_work(item): iexp, exp = item print "Refining crystal", iexp # reflection subset for a single experiment refs = reflections.select(reflections['id'] == iexp) refs['id'] = flex.int(len(refs),0) # DGW commented out as reflections.minimum_number_of_reflections no longer exists #if len(refs) < params.refinement.reflections.minimum_number_of_reflections: # print "Not enough reflections to refine experiment" # return # experiment list for a single experiment exps=ExperimentList() exps.append(exp) try: refiner = RefinerFactory.from_parameters_data_experiments( params, refs, exps) # do refinement refiner.run() except Exception, e: print "Error,", str(e) return
def run(): parser = OptionParser( phil = phil_scope) params, options = parser.parse_args(show_diff_phil=True) assert params.input.single_img is not None assert params.output_dir is not None # load the image img = dxtbx.load(params.input.single_img) imgset = MemImageSet([img]) datablock = DataBlockFactory.from_imageset(imgset)[0] spotfinder = SpotFinderFactory.from_parameters(params) reflections = spotfinder(datablock) base_name = os.path.splitext(params.input.single_img)[0] reflections.as_pickle(os.path.join(params.output_dir, base_name + "_strong.pickle")) # DGW commented out as reflections.minimum_number_of_reflections no longer exists #if len(reflections) < params.refinement.reflections.minimum_number_of_reflections: # print "Not enough spots to index" # return # create the spot finder print "Spotfinder spots found:", len(reflections) if params.indexing.method == "fft3d": from dials.algorithms.indexing.fft3d import indexer_fft3d as indexer elif params.indexing.method == "fft1d": from dials.algorithms.indexing.fft1d import indexer_fft1d as indexer elif params.method == "real_space_grid_search": from dials.algorithms.indexing.real_space_grid_search \ import indexer_real_space_grid_search as indexer try: idxr = indexer(reflections, [imgset], params=params.indexing) except (RuntimeError, Sorry) as e: print str(e) return indexed = idxr.refined_reflections experiments = idxr.refined_experiments #from dxtbx.model.experiment.experiment_list import ExperimentListDumper #dump = ExperimentListDumper(experiments) #dump.as_json(os.path.join(params.output_dir, base_name + "_experiments.json")) indexed.as_pickle(os.path.join(params.output_dir, base_name + "_indexed.pickle")) refiner = RefinerFactory.from_parameters_data_experiments( params, indexed, experiments) refiner.run() refined_experiments = refiner.get_experiments() #dump = ExperimentListDumper(refined_experiments) #dump.as_json(os.path.join(params.output_dir, base_name + "_refined.json")) # Compute the profile model # Predict the reflections # Match the predictions with the reference # Create the integrator reference = indexed reference = process_reference(reference) profile_model = ProfileModelFactory.create(params, refined_experiments, reference) predicted = flex.reflection_table.from_predictions_multi( refined_experiments, dmin=params.prediction.dmin, dmax=params.prediction.dmax, margin=params.prediction.margin, force_static=params.prediction.force_static) predicted.match_with_reference(reference) integrator = IntegratorFactory.create(params, experiments, profile_model, predicted) # Integrate the reflections integrated = integrator.integrate() integrated.as_pickle(os.path.join(params.output_dir, base_name + "_integrated.pickle"))
def __call__(self, experiments, reflections): comm = MPI.COMM_WORLD rank = comm.Get_rank() params = self.working_phil.extract() if rank == 0: data = [] size =comm.Get_size() chunk_size = len(experiments) // size remainder = len(experiments) % size pointer = 0 self.working_phil.show() for i in xrange(size): if i < remainder: sel_range = xrange(pointer,pointer+chunk_size+1) else: sel_range = xrange(pointer,pointer+chunk_size) sel = flex.bool(len(reflections)) for exp_id in sel_range: sel |= reflections['id'] == exp_id if i < remainder: data.append((range(pointer,pointer+chunk_size+1),experiments[pointer:pointer+chunk_size+1],reflections.select(sel))) pointer += 1 else: data.append((range(pointer,pointer+chunk_size),experiments[pointer:pointer+chunk_size],reflections.select(sel))) pointer += chunk_size else: data = None data = comm.scatter(data, root=0) for i, (iexp, exp) in enumerate(zip(data[0],data[1])): print "Refining crystal", iexp # reflection subset for a single experiment refs = data[2].select(data[2]['id'] == iexp) refs['id'] = flex.size_t(len(refs),0) # experiment list for a single experiment exps=ExperimentList() exps.append(exp) refiner = RefinerFactory.from_parameters_data_experiments( params, refs, exps, verbosity=1) # do refinement refiner.run() refined_exps = refiner.get_experiments() # replace this experiment with the refined one data[1][i] = refined_exps[0] data = comm.gather(data, root=0) if rank == 0: for chunk in data: for iexp, experiment in zip(chunk[0], chunk[1]): experiments[iexp] = experiment return experiments else: assert data == None
def run(self): '''Execute the script.''' from time import time import cPickle as pickle from logging import info from dials.util import log from dials.algorithms.refinement import RefinerFactory from dials.util.options import flatten_reflections, flatten_experiments start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) # Try to load the models and data nexp = len(experiments) if nexp == 0: print "No Experiments found in the input" self.parser.print_help() return if len(reflections) == 0: print "No reflection data found in the input" self.parser.print_help() return if len(reflections) > 1: raise Sorry("Only one reflections list can be imported at present") reflections = reflections[0] self.check_input(reflections) # Configure the logging log.config(info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': info('The following parameters have been modified:\n') info(diff_phil) # Modify options if necessary if params.output.correlation_plot.filename is not None: params.refinement.refinery.track_parameter_correlation = True # Get the refiner info('Configuring refiner') refiner = RefinerFactory.from_parameters_data_experiments(params, reflections, experiments) # Refine the geometry if nexp == 1: info('Performing refinement of a single Experiment...') else: info('Performing refinement of {0} Experiments...'.format(nexp)) # Refine and get the refinement history history = refiner.run() if params.output.centroids: info("Writing table of centroids to '{0}'".format( params.output.centroids)) self.write_centroids_table(refiner, params.output.centroids) # Write scan-varying parameters to file, if there were any if params.output.parameter_table: scan = refiner.get_scan() if scan: text = refiner.get_param_reporter().varying_params_vs_image_number( scan.get_array_range()) if text: info("Writing scan-varying parameter table to {0}".format( params.output.parameter_table)) f = open(params.output.parameter_table,"w") f.write(text) f.close() else: info("No scan-varying parameter table to write") # get the refined experiments experiments = refiner.get_experiments() crystals = experiments.crystals() if len(crystals) == 1: # output the refined model for information info('') info('Final refined crystal model:') info(crystals[0]) # Save the refined experiments to file output_experiments_filename = params.output.experiments info('Saving refined experiments to {0}'.format(output_experiments_filename)) from dxtbx.model.experiment.experiment_list import ExperimentListDumper dump = ExperimentListDumper(experiments) dump.as_json(output_experiments_filename) # Save reflections with updated predictions if requested (allow to switch # this off if it is a time-consuming step) if params.output.reflections: # Update predictions for all indexed reflections info('Updating predictions for indexed reflections') preds = refiner.predict_for_indexed() # just copy over the columns of interest, leaving behind things # added by e.g. scan-varying refinement such as 'block', the # U, B and UB matrices and gradients. reflections['s1'] = preds['s1'] reflections['xyzcal.mm'] = preds['xyzcal.mm'] reflections['xyzcal.px'] = preds['xyzcal.px'] if preds.has_key('entering'): reflections['entering'] = preds['entering'] # set used_in_refinement and centroid_outlier flags assert len(preds) == len(reflections) reflections.unset_flags(flex.size_t_range(len(reflections)), reflections.flags.used_in_refinement | reflections.flags.centroid_outlier) mask = preds.get_flags(preds.flags.centroid_outlier) reflections.set_flags(mask, reflections.flags.centroid_outlier) mask = preds.get_flags(preds.flags.used_in_refinement) reflections.set_flags(mask, reflections.flags.used_in_refinement) info('Saving reflections with updated predictions to {0}'.format( params.output.reflections)) if params.output.include_unused_reflections: reflections.as_pickle(params.output.reflections) else: sel = reflections.get_flags(reflections.flags.used_in_refinement) reflections.select(sel).as_pickle(params.output.reflections) # For debugging, if requested save matches to file if params.output.matches: matches = refiner.get_matches() info('Saving matches (use for debugging purposes) to {0}'.format( params.output.matches)) matches.as_pickle(params.output.matches) # Correlation plot if params.output.correlation_plot.filename is not None: from os.path import splitext root, ext = splitext(params.output.correlation_plot.filename) if not ext: ext = ".pdf" steps = params.output.correlation_plot.steps if steps is None: steps = [history.get_nrows()-1] # extract individual column names or indices col_select = params.output.correlation_plot.col_select num_plots = 0 for step in steps: fname_base = root + "_step%02d" % step plot_fname = fname_base + ext corrmat, labels = refiner.get_parameter_correlation_matrix(step, col_select) if [corrmat, labels].count(None) == 0: from dials.algorithms.refinement.refinement_helpers import corrgram plt = corrgram(corrmat, labels) if plt is not None: info('Saving parameter correlation plot to {}'.format(plot_fname)) plt.savefig(plot_fname) num_plots += 1 mat_fname = fname_base + ".pickle" with open(mat_fname, 'wb') as handle: py_mat = corrmat.as_scitbx_matrix() #convert to pickle-friendly form info('Saving parameter correlation matrix to {0}'.format(mat_fname)) pickle.dump({'corrmat':py_mat, 'labels':labels}, handle) if num_plots == 0: msg = "Sorry, no parameter correlation plots were produced. Please set " \ "track_parameter_correlation=True to ensure correlations are " \ "tracked, and make sure correlation_plot.col_select is valid." info(msg) # Write out refinement history, if requested if params.output.history: with open(params.output.history, 'wb') as handle: info('Saving refinement step history to {0}'.format( params.output.history)) pickle.dump(history, handle) # Log the total time taken info("\nTotal time taken: {0:.2f}s".format(time() - start_time)) return
beam.fix=all detector.hierarchy_level=1 sparse=True } target.gradient_calculation_blocksize=10000 reflections.outlier.algorithm=tukey refinery.engine=LBFGScurvs }""") from dials.data.refinement import phil_scope as master_phil working_phil = master_phil.fetch( sources=[user_phil]) working_phil.show() params = working_phil.extract() from dials.algorithms.refinement import RefinerFactory refiner = RefinerFactory.from_parameters_data_experiments( params, reflections, experiments, verbosity=2) refiner.run() # save the refined experiments from dxtbx.model.experiment.experiment_list import ExperimentListDumper experiments = refiner.get_experiments() dump = ExperimentListDumper(experiments) experiments_filename = "refined_experiments.json" dump.as_json(experiments_filename) print "refined geometry written to {0}".format(experiments_filename) # save reflections used in refinement matches = refiner.get_matches() reflections_filename = "refined_reflections.json" matches.as_pickle(reflections_filename)
def test4(): '''Test group restraint with multiple crystals, and a stills refiner''' if not libtbx.env.has_module("dials_regression"): print "Skipping test2 in " + __file__ + " as dials_regression not present" return # The phil scope from dials.algorithms.refinement.refiner import phil_scope user_phil = parse(''' refinement { parameterisation { crystal { unit_cell { restraints { tie_to_group { sigmas=1,0,2,0,0,0 apply_to_all=true } } } } } } ''') working_phil = phil_scope.fetch(source=user_phil) working_params = working_phil.extract() dials_regression = libtbx.env.find_in_repositories( relative_path="dials_regression", test=os.path.isdir) # use the multi stills test data data_dir = os.path.join(dials_regression, "refinement_test_data", "multi_stills") experiments_path = os.path.join(data_dir, "combined_experiments.json") pickle_path = os.path.join(data_dir, "combined_reflections.pickle") experiments = ExperimentListFactory.from_json_file(experiments_path, check_format=False) reflections = flex.reflection_table.from_pickle(pickle_path) refiner = RefinerFactory.from_parameters_data_experiments(working_params, reflections, experiments) # hack to extract the objects needed from the Refiner rp = refiner._target._restraints_parameterisation pred_param = refiner._pred_param # get analytical values and gradients vals, grads, weights = rp.get_residuals_gradients_and_weights() # get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.e-7] * len(p_vals) fd_grad=[] for i in range(len(deltas)): val = p_vals[i] p_vals[i] -= deltas[i] / 2. pred_param.set_param_vals(p_vals) rev_state, foo, bar = rp.get_residuals_gradients_and_weights() rev_state = flex.double(rev_state) p_vals[i] += deltas[i] pred_param.set_param_vals(p_vals) fwd_state, foo, bar = rp.get_residuals_gradients_and_weights() fwd_state = flex.double(fwd_state) p_vals[i] = val fd = (fwd_state - rev_state) / deltas[i] fd_grad.append(fd) # for comparison, fd_grad is a list of flex.doubles, each of which corresponds # to the gradients of the residuals wrt to a single parameter. pnames = pred_param.get_param_names() for i, (pname, fd) in enumerate(zip(pnames, fd_grad)): # extract dense column from the sparse matrix an = grads.col(i).as_dense_vector() #print pname #print list(an.round(6)) #print list(fd.round(6)) #print assert approx_equal(an, fd, eps=1e-5) print "OK" return
def run(): parser = OptionParser(phil=phil_scope) params, options = parser.parse_args(show_diff_phil=True) assert params.input.single_img is not None assert params.output_dir is not None # load the image img = dxtbx.load(params.input.single_img) imgset = MemImageSet([img]) datablock = DataBlockFactory.from_imageset(imgset)[0] spotfinder = SpotFinderFactory.from_parameters(params) reflections = spotfinder(datablock) base_name = os.path.splitext(params.input.single_img)[0] reflections.as_pickle( os.path.join(params.output_dir, base_name + "_strong.pickle")) # DGW commented out as reflections.minimum_number_of_reflections no longer exists # if len(reflections) < params.refinement.reflections.minimum_number_of_reflections: # print "Not enough spots to index" # return # create the spot finder print("Spotfinder spots found:", len(reflections)) if params.indexing.method == "fft3d": from dials.algorithms.indexing.fft3d import indexer_fft3d as indexer elif params.indexing.method == "fft1d": from dials.algorithms.indexing.fft1d import indexer_fft1d as indexer elif params.method == "real_space_grid_search": from dials.algorithms.indexing.real_space_grid_search import ( indexer_real_space_grid_search as indexer, ) try: idxr = indexer(reflections, [imgset], params=params.indexing) except (RuntimeError, Sorry) as e: print(str(e)) return indexed = idxr.refined_reflections experiments = idxr.refined_experiments # from dxtbx.model.experiment.experiment_list import ExperimentListDumper # dump = ExperimentListDumper(experiments) # dump.as_json(os.path.join(params.output_dir, base_name + "_experiments.json")) indexed.as_pickle( os.path.join(params.output_dir, base_name + "_indexed.pickle")) refiner = RefinerFactory.from_parameters_data_experiments( params, indexed, experiments) refiner.run() refined_experiments = refiner.get_experiments() # dump = ExperimentListDumper(refined_experiments) # dump.as_json(os.path.join(params.output_dir, base_name + "_refined.json")) # Compute the profile model # Predict the reflections # Match the predictions with the reference # Create the integrator reference = indexed reference = process_reference(reference) profile_model = ProfileModelFactory.create(params, refined_experiments, reference) predicted = flex.reflection_table.from_predictions_multi( refined_experiments, dmin=params.prediction.dmin, dmax=params.prediction.dmax, margin=params.prediction.margin, force_static=params.prediction.force_static, ) predicted.match_with_reference(reference) integrator = IntegratorFactory.create(params, experiments, profile_model, predicted) # Integrate the reflections integrated = integrator.integrate() integrated.as_pickle( os.path.join(params.output_dir, base_name + "_integrated.pickle"))