amatrixdiridx = [a.find("amatrix_dir")==0 for a in args].index(True) except ValueError: amatrix_dir_prefix = "tmpdir_" else: amatrix_dir_prefix = args.pop(amatrixdiridx).split("=")[1] if (not (rotation_series or stills_process)): raise ValueError,"Must specify image_glob (for rotation series) or index_glob (for stills)." import copy, os import dxtbx from dxtbx.model.experiment_list import ExperimentListFactory from dials.array_family import flex experiments = ExperimentListFactory.from_json_file(metro, check_format=False) beam = experiments[0].beam detector = experiments[0].detector print "s0 from experiment[0]: ",beam.get_s0() print "wavelength from experiment[0]: ",beam.get_wavelength() lab_coordinates = flex.vec3_double() for panel in detector: pixels = flex.vec2_double(panel.get_image_size()) mms = panel.pixel_to_millimeter(pixels) lab_coordinates.extend(panel.get_lab_coord(mms)) # generate s1 vectors s1 = lab_coordinates.each_normalize() * (1/beam.get_wavelength()) # Generate x vectors
def test_refinement_and_compare_with_known_truth(dials_regression, run_in_tmpdir): # use data generated by simulation for this test data_dir = os.path.join(dials_regression, "refinement_test_data", "varying_beam_direction") experiments_path = os.path.join(data_dir, "refined_static.json") pickle_path = os.path.join(data_dir, "refined_static.pickle") for pth in (experiments_path, pickle_path): assert os.path.exists(pth) # Run refinement and load models result = procrunner.run([ "dials.refine", experiments_path, pickle_path, "scan_varying=True", "crystal.orientation.force_static=True", "crystal.unit_cell.force_static=True", "beam.force_static=False", "beam.fix=wavelength", ]) assert not result.returncode and not result.stderr exp = ExperimentListFactory.from_json_file("refined.expt", check_format=False)[0] beam, detector = exp.beam, exp.detector # Beam centre at every scan-point s0_scan_points = [ beam.get_s0_at_scan_point(i) for i in range(beam.num_scan_points) ] bc_scan_points = [ detector[0].get_beam_centre_px(s0) for s0 in s0_scan_points ] # Set up the nanoBragg object as used in the simulation sys.path.append(data_dir) from sim_images import Simulation sim = Simulation() sim.set_varying_beam(along="both") # Simulation beam centre at every scan-point sim_s0_scan_points = [ sim.beam.get_s0_at_scan_point(i) for i in range(sim.beam.num_scan_points) ] sim_bc_scan_points = [ sim.detector[0].get_beam_centre_px(s0) for s0 in sim_s0_scan_points ] assert beam.num_scan_points == sim.beam.num_scan_points # Generate a plot. This shows that the beam centre is worse at the edges of # the scan. This is what we expect as the centroids at the scan edges are # least well determined because of the truncation of found spots. At these # edges the error approaches 0.15 pixels, whilst in the central region of # the scan it is within 0.05 pixels. # # plot_beam_centre_error(sim_bc_scan_points, bc_scan_points) # Compare the results. for bc1, bc2 in zip(sim_bc_scan_points, bc_scan_points): assert bc2 == pytest.approx(bc1, abs=0.15)
def tst_one_monkeypatch(i_exp, spectra, Fmerge, gpu_channels_singleton, rank, params): print("IN MONKEYPATCH") from simtbx.nanoBragg import utils from dxtbx.model.experiment_list import ExperimentListFactory import numpy as np print("Experiment %d" % i_exp, flush=True) outfile = "boop_%d.hdf5" % i_exp from LS49.adse13_187.case_data import retrieve_from_repo experiment_file = retrieve_from_repo(i_exp) cuda = True # False # whether to use cuda omp = False ngpu_on_node = 1 # 8 # number of available GPUs mosaic_spread = 0.07 # degrees mosaic_spread_samples = params.mosaic_spread_samples # number of mosaic blocks sampling mosaicity Ncells_abc = 30, 30, 10 # medians from best stage1 ev_res = 1.5 # resolution of the downsample spectrum total_flux = 1e12 # total flux across channels beamsize_mm = 0.000886226925452758 # sqrt of beam focal area spot_scale = 500. # 5.16324 # median from best stage1 plot_spec = False # plot the downsample spectra before simulating oversample = 1 # oversample factor, 1,2, or 3 probable enough panel_list = None # integer list of panels, usefule for debugging rois_only = False # only set True if you are running openMP, or CPU-only (i.e. not for GPU) include_background = params.include_background # default is to add water background 100 mm thick verbose = 0 # leave as 0, unles debug flat = True # enfore that the camera has 0 thickness #<><><><><><><><> # XXX new code El = ExperimentListFactory.from_json_file(experiment_file, check_format=True) exper = El[0] crystal = exper.crystal detector = exper.detector if flat: from dxtbx_model_ext import SimplePxMmStrategy for panel in detector: panel.set_px_mm_strategy(SimplePxMmStrategy()) panel.set_mu(0) panel.set_thickness(0) beam = exper.beam # XXX new code spec = exper.imageset.get_spectrum(0) energies_raw, weights_raw = spec.get_energies_eV().as_numpy_array(), \ spec.get_weights().as_numpy_array() energies, weights = utils.downsample_spectrum(energies_raw, weights_raw, method=1, total_flux=total_flux, ev_width=ev_res) if flat: assert detector[0].get_thickness() == 0 if panel_list is None: panel_list = list(range(len(detector))) pids_for_rank = panel_list device_Id = 0 if gpu_channels_singleton is not None: device_Id = gpu_channels_singleton.get_deviceID() print("Rank %d will use device %d" % (rank, device_Id)) show_params = False time_panels = (rank == 0) mn_energy = (energies * weights).sum() / weights.sum() mn_wave = utils.ENERGY_CONV / mn_energy jf16m_numpy_array = {} from_gpu_amplitudes_cuda = {} print( "\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>") print("\tBreakdown:") for shapetype in ["gauss_argchk", "gauss"]: BEG = time() print(gpu_channels_singleton.get_deviceID(), "device", shapetype) Famp_is_uninitialized = (gpu_channels_singleton.get_nchannels() == 0) if Famp_is_uninitialized: F_P1 = Fmerge.expand_to_p1() for x in range( 1 ): # in this scenario, amplitudes are independent of lambda gpu_channels_singleton.structure_factors_to_GPU_direct( x, F_P1.indices(), F_P1.data()) assert gpu_channels_singleton.get_nchannels() == 1 JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim( CRYSTAL=crystal, DETECTOR=detector, BEAM=beam, Famp=gpu_channels_singleton, energies=list(energies), fluxes=list(weights), background_wavelengths=[mn_wave], background_wavelength_weights=[1], background_total_flux=total_flux, background_sample_thick_mm=0.5, cuda=True, oversample=oversample, Ncells_abc=Ncells_abc, mos_dom=mosaic_spread_samples, mos_spread=mosaic_spread, beamsize_mm=beamsize_mm, profile=shapetype, show_params=show_params, time_panels=time_panels, verbose=verbose, spot_scale_override=spot_scale, include_background=include_background) TIME_EXA = time() - BEG jf16m_numpy_array[shapetype] = JF16M_numpy_array from_gpu_amplitudes_cuda[shapetype] = TIME_BRAGG print( "\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs" % (TIME_BG, TIME_BRAGG, TIME_EXA)) ratio = from_gpu_amplitudes_cuda["gauss"] / from_gpu_amplitudes_cuda[ "gauss_argchk"] print( "<><><><><><><><><ratio<%.2f><><><><><><><><><><><><><><><><><><><>\n" % (ratio)) # assertion on elapsed time: assert ratio > 1.0, "ratio is %.3f, experiment %d" % (ratio, i_exp) # assertion on equality: abs_diff = np.abs(jf16m_numpy_array["gauss"] - \ jf16m_numpy_array["gauss_argchk"]).max() assert np.allclose(jf16m_numpy_array["gauss"], \ jf16m_numpy_array["gauss_argchk"]), \ "max per-pixel difference: %f photons, experiment %d"%(abs_diff,i_exp)
def run(self): if self._reindexing_operators: logger.debug("Reindexing sweeps for dials.two_theta_refine") from xia2.lib.bits import auto_logfiler from xia2.Wrappers.Dials.Reindex import Reindex self._reindexed_experiments, self._reindexed_reflections = [], [] for e, p, op in zip( self._experiments, self._reflection_files, self._reindexing_operators, ): reindexer = Reindex() reindexer.set_cb_op(op) reindexer.set_experiments_filename(e) reindexer.set_indexed_filename(p) reindexer.set_working_directory(self.get_working_directory()) auto_logfiler(reindexer) reindexer.run() self._reindexed_experiments.append( reindexer.get_reindexed_experiments_filename() ) self._reindexed_reflections.append( reindexer.get_reindexed_reflections_filename() ) logger.debug("Running dials.two_theta_refine") self._output_cif = os.path.join( self.get_working_directory(), "%s_dials.two_theta_refine.cif" % self.get_xpid(), ) self._output_mmcif = os.path.join( self.get_working_directory(), "%s_dials.two_theta_refine.mmcif" % self.get_xpid(), ) if not self._output_p4p: self._output_p4p = os.path.join( self.get_working_directory(), "%s_dials.two_theta_refine.p4p" % self.get_xpid(), ) self._output_correlation_plot = os.path.join( self.get_working_directory(), "%s_dials.two_theta_refine.png" % self.get_xpid(), ) self._output_experiments = os.path.join( self.get_working_directory(), "%s_refined_cell.expt" % self.get_xpid() ) self.clear_command_line() if self._reindexing_operators: for experiment in self._reindexed_experiments: self.add_command_line(experiment) for reflection_file in self._reindexed_reflections: self.add_command_line(reflection_file) else: for experiment in self._experiments: self.add_command_line(experiment) for reflection_file in self._reflection_files: self.add_command_line(reflection_file) self.add_command_line( "combine_crystal_models=%s" % self._combine_crystal_models ) self.add_command_line("output.cif=%s" % self._output_cif) self.add_command_line("output.mmcif=%s" % self._output_mmcif) self.add_command_line("output.p4p=%s" % self._output_p4p) if self._output_correlation_plot is not None: self.add_command_line( "output.correlation_plot.filename=%s" % self._output_correlation_plot ) if self._output_experiments is not None: self.add_command_line( "output.experiments=%s" % self._output_experiments ) if self._phil_file is not None: self.add_command_line(self._phil_file) self.start() self.close_wait() if not os.path.isfile(self._output_cif): logger.warning( "TwoTheta refinement failed, see log file for more details:\n %s", self.get_log_file(), ) raise RuntimeError("unit cell not refined") self.check_for_errors() experiments = ExperimentListFactory.from_json_file( self.get_output_experiments(), check_format=False ) self._crystal = experiments.crystals()[0]
def update_all_data(reflections_path=None, experiments_path=None): dat = InfoData() dat.ref2exp = None if reflections_path is not None: try: refl_tabl = flex.reflection_table.from_file(reflections_path) dat.n_strng = refl_tabl.get_flags( refl_tabl.flags.strong).count(True) dat.n_index = refl_tabl.get_flags( refl_tabl.flags.indexed).count(True) dat.n_refnd = refl_tabl.get_flags( refl_tabl.flags.used_in_refinement).count(True) dat.n_integ_sum = refl_tabl.get_flags( refl_tabl.flags.integrated_sum).count(True) dat.n_integ_prf = refl_tabl.get_flags( refl_tabl.flags.integrated_prf).count(True) except BaseException as e: # We don't want to catch bare exceptions but don't know # what this was supposed to catch. Log it. logger.info(" >> Caught unknown exception type:", type(e).__name__, e, "N###") logger.info("failed to find reflections") logger.info("reflections_path = %s", reflections_path) if experiments_path is not None: try: experiments = ExperimentListFactory.from_json_file( experiments_path, check_format=False) except BaseException as e: # We don't want to catch bare exceptions but don't know # what this was supposed to catch. Log it. logger.info("\n exception #1 %s: %s", type(e).__name__, e) # TODO Maybe the next try block is not needed, consider removing all try: # FIXME here only take the first datablock. What if there are more? datablock = DataBlockFactory.from_serialized_format( experiments_path, check_format=False)[0] # FIXME here only take the first model from each beam = datablock.unique_beams()[0] detector = datablock.unique_detectors()[0] scan = datablock.unique_scans()[0] # build a pseudo ExperimentList (with empty crystals) experiments = ExperimentList() experiments.append( Experiment(beam=beam, detector=detector, scan=scan)) except ValueError: logger.info("failed to read json file") logger.info("experiments_path = %s", experiments_path) return dat try: imageset_tmp = experiments.imagesets()[0] mask_file = imageset_tmp.external_lookup.mask.filename pick_file = open(mask_file, "rb") mask_tup_obj = pickle.load(pick_file) pick_file.close() dat.mask_flex = mask_tup_obj[0] mask_np_arr = dat.mask_flex.as_numpy_array() dat.np_mask = mask_np_arr except IOError: logger.info("No mask in this node") dat.np_mask = None dat.mask_flex = None # FIXME it takes just the first experiment. What if there are more? exp = experiments[0] # Get crystal data if exp.crystal is not None: unit_cell = exp.crystal.get_unit_cell() dat.a, dat.b, dat.c, dat.alpha, dat.beta, dat.gamma = unit_cell.parameters( ) exp_crystal = exp.crystal logger.info("exp_crystal = %s", exp_crystal) b_mat = exp.crystal.get_B() dat.b11 = b_mat[0] dat.b12 = b_mat[1] dat.b13 = b_mat[2] dat.b21 = b_mat[3] dat.b22 = b_mat[4] dat.b23 = b_mat[5] dat.b31 = b_mat[6] dat.b32 = b_mat[7] dat.b33 = b_mat[8] sg = str(exp.crystal.get_space_group().info()) logger.info("spgr = %s", sg) dat.spg_group = sg from scitbx import matrix u_mat = matrix.sqr(exp.crystal.get_U()) dat.u11 = b_mat[0] dat.u12 = b_mat[1] dat.u13 = b_mat[2] dat.u21 = b_mat[3] dat.u22 = b_mat[4] dat.u23 = b_mat[5] dat.u31 = b_mat[6] dat.u32 = b_mat[7] dat.u33 = b_mat[8] rot_angs = u_mat.r3_rotation_matrix_as_x_y_z_angles(deg=True) logger.info("u_mat = %s", u_mat) logger.info("rot_angs = %s", rot_angs) dat.r1, dat.r2, dat.r3 = rot_angs # Get beam data dat.w_lambda = exp.beam.get_wavelength() # Get detector data # assume details for the panel the beam intersects are the same # for the whole detector # TODO fix it for I23 pnl_beam_intersects, (beam_x, beam_y) = exp.detector.get_ray_intersection( exp.beam.get_s0()) pnl = exp.detector[pnl_beam_intersects] logger.info("beam_x, beam_y = %s %s", beam_x, beam_y) dat.n_pan_xb_yb = pnl_beam_intersects dat.xb = beam_x dat.yb = beam_y dist = pnl.get_distance() logger.info("pnl_beam_intersects %s", pnl_beam_intersects) logger.info("dist %s", dist) dat.dd = dist dat.img_ran1, dat.img_ran2 = exp.scan.get_image_range() dat.oscil1, dat.oscil2 = exp.scan.get_oscillation() # is the next line right? check what dials.show does dat.e_time = max(exp.scan.get_exposure_times()) dat.n_pans = len(exp.detector) dat.x_px_size, dat.y_px_size = pnl.get_pixel_size() dat.gain = pnl.get_gain() dat.max_res = exp.detector.get_max_resolution(exp.beam.get_s0()) # manually finding template from experiments_path try: with open(experiments_path) as infile: json_info = json.load(infile) if type(json_info) is dict: imageset = json_info["imageset"] elif type(json_info) is list: imageset = json_info[0]["imageset"] dat.tmpl_str = imageset[0]["template"] logger.info("dat.tmpl_str = %s", dat.tmpl_str) except (KeyError, IndexError): logger.info("failed to find template in JSON file") dat.ref2exp = exp return dat
def process(self, job): """Perform processing tasks for one dataset""" print("Processing job {0}".format(job)) # Import path = os.path.join(job, "*.img") cmd = ("dials.import {0} ").format(path) if self._run_one_job(cmd) is None: return # Spot finding cmd = "dials.find_spots datablock.json filter.d_min={0} nproc={1}".format( self.params.find_spots.d_min, self.params.nproc) if self._run_one_job(cmd) is None: return with open("dials.find_spots.log", "r") as log: while True: line = log.readline() if line.startswith("Histogram"): break for i in range(14): print(log.readline(), end=" ") # Attempt indexing in P1 n_indexed = 0 best_indexing = None cmd = ( "dials.index datablock.json strong.pickle " "output.experiments=P1_experiments.json " "output.reflections=P1_indexed.pickle output.log=P1_dials.index.log" ) if self._run_one_job(cmd) is not None: indexed = flex.reflection_table.from_pickle("P1_indexed.pickle") n_indexed = indexed.get_flags(indexed.flags.indexed).count(True) best_indexing = ("P1_experiments.json", "P1_indexed.pickle") el = ExperimentListFactory.from_json_file("P1_experiments.json") cell = el[0].crystal.get_unit_cell().parameters() cell_s = ", ".join(["{:.3f}".format(e) for e in cell]) print("{0} reflections indexed in P1 with unit cell ({1})".format( n_indexed, cell_s)) # Attempt indexing in P222 cmd = ("dials.index datablock.json strong.pickle " "output.experiments=P222_experiments.json " "output.reflections=P222_indexed.pickle " "output.log=P222_dials.index.log space_group=P222") if self._run_one_job(cmd) is not None: indexed = flex.reflection_table.from_pickle("P222_indexed.pickle") n_indexed2 = indexed.get_flags(indexed.flags.indexed).count(True) if n_indexed2 > n_indexed: best_indexing = ("P222_experiments.json", "P222_indexed.pickle") n_indexed = n_indexed2 el = ExperimentListFactory.from_json_file("P222_experiments.json") cell = el[0].crystal.get_unit_cell().parameters() cell_s = ", ".join(["{:.3f}".format(e) for e in cell]) print( "{0} reflections indexed in P222 with unit cell ({1})".format( n_indexed2, cell_s)) if best_indexing is None: return print("Selecting {0} and {1} for further processing".format( *best_indexing)) # Convert best indexing solution to P1 and P222 versions cmd = ("dials.reindex {0} {1} space_group={2} " "output.experiments=best_experiments_conv_to_{2}.json " "output.reflections=best_reflections_conv_to_{2}.pickle ") if (self._run_one_job( cmd.format(best_indexing[0], best_indexing[1], "P1")) is None): return if (self._run_one_job( cmd.format(best_indexing[0], best_indexing[1], "P222")) is None): return # Further processing in sub directories for sub_job in ("P1", "P222"): try: with cd(sub_job): experiments = "best_experiments_conv_to_{0}.json".format( sub_job) reflections = "best_reflections_conv_to_{0}.pickle".format( sub_job) print("Processing best indexing solution converted to {0}". format(sub_job)) self._refine_onwards(experiments, reflections) except OSError: raise Sorry( "Failed to create sub job directory {0}".format(sub_job)) # Compare summary tables from aimless.log files from each job import difflib logpaths = [ os.path.join("P1", "aimless.log"), os.path.join("P222", "aimless.log"), ] if [os.path.exists(e) for e in logpaths].count(True) == 2: aimlessP1 = self._get_aimless_summary(logpaths[0]) aimlessP222 = self._get_aimless_summary(logpaths[1]) diff = difflib.HtmlDiff(wrapcolumn=80).make_file( aimlessP1, aimlessP222, logpaths[0], logpaths[1]) with open("aimless-diff.html", "w") as f: f.writelines(diff) print() return
def run(self, all_experiments, all_reflections): """ Load all the data using MPI """ from dxtbx.model.experiment_list import ExperimentList from dials.array_family import flex # Both must be none or not none test = [all_experiments is None, all_reflections is None].count(True) assert test in [0,2] if test == 2: all_experiments = ExperimentList() all_reflections = flex.reflection_table() starting_expts_count = starting_refls_count = 0 else: starting_expts_count = len(all_experiments) starting_refls_count = len(all_reflections) self.logger.log("Initial number of experiments: %d; Initial number of reflections: %d"%(starting_expts_count, starting_refls_count)) # Generate and send a list of file paths to each worker if self.mpi_helper.rank == 0: file_list = self.get_list() self.logger.log("Built an input list of %d json/pickle file pairs"%(len(file_list))) self.params.input.path = None # Rank 0 has already parsed the input parameters per_rank_file_list = file_load_calculator(self.params, file_list, self.logger).\ calculate_file_load(available_rank_count = self.mpi_helper.size) self.logger.log('Transmitting a list of %d lists of json/pickle file pairs'%(len(per_rank_file_list))) transmitted = per_rank_file_list else: transmitted = None self.logger.log_step_time("BROADCAST_FILE_LIST") transmitted = self.mpi_helper.comm.bcast(transmitted, root = 0) new_file_list = transmitted[self.mpi_helper.rank] if self.mpi_helper.rank < len(transmitted) else None self.logger.log_step_time("BROADCAST_FILE_LIST", True) # Load the data self.logger.log_step_time("LOAD") if new_file_list is not None: self.logger.log("Received a list of %d json/pickle file pairs"%len(new_file_list)) for experiments_filename, reflections_filename in new_file_list: self.logger.log("Reading %s %s"%(experiments_filename, reflections_filename)) experiments = ExperimentListFactory.from_json_file(experiments_filename, check_format = False) reflections = flex.reflection_table.from_file(reflections_filename) self.logger.log("Data read, prepping") if 'intensity.sum.value' in reflections: reflections['intensity.sum.value.unmodified'] = reflections['intensity.sum.value'] * 1 if 'intensity.sum.variance' in reflections: reflections['intensity.sum.variance.unmodified'] = reflections['intensity.sum.variance'] * 1 new_ids = flex.int(len(reflections), -1) new_identifiers = flex.std_string(len(reflections)) eid = reflections.experiment_identifiers() for k in eid.keys(): del eid[k] for experiment_id, experiment in enumerate(experiments): # select reflections of the current experiment refls_sel = reflections['id'] == experiment_id if refls_sel.count(True) == 0: continue if experiment.identifier is None or len(experiment.identifier) == 0: experiment.identifier = create_experiment_identifier(experiment, experiments_filename, experiment_id) if not self.params.input.keep_imagesets: experiment.imageset = None all_experiments.append(experiment) # Reflection experiment 'id' is unique within this rank; 'exp_id' (i.e. experiment identifier) is unique globally new_identifiers.set_selected(refls_sel, experiment.identifier) new_id = len(all_experiments)-1 eid[new_id] = experiment.identifier new_ids.set_selected(refls_sel, new_id) assert (new_ids < 0).count(True) == 0, "Not all reflections accounted for" reflections['id'] = new_ids reflections['exp_id'] = new_identifiers all_reflections.extend(reflections) else: self.logger.log("Received a list of 0 json/pickle file pairs") self.logger.log_step_time("LOAD", True) self.logger.log('Read %d experiments consisting of %d reflections'%(len(all_experiments)-starting_expts_count, len(all_reflections)-starting_refls_count)) self.logger.log("Memory usage: %d MB"%get_memory_usage()) all_reflections = self.prune_reflection_table_keys(all_reflections) # Do we have any data? from xfel.merging.application.utils.data_counter import data_counter data_counter(self.params).count(all_experiments, all_reflections) return all_experiments, all_reflections
crystal = models.crystal # a hexagonal crystal is a good test case for behaviour of oblique cells do_hexagonal = True if do_hexagonal: import libtbx.load_env, os from dxtbx.model.experiment_list import ExperimentListFactory try: dials_regression = libtbx.env.dist_path('dials_regression') except KeyError: print 'Skipped: dials_regression not configured' exit(0) data_dir = os.path.join(dials_regression, "refinement_test_data", "multi_stills") experiments_path = os.path.join(data_dir, "combined_experiments.json") experiments = ExperimentListFactory.from_json_file(experiments_path, check_format=False) crystal = experiments[0].crystal # derive finite difference gradients of various quantities wrt each param def check_fd_gradients(parameterisation): mp = parameterisation p_vals = mp.get_param_vals() deltas = [1.e-7 for p in p_vals] assert len(deltas) == len(p_vals) fd_grad = [] # get matrix to unset rotations of unit cell vectors Ut = matrix.sqr(mp.get_model().get_U()).transpose()
def read_experiments(filename): from dxtbx.model.experiment_list import ExperimentListFactory return ExperimentListFactory.from_json_file(filename)
def test_nexus_dump_and_reload(dials_regression, tmpdir, filename): path = os.path.join(dials_regression, "nexus_test_data", "shared_models") filename_in = os.path.join(path, f"{filename}.json") filename_out = tmpdir.join(f"{filename}.nxs").strpath experiments = ExperimentListFactory.from_json_file(filename_in) run_single(experiments, filename_out)
print 'OK' def run(): from dxtbx.model.experiment_list import ExperimentListFactory from os.path import join import libtbx.load_env try: dials_regression = libtbx.env.dist_path('dials_regression') except KeyError, e: print 'FAIL: dials_regression not configured' exit(0) path = join(dials_regression, "nexus_test_data", "shared_models") filename_list = [ 'single', 'multiple_unrelated', 'multi_crystal', 'two_colour', 'multiple_sweeps', 'stills' ] for filename in filename_list: filename_in = join(path, "%s.json" % filename) filename_out = "%s.nxs" % filename experiments = ExperimentListFactory.from_json_file(filename_in) run_single(experiments, filename_out) if __name__ == '__main__': from dials.test import cd_auto with cd_auto(__file__): test_polarization_conversion() run()
def test_merge_multi_wavelength(dials_data, tmpdir): """Test that merge handles multi-wavelength data suitably - should be exported into an mtz with seprate columns for each wavelength.""" mean_labels = [ "%sIMEAN_WAVE%s" % (pre, i) for i in [1, 2] for pre in ["", "SIG"] ] anom_labels = [ "%sI_WAVE%s(%s)" % (pre, i, sgn) for i in [1, 2] for pre in ["", "SIG"] for sgn in ["+", "-"] ] amp_labels = [ "%sF_WAVE%s" % (pre, i) for i in [1, 2] for pre in ["", "SIG"] ] anom_amp_labels = [ "%sF_WAVE%s(%s)" % (pre, i, sgn) for i in [1, 2] for pre in ["", "SIG"] for sgn in ["+", "-"] ] location = dials_data("l_cysteine_4_sweeps_scaled") refl1 = location.join("scaled_30.refl").strpath expt1 = location.join("scaled_30.expt").strpath refl2 = location.join("scaled_35.refl").strpath expt2 = location.join("scaled_35.expt").strpath expts1 = ExperimentListFactory.from_json_file(expt1, check_format=False) expts1[0].beam.set_wavelength(0.5) expts2 = ExperimentListFactory.from_json_file(expt2, check_format=False) expts1.extend(expts2) tmp_expt = tmpdir.join("tmp.expt").strpath expts1.as_json(tmp_expt) reflections1 = flex.reflection_table.from_pickle(refl1) reflections2 = flex.reflection_table.from_pickle(refl2) # first need to resolve identifiers - usually done on loading reflections2["id"] = flex.int(reflections2.size(), 1) del reflections2.experiment_identifiers()[0] reflections2.experiment_identifiers()[1] = "3" reflections1.extend(reflections2) tmp_refl = tmpdir.join("tmp.refl").strpath reflections1.as_file(tmp_refl) # Can now run after creating our 'fake' multiwavelength dataset command = [ "dials.merge", tmp_refl, tmp_expt, "truncate=True", "anomalous=True" ] result = procrunner.run(command, working_directory=tmpdir) assert not result.returncode and not result.stderr assert tmpdir.join("merged.mtz").check() m = mtz.object(tmpdir.join("merged.mtz").strpath) labels = [] for ma in m.as_miller_arrays(merge_equivalents=False): labels.extend(ma.info().labels) assert all(x in labels for x in mean_labels) assert all(x in labels for x in anom_labels) assert all(x in labels for x in amp_labels) assert all(x in labels for x in anom_amp_labels) # 5 miller arrays for each dataset assert m.as_miller_arrays()[0].info().wavelength == pytest.approx(0.5) assert m.as_miller_arrays()[5].info().wavelength == pytest.approx(0.6889)
def run(self): MAIN_LOGGER = logging.getLogger("diffBragg.main") assert os.path.exists(self.params.exp_ref_spec_file) input_lines = None best_models = None if COMM.rank == 0: input_lines = open(self.params.exp_ref_spec_file, "r").readlines() if self.params.skip is not None: input_lines = input_lines[self.params.skip:] if self.params.first_n is not None: input_lines = input_lines[:self.params.first_n] if self.params.sanity_test_input: hopper_utils.sanity_test_input_lines(input_lines) if self.params.best_pickle is not None: logging.info("reading pickle %s" % self.params.best_pickle) best_models = pandas.read_pickle(self.params.best_pickle) if self.params.dump_gathers: if self.params.gathers_dir is None: raise ValueError( "Need to provide a file dir path in order to dump_gathers" ) utils.safe_makedirs(self.params.gathers_dir) input_lines = COMM.bcast(input_lines) best_models = COMM.bcast(best_models) if self.params.ignore_existing: exp_names_already = None refl_names_already = None if COMM.rank == 0: exp_names_already = { os.path.basename(f) for f in glob.glob("%s/expers/rank*/*.expt" % self.params.outdir) } refl_names_already = { os.path.basename(f) for f in glob.glob("%s/refls/rank*/*.refl" % self.params.outdir) } exp_names_already = COMM.bcast(exp_names_already) refl_names_already = COMM.bcast(refl_names_already) exp_gatheredRef_spec = [] # optional list of expt, refls, spectra for i_exp, line in enumerate(input_lines): if i_exp == self.params.max_process: break if i_exp % COMM.size != COMM.rank: continue logging.info("COMM.rank %d on shot %d / %d" % (COMM.rank, i_exp + 1, len(input_lines))) line_fields = line.strip().split() assert len(line_fields) in [2, 3] if len(line_fields) == 2: exp, ref = line_fields spec = None else: exp, ref, spec = line_fields if self.params.ignore_existing: basename = os.path.splitext(os.path.basename(exp))[0] opt_exp = "%s_%s_%d.expt" % (self.params.tag, basename, i_exp) opt_refl = opt_exp.replace(".expt", ".refl") if opt_exp in exp_names_already and opt_refl in refl_names_already: continue best = None if best_models is not None: best = best_models.query("exp_name=='%s'" % exp) if len(best) == 0: best = best_models.query("opt_exp_name=='%s'" % exp) if len(best) != 1: raise ValueError( "Should be 1 entry for exp %s in best pickle %s" % (exp, self.params.best_pickle)) self.params.simulator.spectrum.filename = spec Modeler = hopper_utils.DataModeler(self.params) if self.params.load_data_from_refls: gathered = Modeler.GatherFromReflectionTable(exp, ref) else: gathered = Modeler.GatherFromExperiment(exp, ref) if not gathered: logging.warning("No refls in %s; CONTINUE; COMM.rank=%d" % (ref, COMM.rank)) continue MAIN_LOGGER.info("Modeling %s (%d refls)" % (exp, len(Modeler.refls))) if self.params.dump_gathers: output_name = os.path.splitext(os.path.basename(exp))[0] output_name += "_withData.refl" output_name = os.path.join(self.params.gathers_dir, output_name) Modeler.dump_gathered_to_refl( output_name, do_xyobs_sanity_check=True ) # NOTE do this is modelin strong spots only if self.params.test_gathered_file: all_data = Modeler.all_data.copy() all_roi_id = Modeler.roi_id.copy() all_bg = Modeler.all_background.copy() all_trusted = Modeler.all_trusted.copy() all_pids = np.array(Modeler.pids) all_rois = np.array(Modeler.rois) new_Modeler = hopper_utils.DataModeler(self.params) assert new_Modeler.GatherFromReflectionTable( exp, output_name) assert np.allclose(new_Modeler.all_data, all_data) assert np.allclose(new_Modeler.all_background, all_bg) assert np.allclose(new_Modeler.rois, all_rois) assert np.allclose(new_Modeler.pids, all_pids) assert np.allclose(new_Modeler.all_trusted, all_trusted) assert np.allclose(new_Modeler.roi_id, all_roi_id) exp_gatheredRef_spec.append( (exp, os.path.abspath(output_name), spec)) if self.params.only_dump_gathers: continue if self.params.refiner.reference_geom is not None: detector = ExperimentListFactory.from_json_file( self.params.refiner.reference_geom, check_format=False)[0].detector Modeler.E.detector = detector Modeler.SimulatorFromExperiment(best) Modeler.SIM.D.store_ave_wavelength_image = self.params.store_wavelength_images if self.params.refiner.verbose is not None and COMM.rank == 0: Modeler.SIM.D.verbose = self.params.refiner.verbose if self.params.profile: Modeler.SIM.record_timings = True if self.params.use_float32: Modeler.all_data = Modeler.all_data.astype(np.float32) Modeler.all_background = Modeler.all_background.astype( np.float32) if self.params.refiner.randomize_devices: dev = np.random.choice(self.params.refiner.num_devices) logging.info( "Rank %d will use randomly chosen device %d on host %s" % (COMM.rank, dev, socket.gethostname())) else: dev = COMM.rank % self.params.refiner.num_devices logging.info("Rank %d will use device %d on host %s" % (COMM.rank, dev, socket.gethostname())) print("RANK%d, DEV=%d" % (COMM.rank, dev)) Modeler.SIM.D.device_Id = dev nparam = len(Modeler.SIM.P) x0 = [1] * nparam try: x = Modeler.Minimize(x0) except StopIteration: x = Modeler.target.x0 if self.params.profile: Modeler.SIM.D.show_timings(COMM.rank) #, out) save_up(Modeler, x, exp, i_exp, ref) if self.params.dump_gathers and self.params.gathered_output_file is not None: exp_gatheredRef_spec = COMM.reduce(exp_gatheredRef_spec) if COMM.rank == 0: o = open(self.params.gathered_output_file, "w") for e, r, s in exp_gatheredRef_spec: if s is not None: o.write("%s %s %s\n" % (e, r, s)) else: o.write("%s %s\n" % (e, r)) o.close()
exp_name = sys.argv[1] data_name = sys.argv[2] tag = sys.argv[3] hkl_tol = .15 ENERGIES = [parameters.ENERGY_LOW, parameters.ENERGY_HIGH] # colors of the beams FF = [5000, None] # Setting structure factors takes long time in nanoBragg, so # unless you want energy-dependent structure factors # you need only provide one number -or- one structure factor flex miller table # and the computer will know to preserve that for all beam colors FLUX = [1e14, 1e14] # fluxes of the beams waveA = parameters.ENERGY_CONV / ENERGIES[0] waveB = parameters.ENERGY_CONV / ENERGIES[1] exp_lst = ExperimentListFactory.from_json_file(exp_name) iset = exp_lst.imagesets()[0] detector = iset.get_detector(0) data = utils.open_flex(data_name) beamA = deepcopy(iset.get_beam()) beamB = deepcopy(iset.get_beam()) beamA.set_wavelength(waveA) beamB.set_wavelength(waveB) refls_strong = data["refls_strong"] crystalAB = data["crystalAB"] reflsPP = spot_utils.refls_by_panelname(refls_strong) pids = [i for i in reflsPP if len(reflsPP[i]) > 0] # refine on these panels only pan_imgs = [iset.get_raw_data(0)[pid].as_numpy_array() for pid in pids]
def refine_expanding(params, merged_scope, combine_phil): assert params.start_at_hierarchy_level == 0 if params.rmsd_filter.enable: input_name = "filtered" command = "cctbx.xfel.filter_experiments_by_rmsd %s %s output.filtered_experiments=%s output.filtered_reflections=%s" command = command % ( "%s_combined.expt" % params.tag, "%s_combined.refl" % params.tag, "%s_filtered.expt" % params.tag, "%s_filtered.refl" % params.tag) command += " iqr_multiplier=%f" % params.rmsd_filter.iqr_multiplier print(command) result = easy_run.fully_buffered(command=command).raise_if_errors() result.show_stdout() else: input_name = "combined" # -------------------------- if params.panel_filter is not None: from libtbx import easy_pickle print("Filtering out all reflections except those on panels %s" % (", ".join(["%d" % p for p in params.panel_filter]))) combined_path = "%s_combined.refl" % params.tag data = easy_pickle.load(combined_path) sel = None for panel_id in params.panel_filter: if sel is None: sel = data['panel'] == panel_id else: sel |= data['panel'] == panel_id print("Retaining", len(data.select(sel)), "out of", len(data), "reflections") easy_pickle.dump(combined_path, data.select(sel)) # ---------------------------------- # this is the order to refine the CSPAD in steps = {} steps[0] = [2, 3] steps[1] = steps[0] + [0, 1] steps[2] = steps[1] + [14, 15] steps[3] = steps[2] + [6, 7] steps[4] = steps[3] + [4, 5] steps[5] = steps[4] + [12, 13] steps[6] = steps[5] + [8, 9] steps[7] = steps[6] + [10, 11] for s, panels in six.iteritems(steps): rest = [] for p in panels: rest.append(p + 16) rest.append(p + 32) rest.append(p + 48) panels.extend(rest) levels = {0: (0, 1)} # levels 0 and 1 for i in range(7): levels[i + 1] = (2, ) # level 2 previous_step_and_level = None for j in range(8): from libtbx import easy_pickle print("Filtering out all reflections except those on panels %s" % (", ".join(["%d" % p for p in steps[j]]))) combined_path = "%s_%s.refl" % (params.tag, input_name) output_path = "%s_step%d.refl" % (params.tag, j) data = easy_pickle.load(combined_path) sel = None for panel_id in steps[j]: if sel is None: sel = data['panel'] == panel_id else: sel |= data['panel'] == panel_id print("Retaining", len(data.select(sel)), "out of", len(data), "reflections") easy_pickle.dump(output_path, data.select(sel)) for i in levels[j]: print("Step", j, "refining at hierarchy level", i) refine_phil_file = "%s_refine_step%d_level%d.phil" % (params.tag, j, i) if i == 0: if params.refine_distance: diff_phil = "refinement.parameterisation.detector.fix_list=Tau1" # fix detector rotz else: diff_phil = "refinement.parameterisation.detector.fix_list=Dist,Tau1" # fix detector rotz, distance if params.flat_refinement: diff_phil += ",Tau2,Tau3" # Also fix x and y rotations diff_phil += "\n" if params.refine_energy: diff_phil += "refinement.parameterisation.beam.fix=in_spindle_plane+out_spindle_plane\n" # allow energy to refine else: # Note, always need to fix something, so pick a panel group and fix its Tau1 (rotation around Z) always if params.flat_refinement and params.flat_refinement_with_distance: diff_phil = "refinement.parameterisation.detector.fix_list=Group1Tau1,Tau2,Tau3\n" # refine distance, rotz and xy translation diff_phil += "refinement.parameterisation.detector.constraints.parameter=Dist\n" # constrain distance to be refined identically for all panels at this hierarchy level elif params.flat_refinement: diff_phil = "refinement.parameterisation.detector.fix_list=Dist,Group1Tau1,Tau2,Tau3\n" # refine only rotz and xy translation else: diff_phil = "refinement.parameterisation.detector.fix_list=Group1Tau1\n" # refine almost everything if previous_step_and_level is None: command = "dials.refine %s %s_%s.expt %s_step%d.refl"%( \ refine_phil_file, params.tag, input_name, params.tag, j) else: p_step, p_level = previous_step_and_level if p_step == j: command = "dials.refine %s %s_refined_step%d_level%d.expt %s_refined_step%d_level%d.refl"%( \ refine_phil_file, params.tag, p_step, p_level, params.tag, p_step, p_level) else: command = "dials.refine %s %s_refined_step%d_level%d.expt %s_step%d.refl"%( \ refine_phil_file, params.tag, p_step, p_level, params.tag, j) diff_phil += "refinement.parameterisation.detector.hierarchy_level=%d\n" % i output_experiments = "%s_refined_step%d_level%d.expt" % ( params.tag, j, i) command += " output.experiments=%s output.reflections=%s_refined_step%d_level%d.refl"%( \ output_experiments, params.tag, j, i) scope = merged_scope.fetch(parse(diff_phil)) f = open(refine_phil_file, 'w') f.write(refine_scope.fetch_diff(scope).as_str()) f.close() print(command) result = easy_run.fully_buffered(command=command).raise_if_errors() result.show_stdout() # In expanding mode, if using flat refinement with distance, after having refined this step as a block, unrefined # panels will have been left behind. Read back the new metrology, compute the shift applied to the panels refined # in this step,and apply that shift to the unrefined panels in this step if params.flat_refinement and params.flat_refinement_with_distance and i > 0: from dxtbx.model.experiment_list import ExperimentListFactory from xfel.command_line.cspad_detector_congruence import iterate_detector_at_level, iterate_panels from scitbx.array_family import flex from scitbx.matrix import col from libtbx.test_utils import approx_equal experiments = ExperimentListFactory.from_json_file( output_experiments, check_format=False) assert len(experiments.detectors()) == 1 detector = experiments.detectors()[0] # Displacements: deltas along the vector normal to the detector displacements = flex.double() # Iterate through the panel groups at this level for panel_group in iterate_detector_at_level( detector.hierarchy(), 0, i): # Were there panels refined in this step in this panel group? if params.panel_filter: test = [ list(detector).index(panel) in steps[j] for panel in iterate_panels(panel_group) if list( detector).index(panel) in params.panel_filter ] else: test = [ list(detector).index(panel) in steps[j] for panel in iterate_panels(panel_group) ] if not any(test): continue # Compute the translation along the normal of this panel group. This is defined as distance in dials.refine displacements.append( col(panel_group.get_local_fast_axis()).cross( col(panel_group.get_local_slow_axis())).dot( col(panel_group.get_local_origin()))) # Even though the panels are constrained to move the same amount, there is a bit a variation. stats = flex.mean_and_variance(displacements) displacement = stats.mean() print("Average displacement along normals: %f +/- %f" % (stats.mean(), stats.unweighted_sample_standard_deviation())) # Verify the variation isn't significant for k in range(1, len(displacements)): assert approx_equal(displacements[0], displacements[k]) # If all of the panel groups in this level moved, no need to do anything. if len(displacements) != len( list( iterate_detector_at_level(detector.hierarchy(), 0, i))): for panel_group in iterate_detector_at_level( detector.hierarchy(), 0, i): if params.panel_filter: test = [ list(detector).index(panel) in steps[j] and list(detector).index(panel) in params.panel_filter for panel in iterate_panels(panel_group) ] else: test = [ list(detector).index(panel) in steps[j] for panel in iterate_panels(panel_group) ] # If any of the panels in this panel group moved, no need to do anything if any(test): continue # None of the panels in this panel group moved in this step, so need to apply displacement from other panel # groups at this level fast = col(panel_group.get_local_fast_axis()) slow = col(panel_group.get_local_slow_axis()) ori = col(panel_group.get_local_origin()) normal = fast.cross(slow) panel_group.set_local_frame( fast, slow, (ori.dot(fast) * fast) + (ori.dot(slow) * slow) + (normal * displacement)) # Check the new displacements. Should be the same across all panels. displacements = [] for panel_group in iterate_detector_at_level( detector.hierarchy(), 0, i): displacements.append( col(panel_group.get_local_fast_axis()).cross( col(panel_group.get_local_slow_axis())).dot( col(panel_group.get_local_origin()))) for k in range(1, len(displacements)): assert approx_equal(displacements[0], displacements[k]) experiments.as_file(output_experiments) previous_step_and_level = j, i output_geometry(params)
def get_dials_matrix(experiments_json): from dxtbx.model.experiment_list import ExperimentListFactory experiments = ExperimentListFactory.from_json_file(experiments_json) return experiments[0].crystal.get_A()
def test1(): dials_regression = libtbx.env.find_in_repositories( relative_path="dials_regression", test=os.path.isdir) data_dir = os.path.join(dials_regression, "refinement_test_data", "multi_narrow_wedges") # work in a temporary directory cwd = os.path.abspath(os.curdir) tmp_dir = open_tmp_directory( suffix="tst_combine_experiments_and_reflections") os.chdir(tmp_dir) input_phil = phil_input.format(data_dir) + """ reference_from_experiment.beam=0 reference_from_experiment.scan=0 reference_from_experiment.goniometer=0 reference_from_experiment.detector=0 """ with open("input.phil", "w") as phil_file: phil_file.writelines(input_phil) cmd = "dials.combine_experiments input.phil" #print cmd result = easy_run.fully_buffered(command=cmd).raise_if_errors() # load results exp = ExperimentListFactory.from_json_file("combined_experiments.json", check_format=False) ref = flex.reflection_table.from_pickle("combined_reflections.pickle") # test the experiments assert len(exp) == 103 assert len(exp.crystals()) == 103 assert len(exp.beams()) == 1 assert len(exp.scans()) == 1 assert len(exp.detectors()) == 1 assert len(exp.goniometers()) == 1 for e in exp: assert e.imageset is not None # test the reflections assert len(ref) == 11689 cmd = " ".join([ "dials.split_experiments", "combined_experiments.json", "combined_reflections.pickle" ]) result = easy_run.fully_buffered(command=cmd).raise_if_errors() for i in range(len(exp)): assert os.path.exists("experiments_%03d.json" % i) assert os.path.exists("reflections_%03d.pickle" % i) exp_single = ExperimentListFactory.from_json_file( "experiments_%03d.json" % i, check_format=False) ref_single = flex.reflection_table.from_pickle( "reflections_%03d.pickle" % i) assert len(exp_single) == 1 assert exp_single[0].crystal == exp[i].crystal assert exp_single[0].beam == exp[i].beam assert exp_single[0].detector == exp[i].detector assert exp_single[0].scan == exp[i].scan assert exp_single[0].goniometer == exp[i].goniometer assert exp_single[0].imageset == exp[i].imageset assert len(ref_single) == len(ref.select(ref['id'] == i)) assert ref_single['id'].all_eq(0) cmd = " ".join([ "dials.split_experiments", "combined_experiments.json", "output.experiments_prefix=test" ]) result = easy_run.fully_buffered(command=cmd).raise_if_errors() for i in range(len(exp)): assert os.path.exists("test_%03d.json" % i) return
def get_dials_coordinate_frame(experiments_json): from dxtbx.model.experiment_list import ExperimentListFactory experiments = ExperimentListFactory.from_json_file(experiments_json) return experiments[0].beam.get_direction(), \ experiments[0].goniometer.get_rotation_axis()
def _refine_onwards(self, experiments, reflections): """Perform processing tasks in a sub-directory for files matching the base_name in the parent directory""" exp_path = os.path.join("..", experiments) ref_path = os.path.join("..", reflections) # Static refinement, two rounds to improve outlier rejection. Increase # max_iterations from the default cmd = ("dials.refine {0} {1} max_iterations=100 " "output.experiments=static_01_refined_experiments.json " "output.reflections=static_01_refined_reflections.pickle " "output.log=static_01_dials.refine.log").format( exp_path, ref_path) if self._run_one_job(cmd) is None: return cmd = ("dials.refine static_01_refined_experiments.json " "static_01_refined_reflections.pickle " "max_iterations=100 " "output.experiments=static_02_refined_experiments.json " "output.reflections=static_02_refined_reflections.pickle " "output.log=static_02_dials.refine.log") if self._run_one_job(cmd) is None: return # Print cell and detector geometry el = ExperimentListFactory.from_json_file( "static_02_refined_experiments.json") cell = el[0].crystal.get_unit_cell().parameters() cell_s = ", ".join(["{:.3f}".format(e) for e in cell]) print("Refined unit cell: ({0})".format(cell_s)) d = self._get_detector_geometry(el[0].detector) print("Detector origin: ({0:.5f}, {1:.5f}, {2:.5f})".format( *d["origin"].elems)) print("Detector fast ax: ({0:.5f}, {1:.5f}, {2:.5f})".format( *d["fast"].elems)) print("Detector slow ax: ({0:.5f}, {1:.5f}, {2:.5f})".format( *d["slow"].elems)) print("Detector distance: {0}".format(d["distance"])) # Scan-varying refinement cmd = ("dials.refine static_02_refined_experiments.json " "static_02_refined_reflections.pickle " "scan_varying=true " "max_iterations=100 " "output.experiments=sv_refined_experiments.json " "output.reflections=sv_refined_reflections.pickle " "output.log=sv_dials.refine.log") if self._run_one_job(cmd) is None: return # Plot scan-varying crystal cmd = "dials.plot_scan_varying_crystal sv_refined_experiments.json" self._run_one_job(cmd) # Print scan-varying cell and new detector geometry el = ExperimentListFactory.from_json_file( "sv_refined_experiments.json") xl = el[0].crystal abc = flex.vec3_double() angles = flex.vec3_double() for n in range(xl.num_scan_points): a, b, c, alpha, beta, gamma = xl.get_unit_cell_at_scan_point( n).parameters() abc.append((a, b, c)) angles.append((alpha, beta, gamma)) a, b, c = abc.mean() alpha, beta, gamma = angles.mean() cell_s = ", ".join( ["{:.3f}".format(e) for e in [a, b, c, alpha, beta, gamma]]) print("Average refined unit cell: ({0})".format(cell_s)) d = self._get_detector_geometry(el[0].detector) print("Detector origin: ({0:.5f}, {1:.5f}, {2:.5f})".format( *d["origin"].elems)) print("Detector fast ax: ({0:.5f}, {1:.5f}, {2:.5f})".format( *d["fast"].elems)) print("Detector slow ax: ({0:.5f}, {1:.5f}, {2:.5f})".format( *d["slow"].elems)) print("Detector distance: {0}".format(d["distance"])) # integrate cmd = ("dials.integrate sv_refined_experiments.json " "sv_refined_reflections.pickle nproc={0} " "prediction.d_min={1}").format(self.params.nproc, self.params.integrate.d_min) if self._run_one_job(cmd) is None: return # FIXME print out some statistics from integration # export cmd = ("dials.export integrated_experiments.json integrated.pickle " "mtz.ignore_panels=true") if self._run_one_job(cmd) is None: return # pointless cmd = "pointless hklin integrated.mtz hklout sorted.mtz > pointless.log" if self._run_one_job(cmd) is None: return # aimless cmd = "aimless hklin sorted.mtz hklout scaled.mtz > aimless.log" keywords = ["resolution 80.0 2.0"] if self._run_one_job(cmd, keywords=keywords) is None: return print("DONE") return
z = self.im.get_array()[int(y), int(x)] print(z) return "x={:.01f}, y={:.01f}, z={:f}".format(x, y, z) from matplotlib import pylab fig, ax = pylab.subplots() im = ax.imshow(data.as_numpy_array(), interpolation="none") ax.format_coord = Formatter(im) pylab.show() if __name__ == "__main__": import sys from dxtbx.model.experiment_list import ExperimentListFactory experiments = ExperimentListFactory.from_json_file(sys.argv[1]) # volume1 = pixel_volume_all_pixels(experiments[0]) volume2 = monte_carlo_pixel_volume_all_pixels(experiments[0]) import cPickle as pickle # pickle.dump(volume1, open("volume.pickle", "w")) # pickle.dump(volume2, open("monte_carlo_volume.pickle", "w")) # volume1 = pickle.load(open("volume.pickle")) # display(volume1)
def test_dials_cluster_unit_cell_command_line(dials_regression, run_in_tmpdir): pytest.importorskip("scipy") pytest.importorskip("xfel") data_dir = os.path.join(dials_regression, "refinement_test_data", "multi_narrow_wedges") import glob experiments = glob.glob( os.path.join(data_dir, "data/sweep_*/experiments.json")) import procrunner result = procrunner.run( command=["dials.cluster_unit_cell", "plot.show=False"] + experiments, print_stdout=False, ) assert not result.returncode # print result assert os.path.exists("cluster_unit_cell.png") from dxtbx.model.experiment_list import ExperimentList, ExperimentListFactory from dials.command_line import cluster_unit_cell experiments = ExperimentList([ ExperimentListFactory.from_json_file(expt, check_format=False)[0] for expt in experiments ]) from cctbx import crystal crystal_symmetries = [ crystal.symmetry( unit_cell=expt.crystal.get_unit_cell(), space_group=expt.crystal.get_space_group(), ) for expt in experiments ] params = cluster_unit_cell.phil_scope.extract() params.plot.show = False params.plot.name = None clusters = cluster_unit_cell.do_cluster_analysis(crystal_symmetries, params) assert len(clusters) == 1 cluster = clusters[0] assert len(cluster.members) == 40 assert cluster.medians == pytest.approx( [ 90.9430182020995, 90.9430182020995, 90.9430182020995, 109.47122063449069, 109.47122063449069, 109.47122063449069, ], abs=1e-6, ) assert cluster.stdevs == pytest.approx( [ 0.09509739126548639, 0.09509739126548526, 0.0950973912654865, 0, 0, 0 ], abs=1e-6, )
except ImportError: size = 1 rank = 0 has_mpi = False if rank == 0: print(args) if args.sanityplots: import pylab as plt fig = plt.figure() ax = plt.gca() # Load in the reflection tables and experiment lists if rank == 0: print("Reading in the files") El = ExperimentListFactory.from_json_file( args.filteredexpt, check_format=True) # check format is it slow ?? Rmaster_strong = flex.reflection_table.from_file(args.filteredstrong) Rmaster_int = flex.reflection_table.from_file(args.filteredint) Nexper = len(El) print("Read int %d experiments" % Nexper) # get the original indexing directory name # NOTE : forgot what this was doing??? indexdirname = args.indexdirname if args.indexdirname is None: indexdirname = os.path.dirname(args.filteredexpt) if args.noiseless: GAIN = 1 if not os.path.exists(args.o) and rank == 0: os.makedirs(args.o)
from two_color.two_color_phil import params as index_params from cxid9114.utils import open_flex num_imgs = 1 import os datadir = os.environ.get("DD3") #DD = "/net/dials/raid1/dermen/cyto/" ###################### # LOAD JUNGFRAU MODEL ###################### print("Load the big azz detector") expList_for_DET = ExperimentListFactory.from_json_file(os.path.join( datadir, "refined_final.expt"), check_format=False) DETECTOR = expList_for_DET.detectors()[0] image_shape = (len(DETECTOR), ) + DETECTOR[0].get_image_size() ### # load the beam and the crystal from the refined experiment print("Opening refined exp") refined_El = ExperimentListFactory.from_json_file( "/net/dials/raid1/dermen/cyto/idx-run_000795.JF07T32V01_master_00344_integrated.expt" ) BEAM = refined_El.beams()[0] iset = refined_El.imagesets()[0] master_file_path = iset.get_path(0) print("Opening master load") master_loader = dxtbx.load(master_file_path)
def test3(dials_regression, tmpdir): """Strict check for scan-varying refinement using automated outlier rejection block width and interval width setting""" # use the i04_weak_data for this test data_dir = os.path.join(dials_regression, "refinement_test_data", "centroid") experiments_path = os.path.join(data_dir, "experiments_XPARM_REGULARIZED.json") pickle_path = os.path.join(data_dir, "spot_all_xds.pickle") for pth in (experiments_path, pickle_path): assert os.path.exists(pth) result = procrunner.run( ( "dials.refine", experiments_path, pickle_path, "scan_varying=true", "max_iterations=5", "output.history=history.json", "crystal.orientation.smoother.interval_width_degrees=auto", "crystal.unit_cell.smoother.interval_width_degrees=auto", ), working_directory=tmpdir, ) assert not result.returncode and not result.stderr # load and check results history = Journal.from_json_file(tmpdir.join("history.json").strpath) expected_rmsds = [ [0.619507829, 0.351326044, 0.006955399], [0.174024575, 0.113486044, 0.004704006], [0.098351363, 0.084052519, 0.002660408], [0.069202909, 0.072796782, 0.001451734], [0.064305277, 0.071560831, 0.001165639], [0.062955462, 0.071315612, 0.001074453], ] for a, b in zip(history["rmsd"], expected_rmsds): assert a == pytest.approx(b, abs=1e-6) # check the refined unit cell ref_exp = ExperimentListFactory.from_json_file( tmpdir.join("refined.expt").strpath, check_format=False)[0] unit_cell = ref_exp.crystal.get_unit_cell().parameters() assert unit_cell == pytest.approx( [42.27482, 42.27482, 39.66893, 90.00000, 90.00000, 90.00000], abs=1e-3) refined_refl = flex.reflection_table.from_file( tmpdir.join("refined.refl").strpath) # re-predict reflections using the refined experiments predicted = flex.reflection_table.from_predictions_multi([ref_exp]) matched, reference, unmatched = predicted.match_with_reference( refined_refl) # assert most refined reflections are matched with predictions assert reference.size() > (0.997 * refined_refl.size()) # second check with nearest neighbour matching that the predictions match up ann = AnnAdaptor(data=predicted["xyzcal.px"].as_double(), dim=3, k=1) ann.query(refined_refl["xyzcal.px"].as_double()) assert (ann.distances < 0.5).count(True) > (0.998 * refined_refl.size())
Ntrial = 25 outlier_cutoff = 0.6 # probability to not be an outlier.. szx = szy = 5 Nmos_dom = 100 exp_name = sys.argv[1] data_name = sys.argv[2] cuda = int(sys.argv[3]) outdir = sys.argv[4] if not os.path.exists(outdir): os.makedirs(outdir) output_basename = os.path.basename(data_name).replace(".pkl", "_jitt.pkl") outputname = os.path.join(outdir, output_basename) exp_lst = ExperimentListFactory.from_json_file( exp_name) #, check_format=False) iset = exp_lst.imagesets()[0] data = utils.open_flex(data_name) waveA = parameters.ENERGY_CONV / ENERGIES[0] waveB = parameters.ENERGY_CONV / ENERGIES[1] ENERGIES = [parameters.ENERGY_LOW, parameters.ENERGY_HIGH] # colors of the beams FF = [10000, None] # Setting structure factors takes long time in nanoBragg, so # unless you want energy-dependent structure factors # you need only provide one number -or- one structure factor flex miller table # and the computer will know to preserve that for all beam colors FLUX = [1e12, 1e12] # fluxes of the beams waveA = parameters.ENERGY_CONV / ENERGIES[0]
def test(dials_regression): from scitbx import matrix from libtbx.phil import parse from libtbx.test_utils import approx_equal from scitbx.array_family import flex # Get modules to build models and minimiser using PHIL from dials.test.algorithms.refinement import setup_geometry from dials.test.algorithms.refinement import setup_minimiser from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalOrientationParameterisation, CrystalUnitCellParameterisation # Symmetry constrained parameterisation for the unit cell from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import \ symmetrize_reduce_enlarge DEG2RAD = math.pi/180.0 RAD2DEG = 180.0/math.pi master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True) # make cell more oblique args=["a.direction.close_to.sd=5","b.direction.close_to.sd=5","c.direction.close_to.sd=5"] models = setup_geometry.Extract(master_phil, cmdline_args = args) crystal = models.crystal # a hexagonal crystal is a good test case for behaviour of oblique cells do_hexagonal = True if do_hexagonal: from dxtbx.model.experiment_list import ExperimentListFactory experiments = ExperimentListFactory.from_json_file( os.path.join(dials_regression, "refinement_test_data", "multi_stills", "combined_experiments.json"), check_format=False) crystal = experiments[0].crystal # derive finite difference gradients of various quantities wrt each param def check_fd_gradients(parameterisation): mp = parameterisation p_vals = mp.get_param_vals() deltas = [1.e-7 for p in p_vals] assert len(deltas) == len(p_vals) fd_grad = [] # get matrix to unset rotations of unit cell vectors Ut = matrix.sqr(mp.get_model().get_U()).transpose() for i, delta in enumerate(deltas): val = p_vals[i] p_vals[i] -= delta / 2. mp.set_param_vals(p_vals) rev_uc = mp.get_model().get_unit_cell().parameters() rev_vec = mp.get_model().get_real_space_vectors() rev_vec = [Ut * vec for vec in rev_vec] rev_B = matrix.sqr(mp.get_model().get_B()) rev_O = rev_B.transpose().inverse() p_vals[i] += delta mp.set_param_vals(p_vals) fwd_uc = mp.get_model().get_unit_cell().parameters() fwd_vec = mp.get_model().get_real_space_vectors() fwd_vec = [Ut * vec for vec in fwd_vec] fwd_B = matrix.sqr(mp.get_model().get_B()) fwd_O = fwd_B.transpose().inverse() fd_uc = [(f - r) / delta for f,r in zip(fwd_uc, rev_uc)] fd_vec = [(f - r) / delta for f,r in zip(fwd_vec, rev_vec)] fd_B = (fwd_B - rev_B) / delta fd_O = (fwd_O - rev_O) / delta fd_grad.append({'da_dp':fd_uc[0], 'db_dp':fd_uc[1], 'dc_dp':fd_uc[2], 'daa_dp':fd_uc[3], 'dbb_dp':fd_uc[4], 'dcc_dp':fd_uc[5], 'davec_dp':fd_vec[0], 'dbvec_dp':fd_vec[1], 'dcvec_dp':fd_vec[2], 'dB_dp':fd_B, 'dO_dp':fd_O}) p_vals[i] = val # return to the initial state mp.set_param_vals(p_vals) return fd_grad xlo_param = CrystalOrientationParameterisation(crystal) xluc_param = CrystalUnitCellParameterisation(crystal) from dials.algorithms.refinement.restraints.restraints import SingleUnitCellTie uct = SingleUnitCellTie(xluc_param, [None]*6, [None]*6) from scitbx.math import angle_derivative_wrt_vectors B = matrix.sqr(crystal.get_B()) O = (B.transpose()).inverse() a, b, c, aa, bb, cc = crystal.get_unit_cell().parameters() aa *= DEG2RAD bb *= DEG2RAD cc *= DEG2RAD Ut = matrix.sqr(crystal.get_U()).transpose() avec, bvec, cvec = [Ut * vec for vec in crystal.get_real_space_vectors()] # calculate d[B^T]/dp dB_dp = xluc_param.get_ds_dp() dBT_dp = [dB.transpose() for dB in dB_dp] # calculate d[O]/dp dO_dp = [-O * dBT * O for dBT in dBT_dp] # function to get analytical derivative of angles wrt vectors def dangle(u, v): return [matrix.col(e) for e in angle_derivative_wrt_vectors(u,v)] dalpha_db, dalpha_dc = dangle(bvec, cvec) dbeta_da, dbeta_dc = dangle(avec, cvec) dgamma_da, dgamma_db = dangle(avec, bvec) # get all FD derivatives fd_grad = check_fd_gradients(xluc_param) # look at each parameter for i, dO in enumerate(dO_dp): #print #print "***** PARAMETER {0} *****".format(i) #print "dB_dp analytical" #print dB_dp[i] #print "dB_dp FD" #print fd_grad[i]['dB_dp'] #print # dB_dp is good. What about dO_dp? #print "O MATRIX" #print "dO_dp analytical" #print dO.round(6) #print "dO_dp FD" #print fd_grad[i]['dO_dp'].round(6) #print assert approx_equal(dO, fd_grad[i]['dO_dp']) # extract derivatives of each unit cell vector wrt p dav_dp, dbv_dp, dcv_dp = dO.transpose().as_list_of_lists() dav_dp = matrix.col(dav_dp) dbv_dp = matrix.col(dbv_dp) dcv_dp = matrix.col(dcv_dp) # check these are correct vs FD #print "CELL VECTORS" #diff = dav_dp - fd_grad[i]['davec_dp'] #print 2 * diff.length() / (dav_dp.length() + fd_grad[i]['davec_dp'].length()) * 100 #print 'davec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dav_dp.elems) #print 'davec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['davec_dp'].elems) assert approx_equal(dav_dp, fd_grad[i]['davec_dp']) #diff = dbv_dp - fd_grad[i]['dbvec_dp'] #print 2 * diff.length() / (dbv_dp.length() + fd_grad[i]['dbvec_dp'].length()) * 100 #print 'dbvec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dbv_dp.elems) #print 'dbvec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['dbvec_dp'].elems) assert approx_equal(dbv_dp, fd_grad[i]['dbvec_dp']) #diff = dcv_dp - fd_grad[i]['dcvec_dp'] #print 2 * diff.length() / (dcv_dp.length() + fd_grad[i]['dcvec_dp'].length()) * 100 #print 'dcvec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dcv_dp.elems) #print 'dcvec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['dcvec_dp'].elems) #print assert approx_equal(dcv_dp, fd_grad[i]['dcvec_dp']) #print "CELL LENGTHS" da_dp = 1./a * avec.dot(dav_dp) #print "d[a]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(da_dp, fd_grad[i]['da_dp'], i) assert approx_equal(da_dp, fd_grad[i]['da_dp']) db_dp = 1./b * bvec.dot(dbv_dp) #print "d[b]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(db_dp, fd_grad[i]['db_dp'], i) assert approx_equal(db_dp, fd_grad[i]['db_dp']) dc_dp = 1./c * cvec.dot(dcv_dp) #print "d[c]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dc_dp, fd_grad[i]['dc_dp'], i) assert approx_equal(dc_dp, fd_grad[i]['dc_dp']) #print #print "CELL ANGLES" daa_dp = RAD2DEG * (dbv_dp.dot(dalpha_db) + dcv_dp.dot(dalpha_dc)) dbb_dp = RAD2DEG * (dav_dp.dot(dbeta_da) + dcv_dp.dot(dbeta_dc)) dcc_dp = RAD2DEG * (dav_dp.dot(dgamma_da) + dbv_dp.dot(dgamma_db)) #print "d[alpha]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(daa_dp, fd_grad[i]['daa_dp'], i) #print "d[beta]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dbb_dp, fd_grad[i]['dbb_dp'], i) #print "d[gamma]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dcc_dp, fd_grad[i]['dcc_dp'], i) assert approx_equal(daa_dp, fd_grad[i]['daa_dp']) assert approx_equal(dbb_dp, fd_grad[i]['dbb_dp']) assert approx_equal(dcc_dp, fd_grad[i]['dcc_dp'])
def test(dials_data): from dxtbx.model.experiment_list import ExperimentListFactory from dxtbx.model.experiment_list import ExperimentList exlist = ExperimentListFactory.from_json_file( dials_data("centroid_test_data").join( "fake_long_experiments.json").strpath) assert len(exlist) == 1 experiment = exlist[0] # Set the delta_divergence/mosaicity n_sigma = 5 sigma_b = 0.060 * math.pi / 180 sigma_m = 0.154 * math.pi / 180 from dials.algorithms.profile_model.gaussian_rs import Model profile_model = Model(None, n_sigma, sigma_b, sigma_m) experiment.profile = profile_model experiments = ExperimentList() experiments.append(experiment) from dials.algorithms.profile_model.gaussian_rs import PartialityCalculator3D from dials.array_family import flex calculator = PartialityCalculator3D(experiment.beam, experiment.goniometer, experiment.scan, sigma_m) predicted = flex.reflection_table.from_predictions_multi(experiments) predicted["bbox"] = predicted.compute_bbox(experiments) # Remove any touching edges of scan to get only fully recorded x0, x1, y0, y1, z0, z1 = predicted["bbox"].parts() predicted = predicted.select((z0 > 0) & (z1 < 100)) assert len(predicted) > 0 # Compute partiality partiality = calculator(predicted["s1"], predicted["xyzcal.px"].parts()[2], predicted["bbox"]) # Should have all fully recorded assert len(partiality) == len(predicted) three_sigma = 0.5 * (math.erf(3.0 / math.sqrt(2.0)) - math.erf(-3.0 / math.sqrt(2.0))) assert partiality.all_gt(three_sigma) # Trim bounding boxes x0, x1, y0, y1, z0, z1 = predicted["bbox"].parts() z0 = z0 + 1 z1 = z1 - 1 predicted["bbox"] = flex.int6(x0, x1, y0, y1, z0, z1) predicted = predicted.select(z1 > z0) assert len(predicted) > 0 # Compute partiality partiality = calculator(predicted["s1"], predicted["xyzcal.px"].parts()[2], predicted["bbox"]) # Should have all partials assert len(partiality) == len(predicted) assert partiality.all_le(1.0) and partiality.all_gt(0)
def test_run(dials_regression, run_in_tmpdir): from dials.util.nexus import dump, load from dxtbx.model.experiment_list import ExperimentListFactory from dials.array_family import flex from os.path import join path = join(dials_regression, "nexus_test_data") # Read the experiments experiments1 = ExperimentListFactory.from_json_file( join(path, "refined_experiments.json") ) # Read the reflections reflections1 = flex.reflection_table.from_file(join(path, "integrated.pickle")) # Delete some columns for the test del reflections1["s1"] del reflections1["zeta"] del reflections1["background.mse"] # Dump the reflections dump(experiments1, reflections1, "hklout.nxs") # Load them again experiments2, reflections2 = load("hklout.nxs") EPS = 1e-7 # Check the reflections are OK assert reflections1.nrows() == reflections2.nrows() assert reflections1.ncols() == reflections2.ncols() for key in reflections1.keys(): data1 = reflections1[key] data2 = reflections2[key] assert data1.__class__ == data2.__class__ if isinstance(data1, flex.double): assert data1.all_approx_equal(data2) elif isinstance(data1, flex.int6): for p1, p2 in zip(data1.parts(), data2.parts()): assert p1.all_eq(p2) elif isinstance(data1, flex.vec3_double): for p1, p2 in zip(data1.parts(), data2.parts()): assert p1.all_approx_equal(p2) else: assert data1.all_eq(data2) # Test passed # Check the experiments are ok assert len(experiments1) == len(experiments2) exp1 = experiments1[0] exp2 = experiments2[0] # Check the beam b1 = exp1.beam b2 = exp2.beam assert all( abs(d1 - d2) < EPS for d1, d2 in zip( b1.get_sample_to_source_direction(), b2.get_sample_to_source_direction() ) ) assert abs(b1.get_wavelength() - b2.get_wavelength()) < EPS assert abs(b1.get_polarization_fraction() - b2.get_polarization_fraction()) < EPS assert all( abs(d1 - d2) < EPS for d1, d2 in zip(b1.get_polarization_normal(), b2.get_polarization_normal()) ) # Check the goniometer g1 = exp1.goniometer g2 = exp2.goniometer assert all( abs(d1 - d2) < EPS for d1, d2 in zip(g1.get_rotation_axis(), g2.get_rotation_axis()) ) assert all( abs(d1 - d2) < EPS for d1, d2 in zip(g1.get_fixed_rotation(), g2.get_fixed_rotation()) ) assert all( abs(d1 - d2) < EPS for d1, d2 in zip(g1.get_setting_rotation(), g2.get_setting_rotation()) ) # Check the scan s1 = exp1.scan s2 = exp2.scan assert len(s1) == len(s2) assert s1.get_image_range() == s2.get_image_range() assert abs(s1.get_oscillation()[0] - s1.get_oscillation()[0]) < EPS assert abs(s1.get_oscillation()[1] - s1.get_oscillation()[1]) < EPS for e1, e2 in zip(s1.get_exposure_times(), s2.get_exposure_times()): assert abs(e1 - e2) < EPS for e1, e2 in zip(s1.get_epochs(), s2.get_epochs()): assert abs(e1 - e2) < EPS # Check the detector d1 = exp1.detector d2 = exp2.detector assert len(d1) == len(d2) for p1, p2 in zip(d1, d2): assert p1.get_type() == p2.get_type() assert p1.get_material() == p2.get_material() assert p1.get_thickness() == p2.get_thickness() assert p1.get_image_size() == p2.get_image_size() assert p1.get_pixel_size() == p2.get_pixel_size() assert p1.get_trusted_range() == p2.get_trusted_range() for x1, x2 in zip(p1.get_fast_axis(), p2.get_fast_axis()): assert abs(x1 - x2) < EPS for x1, x2 in zip(p1.get_slow_axis(), p2.get_slow_axis()): assert abs(x1 - x2) < EPS for x1, x2 in zip(p1.get_origin(), p2.get_origin()): assert abs(x1 - x2) < EPS # Check the crystal c1 = exp1.crystal c2 = exp2.crystal assert c1.get_space_group() == c2.get_space_group() for p1, p2 in zip(c1.get_unit_cell().parameters(), c2.get_unit_cell().parameters()): assert abs(p1 - p2) < EPS for p1, p2 in zip(c1.get_A(), c2.get_A()): assert abs(p1 - p2) < EPS assert c1.num_scan_points == c2.num_scan_points for i in range(c1.num_scan_points): A1 = c1.get_A_at_scan_point(i) A2 = c2.get_A_at_scan_point(i) for a1, a2 in zip(A1, A2): assert abs(a1 - a2) < EPS uc1 = c1.get_unit_cell_at_scan_point(i) uc2 = c2.get_unit_cell_at_scan_point(i) for p1, p2 in zip(uc1.parameters(), uc2.parameters()): assert abs(p1 - p2) < EPS
def build_hist(nproc=1): from scitbx.array_family import flex from libtbx import easy_mp from collections import Counter # FIXME use proper optionparser here. This works for now if len(sys.argv) >= 2 and sys.argv[1].startswith("nproc="): nproc = int(sys.argv[1][6:]) sys.argv = sys.argv[1:] if len(sys.argv) == 2 and sys.argv[1].endswith((".expt", ".json")): from dxtbx.model.experiment_list import ExperimentListFactory experiments = ExperimentListFactory.from_json_file(sys.argv[1]) image_list = experiments.imagesets()[0].paths() else: image_list = sys.argv[1:] image_count = len(image_list) # Faster, yet still less than ideal and wasting a lot of resources. limit = get_overload(image_list[0]) binfactor = 5 # register up to 500% counts histmax = (limit * binfactor) + 0.0 histbins = int(limit * binfactor) + 1 use_python_counter = histbins > 90000000 # empirically determined print("Processing %d images in %d processes using %s\n" % ( image_count, nproc, "python Counter" if use_python_counter else "flex arrays", )) def process_image(process): last_update = start = timeit.default_timer() i = process if use_python_counter: local_hist = Counter() else: local_hist = flex.histogram(flex.double(), data_min=0.0, data_max=histmax, n_slots=histbins) max_images = image_count // nproc if process >= image_count % nproc: max_images += 1 while i < image_count: data = read_cbf_image(image_list[i]) if not use_python_counter: data = flex.histogram( data.as_double().as_1d(), data_min=0.0, data_max=histmax, n_slots=histbins, ) local_hist.update(data) i = i + nproc if process == 0: if timeit.default_timer() > (last_update + 3): last_update = timeit.default_timer() if sys.stdout.isatty(): sys.stdout.write("\033[A") print("Processed %d%% (%d seconds remain) " % ( 100 * i // image_count, round((image_count - i) * (last_update - start) / (i + 1)), )) return local_hist results = easy_mp.parallel_map( func=process_image, iterable=range(nproc), processes=nproc, preserve_exception_message=True, ) print("Merging results") result_hist = None for hist in results: if result_hist is None: result_hist = hist else: result_hist.update(hist) if not use_python_counter: # reformat histogram into dictionary result = list(result_hist.slots()) result_hist = {b: count for b, count in enumerate(result) if count > 0} results = { "scale_factor": 1 / limit, "overload_limit": limit, "counts": result_hist, } print("Writing results to overload.json") with open("overload.json", "w") as fh: json.dump(results, fh, indent=1, sort_keys=True)
def test_stills_indexer_multi_lattice_bug_MosaicSauter2014( dials_regression, tmpdir): """Problem: In stills_indexer, before calling the refine function, the experiment list contains a list of dxtbx crystal models (that are not MosaicSauter2014 models). The conversion to MosaicSauter2014 is made during the refine step when functions from nave_parameters is called. If the experiment list contains more than 1 experiment, for eg. multiple lattices, only the first crystal gets assigned mosaicity. In actuality, all crystal models should be assigned mosaicity. This test only compares whether or not all crystal models have been assigned a MosaicSauter2014 model.""" import dxtbx.model from dxtbx.model.experiment_list import ExperimentListFactory from dxtbx.model.experiment_list import Experiment, ExperimentList from dials.array_family import flex from dxtbx.model import Crystal from dials.algorithms.indexing.stills_indexer import StillsIndexer from dials.command_line.stills_process import ( phil_scope as stills_process_phil_scope, ) experiment_data = os.path.join( dials_regression, "refinement_test_data", "cspad_refinement", "cspad_refined_experiments_step6_level2_300.json", ) reflection_data = os.path.join( dials_regression, "refinement_test_data", "cspad_refinement", "cspad_reflections_step7_300.pickle", ) refl = flex.reflection_table.from_file(reflection_data) explist = ExperimentListFactory.from_json_file(experiment_data, check_format=False)[0:2] reflist = refl.select( refl["id"] < 2) # Only use the first 2 for convenience # Construct crystal models that don't have mosaicity. These A,B,C values are the same # as read in from the dials_regression folder # Crystal-0 cs0 = Crystal(explist[0].crystal) exp0 = Experiment( imageset=explist[0].imageset, beam=explist[0].beam, detector=explist[0].detector, goniometer=None, scan=None, crystal=cs0, ) # Crystal-1 cs1 = Crystal(explist[1].crystal) exp1 = Experiment( imageset=explist[1].imageset, beam=explist[1].beam, detector=explist[1].detector, goniometer=None, scan=None, crystal=cs1, ) # Construct a new experiment_list that will be passed on for refinement unrefined_explist = ExperimentList([exp0, exp1]) # Get default params from stills_process and construct StillsIndexer, then run refinement params = stills_process_phil_scope.extract() SI = StillsIndexer(reflist, unrefined_explist, params=params) refined_explist, new_reflist = SI.refine(unrefined_explist, reflist) # Now check whether the models have mosaicity after stills_indexer refinement # Also check that mosaicity values are within expected limits for ii, crys in enumerate(refined_explist.crystals()): assert isinstance(crys, dxtbx.model.MosaicCrystalSauter2014) if ii == 0: assert crys.get_domain_size_ang() == pytest.approx(2242.0, rel=0.1) if ii == 1: assert crys.get_domain_size_ang() == pytest.approx(2689.0, rel=0.1)
# Output datasize try: datasizeidx = [a.find("datasize")==0 for a in args].index(True) except ValueError: print "You really should supply datasize, but ok" # raise ValueError,"Output datasze file must be specified." else: datasize = args.pop(datasizeidx).split("=")[1] import copy, os import dxtbx from dxtbx.model.experiment_list import ExperimentListFactory from dials.array_family import flex experiments = ExperimentListFactory.from_json_file(json, check_format=False) beam = experiments[0].beam detector = experiments[0].detector lab_coordinates = flex.vec3_double() for panel in detector: pixels = flex.vec2_double(panel.get_image_size()) mms = panel.pixel_to_millimeter(pixels) lab_coordinates.extend(panel.get_lab_coord(mms)) # generate s1 vectors s1 = lab_coordinates.each_normalize() * (1/beam.get_wavelength()) # Generate x vectors x = np.asarray(s1 - beam.get_s0()) # DATAsize = np.asarray(detector[0].get_image_size())
def tst_one_monkeypatch(i_exp, spectra, Fmerge, gpu_channels_singleton, rank, params): print("IN MONKEYPATCH") from simtbx.nanoBragg import utils from dxtbx.model.experiment_list import ExperimentListFactory import numpy as np print("Experiment %d" % i_exp, flush=True) outfile = "boop_%d.hdf5" % i_exp from LS49.adse13_187.case_data import retrieve_from_repo experiment_file = retrieve_from_repo(i_exp) cuda = True # False # whether to use cuda omp = False ngpu_on_node = 1 # 8 # number of available GPUs mosaic_spread = 0.07 # degrees mosaic_spread_samples = params.mosaic_spread_samples # number of mosaic blocks sampling mosaicity Ncells_abc = 30, 30, 10 # medians from best stage1 ev_res = 1.5 # resolution of the downsample spectrum total_flux = 1e12 # total flux across channels beamsize_mm = 0.000886226925452758 # sqrt of beam focal area spot_scale = 500. # 5.16324 # median from best stage1 plot_spec = False # plot the downsample spectra before simulating oversample = 1 # oversample factor, 1,2, or 3 probable enough panel_list = None # integer list of panels, usefule for debugging rois_only = False # only set True if you are running openMP, or CPU-only (i.e. not for GPU) include_background = params.include_background # default is to add water background 100 mm thick verbose = 0 # leave as 0, unles debug flat = True # enfore that the camera has 0 thickness #<><><><><><><><> # XXX new code El = ExperimentListFactory.from_json_file(experiment_file, check_format=True) exper = El[0] crystal = exper.crystal detector = exper.detector if flat: from dxtbx_model_ext import SimplePxMmStrategy for panel in detector: panel.set_px_mm_strategy(SimplePxMmStrategy()) panel.set_mu(0) panel.set_thickness(0) beam = exper.beam # XXX new code spec = exper.imageset.get_spectrum(0) energies_raw, weights_raw = spec.get_energies_eV().as_numpy_array(), \ spec.get_weights().as_numpy_array() energies, weights = utils.downsample_spectrum(energies_raw, weights_raw, method=1, total_flux=total_flux, ev_width=ev_res) if flat: assert detector[0].get_thickness() == 0 if panel_list is None: panel_list = list(range(len(detector))) pids_for_rank = panel_list device_Id = 0 if gpu_channels_singleton is not None: device_Id = gpu_channels_singleton.get_deviceID() print("Rank %d will use device %d" % (rank, device_Id)) show_params = False time_panels = (rank == 0) mn_energy = (energies * weights).sum() / weights.sum() mn_wave = utils.ENERGY_CONV / mn_energy print( "\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>") print("\tBreakdown:") for shapetype in ["gauss_argchk"]: BEG = time() print(gpu_channels_singleton.get_deviceID(), "device", shapetype) Famp_is_uninitialized = (gpu_channels_singleton.get_nchannels() == 0) if Famp_is_uninitialized: F_P1 = Fmerge.expand_to_p1() for x in range( 1 ): # in this scenario, amplitudes are independent of lambda gpu_channels_singleton.structure_factors_to_GPU_direct( x, F_P1.indices(), F_P1.data()) assert gpu_channels_singleton.get_nchannels() == 1 JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim( CRYSTAL=crystal, DETECTOR=detector, BEAM=beam, Famp=gpu_channels_singleton, energies=list(energies), fluxes=list(weights), background_wavelengths=[mn_wave], background_wavelength_weights=[1], background_total_flux=total_flux, background_sample_thick_mm=0.5, cuda=True, oversample=oversample, Ncells_abc=Ncells_abc, mos_dom=mosaic_spread_samples, mos_spread=mosaic_spread, mosaic_method="double_random", beamsize_mm=beamsize_mm, profile=shapetype, show_params=show_params, time_panels=time_panels, verbose=verbose, spot_scale_override=spot_scale, include_background=include_background, mask_file=params.mask_file) TIME_EXA = time() - BEG print( "\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs" % (TIME_BG, TIME_BRAGG, TIME_EXA)) print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>\n") if params.write_output: if params.write_experimental_data: data = exper.imageset.get_raw_data(0) img_sh = JF16M_numpy_array.shape assert img_sh == (256, 254, 254) num_output_images = 1 + int(params.write_experimental_data) print("Saving exascale output data of shape", img_sh) beam_dict = beam.to_dict() det_dict = detector.to_dict() try: beam_dict.pop("spectrum_energies") beam_dict.pop("spectrum_weights") except Exception: pass with utils.H5AttributeGeomWriter( os.path.join(params.log.outdir, "exap_%d.hdf5" % i_exp), image_shape=img_sh, num_images=num_output_images, detector=det_dict, beam=beam_dict, detector_and_beam_are_dicts=True) as writer: writer.add_image(JF16M_numpy_array) if params.write_experimental_data: data = [data[pid].as_numpy_array() for pid in panel_list] writer.add_image(data) print("Saved output to file %s" % ("exap_%d.hdf5" % i_exp)) if not params.write_output: # ability to read in the special file format # note to end-user: The special file format can be installed permanently into a # developmental version of dials/cctbx: # dxtbx.install_format ./FormatHDF5AttributeGeometry.py --global # writes to build directory # or alternatively to the user's account: # dxtbx.install_format ./FormatHDF5AttributeGeometry.py --user # writes to ~/.dxtbx from LS49.adse13_187.FormatHDF5AttributeGeometry import FormatHDF5AttributeGeometry as format_instance from LS49 import ls49_big_data filename = os.path.join(ls49_big_data, "adse13_228", "exap_%d.hdf5" % i_exp) instance = format_instance(filename) reference = [D.as_numpy_array() for D in instance.get_raw_data()] print("reference length for %s is %d" % ("exap_%d.hdf5" % i_exp, len(reference))) # assertion on equality: abs_diff = np.abs(JF16M_numpy_array - reference).max() assert np.allclose(JF16M_numpy_array, reference), \ "max per-pixel difference: %f photons, experiment %d"%(abs_diff,i_exp)