amatrixdiridx = [a.find("amatrix_dir")==0 for a in args].index(True)
except ValueError:
amatrix_dir_prefix = "tmpdir_"
else:
amatrix_dir_prefix = args.pop(amatrixdiridx).split("=")[1]
if (not (rotation_series or stills_process)):
raise ValueError,"Must specify image_glob (for rotation series) or index_glob (for stills)."
import copy, os
import dxtbx
from dxtbx.model.experiment_list import ExperimentListFactory
from dials.array_family import flex
experiments = ExperimentListFactory.from_json_file(metro, check_format=False)
beam = experiments[0].beam
detector = experiments[0].detector
print "s0 from experiment[0]: ",beam.get_s0()
print "wavelength from experiment[0]: ",beam.get_wavelength()
lab_coordinates = flex.vec3_double()
for panel in detector:
pixels = flex.vec2_double(panel.get_image_size())
mms = panel.pixel_to_millimeter(pixels)
lab_coordinates.extend(panel.get_lab_coord(mms))
# generate s1 vectors
s1 = lab_coordinates.each_normalize() * (1/beam.get_wavelength())
# Generate x vectors
def test_refinement_and_compare_with_known_truth(dials_regression,
run_in_tmpdir):
# use data generated by simulation for this test
data_dir = os.path.join(dials_regression, "refinement_test_data",
"varying_beam_direction")
experiments_path = os.path.join(data_dir, "refined_static.json")
pickle_path = os.path.join(data_dir, "refined_static.pickle")
for pth in (experiments_path, pickle_path):
assert os.path.exists(pth)
# Run refinement and load models
result = procrunner.run([
"dials.refine",
experiments_path,
pickle_path,
"scan_varying=True",
"crystal.orientation.force_static=True",
"crystal.unit_cell.force_static=True",
"beam.force_static=False",
"beam.fix=wavelength",
])
assert not result.returncode and not result.stderr
exp = ExperimentListFactory.from_json_file("refined.expt",
check_format=False)[0]
beam, detector = exp.beam, exp.detector
# Beam centre at every scan-point
s0_scan_points = [
beam.get_s0_at_scan_point(i) for i in range(beam.num_scan_points)
]
bc_scan_points = [
detector[0].get_beam_centre_px(s0) for s0 in s0_scan_points
]
# Set up the nanoBragg object as used in the simulation
sys.path.append(data_dir)
from sim_images import Simulation
sim = Simulation()
sim.set_varying_beam(along="both")
# Simulation beam centre at every scan-point
sim_s0_scan_points = [
sim.beam.get_s0_at_scan_point(i)
for i in range(sim.beam.num_scan_points)
]
sim_bc_scan_points = [
sim.detector[0].get_beam_centre_px(s0) for s0 in sim_s0_scan_points
]
assert beam.num_scan_points == sim.beam.num_scan_points
# Generate a plot. This shows that the beam centre is worse at the edges of
# the scan. This is what we expect as the centroids at the scan edges are
# least well determined because of the truncation of found spots. At these
# edges the error approaches 0.15 pixels, whilst in the central region of
# the scan it is within 0.05 pixels.
#
# plot_beam_centre_error(sim_bc_scan_points, bc_scan_points)
# Compare the results.
for bc1, bc2 in zip(sim_bc_scan_points, bc_scan_points):
assert bc2 == pytest.approx(bc1, abs=0.15)
def tst_one_monkeypatch(i_exp, spectra, Fmerge, gpu_channels_singleton, rank,
params):
print("IN MONKEYPATCH")
from simtbx.nanoBragg import utils
from dxtbx.model.experiment_list import ExperimentListFactory
import numpy as np
print("Experiment %d" % i_exp, flush=True)
outfile = "boop_%d.hdf5" % i_exp
from LS49.adse13_187.case_data import retrieve_from_repo
experiment_file = retrieve_from_repo(i_exp)
cuda = True # False # whether to use cuda
omp = False
ngpu_on_node = 1 # 8 # number of available GPUs
mosaic_spread = 0.07 # degrees
mosaic_spread_samples = params.mosaic_spread_samples # number of mosaic blocks sampling mosaicity
Ncells_abc = 30, 30, 10 # medians from best stage1
ev_res = 1.5 # resolution of the downsample spectrum
total_flux = 1e12 # total flux across channels
beamsize_mm = 0.000886226925452758 # sqrt of beam focal area
spot_scale = 500. # 5.16324 # median from best stage1
plot_spec = False # plot the downsample spectra before simulating
oversample = 1 # oversample factor, 1,2, or 3 probable enough
panel_list = None # integer list of panels, usefule for debugging
rois_only = False # only set True if you are running openMP, or CPU-only (i.e. not for GPU)
include_background = params.include_background # default is to add water background 100 mm thick
verbose = 0 # leave as 0, unles debug
flat = True # enfore that the camera has 0 thickness
#<><><><><><><><>
# XXX new code
El = ExperimentListFactory.from_json_file(experiment_file,
check_format=True)
exper = El[0]
crystal = exper.crystal
detector = exper.detector
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
beam = exper.beam
# XXX new code
spec = exper.imageset.get_spectrum(0)
energies_raw, weights_raw = spec.get_energies_eV().as_numpy_array(), \
spec.get_weights().as_numpy_array()
energies, weights = utils.downsample_spectrum(energies_raw,
weights_raw,
method=1,
total_flux=total_flux,
ev_width=ev_res)
if flat:
assert detector[0].get_thickness() == 0
if panel_list is None:
panel_list = list(range(len(detector)))
pids_for_rank = panel_list
device_Id = 0
if gpu_channels_singleton is not None:
device_Id = gpu_channels_singleton.get_deviceID()
print("Rank %d will use device %d" % (rank, device_Id))
show_params = False
time_panels = (rank == 0)
mn_energy = (energies * weights).sum() / weights.sum()
mn_wave = utils.ENERGY_CONV / mn_energy
jf16m_numpy_array = {}
from_gpu_amplitudes_cuda = {}
print(
"\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print("\tBreakdown:")
for shapetype in ["gauss_argchk", "gauss"]:
BEG = time()
print(gpu_channels_singleton.get_deviceID(), "device", shapetype)
Famp_is_uninitialized = (gpu_channels_singleton.get_nchannels() == 0)
if Famp_is_uninitialized:
F_P1 = Fmerge.expand_to_p1()
for x in range(
1
): # in this scenario, amplitudes are independent of lambda
gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert gpu_channels_singleton.get_nchannels() == 1
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=crystal,
DETECTOR=detector,
BEAM=beam,
Famp=gpu_channels_singleton,
energies=list(energies),
fluxes=list(weights),
background_wavelengths=[mn_wave],
background_wavelength_weights=[1],
background_total_flux=total_flux,
background_sample_thick_mm=0.5,
cuda=True,
oversample=oversample,
Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples,
mos_spread=mosaic_spread,
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=show_params,
time_panels=time_panels,
verbose=verbose,
spot_scale_override=spot_scale,
include_background=include_background)
TIME_EXA = time() - BEG
jf16m_numpy_array[shapetype] = JF16M_numpy_array
from_gpu_amplitudes_cuda[shapetype] = TIME_BRAGG
print(
"\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs"
% (TIME_BG, TIME_BRAGG, TIME_EXA))
ratio = from_gpu_amplitudes_cuda["gauss"] / from_gpu_amplitudes_cuda[
"gauss_argchk"]
print(
"<><><><><><><><><ratio<%.2f><><><><><><><><><><><><><><><><><><><>\n"
% (ratio))
# assertion on elapsed time:
assert ratio > 1.0, "ratio is %.3f, experiment %d" % (ratio, i_exp)
# assertion on equality:
abs_diff = np.abs(jf16m_numpy_array["gauss"] - \
jf16m_numpy_array["gauss_argchk"]).max()
assert np.allclose(jf16m_numpy_array["gauss"], \
jf16m_numpy_array["gauss_argchk"]), \
"max per-pixel difference: %f photons, experiment %d"%(abs_diff,i_exp)
def run(self):
if self._reindexing_operators:
logger.debug("Reindexing sweeps for dials.two_theta_refine")
from xia2.lib.bits import auto_logfiler
from xia2.Wrappers.Dials.Reindex import Reindex
self._reindexed_experiments, self._reindexed_reflections = [], []
for e, p, op in zip(
self._experiments,
self._reflection_files,
self._reindexing_operators,
):
reindexer = Reindex()
reindexer.set_cb_op(op)
reindexer.set_experiments_filename(e)
reindexer.set_indexed_filename(p)
reindexer.set_working_directory(self.get_working_directory())
auto_logfiler(reindexer)
reindexer.run()
self._reindexed_experiments.append(
reindexer.get_reindexed_experiments_filename()
)
self._reindexed_reflections.append(
reindexer.get_reindexed_reflections_filename()
)
logger.debug("Running dials.two_theta_refine")
self._output_cif = os.path.join(
self.get_working_directory(),
"%s_dials.two_theta_refine.cif" % self.get_xpid(),
)
self._output_mmcif = os.path.join(
self.get_working_directory(),
"%s_dials.two_theta_refine.mmcif" % self.get_xpid(),
)
if not self._output_p4p:
self._output_p4p = os.path.join(
self.get_working_directory(),
"%s_dials.two_theta_refine.p4p" % self.get_xpid(),
)
self._output_correlation_plot = os.path.join(
self.get_working_directory(),
"%s_dials.two_theta_refine.png" % self.get_xpid(),
)
self._output_experiments = os.path.join(
self.get_working_directory(), "%s_refined_cell.expt" % self.get_xpid()
)
self.clear_command_line()
if self._reindexing_operators:
for experiment in self._reindexed_experiments:
self.add_command_line(experiment)
for reflection_file in self._reindexed_reflections:
self.add_command_line(reflection_file)
else:
for experiment in self._experiments:
self.add_command_line(experiment)
for reflection_file in self._reflection_files:
self.add_command_line(reflection_file)
self.add_command_line(
"combine_crystal_models=%s" % self._combine_crystal_models
)
self.add_command_line("output.cif=%s" % self._output_cif)
self.add_command_line("output.mmcif=%s" % self._output_mmcif)
self.add_command_line("output.p4p=%s" % self._output_p4p)
if self._output_correlation_plot is not None:
self.add_command_line(
"output.correlation_plot.filename=%s"
% self._output_correlation_plot
)
if self._output_experiments is not None:
self.add_command_line(
"output.experiments=%s" % self._output_experiments
)
if self._phil_file is not None:
self.add_command_line(self._phil_file)
self.start()
self.close_wait()
if not os.path.isfile(self._output_cif):
logger.warning(
"TwoTheta refinement failed, see log file for more details:\n %s",
self.get_log_file(),
)
raise RuntimeError("unit cell not refined")
self.check_for_errors()
experiments = ExperimentListFactory.from_json_file(
self.get_output_experiments(), check_format=False
)
self._crystal = experiments.crystals()[0]
def update_all_data(reflections_path=None, experiments_path=None):
dat = InfoData()
dat.ref2exp = None
if reflections_path is not None:
try:
refl_tabl = flex.reflection_table.from_file(reflections_path)
dat.n_strng = refl_tabl.get_flags(
refl_tabl.flags.strong).count(True)
dat.n_index = refl_tabl.get_flags(
refl_tabl.flags.indexed).count(True)
dat.n_refnd = refl_tabl.get_flags(
refl_tabl.flags.used_in_refinement).count(True)
dat.n_integ_sum = refl_tabl.get_flags(
refl_tabl.flags.integrated_sum).count(True)
dat.n_integ_prf = refl_tabl.get_flags(
refl_tabl.flags.integrated_prf).count(True)
except BaseException as e:
# We don't want to catch bare exceptions but don't know
# what this was supposed to catch. Log it.
logger.info(" >> Caught unknown exception type:",
type(e).__name__, e, "N###")
logger.info("failed to find reflections")
logger.info("reflections_path = %s", reflections_path)
if experiments_path is not None:
try:
experiments = ExperimentListFactory.from_json_file(
experiments_path, check_format=False)
except BaseException as e:
# We don't want to catch bare exceptions but don't know
# what this was supposed to catch. Log it.
logger.info("\n exception #1 %s: %s", type(e).__name__, e)
# TODO Maybe the next try block is not needed, consider removing all
try:
# FIXME here only take the first datablock. What if there are more?
datablock = DataBlockFactory.from_serialized_format(
experiments_path, check_format=False)[0]
# FIXME here only take the first model from each
beam = datablock.unique_beams()[0]
detector = datablock.unique_detectors()[0]
scan = datablock.unique_scans()[0]
# build a pseudo ExperimentList (with empty crystals)
experiments = ExperimentList()
experiments.append(
Experiment(beam=beam, detector=detector, scan=scan))
except ValueError:
logger.info("failed to read json file")
logger.info("experiments_path = %s", experiments_path)
return dat
try:
imageset_tmp = experiments.imagesets()[0]
mask_file = imageset_tmp.external_lookup.mask.filename
pick_file = open(mask_file, "rb")
mask_tup_obj = pickle.load(pick_file)
pick_file.close()
dat.mask_flex = mask_tup_obj[0]
mask_np_arr = dat.mask_flex.as_numpy_array()
dat.np_mask = mask_np_arr
except IOError:
logger.info("No mask in this node")
dat.np_mask = None
dat.mask_flex = None
# FIXME it takes just the first experiment. What if there are more?
exp = experiments[0]
# Get crystal data
if exp.crystal is not None:
unit_cell = exp.crystal.get_unit_cell()
dat.a, dat.b, dat.c, dat.alpha, dat.beta, dat.gamma = unit_cell.parameters(
)
exp_crystal = exp.crystal
logger.info("exp_crystal = %s", exp_crystal)
b_mat = exp.crystal.get_B()
dat.b11 = b_mat[0]
dat.b12 = b_mat[1]
dat.b13 = b_mat[2]
dat.b21 = b_mat[3]
dat.b22 = b_mat[4]
dat.b23 = b_mat[5]
dat.b31 = b_mat[6]
dat.b32 = b_mat[7]
dat.b33 = b_mat[8]
sg = str(exp.crystal.get_space_group().info())
logger.info("spgr = %s", sg)
dat.spg_group = sg
from scitbx import matrix
u_mat = matrix.sqr(exp.crystal.get_U())
dat.u11 = b_mat[0]
dat.u12 = b_mat[1]
dat.u13 = b_mat[2]
dat.u21 = b_mat[3]
dat.u22 = b_mat[4]
dat.u23 = b_mat[5]
dat.u31 = b_mat[6]
dat.u32 = b_mat[7]
dat.u33 = b_mat[8]
rot_angs = u_mat.r3_rotation_matrix_as_x_y_z_angles(deg=True)
logger.info("u_mat = %s", u_mat)
logger.info("rot_angs = %s", rot_angs)
dat.r1, dat.r2, dat.r3 = rot_angs
# Get beam data
dat.w_lambda = exp.beam.get_wavelength()
# Get detector data
# assume details for the panel the beam intersects are the same
# for the whole detector
# TODO fix it for I23
pnl_beam_intersects, (beam_x,
beam_y) = exp.detector.get_ray_intersection(
exp.beam.get_s0())
pnl = exp.detector[pnl_beam_intersects]
logger.info("beam_x, beam_y = %s %s", beam_x, beam_y)
dat.n_pan_xb_yb = pnl_beam_intersects
dat.xb = beam_x
dat.yb = beam_y
dist = pnl.get_distance()
logger.info("pnl_beam_intersects %s", pnl_beam_intersects)
logger.info("dist %s", dist)
dat.dd = dist
dat.img_ran1, dat.img_ran2 = exp.scan.get_image_range()
dat.oscil1, dat.oscil2 = exp.scan.get_oscillation()
# is the next line right? check what dials.show does
dat.e_time = max(exp.scan.get_exposure_times())
dat.n_pans = len(exp.detector)
dat.x_px_size, dat.y_px_size = pnl.get_pixel_size()
dat.gain = pnl.get_gain()
dat.max_res = exp.detector.get_max_resolution(exp.beam.get_s0())
# manually finding template from experiments_path
try:
with open(experiments_path) as infile:
json_info = json.load(infile)
if type(json_info) is dict:
imageset = json_info["imageset"]
elif type(json_info) is list:
imageset = json_info[0]["imageset"]
dat.tmpl_str = imageset[0]["template"]
logger.info("dat.tmpl_str = %s", dat.tmpl_str)
except (KeyError, IndexError):
logger.info("failed to find template in JSON file")
dat.ref2exp = exp
return dat
def process(self, job):
"""Perform processing tasks for one dataset"""
print("Processing job {0}".format(job))
# Import
path = os.path.join(job, "*.img")
cmd = ("dials.import {0} ").format(path)
if self._run_one_job(cmd) is None:
return
# Spot finding
cmd = "dials.find_spots datablock.json filter.d_min={0} nproc={1}".format(
self.params.find_spots.d_min, self.params.nproc)
if self._run_one_job(cmd) is None:
return
with open("dials.find_spots.log", "r") as log:
while True:
line = log.readline()
if line.startswith("Histogram"):
break
for i in range(14):
print(log.readline(), end=" ")
# Attempt indexing in P1
n_indexed = 0
best_indexing = None
cmd = (
"dials.index datablock.json strong.pickle "
"output.experiments=P1_experiments.json "
"output.reflections=P1_indexed.pickle output.log=P1_dials.index.log"
)
if self._run_one_job(cmd) is not None:
indexed = flex.reflection_table.from_pickle("P1_indexed.pickle")
n_indexed = indexed.get_flags(indexed.flags.indexed).count(True)
best_indexing = ("P1_experiments.json", "P1_indexed.pickle")
el = ExperimentListFactory.from_json_file("P1_experiments.json")
cell = el[0].crystal.get_unit_cell().parameters()
cell_s = ", ".join(["{:.3f}".format(e) for e in cell])
print("{0} reflections indexed in P1 with unit cell ({1})".format(
n_indexed, cell_s))
# Attempt indexing in P222
cmd = ("dials.index datablock.json strong.pickle "
"output.experiments=P222_experiments.json "
"output.reflections=P222_indexed.pickle "
"output.log=P222_dials.index.log space_group=P222")
if self._run_one_job(cmd) is not None:
indexed = flex.reflection_table.from_pickle("P222_indexed.pickle")
n_indexed2 = indexed.get_flags(indexed.flags.indexed).count(True)
if n_indexed2 > n_indexed:
best_indexing = ("P222_experiments.json",
"P222_indexed.pickle")
n_indexed = n_indexed2
el = ExperimentListFactory.from_json_file("P222_experiments.json")
cell = el[0].crystal.get_unit_cell().parameters()
cell_s = ", ".join(["{:.3f}".format(e) for e in cell])
print(
"{0} reflections indexed in P222 with unit cell ({1})".format(
n_indexed2, cell_s))
if best_indexing is None:
return
print("Selecting {0} and {1} for further processing".format(
*best_indexing))
# Convert best indexing solution to P1 and P222 versions
cmd = ("dials.reindex {0} {1} space_group={2} "
"output.experiments=best_experiments_conv_to_{2}.json "
"output.reflections=best_reflections_conv_to_{2}.pickle ")
if (self._run_one_job(
cmd.format(best_indexing[0], best_indexing[1], "P1")) is None):
return
if (self._run_one_job(
cmd.format(best_indexing[0], best_indexing[1], "P222")) is
None):
return
# Further processing in sub directories
for sub_job in ("P1", "P222"):
try:
with cd(sub_job):
experiments = "best_experiments_conv_to_{0}.json".format(
sub_job)
reflections = "best_reflections_conv_to_{0}.pickle".format(
sub_job)
print("Processing best indexing solution converted to {0}".
format(sub_job))
self._refine_onwards(experiments, reflections)
except OSError:
raise Sorry(
"Failed to create sub job directory {0}".format(sub_job))
# Compare summary tables from aimless.log files from each job
import difflib
logpaths = [
os.path.join("P1", "aimless.log"),
os.path.join("P222", "aimless.log"),
]
if [os.path.exists(e) for e in logpaths].count(True) == 2:
aimlessP1 = self._get_aimless_summary(logpaths[0])
aimlessP222 = self._get_aimless_summary(logpaths[1])
diff = difflib.HtmlDiff(wrapcolumn=80).make_file(
aimlessP1, aimlessP222, logpaths[0], logpaths[1])
with open("aimless-diff.html", "w") as f:
f.writelines(diff)
print()
return
def run(self, all_experiments, all_reflections):
""" Load all the data using MPI """
from dxtbx.model.experiment_list import ExperimentList
from dials.array_family import flex
# Both must be none or not none
test = [all_experiments is None, all_reflections is None].count(True)
assert test in [0,2]
if test == 2:
all_experiments = ExperimentList()
all_reflections = flex.reflection_table()
starting_expts_count = starting_refls_count = 0
else:
starting_expts_count = len(all_experiments)
starting_refls_count = len(all_reflections)
self.logger.log("Initial number of experiments: %d; Initial number of reflections: %d"%(starting_expts_count, starting_refls_count))
# Generate and send a list of file paths to each worker
if self.mpi_helper.rank == 0:
file_list = self.get_list()
self.logger.log("Built an input list of %d json/pickle file pairs"%(len(file_list)))
self.params.input.path = None # Rank 0 has already parsed the input parameters
per_rank_file_list = file_load_calculator(self.params, file_list, self.logger).\
calculate_file_load(available_rank_count = self.mpi_helper.size)
self.logger.log('Transmitting a list of %d lists of json/pickle file pairs'%(len(per_rank_file_list)))
transmitted = per_rank_file_list
else:
transmitted = None
self.logger.log_step_time("BROADCAST_FILE_LIST")
transmitted = self.mpi_helper.comm.bcast(transmitted, root = 0)
new_file_list = transmitted[self.mpi_helper.rank] if self.mpi_helper.rank < len(transmitted) else None
self.logger.log_step_time("BROADCAST_FILE_LIST", True)
# Load the data
self.logger.log_step_time("LOAD")
if new_file_list is not None:
self.logger.log("Received a list of %d json/pickle file pairs"%len(new_file_list))
for experiments_filename, reflections_filename in new_file_list:
self.logger.log("Reading %s %s"%(experiments_filename, reflections_filename))
experiments = ExperimentListFactory.from_json_file(experiments_filename, check_format = False)
reflections = flex.reflection_table.from_file(reflections_filename)
self.logger.log("Data read, prepping")
if 'intensity.sum.value' in reflections:
reflections['intensity.sum.value.unmodified'] = reflections['intensity.sum.value'] * 1
if 'intensity.sum.variance' in reflections:
reflections['intensity.sum.variance.unmodified'] = reflections['intensity.sum.variance'] * 1
new_ids = flex.int(len(reflections), -1)
new_identifiers = flex.std_string(len(reflections))
eid = reflections.experiment_identifiers()
for k in eid.keys():
del eid[k]
for experiment_id, experiment in enumerate(experiments):
# select reflections of the current experiment
refls_sel = reflections['id'] == experiment_id
if refls_sel.count(True) == 0: continue
if experiment.identifier is None or len(experiment.identifier) == 0:
experiment.identifier = create_experiment_identifier(experiment, experiments_filename, experiment_id)
if not self.params.input.keep_imagesets:
experiment.imageset = None
all_experiments.append(experiment)
# Reflection experiment 'id' is unique within this rank; 'exp_id' (i.e. experiment identifier) is unique globally
new_identifiers.set_selected(refls_sel, experiment.identifier)
new_id = len(all_experiments)-1
eid[new_id] = experiment.identifier
new_ids.set_selected(refls_sel, new_id)
assert (new_ids < 0).count(True) == 0, "Not all reflections accounted for"
reflections['id'] = new_ids
reflections['exp_id'] = new_identifiers
all_reflections.extend(reflections)
else:
self.logger.log("Received a list of 0 json/pickle file pairs")
self.logger.log_step_time("LOAD", True)
self.logger.log('Read %d experiments consisting of %d reflections'%(len(all_experiments)-starting_expts_count, len(all_reflections)-starting_refls_count))
self.logger.log("Memory usage: %d MB"%get_memory_usage())
all_reflections = self.prune_reflection_table_keys(all_reflections)
# Do we have any data?
from xfel.merging.application.utils.data_counter import data_counter
data_counter(self.params).count(all_experiments, all_reflections)
return all_experiments, all_reflections
crystal = models.crystal
# a hexagonal crystal is a good test case for behaviour of oblique cells
do_hexagonal = True
if do_hexagonal:
import libtbx.load_env, os
from dxtbx.model.experiment_list import ExperimentListFactory
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError:
print 'Skipped: dials_regression not configured'
exit(0)
data_dir = os.path.join(dials_regression, "refinement_test_data",
"multi_stills")
experiments_path = os.path.join(data_dir, "combined_experiments.json")
experiments = ExperimentListFactory.from_json_file(experiments_path,
check_format=False)
crystal = experiments[0].crystal
# derive finite difference gradients of various quantities wrt each param
def check_fd_gradients(parameterisation):
mp = parameterisation
p_vals = mp.get_param_vals()
deltas = [1.e-7 for p in p_vals]
assert len(deltas) == len(p_vals)
fd_grad = []
# get matrix to unset rotations of unit cell vectors
Ut = matrix.sqr(mp.get_model().get_U()).transpose()
def read_experiments(filename):
from dxtbx.model.experiment_list import ExperimentListFactory
return ExperimentListFactory.from_json_file(filename)
def test_nexus_dump_and_reload(dials_regression, tmpdir, filename):
path = os.path.join(dials_regression, "nexus_test_data", "shared_models")
filename_in = os.path.join(path, f"{filename}.json")
filename_out = tmpdir.join(f"{filename}.nxs").strpath
experiments = ExperimentListFactory.from_json_file(filename_in)
run_single(experiments, filename_out)
print 'OK'
def run():
from dxtbx.model.experiment_list import ExperimentListFactory
from os.path import join
import libtbx.load_env
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'FAIL: dials_regression not configured'
exit(0)
path = join(dials_regression, "nexus_test_data", "shared_models")
filename_list = [
'single', 'multiple_unrelated', 'multi_crystal', 'two_colour',
'multiple_sweeps', 'stills'
]
for filename in filename_list:
filename_in = join(path, "%s.json" % filename)
filename_out = "%s.nxs" % filename
experiments = ExperimentListFactory.from_json_file(filename_in)
run_single(experiments, filename_out)
if __name__ == '__main__':
from dials.test import cd_auto
with cd_auto(__file__):
test_polarization_conversion()
run()
def test_merge_multi_wavelength(dials_data, tmpdir):
"""Test that merge handles multi-wavelength data suitably - should be
exported into an mtz with seprate columns for each wavelength."""
mean_labels = [
"%sIMEAN_WAVE%s" % (pre, i) for i in [1, 2] for pre in ["", "SIG"]
]
anom_labels = [
"%sI_WAVE%s(%s)" % (pre, i, sgn) for i in [1, 2]
for pre in ["", "SIG"] for sgn in ["+", "-"]
]
amp_labels = [
"%sF_WAVE%s" % (pre, i) for i in [1, 2] for pre in ["", "SIG"]
]
anom_amp_labels = [
"%sF_WAVE%s(%s)" % (pre, i, sgn) for i in [1, 2]
for pre in ["", "SIG"] for sgn in ["+", "-"]
]
location = dials_data("l_cysteine_4_sweeps_scaled")
refl1 = location.join("scaled_30.refl").strpath
expt1 = location.join("scaled_30.expt").strpath
refl2 = location.join("scaled_35.refl").strpath
expt2 = location.join("scaled_35.expt").strpath
expts1 = ExperimentListFactory.from_json_file(expt1, check_format=False)
expts1[0].beam.set_wavelength(0.5)
expts2 = ExperimentListFactory.from_json_file(expt2, check_format=False)
expts1.extend(expts2)
tmp_expt = tmpdir.join("tmp.expt").strpath
expts1.as_json(tmp_expt)
reflections1 = flex.reflection_table.from_pickle(refl1)
reflections2 = flex.reflection_table.from_pickle(refl2)
# first need to resolve identifiers - usually done on loading
reflections2["id"] = flex.int(reflections2.size(), 1)
del reflections2.experiment_identifiers()[0]
reflections2.experiment_identifiers()[1] = "3"
reflections1.extend(reflections2)
tmp_refl = tmpdir.join("tmp.refl").strpath
reflections1.as_file(tmp_refl)
# Can now run after creating our 'fake' multiwavelength dataset
command = [
"dials.merge", tmp_refl, tmp_expt, "truncate=True", "anomalous=True"
]
result = procrunner.run(command, working_directory=tmpdir)
assert not result.returncode and not result.stderr
assert tmpdir.join("merged.mtz").check()
m = mtz.object(tmpdir.join("merged.mtz").strpath)
labels = []
for ma in m.as_miller_arrays(merge_equivalents=False):
labels.extend(ma.info().labels)
assert all(x in labels for x in mean_labels)
assert all(x in labels for x in anom_labels)
assert all(x in labels for x in amp_labels)
assert all(x in labels for x in anom_amp_labels)
# 5 miller arrays for each dataset
assert m.as_miller_arrays()[0].info().wavelength == pytest.approx(0.5)
assert m.as_miller_arrays()[5].info().wavelength == pytest.approx(0.6889)
def run(self):
MAIN_LOGGER = logging.getLogger("diffBragg.main")
assert os.path.exists(self.params.exp_ref_spec_file)
input_lines = None
best_models = None
if COMM.rank == 0:
input_lines = open(self.params.exp_ref_spec_file, "r").readlines()
if self.params.skip is not None:
input_lines = input_lines[self.params.skip:]
if self.params.first_n is not None:
input_lines = input_lines[:self.params.first_n]
if self.params.sanity_test_input:
hopper_utils.sanity_test_input_lines(input_lines)
if self.params.best_pickle is not None:
logging.info("reading pickle %s" % self.params.best_pickle)
best_models = pandas.read_pickle(self.params.best_pickle)
if self.params.dump_gathers:
if self.params.gathers_dir is None:
raise ValueError(
"Need to provide a file dir path in order to dump_gathers"
)
utils.safe_makedirs(self.params.gathers_dir)
input_lines = COMM.bcast(input_lines)
best_models = COMM.bcast(best_models)
if self.params.ignore_existing:
exp_names_already = None
refl_names_already = None
if COMM.rank == 0:
exp_names_already = {
os.path.basename(f)
for f in glob.glob("%s/expers/rank*/*.expt" %
self.params.outdir)
}
refl_names_already = {
os.path.basename(f)
for f in glob.glob("%s/refls/rank*/*.refl" %
self.params.outdir)
}
exp_names_already = COMM.bcast(exp_names_already)
refl_names_already = COMM.bcast(refl_names_already)
exp_gatheredRef_spec = [] # optional list of expt, refls, spectra
for i_exp, line in enumerate(input_lines):
if i_exp == self.params.max_process:
break
if i_exp % COMM.size != COMM.rank:
continue
logging.info("COMM.rank %d on shot %d / %d" %
(COMM.rank, i_exp + 1, len(input_lines)))
line_fields = line.strip().split()
assert len(line_fields) in [2, 3]
if len(line_fields) == 2:
exp, ref = line_fields
spec = None
else:
exp, ref, spec = line_fields
if self.params.ignore_existing:
basename = os.path.splitext(os.path.basename(exp))[0]
opt_exp = "%s_%s_%d.expt" % (self.params.tag, basename, i_exp)
opt_refl = opt_exp.replace(".expt", ".refl")
if opt_exp in exp_names_already and opt_refl in refl_names_already:
continue
best = None
if best_models is not None:
best = best_models.query("exp_name=='%s'" % exp)
if len(best) == 0:
best = best_models.query("opt_exp_name=='%s'" % exp)
if len(best) != 1:
raise ValueError(
"Should be 1 entry for exp %s in best pickle %s" %
(exp, self.params.best_pickle))
self.params.simulator.spectrum.filename = spec
Modeler = hopper_utils.DataModeler(self.params)
if self.params.load_data_from_refls:
gathered = Modeler.GatherFromReflectionTable(exp, ref)
else:
gathered = Modeler.GatherFromExperiment(exp, ref)
if not gathered:
logging.warning("No refls in %s; CONTINUE; COMM.rank=%d" %
(ref, COMM.rank))
continue
MAIN_LOGGER.info("Modeling %s (%d refls)" %
(exp, len(Modeler.refls)))
if self.params.dump_gathers:
output_name = os.path.splitext(os.path.basename(exp))[0]
output_name += "_withData.refl"
output_name = os.path.join(self.params.gathers_dir,
output_name)
Modeler.dump_gathered_to_refl(
output_name, do_xyobs_sanity_check=True
) # NOTE do this is modelin strong spots only
if self.params.test_gathered_file:
all_data = Modeler.all_data.copy()
all_roi_id = Modeler.roi_id.copy()
all_bg = Modeler.all_background.copy()
all_trusted = Modeler.all_trusted.copy()
all_pids = np.array(Modeler.pids)
all_rois = np.array(Modeler.rois)
new_Modeler = hopper_utils.DataModeler(self.params)
assert new_Modeler.GatherFromReflectionTable(
exp, output_name)
assert np.allclose(new_Modeler.all_data, all_data)
assert np.allclose(new_Modeler.all_background, all_bg)
assert np.allclose(new_Modeler.rois, all_rois)
assert np.allclose(new_Modeler.pids, all_pids)
assert np.allclose(new_Modeler.all_trusted, all_trusted)
assert np.allclose(new_Modeler.roi_id, all_roi_id)
exp_gatheredRef_spec.append(
(exp, os.path.abspath(output_name), spec))
if self.params.only_dump_gathers:
continue
if self.params.refiner.reference_geom is not None:
detector = ExperimentListFactory.from_json_file(
self.params.refiner.reference_geom,
check_format=False)[0].detector
Modeler.E.detector = detector
Modeler.SimulatorFromExperiment(best)
Modeler.SIM.D.store_ave_wavelength_image = self.params.store_wavelength_images
if self.params.refiner.verbose is not None and COMM.rank == 0:
Modeler.SIM.D.verbose = self.params.refiner.verbose
if self.params.profile:
Modeler.SIM.record_timings = True
if self.params.use_float32:
Modeler.all_data = Modeler.all_data.astype(np.float32)
Modeler.all_background = Modeler.all_background.astype(
np.float32)
if self.params.refiner.randomize_devices:
dev = np.random.choice(self.params.refiner.num_devices)
logging.info(
"Rank %d will use randomly chosen device %d on host %s" %
(COMM.rank, dev, socket.gethostname()))
else:
dev = COMM.rank % self.params.refiner.num_devices
logging.info("Rank %d will use device %d on host %s" %
(COMM.rank, dev, socket.gethostname()))
print("RANK%d, DEV=%d" % (COMM.rank, dev))
Modeler.SIM.D.device_Id = dev
nparam = len(Modeler.SIM.P)
x0 = [1] * nparam
try:
x = Modeler.Minimize(x0)
except StopIteration:
x = Modeler.target.x0
if self.params.profile:
Modeler.SIM.D.show_timings(COMM.rank) #, out)
save_up(Modeler, x, exp, i_exp, ref)
if self.params.dump_gathers and self.params.gathered_output_file is not None:
exp_gatheredRef_spec = COMM.reduce(exp_gatheredRef_spec)
if COMM.rank == 0:
o = open(self.params.gathered_output_file, "w")
for e, r, s in exp_gatheredRef_spec:
if s is not None:
o.write("%s %s %s\n" % (e, r, s))
else:
o.write("%s %s\n" % (e, r))
o.close()
exp_name = sys.argv[1]
data_name = sys.argv[2]
tag = sys.argv[3]
hkl_tol = .15
ENERGIES = [parameters.ENERGY_LOW,
parameters.ENERGY_HIGH] # colors of the beams
FF = [5000, None] # Setting structure factors takes long time in nanoBragg, so
# unless you want energy-dependent structure factors
# you need only provide one number -or- one structure factor flex miller table
# and the computer will know to preserve that for all beam colors
FLUX = [1e14, 1e14] # fluxes of the beams
waveA = parameters.ENERGY_CONV / ENERGIES[0]
waveB = parameters.ENERGY_CONV / ENERGIES[1]
exp_lst = ExperimentListFactory.from_json_file(exp_name)
iset = exp_lst.imagesets()[0]
detector = iset.get_detector(0)
data = utils.open_flex(data_name)
beamA = deepcopy(iset.get_beam())
beamB = deepcopy(iset.get_beam())
beamA.set_wavelength(waveA)
beamB.set_wavelength(waveB)
refls_strong = data["refls_strong"]
crystalAB = data["crystalAB"]
reflsPP = spot_utils.refls_by_panelname(refls_strong)
pids = [i for i in reflsPP
if len(reflsPP[i]) > 0] # refine on these panels only
pan_imgs = [iset.get_raw_data(0)[pid].as_numpy_array() for pid in pids]
def refine_expanding(params, merged_scope, combine_phil):
assert params.start_at_hierarchy_level == 0
if params.rmsd_filter.enable:
input_name = "filtered"
command = "cctbx.xfel.filter_experiments_by_rmsd %s %s output.filtered_experiments=%s output.filtered_reflections=%s"
command = command % (
"%s_combined.expt" % params.tag, "%s_combined.refl" % params.tag,
"%s_filtered.expt" % params.tag, "%s_filtered.refl" % params.tag)
command += " iqr_multiplier=%f" % params.rmsd_filter.iqr_multiplier
print(command)
result = easy_run.fully_buffered(command=command).raise_if_errors()
result.show_stdout()
else:
input_name = "combined"
# --------------------------
if params.panel_filter is not None:
from libtbx import easy_pickle
print("Filtering out all reflections except those on panels %s" %
(", ".join(["%d" % p for p in params.panel_filter])))
combined_path = "%s_combined.refl" % params.tag
data = easy_pickle.load(combined_path)
sel = None
for panel_id in params.panel_filter:
if sel is None:
sel = data['panel'] == panel_id
else:
sel |= data['panel'] == panel_id
print("Retaining", len(data.select(sel)), "out of", len(data),
"reflections")
easy_pickle.dump(combined_path, data.select(sel))
# ----------------------------------
# this is the order to refine the CSPAD in
steps = {}
steps[0] = [2, 3]
steps[1] = steps[0] + [0, 1]
steps[2] = steps[1] + [14, 15]
steps[3] = steps[2] + [6, 7]
steps[4] = steps[3] + [4, 5]
steps[5] = steps[4] + [12, 13]
steps[6] = steps[5] + [8, 9]
steps[7] = steps[6] + [10, 11]
for s, panels in six.iteritems(steps):
rest = []
for p in panels:
rest.append(p + 16)
rest.append(p + 32)
rest.append(p + 48)
panels.extend(rest)
levels = {0: (0, 1)} # levels 0 and 1
for i in range(7):
levels[i + 1] = (2, ) # level 2
previous_step_and_level = None
for j in range(8):
from libtbx import easy_pickle
print("Filtering out all reflections except those on panels %s" %
(", ".join(["%d" % p for p in steps[j]])))
combined_path = "%s_%s.refl" % (params.tag, input_name)
output_path = "%s_step%d.refl" % (params.tag, j)
data = easy_pickle.load(combined_path)
sel = None
for panel_id in steps[j]:
if sel is None:
sel = data['panel'] == panel_id
else:
sel |= data['panel'] == panel_id
print("Retaining", len(data.select(sel)), "out of", len(data),
"reflections")
easy_pickle.dump(output_path, data.select(sel))
for i in levels[j]:
print("Step", j, "refining at hierarchy level", i)
refine_phil_file = "%s_refine_step%d_level%d.phil" % (params.tag,
j, i)
if i == 0:
if params.refine_distance:
diff_phil = "refinement.parameterisation.detector.fix_list=Tau1" # fix detector rotz
else:
diff_phil = "refinement.parameterisation.detector.fix_list=Dist,Tau1" # fix detector rotz, distance
if params.flat_refinement:
diff_phil += ",Tau2,Tau3" # Also fix x and y rotations
diff_phil += "\n"
if params.refine_energy:
diff_phil += "refinement.parameterisation.beam.fix=in_spindle_plane+out_spindle_plane\n" # allow energy to refine
else:
# Note, always need to fix something, so pick a panel group and fix its Tau1 (rotation around Z) always
if params.flat_refinement and params.flat_refinement_with_distance:
diff_phil = "refinement.parameterisation.detector.fix_list=Group1Tau1,Tau2,Tau3\n" # refine distance, rotz and xy translation
diff_phil += "refinement.parameterisation.detector.constraints.parameter=Dist\n" # constrain distance to be refined identically for all panels at this hierarchy level
elif params.flat_refinement:
diff_phil = "refinement.parameterisation.detector.fix_list=Dist,Group1Tau1,Tau2,Tau3\n" # refine only rotz and xy translation
else:
diff_phil = "refinement.parameterisation.detector.fix_list=Group1Tau1\n" # refine almost everything
if previous_step_and_level is None:
command = "dials.refine %s %s_%s.expt %s_step%d.refl"%( \
refine_phil_file, params.tag, input_name, params.tag, j)
else:
p_step, p_level = previous_step_and_level
if p_step == j:
command = "dials.refine %s %s_refined_step%d_level%d.expt %s_refined_step%d_level%d.refl"%( \
refine_phil_file, params.tag, p_step, p_level, params.tag, p_step, p_level)
else:
command = "dials.refine %s %s_refined_step%d_level%d.expt %s_step%d.refl"%( \
refine_phil_file, params.tag, p_step, p_level, params.tag, j)
diff_phil += "refinement.parameterisation.detector.hierarchy_level=%d\n" % i
output_experiments = "%s_refined_step%d_level%d.expt" % (
params.tag, j, i)
command += " output.experiments=%s output.reflections=%s_refined_step%d_level%d.refl"%( \
output_experiments, params.tag, j, i)
scope = merged_scope.fetch(parse(diff_phil))
f = open(refine_phil_file, 'w')
f.write(refine_scope.fetch_diff(scope).as_str())
f.close()
print(command)
result = easy_run.fully_buffered(command=command).raise_if_errors()
result.show_stdout()
# In expanding mode, if using flat refinement with distance, after having refined this step as a block, unrefined
# panels will have been left behind. Read back the new metrology, compute the shift applied to the panels refined
# in this step,and apply that shift to the unrefined panels in this step
if params.flat_refinement and params.flat_refinement_with_distance and i > 0:
from dxtbx.model.experiment_list import ExperimentListFactory
from xfel.command_line.cspad_detector_congruence import iterate_detector_at_level, iterate_panels
from scitbx.array_family import flex
from scitbx.matrix import col
from libtbx.test_utils import approx_equal
experiments = ExperimentListFactory.from_json_file(
output_experiments, check_format=False)
assert len(experiments.detectors()) == 1
detector = experiments.detectors()[0]
# Displacements: deltas along the vector normal to the detector
displacements = flex.double()
# Iterate through the panel groups at this level
for panel_group in iterate_detector_at_level(
detector.hierarchy(), 0, i):
# Were there panels refined in this step in this panel group?
if params.panel_filter:
test = [
list(detector).index(panel) in steps[j]
for panel in iterate_panels(panel_group) if list(
detector).index(panel) in params.panel_filter
]
else:
test = [
list(detector).index(panel) in steps[j]
for panel in iterate_panels(panel_group)
]
if not any(test): continue
# Compute the translation along the normal of this panel group. This is defined as distance in dials.refine
displacements.append(
col(panel_group.get_local_fast_axis()).cross(
col(panel_group.get_local_slow_axis())).dot(
col(panel_group.get_local_origin())))
# Even though the panels are constrained to move the same amount, there is a bit a variation.
stats = flex.mean_and_variance(displacements)
displacement = stats.mean()
print("Average displacement along normals: %f +/- %f" %
(stats.mean(),
stats.unweighted_sample_standard_deviation()))
# Verify the variation isn't significant
for k in range(1, len(displacements)):
assert approx_equal(displacements[0], displacements[k])
# If all of the panel groups in this level moved, no need to do anything.
if len(displacements) != len(
list(
iterate_detector_at_level(detector.hierarchy(), 0,
i))):
for panel_group in iterate_detector_at_level(
detector.hierarchy(), 0, i):
if params.panel_filter:
test = [
list(detector).index(panel) in steps[j]
and list(detector).index(panel)
in params.panel_filter
for panel in iterate_panels(panel_group)
]
else:
test = [
list(detector).index(panel) in steps[j]
for panel in iterate_panels(panel_group)
]
# If any of the panels in this panel group moved, no need to do anything
if any(test): continue
# None of the panels in this panel group moved in this step, so need to apply displacement from other panel
# groups at this level
fast = col(panel_group.get_local_fast_axis())
slow = col(panel_group.get_local_slow_axis())
ori = col(panel_group.get_local_origin())
normal = fast.cross(slow)
panel_group.set_local_frame(
fast, slow, (ori.dot(fast) * fast) +
(ori.dot(slow) * slow) + (normal * displacement))
# Check the new displacements. Should be the same across all panels.
displacements = []
for panel_group in iterate_detector_at_level(
detector.hierarchy(), 0, i):
displacements.append(
col(panel_group.get_local_fast_axis()).cross(
col(panel_group.get_local_slow_axis())).dot(
col(panel_group.get_local_origin())))
for k in range(1, len(displacements)):
assert approx_equal(displacements[0], displacements[k])
experiments.as_file(output_experiments)
previous_step_and_level = j, i
output_geometry(params)
def get_dials_matrix(experiments_json):
from dxtbx.model.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(experiments_json)
return experiments[0].crystal.get_A()
def test1():
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression", test=os.path.isdir)
data_dir = os.path.join(dials_regression, "refinement_test_data",
"multi_narrow_wedges")
# work in a temporary directory
cwd = os.path.abspath(os.curdir)
tmp_dir = open_tmp_directory(
suffix="tst_combine_experiments_and_reflections")
os.chdir(tmp_dir)
input_phil = phil_input.format(data_dir) + """
reference_from_experiment.beam=0
reference_from_experiment.scan=0
reference_from_experiment.goniometer=0
reference_from_experiment.detector=0
"""
with open("input.phil", "w") as phil_file:
phil_file.writelines(input_phil)
cmd = "dials.combine_experiments input.phil"
#print cmd
result = easy_run.fully_buffered(command=cmd).raise_if_errors()
# load results
exp = ExperimentListFactory.from_json_file("combined_experiments.json",
check_format=False)
ref = flex.reflection_table.from_pickle("combined_reflections.pickle")
# test the experiments
assert len(exp) == 103
assert len(exp.crystals()) == 103
assert len(exp.beams()) == 1
assert len(exp.scans()) == 1
assert len(exp.detectors()) == 1
assert len(exp.goniometers()) == 1
for e in exp:
assert e.imageset is not None
# test the reflections
assert len(ref) == 11689
cmd = " ".join([
"dials.split_experiments", "combined_experiments.json",
"combined_reflections.pickle"
])
result = easy_run.fully_buffered(command=cmd).raise_if_errors()
for i in range(len(exp)):
assert os.path.exists("experiments_%03d.json" % i)
assert os.path.exists("reflections_%03d.pickle" % i)
exp_single = ExperimentListFactory.from_json_file(
"experiments_%03d.json" % i, check_format=False)
ref_single = flex.reflection_table.from_pickle(
"reflections_%03d.pickle" % i)
assert len(exp_single) == 1
assert exp_single[0].crystal == exp[i].crystal
assert exp_single[0].beam == exp[i].beam
assert exp_single[0].detector == exp[i].detector
assert exp_single[0].scan == exp[i].scan
assert exp_single[0].goniometer == exp[i].goniometer
assert exp_single[0].imageset == exp[i].imageset
assert len(ref_single) == len(ref.select(ref['id'] == i))
assert ref_single['id'].all_eq(0)
cmd = " ".join([
"dials.split_experiments", "combined_experiments.json",
"output.experiments_prefix=test"
])
result = easy_run.fully_buffered(command=cmd).raise_if_errors()
for i in range(len(exp)):
assert os.path.exists("test_%03d.json" % i)
return
def get_dials_coordinate_frame(experiments_json):
from dxtbx.model.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(experiments_json)
return experiments[0].beam.get_direction(), \
experiments[0].goniometer.get_rotation_axis()
def _refine_onwards(self, experiments, reflections):
"""Perform processing tasks in a sub-directory for files matching the
base_name in the parent directory"""
exp_path = os.path.join("..", experiments)
ref_path = os.path.join("..", reflections)
# Static refinement, two rounds to improve outlier rejection. Increase
# max_iterations from the default
cmd = ("dials.refine {0} {1} max_iterations=100 "
"output.experiments=static_01_refined_experiments.json "
"output.reflections=static_01_refined_reflections.pickle "
"output.log=static_01_dials.refine.log").format(
exp_path, ref_path)
if self._run_one_job(cmd) is None:
return
cmd = ("dials.refine static_01_refined_experiments.json "
"static_01_refined_reflections.pickle "
"max_iterations=100 "
"output.experiments=static_02_refined_experiments.json "
"output.reflections=static_02_refined_reflections.pickle "
"output.log=static_02_dials.refine.log")
if self._run_one_job(cmd) is None:
return
# Print cell and detector geometry
el = ExperimentListFactory.from_json_file(
"static_02_refined_experiments.json")
cell = el[0].crystal.get_unit_cell().parameters()
cell_s = ", ".join(["{:.3f}".format(e) for e in cell])
print("Refined unit cell: ({0})".format(cell_s))
d = self._get_detector_geometry(el[0].detector)
print("Detector origin: ({0:.5f}, {1:.5f}, {2:.5f})".format(
*d["origin"].elems))
print("Detector fast ax: ({0:.5f}, {1:.5f}, {2:.5f})".format(
*d["fast"].elems))
print("Detector slow ax: ({0:.5f}, {1:.5f}, {2:.5f})".format(
*d["slow"].elems))
print("Detector distance: {0}".format(d["distance"]))
# Scan-varying refinement
cmd = ("dials.refine static_02_refined_experiments.json "
"static_02_refined_reflections.pickle "
"scan_varying=true "
"max_iterations=100 "
"output.experiments=sv_refined_experiments.json "
"output.reflections=sv_refined_reflections.pickle "
"output.log=sv_dials.refine.log")
if self._run_one_job(cmd) is None:
return
# Plot scan-varying crystal
cmd = "dials.plot_scan_varying_crystal sv_refined_experiments.json"
self._run_one_job(cmd)
# Print scan-varying cell and new detector geometry
el = ExperimentListFactory.from_json_file(
"sv_refined_experiments.json")
xl = el[0].crystal
abc = flex.vec3_double()
angles = flex.vec3_double()
for n in range(xl.num_scan_points):
a, b, c, alpha, beta, gamma = xl.get_unit_cell_at_scan_point(
n).parameters()
abc.append((a, b, c))
angles.append((alpha, beta, gamma))
a, b, c = abc.mean()
alpha, beta, gamma = angles.mean()
cell_s = ", ".join(
["{:.3f}".format(e) for e in [a, b, c, alpha, beta, gamma]])
print("Average refined unit cell: ({0})".format(cell_s))
d = self._get_detector_geometry(el[0].detector)
print("Detector origin: ({0:.5f}, {1:.5f}, {2:.5f})".format(
*d["origin"].elems))
print("Detector fast ax: ({0:.5f}, {1:.5f}, {2:.5f})".format(
*d["fast"].elems))
print("Detector slow ax: ({0:.5f}, {1:.5f}, {2:.5f})".format(
*d["slow"].elems))
print("Detector distance: {0}".format(d["distance"]))
# integrate
cmd = ("dials.integrate sv_refined_experiments.json "
"sv_refined_reflections.pickle nproc={0} "
"prediction.d_min={1}").format(self.params.nproc,
self.params.integrate.d_min)
if self._run_one_job(cmd) is None:
return
# FIXME print out some statistics from integration
# export
cmd = ("dials.export integrated_experiments.json integrated.pickle "
"mtz.ignore_panels=true")
if self._run_one_job(cmd) is None:
return
# pointless
cmd = "pointless hklin integrated.mtz hklout sorted.mtz > pointless.log"
if self._run_one_job(cmd) is None:
return
# aimless
cmd = "aimless hklin sorted.mtz hklout scaled.mtz > aimless.log"
keywords = ["resolution 80.0 2.0"]
if self._run_one_job(cmd, keywords=keywords) is None:
return
print("DONE")
return
z = self.im.get_array()[int(y), int(x)]
print(z)
return "x={:.01f}, y={:.01f}, z={:f}".format(x, y, z)
from matplotlib import pylab
fig, ax = pylab.subplots()
im = ax.imshow(data.as_numpy_array(), interpolation="none")
ax.format_coord = Formatter(im)
pylab.show()
if __name__ == "__main__":
import sys
from dxtbx.model.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(sys.argv[1])
# volume1 = pixel_volume_all_pixels(experiments[0])
volume2 = monte_carlo_pixel_volume_all_pixels(experiments[0])
import cPickle as pickle
# pickle.dump(volume1, open("volume.pickle", "w"))
# pickle.dump(volume2, open("monte_carlo_volume.pickle", "w"))
# volume1 = pickle.load(open("volume.pickle"))
# display(volume1)
def test_dials_cluster_unit_cell_command_line(dials_regression, run_in_tmpdir):
pytest.importorskip("scipy")
pytest.importorskip("xfel")
data_dir = os.path.join(dials_regression, "refinement_test_data",
"multi_narrow_wedges")
import glob
experiments = glob.glob(
os.path.join(data_dir, "data/sweep_*/experiments.json"))
import procrunner
result = procrunner.run(
command=["dials.cluster_unit_cell", "plot.show=False"] + experiments,
print_stdout=False,
)
assert not result.returncode
# print result
assert os.path.exists("cluster_unit_cell.png")
from dxtbx.model.experiment_list import ExperimentList, ExperimentListFactory
from dials.command_line import cluster_unit_cell
experiments = ExperimentList([
ExperimentListFactory.from_json_file(expt, check_format=False)[0]
for expt in experiments
])
from cctbx import crystal
crystal_symmetries = [
crystal.symmetry(
unit_cell=expt.crystal.get_unit_cell(),
space_group=expt.crystal.get_space_group(),
) for expt in experiments
]
params = cluster_unit_cell.phil_scope.extract()
params.plot.show = False
params.plot.name = None
clusters = cluster_unit_cell.do_cluster_analysis(crystal_symmetries,
params)
assert len(clusters) == 1
cluster = clusters[0]
assert len(cluster.members) == 40
assert cluster.medians == pytest.approx(
[
90.9430182020995,
90.9430182020995,
90.9430182020995,
109.47122063449069,
109.47122063449069,
109.47122063449069,
],
abs=1e-6,
)
assert cluster.stdevs == pytest.approx(
[
0.09509739126548639, 0.09509739126548526, 0.0950973912654865, 0, 0,
0
],
abs=1e-6,
)
except ImportError:
size = 1
rank = 0
has_mpi = False
if rank == 0:
print(args)
if args.sanityplots:
import pylab as plt
fig = plt.figure()
ax = plt.gca()
# Load in the reflection tables and experiment lists
if rank == 0:
print("Reading in the files")
El = ExperimentListFactory.from_json_file(
args.filteredexpt, check_format=True) # check format is it slow ??
Rmaster_strong = flex.reflection_table.from_file(args.filteredstrong)
Rmaster_int = flex.reflection_table.from_file(args.filteredint)
Nexper = len(El)
print("Read int %d experiments" % Nexper)
# get the original indexing directory name
# NOTE : forgot what this was doing???
indexdirname = args.indexdirname
if args.indexdirname is None:
indexdirname = os.path.dirname(args.filteredexpt)
if args.noiseless:
GAIN = 1
if not os.path.exists(args.o) and rank == 0:
os.makedirs(args.o)
from two_color.two_color_phil import params as index_params
from cxid9114.utils import open_flex
num_imgs = 1
import os
datadir = os.environ.get("DD3")
#DD = "/net/dials/raid1/dermen/cyto/"
######################
# LOAD JUNGFRAU MODEL
######################
print("Load the big azz detector")
expList_for_DET = ExperimentListFactory.from_json_file(os.path.join(
datadir, "refined_final.expt"),
check_format=False)
DETECTOR = expList_for_DET.detectors()[0]
image_shape = (len(DETECTOR), ) + DETECTOR[0].get_image_size()
###
# load the beam and the crystal from the refined experiment
print("Opening refined exp")
refined_El = ExperimentListFactory.from_json_file(
"/net/dials/raid1/dermen/cyto/idx-run_000795.JF07T32V01_master_00344_integrated.expt"
)
BEAM = refined_El.beams()[0]
iset = refined_El.imagesets()[0]
master_file_path = iset.get_path(0)
print("Opening master load")
master_loader = dxtbx.load(master_file_path)
def test3(dials_regression, tmpdir):
"""Strict check for scan-varying refinement using automated outlier rejection
block width and interval width setting"""
# use the i04_weak_data for this test
data_dir = os.path.join(dials_regression, "refinement_test_data",
"centroid")
experiments_path = os.path.join(data_dir,
"experiments_XPARM_REGULARIZED.json")
pickle_path = os.path.join(data_dir, "spot_all_xds.pickle")
for pth in (experiments_path, pickle_path):
assert os.path.exists(pth)
result = procrunner.run(
(
"dials.refine",
experiments_path,
pickle_path,
"scan_varying=true",
"max_iterations=5",
"output.history=history.json",
"crystal.orientation.smoother.interval_width_degrees=auto",
"crystal.unit_cell.smoother.interval_width_degrees=auto",
),
working_directory=tmpdir,
)
assert not result.returncode and not result.stderr
# load and check results
history = Journal.from_json_file(tmpdir.join("history.json").strpath)
expected_rmsds = [
[0.619507829, 0.351326044, 0.006955399],
[0.174024575, 0.113486044, 0.004704006],
[0.098351363, 0.084052519, 0.002660408],
[0.069202909, 0.072796782, 0.001451734],
[0.064305277, 0.071560831, 0.001165639],
[0.062955462, 0.071315612, 0.001074453],
]
for a, b in zip(history["rmsd"], expected_rmsds):
assert a == pytest.approx(b, abs=1e-6)
# check the refined unit cell
ref_exp = ExperimentListFactory.from_json_file(
tmpdir.join("refined.expt").strpath, check_format=False)[0]
unit_cell = ref_exp.crystal.get_unit_cell().parameters()
assert unit_cell == pytest.approx(
[42.27482, 42.27482, 39.66893, 90.00000, 90.00000, 90.00000], abs=1e-3)
refined_refl = flex.reflection_table.from_file(
tmpdir.join("refined.refl").strpath)
# re-predict reflections using the refined experiments
predicted = flex.reflection_table.from_predictions_multi([ref_exp])
matched, reference, unmatched = predicted.match_with_reference(
refined_refl)
# assert most refined reflections are matched with predictions
assert reference.size() > (0.997 * refined_refl.size())
# second check with nearest neighbour matching that the predictions match up
ann = AnnAdaptor(data=predicted["xyzcal.px"].as_double(), dim=3, k=1)
ann.query(refined_refl["xyzcal.px"].as_double())
assert (ann.distances < 0.5).count(True) > (0.998 * refined_refl.size())
Ntrial = 25
outlier_cutoff = 0.6 # probability to not be an outlier..
szx = szy = 5
Nmos_dom = 100
exp_name = sys.argv[1]
data_name = sys.argv[2]
cuda = int(sys.argv[3])
outdir = sys.argv[4]
if not os.path.exists(outdir):
os.makedirs(outdir)
output_basename = os.path.basename(data_name).replace(".pkl", "_jitt.pkl")
outputname = os.path.join(outdir, output_basename)
exp_lst = ExperimentListFactory.from_json_file(
exp_name) #, check_format=False)
iset = exp_lst.imagesets()[0]
data = utils.open_flex(data_name)
waveA = parameters.ENERGY_CONV / ENERGIES[0]
waveB = parameters.ENERGY_CONV / ENERGIES[1]
ENERGIES = [parameters.ENERGY_LOW,
parameters.ENERGY_HIGH] # colors of the beams
FF = [10000,
None] # Setting structure factors takes long time in nanoBragg, so
# unless you want energy-dependent structure factors
# you need only provide one number -or- one structure factor flex miller table
# and the computer will know to preserve that for all beam colors
FLUX = [1e12, 1e12] # fluxes of the beams
waveA = parameters.ENERGY_CONV / ENERGIES[0]
def test(dials_regression):
from scitbx import matrix
from libtbx.phil import parse
from libtbx.test_utils import approx_equal
from scitbx.array_family import flex
# Get modules to build models and minimiser using PHIL
from dials.test.algorithms.refinement import setup_geometry
from dials.test.algorithms.refinement import setup_minimiser
from dials.algorithms.refinement.parameterisation.crystal_parameters import \
CrystalOrientationParameterisation, CrystalUnitCellParameterisation
# Symmetry constrained parameterisation for the unit cell
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import \
symmetrize_reduce_enlarge
DEG2RAD = math.pi/180.0
RAD2DEG = 180.0/math.pi
master_phil = parse("""
include scope dials.test.algorithms.refinement.geometry_phil
include scope dials.test.algorithms.refinement.minimiser_phil
""", process_includes=True)
# make cell more oblique
args=["a.direction.close_to.sd=5","b.direction.close_to.sd=5","c.direction.close_to.sd=5"]
models = setup_geometry.Extract(master_phil, cmdline_args = args)
crystal = models.crystal
# a hexagonal crystal is a good test case for behaviour of oblique cells
do_hexagonal = True
if do_hexagonal:
from dxtbx.model.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(
os.path.join(dials_regression, "refinement_test_data", "multi_stills", "combined_experiments.json"),
check_format=False)
crystal = experiments[0].crystal
# derive finite difference gradients of various quantities wrt each param
def check_fd_gradients(parameterisation):
mp = parameterisation
p_vals = mp.get_param_vals()
deltas = [1.e-7 for p in p_vals]
assert len(deltas) == len(p_vals)
fd_grad = []
# get matrix to unset rotations of unit cell vectors
Ut = matrix.sqr(mp.get_model().get_U()).transpose()
for i, delta in enumerate(deltas):
val = p_vals[i]
p_vals[i] -= delta / 2.
mp.set_param_vals(p_vals)
rev_uc = mp.get_model().get_unit_cell().parameters()
rev_vec = mp.get_model().get_real_space_vectors()
rev_vec = [Ut * vec for vec in rev_vec]
rev_B = matrix.sqr(mp.get_model().get_B())
rev_O = rev_B.transpose().inverse()
p_vals[i] += delta
mp.set_param_vals(p_vals)
fwd_uc = mp.get_model().get_unit_cell().parameters()
fwd_vec = mp.get_model().get_real_space_vectors()
fwd_vec = [Ut * vec for vec in fwd_vec]
fwd_B = matrix.sqr(mp.get_model().get_B())
fwd_O = fwd_B.transpose().inverse()
fd_uc = [(f - r) / delta for f,r in zip(fwd_uc, rev_uc)]
fd_vec = [(f - r) / delta for f,r in zip(fwd_vec, rev_vec)]
fd_B = (fwd_B - rev_B) / delta
fd_O = (fwd_O - rev_O) / delta
fd_grad.append({'da_dp':fd_uc[0],
'db_dp':fd_uc[1],
'dc_dp':fd_uc[2],
'daa_dp':fd_uc[3],
'dbb_dp':fd_uc[4],
'dcc_dp':fd_uc[5],
'davec_dp':fd_vec[0],
'dbvec_dp':fd_vec[1],
'dcvec_dp':fd_vec[2],
'dB_dp':fd_B,
'dO_dp':fd_O})
p_vals[i] = val
# return to the initial state
mp.set_param_vals(p_vals)
return fd_grad
xlo_param = CrystalOrientationParameterisation(crystal)
xluc_param = CrystalUnitCellParameterisation(crystal)
from dials.algorithms.refinement.restraints.restraints import SingleUnitCellTie
uct = SingleUnitCellTie(xluc_param, [None]*6, [None]*6)
from scitbx.math import angle_derivative_wrt_vectors
B = matrix.sqr(crystal.get_B())
O = (B.transpose()).inverse()
a, b, c, aa, bb, cc = crystal.get_unit_cell().parameters()
aa *= DEG2RAD
bb *= DEG2RAD
cc *= DEG2RAD
Ut = matrix.sqr(crystal.get_U()).transpose()
avec, bvec, cvec = [Ut * vec for vec in crystal.get_real_space_vectors()]
# calculate d[B^T]/dp
dB_dp = xluc_param.get_ds_dp()
dBT_dp = [dB.transpose() for dB in dB_dp]
# calculate d[O]/dp
dO_dp = [-O * dBT * O for dBT in dBT_dp]
# function to get analytical derivative of angles wrt vectors
def dangle(u, v):
return [matrix.col(e) for e in angle_derivative_wrt_vectors(u,v)]
dalpha_db, dalpha_dc = dangle(bvec, cvec)
dbeta_da, dbeta_dc = dangle(avec, cvec)
dgamma_da, dgamma_db = dangle(avec, bvec)
# get all FD derivatives
fd_grad = check_fd_gradients(xluc_param)
# look at each parameter
for i, dO in enumerate(dO_dp):
#print
#print "***** PARAMETER {0} *****".format(i)
#print "dB_dp analytical"
#print dB_dp[i]
#print "dB_dp FD"
#print fd_grad[i]['dB_dp']
#print
# dB_dp is good. What about dO_dp?
#print "O MATRIX"
#print "dO_dp analytical"
#print dO.round(6)
#print "dO_dp FD"
#print fd_grad[i]['dO_dp'].round(6)
#print
assert approx_equal(dO, fd_grad[i]['dO_dp'])
# extract derivatives of each unit cell vector wrt p
dav_dp, dbv_dp, dcv_dp = dO.transpose().as_list_of_lists()
dav_dp = matrix.col(dav_dp)
dbv_dp = matrix.col(dbv_dp)
dcv_dp = matrix.col(dcv_dp)
# check these are correct vs FD
#print "CELL VECTORS"
#diff = dav_dp - fd_grad[i]['davec_dp']
#print 2 * diff.length() / (dav_dp.length() + fd_grad[i]['davec_dp'].length()) * 100
#print 'davec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dav_dp.elems)
#print 'davec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['davec_dp'].elems)
assert approx_equal(dav_dp, fd_grad[i]['davec_dp'])
#diff = dbv_dp - fd_grad[i]['dbvec_dp']
#print 2 * diff.length() / (dbv_dp.length() + fd_grad[i]['dbvec_dp'].length()) * 100
#print 'dbvec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dbv_dp.elems)
#print 'dbvec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['dbvec_dp'].elems)
assert approx_equal(dbv_dp, fd_grad[i]['dbvec_dp'])
#diff = dcv_dp - fd_grad[i]['dcvec_dp']
#print 2 * diff.length() / (dcv_dp.length() + fd_grad[i]['dcvec_dp'].length()) * 100
#print 'dcvec_dp analytical: {0:.5f} {1:.5f} {2:.5f}'.format(*dcv_dp.elems)
#print 'dcvec_dp finite diff: {0:.5f} {1:.5f} {2:.5f}'.format(*fd_grad[i]['dcvec_dp'].elems)
#print
assert approx_equal(dcv_dp, fd_grad[i]['dcvec_dp'])
#print "CELL LENGTHS"
da_dp = 1./a * avec.dot(dav_dp)
#print "d[a]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(da_dp, fd_grad[i]['da_dp'], i)
assert approx_equal(da_dp, fd_grad[i]['da_dp'])
db_dp = 1./b * bvec.dot(dbv_dp)
#print "d[b]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(db_dp, fd_grad[i]['db_dp'], i)
assert approx_equal(db_dp, fd_grad[i]['db_dp'])
dc_dp = 1./c * cvec.dot(dcv_dp)
#print "d[c]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dc_dp, fd_grad[i]['dc_dp'], i)
assert approx_equal(dc_dp, fd_grad[i]['dc_dp'])
#print
#print "CELL ANGLES"
daa_dp = RAD2DEG * (dbv_dp.dot(dalpha_db) + dcv_dp.dot(dalpha_dc))
dbb_dp = RAD2DEG * (dav_dp.dot(dbeta_da) + dcv_dp.dot(dbeta_dc))
dcc_dp = RAD2DEG * (dav_dp.dot(dgamma_da) + dbv_dp.dot(dgamma_db))
#print "d[alpha]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(daa_dp, fd_grad[i]['daa_dp'], i)
#print "d[beta]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dbb_dp, fd_grad[i]['dbb_dp'], i)
#print "d[gamma]/dp{2} analytical: {0:.5f} FD: {1:.5f}".format(dcc_dp, fd_grad[i]['dcc_dp'], i)
assert approx_equal(daa_dp, fd_grad[i]['daa_dp'])
assert approx_equal(dbb_dp, fd_grad[i]['dbb_dp'])
assert approx_equal(dcc_dp, fd_grad[i]['dcc_dp'])
def test(dials_data):
from dxtbx.model.experiment_list import ExperimentListFactory
from dxtbx.model.experiment_list import ExperimentList
exlist = ExperimentListFactory.from_json_file(
dials_data("centroid_test_data").join(
"fake_long_experiments.json").strpath)
assert len(exlist) == 1
experiment = exlist[0]
# Set the delta_divergence/mosaicity
n_sigma = 5
sigma_b = 0.060 * math.pi / 180
sigma_m = 0.154 * math.pi / 180
from dials.algorithms.profile_model.gaussian_rs import Model
profile_model = Model(None, n_sigma, sigma_b, sigma_m)
experiment.profile = profile_model
experiments = ExperimentList()
experiments.append(experiment)
from dials.algorithms.profile_model.gaussian_rs import PartialityCalculator3D
from dials.array_family import flex
calculator = PartialityCalculator3D(experiment.beam, experiment.goniometer,
experiment.scan, sigma_m)
predicted = flex.reflection_table.from_predictions_multi(experiments)
predicted["bbox"] = predicted.compute_bbox(experiments)
# Remove any touching edges of scan to get only fully recorded
x0, x1, y0, y1, z0, z1 = predicted["bbox"].parts()
predicted = predicted.select((z0 > 0) & (z1 < 100))
assert len(predicted) > 0
# Compute partiality
partiality = calculator(predicted["s1"], predicted["xyzcal.px"].parts()[2],
predicted["bbox"])
# Should have all fully recorded
assert len(partiality) == len(predicted)
three_sigma = 0.5 * (math.erf(3.0 / math.sqrt(2.0)) -
math.erf(-3.0 / math.sqrt(2.0)))
assert partiality.all_gt(three_sigma)
# Trim bounding boxes
x0, x1, y0, y1, z0, z1 = predicted["bbox"].parts()
z0 = z0 + 1
z1 = z1 - 1
predicted["bbox"] = flex.int6(x0, x1, y0, y1, z0, z1)
predicted = predicted.select(z1 > z0)
assert len(predicted) > 0
# Compute partiality
partiality = calculator(predicted["s1"], predicted["xyzcal.px"].parts()[2],
predicted["bbox"])
# Should have all partials
assert len(partiality) == len(predicted)
assert partiality.all_le(1.0) and partiality.all_gt(0)
def test_run(dials_regression, run_in_tmpdir):
from dials.util.nexus import dump, load
from dxtbx.model.experiment_list import ExperimentListFactory
from dials.array_family import flex
from os.path import join
path = join(dials_regression, "nexus_test_data")
# Read the experiments
experiments1 = ExperimentListFactory.from_json_file(
join(path, "refined_experiments.json")
)
# Read the reflections
reflections1 = flex.reflection_table.from_file(join(path, "integrated.pickle"))
# Delete some columns for the test
del reflections1["s1"]
del reflections1["zeta"]
del reflections1["background.mse"]
# Dump the reflections
dump(experiments1, reflections1, "hklout.nxs")
# Load them again
experiments2, reflections2 = load("hklout.nxs")
EPS = 1e-7
# Check the reflections are OK
assert reflections1.nrows() == reflections2.nrows()
assert reflections1.ncols() == reflections2.ncols()
for key in reflections1.keys():
data1 = reflections1[key]
data2 = reflections2[key]
assert data1.__class__ == data2.__class__
if isinstance(data1, flex.double):
assert data1.all_approx_equal(data2)
elif isinstance(data1, flex.int6):
for p1, p2 in zip(data1.parts(), data2.parts()):
assert p1.all_eq(p2)
elif isinstance(data1, flex.vec3_double):
for p1, p2 in zip(data1.parts(), data2.parts()):
assert p1.all_approx_equal(p2)
else:
assert data1.all_eq(data2)
# Test passed
# Check the experiments are ok
assert len(experiments1) == len(experiments2)
exp1 = experiments1[0]
exp2 = experiments2[0]
# Check the beam
b1 = exp1.beam
b2 = exp2.beam
assert all(
abs(d1 - d2) < EPS
for d1, d2 in zip(
b1.get_sample_to_source_direction(), b2.get_sample_to_source_direction()
)
)
assert abs(b1.get_wavelength() - b2.get_wavelength()) < EPS
assert abs(b1.get_polarization_fraction() - b2.get_polarization_fraction()) < EPS
assert all(
abs(d1 - d2) < EPS
for d1, d2 in zip(b1.get_polarization_normal(), b2.get_polarization_normal())
)
# Check the goniometer
g1 = exp1.goniometer
g2 = exp2.goniometer
assert all(
abs(d1 - d2) < EPS
for d1, d2 in zip(g1.get_rotation_axis(), g2.get_rotation_axis())
)
assert all(
abs(d1 - d2) < EPS
for d1, d2 in zip(g1.get_fixed_rotation(), g2.get_fixed_rotation())
)
assert all(
abs(d1 - d2) < EPS
for d1, d2 in zip(g1.get_setting_rotation(), g2.get_setting_rotation())
)
# Check the scan
s1 = exp1.scan
s2 = exp2.scan
assert len(s1) == len(s2)
assert s1.get_image_range() == s2.get_image_range()
assert abs(s1.get_oscillation()[0] - s1.get_oscillation()[0]) < EPS
assert abs(s1.get_oscillation()[1] - s1.get_oscillation()[1]) < EPS
for e1, e2 in zip(s1.get_exposure_times(), s2.get_exposure_times()):
assert abs(e1 - e2) < EPS
for e1, e2 in zip(s1.get_epochs(), s2.get_epochs()):
assert abs(e1 - e2) < EPS
# Check the detector
d1 = exp1.detector
d2 = exp2.detector
assert len(d1) == len(d2)
for p1, p2 in zip(d1, d2):
assert p1.get_type() == p2.get_type()
assert p1.get_material() == p2.get_material()
assert p1.get_thickness() == p2.get_thickness()
assert p1.get_image_size() == p2.get_image_size()
assert p1.get_pixel_size() == p2.get_pixel_size()
assert p1.get_trusted_range() == p2.get_trusted_range()
for x1, x2 in zip(p1.get_fast_axis(), p2.get_fast_axis()):
assert abs(x1 - x2) < EPS
for x1, x2 in zip(p1.get_slow_axis(), p2.get_slow_axis()):
assert abs(x1 - x2) < EPS
for x1, x2 in zip(p1.get_origin(), p2.get_origin()):
assert abs(x1 - x2) < EPS
# Check the crystal
c1 = exp1.crystal
c2 = exp2.crystal
assert c1.get_space_group() == c2.get_space_group()
for p1, p2 in zip(c1.get_unit_cell().parameters(), c2.get_unit_cell().parameters()):
assert abs(p1 - p2) < EPS
for p1, p2 in zip(c1.get_A(), c2.get_A()):
assert abs(p1 - p2) < EPS
assert c1.num_scan_points == c2.num_scan_points
for i in range(c1.num_scan_points):
A1 = c1.get_A_at_scan_point(i)
A2 = c2.get_A_at_scan_point(i)
for a1, a2 in zip(A1, A2):
assert abs(a1 - a2) < EPS
uc1 = c1.get_unit_cell_at_scan_point(i)
uc2 = c2.get_unit_cell_at_scan_point(i)
for p1, p2 in zip(uc1.parameters(), uc2.parameters()):
assert abs(p1 - p2) < EPS
def build_hist(nproc=1):
from scitbx.array_family import flex
from libtbx import easy_mp
from collections import Counter
# FIXME use proper optionparser here. This works for now
if len(sys.argv) >= 2 and sys.argv[1].startswith("nproc="):
nproc = int(sys.argv[1][6:])
sys.argv = sys.argv[1:]
if len(sys.argv) == 2 and sys.argv[1].endswith((".expt", ".json")):
from dxtbx.model.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(sys.argv[1])
image_list = experiments.imagesets()[0].paths()
else:
image_list = sys.argv[1:]
image_count = len(image_list)
# Faster, yet still less than ideal and wasting a lot of resources.
limit = get_overload(image_list[0])
binfactor = 5 # register up to 500% counts
histmax = (limit * binfactor) + 0.0
histbins = int(limit * binfactor) + 1
use_python_counter = histbins > 90000000 # empirically determined
print("Processing %d images in %d processes using %s\n" % (
image_count,
nproc,
"python Counter" if use_python_counter else "flex arrays",
))
def process_image(process):
last_update = start = timeit.default_timer()
i = process
if use_python_counter:
local_hist = Counter()
else:
local_hist = flex.histogram(flex.double(),
data_min=0.0,
data_max=histmax,
n_slots=histbins)
max_images = image_count // nproc
if process >= image_count % nproc:
max_images += 1
while i < image_count:
data = read_cbf_image(image_list[i])
if not use_python_counter:
data = flex.histogram(
data.as_double().as_1d(),
data_min=0.0,
data_max=histmax,
n_slots=histbins,
)
local_hist.update(data)
i = i + nproc
if process == 0:
if timeit.default_timer() > (last_update + 3):
last_update = timeit.default_timer()
if sys.stdout.isatty():
sys.stdout.write("\033[A")
print("Processed %d%% (%d seconds remain) " % (
100 * i // image_count,
round((image_count - i) * (last_update - start) /
(i + 1)),
))
return local_hist
results = easy_mp.parallel_map(
func=process_image,
iterable=range(nproc),
processes=nproc,
preserve_exception_message=True,
)
print("Merging results")
result_hist = None
for hist in results:
if result_hist is None:
result_hist = hist
else:
result_hist.update(hist)
if not use_python_counter:
# reformat histogram into dictionary
result = list(result_hist.slots())
result_hist = {b: count for b, count in enumerate(result) if count > 0}
results = {
"scale_factor": 1 / limit,
"overload_limit": limit,
"counts": result_hist,
}
print("Writing results to overload.json")
with open("overload.json", "w") as fh:
json.dump(results, fh, indent=1, sort_keys=True)
def test_stills_indexer_multi_lattice_bug_MosaicSauter2014(
dials_regression, tmpdir):
"""Problem: In stills_indexer, before calling the refine function, the
experiment list contains a list of dxtbx crystal models (that are not
MosaicSauter2014 models). The conversion to MosaicSauter2014 is made
during the refine step when functions from nave_parameters is called.
If the experiment list contains more than 1 experiment, for eg.
multiple lattices, only the first crystal gets assigned mosaicity. In
actuality, all crystal models should be assigned mosaicity. This test
only compares whether or not all crystal models have been assigned a
MosaicSauter2014 model."""
import dxtbx.model
from dxtbx.model.experiment_list import ExperimentListFactory
from dxtbx.model.experiment_list import Experiment, ExperimentList
from dials.array_family import flex
from dxtbx.model import Crystal
from dials.algorithms.indexing.stills_indexer import StillsIndexer
from dials.command_line.stills_process import (
phil_scope as stills_process_phil_scope, )
experiment_data = os.path.join(
dials_regression,
"refinement_test_data",
"cspad_refinement",
"cspad_refined_experiments_step6_level2_300.json",
)
reflection_data = os.path.join(
dials_regression,
"refinement_test_data",
"cspad_refinement",
"cspad_reflections_step7_300.pickle",
)
refl = flex.reflection_table.from_file(reflection_data)
explist = ExperimentListFactory.from_json_file(experiment_data,
check_format=False)[0:2]
reflist = refl.select(
refl["id"] < 2) # Only use the first 2 for convenience
# Construct crystal models that don't have mosaicity. These A,B,C values are the same
# as read in from the dials_regression folder
# Crystal-0
cs0 = Crystal(explist[0].crystal)
exp0 = Experiment(
imageset=explist[0].imageset,
beam=explist[0].beam,
detector=explist[0].detector,
goniometer=None,
scan=None,
crystal=cs0,
)
# Crystal-1
cs1 = Crystal(explist[1].crystal)
exp1 = Experiment(
imageset=explist[1].imageset,
beam=explist[1].beam,
detector=explist[1].detector,
goniometer=None,
scan=None,
crystal=cs1,
)
# Construct a new experiment_list that will be passed on for refinement
unrefined_explist = ExperimentList([exp0, exp1])
# Get default params from stills_process and construct StillsIndexer, then run refinement
params = stills_process_phil_scope.extract()
SI = StillsIndexer(reflist, unrefined_explist, params=params)
refined_explist, new_reflist = SI.refine(unrefined_explist, reflist)
# Now check whether the models have mosaicity after stills_indexer refinement
# Also check that mosaicity values are within expected limits
for ii, crys in enumerate(refined_explist.crystals()):
assert isinstance(crys, dxtbx.model.MosaicCrystalSauter2014)
if ii == 0:
assert crys.get_domain_size_ang() == pytest.approx(2242.0, rel=0.1)
if ii == 1:
assert crys.get_domain_size_ang() == pytest.approx(2689.0, rel=0.1)
# Output datasize
try:
datasizeidx = [a.find("datasize")==0 for a in args].index(True)
except ValueError:
print "You really should supply datasize, but ok"
# raise ValueError,"Output datasze file must be specified."
else:
datasize = args.pop(datasizeidx).split("=")[1]
import copy, os
import dxtbx
from dxtbx.model.experiment_list import ExperimentListFactory
from dials.array_family import flex
experiments = ExperimentListFactory.from_json_file(json, check_format=False)
beam = experiments[0].beam
detector = experiments[0].detector
lab_coordinates = flex.vec3_double()
for panel in detector:
pixels = flex.vec2_double(panel.get_image_size())
mms = panel.pixel_to_millimeter(pixels)
lab_coordinates.extend(panel.get_lab_coord(mms))
# generate s1 vectors
s1 = lab_coordinates.each_normalize() * (1/beam.get_wavelength())
# Generate x vectors
x = np.asarray(s1 - beam.get_s0())
# DATAsize = np.asarray(detector[0].get_image_size())
def tst_one_monkeypatch(i_exp, spectra, Fmerge, gpu_channels_singleton, rank,
params):
print("IN MONKEYPATCH")
from simtbx.nanoBragg import utils
from dxtbx.model.experiment_list import ExperimentListFactory
import numpy as np
print("Experiment %d" % i_exp, flush=True)
outfile = "boop_%d.hdf5" % i_exp
from LS49.adse13_187.case_data import retrieve_from_repo
experiment_file = retrieve_from_repo(i_exp)
cuda = True # False # whether to use cuda
omp = False
ngpu_on_node = 1 # 8 # number of available GPUs
mosaic_spread = 0.07 # degrees
mosaic_spread_samples = params.mosaic_spread_samples # number of mosaic blocks sampling mosaicity
Ncells_abc = 30, 30, 10 # medians from best stage1
ev_res = 1.5 # resolution of the downsample spectrum
total_flux = 1e12 # total flux across channels
beamsize_mm = 0.000886226925452758 # sqrt of beam focal area
spot_scale = 500. # 5.16324 # median from best stage1
plot_spec = False # plot the downsample spectra before simulating
oversample = 1 # oversample factor, 1,2, or 3 probable enough
panel_list = None # integer list of panels, usefule for debugging
rois_only = False # only set True if you are running openMP, or CPU-only (i.e. not for GPU)
include_background = params.include_background # default is to add water background 100 mm thick
verbose = 0 # leave as 0, unles debug
flat = True # enfore that the camera has 0 thickness
#<><><><><><><><>
# XXX new code
El = ExperimentListFactory.from_json_file(experiment_file,
check_format=True)
exper = El[0]
crystal = exper.crystal
detector = exper.detector
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
beam = exper.beam
# XXX new code
spec = exper.imageset.get_spectrum(0)
energies_raw, weights_raw = spec.get_energies_eV().as_numpy_array(), \
spec.get_weights().as_numpy_array()
energies, weights = utils.downsample_spectrum(energies_raw,
weights_raw,
method=1,
total_flux=total_flux,
ev_width=ev_res)
if flat:
assert detector[0].get_thickness() == 0
if panel_list is None:
panel_list = list(range(len(detector)))
pids_for_rank = panel_list
device_Id = 0
if gpu_channels_singleton is not None:
device_Id = gpu_channels_singleton.get_deviceID()
print("Rank %d will use device %d" % (rank, device_Id))
show_params = False
time_panels = (rank == 0)
mn_energy = (energies * weights).sum() / weights.sum()
mn_wave = utils.ENERGY_CONV / mn_energy
print(
"\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print("\tBreakdown:")
for shapetype in ["gauss_argchk"]:
BEG = time()
print(gpu_channels_singleton.get_deviceID(), "device", shapetype)
Famp_is_uninitialized = (gpu_channels_singleton.get_nchannels() == 0)
if Famp_is_uninitialized:
F_P1 = Fmerge.expand_to_p1()
for x in range(
1
): # in this scenario, amplitudes are independent of lambda
gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert gpu_channels_singleton.get_nchannels() == 1
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=crystal,
DETECTOR=detector,
BEAM=beam,
Famp=gpu_channels_singleton,
energies=list(energies),
fluxes=list(weights),
background_wavelengths=[mn_wave],
background_wavelength_weights=[1],
background_total_flux=total_flux,
background_sample_thick_mm=0.5,
cuda=True,
oversample=oversample,
Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples,
mos_spread=mosaic_spread,
mosaic_method="double_random",
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=show_params,
time_panels=time_panels,
verbose=verbose,
spot_scale_override=spot_scale,
include_background=include_background,
mask_file=params.mask_file)
TIME_EXA = time() - BEG
print(
"\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs"
% (TIME_BG, TIME_BRAGG, TIME_EXA))
print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>\n")
if params.write_output:
if params.write_experimental_data:
data = exper.imageset.get_raw_data(0)
img_sh = JF16M_numpy_array.shape
assert img_sh == (256, 254, 254)
num_output_images = 1 + int(params.write_experimental_data)
print("Saving exascale output data of shape", img_sh)
beam_dict = beam.to_dict()
det_dict = detector.to_dict()
try:
beam_dict.pop("spectrum_energies")
beam_dict.pop("spectrum_weights")
except Exception:
pass
with utils.H5AttributeGeomWriter(
os.path.join(params.log.outdir, "exap_%d.hdf5" % i_exp),
image_shape=img_sh,
num_images=num_output_images,
detector=det_dict,
beam=beam_dict,
detector_and_beam_are_dicts=True) as writer:
writer.add_image(JF16M_numpy_array)
if params.write_experimental_data:
data = [data[pid].as_numpy_array() for pid in panel_list]
writer.add_image(data)
print("Saved output to file %s" % ("exap_%d.hdf5" % i_exp))
if not params.write_output:
# ability to read in the special file format
# note to end-user: The special file format can be installed permanently into a
# developmental version of dials/cctbx:
# dxtbx.install_format ./FormatHDF5AttributeGeometry.py --global # writes to build directory
# or alternatively to the user's account:
# dxtbx.install_format ./FormatHDF5AttributeGeometry.py --user # writes to ~/.dxtbx
from LS49.adse13_187.FormatHDF5AttributeGeometry import FormatHDF5AttributeGeometry as format_instance
from LS49 import ls49_big_data
filename = os.path.join(ls49_big_data, "adse13_228",
"exap_%d.hdf5" % i_exp)
instance = format_instance(filename)
reference = [D.as_numpy_array() for D in instance.get_raw_data()]
print("reference length for %s is %d" %
("exap_%d.hdf5" % i_exp, len(reference)))
# assertion on equality:
abs_diff = np.abs(JF16M_numpy_array - reference).max()
assert np.allclose(JF16M_numpy_array, reference), \
"max per-pixel difference: %f photons, experiment %d"%(abs_diff,i_exp)
