def __init__(
self,
crystal,
beam,
detector,
goniometer,
scan,
reflections,
n_macro_cycles=10,
):
from scitbx import simplex
# Get the oscillation width
dphi2 = scan.get_oscillation(deg=False)[1] / 2.0
# Calculate a list of angles and zeta's
tau, zeta, n, indices = self._calculate_tau_and_zeta(
crystal, beam, detector, goniometer, scan, reflections)
# Calculate zeta * (tau +- dphi / 2) / math.sqrt(2)
self.e1 = (tau + dphi2) * flex.abs(zeta) / math.sqrt(2.0)
self.e2 = (tau - dphi2) * flex.abs(zeta) / math.sqrt(2.0)
self.indices = indices
if len(self.e1) == 0:
raise RuntimeError(
"Something went wrong. Zero pixels selected for estimation of profile parameters."
)
# Compute intensity
self.K = flex.double()
self.nj = []
for i0, i1 in zip(self.indices[:-1], self.indices[1:]):
nj = n[i0:i1]
self.K.append(flex.sum(nj))
self.nj.append(nj)
# Set the starting values to try 1, 3 degrees seems sensible for
# crystal mosaic spread
start = math.log(0.1 * math.pi / 180)
stop = math.log(1 * math.pi / 180)
starting_simplex = [flex.double([start]), flex.double([stop])]
# Initialise the optimizer
optimizer = simplex.simplex_opt(1,
matrix=starting_simplex,
evaluator=self,
tolerance=1e-3)
# Get the solution
sigma = math.exp(optimizer.get_solution()[0])
# Save the result
self.sigma = sigma
def lorentz_callable(self, values):
Rh = self.get_Rh_array(values)
Rs = flex.double(len(self.MILLER), 1. / values.DEFF) + flex.double(
len(self.MILLER), values.ETA / 2.) / self.DVEC
ratio = Rh / Rs
ratio_abs = flex.abs(ratio)
return ratio_abs
def _reorient_coordinate_frame(self):
"""Align a DIALS experiment and data in a reflection table to the
PETS coordinate system. In that system, the s0 vector is aligned with
-Z, while the rotation axis is in the X-Z plane, close to +X"""
axis = matrix.col(self.experiment.goniometer.get_rotation_axis())
us0 = matrix.col(self.experiment.beam.get_unit_s0())
normal = us0.cross(-axis).normalize()
orthogonalised_axis = us0.cross(normal).normalize()
# Keep track of s1.us0 values to check reorientation
s1_dot_us0 = self.reflections["s1"].dot(us0)
R = align_reference_frame(us0, (0, 0, -1), orthogonalised_axis,
(1, 0, 0))
axis_angle = r3_rotation_axis_and_angle_from_matrix(R)
axis = axis_angle.axis
angle = axis_angle.angle(deg=False)
logger.info(
"Rotating experiment about axis ({:.4f}, {:.4f}, {:.4f}) by {:.3f}°"
.format(*axis, np.degrees(angle)))
self.experiment.detector.rotate_around_origin(axis, angle, deg=False)
self.experiment.crystal = rotate_crystal(self.experiment.crystal, R,
axis, angle)
# Following does not work (https://github.com/cctbx/dxtbx/issues/454)
# self.experiment.beam.rotate_around_origin(axis, angle, deg=False)
# Set unit s0 and polarization normal directly instead (preserving inconsistent
# beam, if that's what we have).
new_us0 = (R *
matrix.col(self.experiment.beam.get_unit_s0())).normalize()
new_p_norm = (R * matrix.col(
self.experiment.beam.get_polarization_normal())).normalize()
self.experiment.beam.set_unit_s0(new_us0)
self.experiment.beam.set_polarization_normal(new_p_norm)
# Rotating the goniometer is also complicated. See https://github.com/cctbx/dxtbx/pull/451
new_datum = (R * matrix.col(
self.experiment.goniometer.get_rotation_axis_datum())).normalize()
new_F = (R *
matrix.sqr(self.experiment.goniometer.get_fixed_rotation()) *
R.transpose())
new_S = (
R * matrix.sqr(self.experiment.goniometer.get_setting_rotation()) *
R.transpose())
self.experiment.goniometer.set_rotation_axis_datum(new_datum)
self.experiment.goniometer.set_setting_rotation(new_S)
self.experiment.goniometer.set_fixed_rotation(new_F)
# Re-calculate s1 vectors and reciprocal lattice points with new geometry
el = ExperimentList()
el.append(self.experiment)
self.reflections.map_centroids_to_reciprocal_space(el, calculated=True)
error = flex.abs(s1_dot_us0 - self.reflections["s1"].dot(new_us0))
if flex.max(error) > 1e-10:
raise RuntimeError("Failed to rotate experiment correctly")
def _filter_reflections(self, params, experiments, reflections):
''' Filter the reflections to integrate. '''
from dials.util.command_line import Command
from dials.algorithms import filtering
from dials.array_family import flex
# Set all reflections which overlap bad pixels to zero
Command.start('Filtering reflections by detector mask')
if experiments[0].scan == None:
array_range = 1
else:
array_range = experiments[0].scan.get_array_range()
mask = filtering.by_detector_mask(
reflections['bbox'],
experiments[0].imageset.get_raw_data(0)[0] >= 0, array_range)
reflections.del_selected(not mask)
Command.end('Filtered %d reflections by detector mask' %
len(reflections))
# Filter the reflections by zeta
min_zeta = params.integration.filter.by_zeta
if min_zeta > 0:
Command.start('Filtering reflections by zeta >= %f' % min_zeta)
zeta = reflections.compute_zeta(experiments[0])
reflections.del_selected(flex.abs(zeta) < min_zeta)
n = len(reflections)
Command.end('Filtered %d reflections by zeta >= %f' %
(n, min_zeta))
return reflections
def test_multi_sweep(dials_regression, tmpdir):
tmpdir.chdir()
# Call dials.integrate
result = procrunner.run_process([
'dials.integrate',
os.path.join(dials_regression, "integration_test_data", 'multi_sweep', 'experiments.json'),
os.path.join(dials_regression, "integration_test_data", 'multi_sweep', 'indexed.pickle'),
'prediction.padding=0',
])
assert result['exitcode'] == 0
assert result['stderr'] == ''
assert os.path.exists('integrated.pickle')
with open('integrated.pickle', 'rb') as fh:
table = pickle.load(fh)
assert len(table) == 4020
# Check the results
T1 = table[:2010]
T2 = table[2010:]
ID1 = list(set(T1['id']))
ID2 = list(set(T2['id']))
assert len(ID1) == 1
assert len(ID2) == 1
assert ID1[0] == 0
assert ID2[0] == 1
I1 = T1['intensity.prf.value']
I2 = T2['intensity.prf.value']
F1 = T1.get_flags(T1.flags.integrated_prf)
F2 = T2.get_flags(T2.flags.integrated_prf)
assert F1 == F2
I1 = I1.select(F1)
I2 = I2.select(F2)
assert flex.abs(I1 - I2) < 1e-6
def tst_flatten(self):
from dials.array_family import flex
from dials.algorithms.shoebox import MaskCode
for shoebox, (XC, I) in self.random_shoeboxes(10, mask=True):
assert (not shoebox.flat)
zs = shoebox.zsize()
ys = shoebox.ysize()
xs = shoebox.xsize()
expected_data = flex.real(flex.grid(1, ys, xs), 0)
expected_mask = flex.int(flex.grid(1, ys, xs), 0)
for k in range(zs):
for j in range(ys):
for i in range(xs):
expected_data[0, j, i] += shoebox.data[k, j, i]
expected_mask[0, j, i] |= shoebox.mask[k, j, i]
if (not (expected_mask[0, j, i] & MaskCode.Valid) or
not (shoebox.mask[k, j, i] & MaskCode.Valid)):
expected_mask[0, j, i] &= ~MaskCode.Valid
shoebox.flatten()
diff = expected_data.as_double() - shoebox.data.as_double()
max_diff = flex.max(flex.abs(diff))
assert (max_diff < 1e-7)
assert (expected_mask.all_eq(shoebox.mask))
assert (shoebox.flat)
assert (shoebox.is_consistent())
print 'OK'
def test_run(dials_regression):
filename = os.path.join(dials_regression, 'centroid_test_data',
'experiments.json')
from dxtbx.model.experiment_list import ExperimentListFactory
exlist = ExperimentListFactory.from_json_file(filename)
assert len(exlist) == 1
from dials.array_family import flex
rlist = flex.reflection_table.from_predictions_multi(exlist)
from dials.algorithms.integration import CorrectionsMulti, Corrections
corrector = CorrectionsMulti()
for experiment in exlist:
corrector.append(
Corrections(experiment.beam, experiment.goniometer,
experiment.detector))
lp1 = corrector.lp(rlist['id'], rlist['s1'])
lp2 = flex.double([
LP_calculations(exlist[i], s1)
for i, s1 in zip(rlist['id'], rlist['s1'])
])
diff = flex.abs(lp1 - lp2)
assert diff.all_lt(1e-7)
def onclick(event): import math ts = event.xdata if ts is None: return diffs = flex.abs(t - ts) ts = t[flex.first_index(diffs, flex.min(diffs))] print(get_paths_from_timestamps([ts], tag="shot", ext=ext)[0])
def test_simple(dials_data, model, tmpdir):
path = dials_data("centroid_test_data")
experiments = path.join("experiments.json")
reflns_simple = tmpdir.join("simple").join("observations.refl")
reflns_g_simple = tmpdir.join("gmodel_simple").join("observations.refl")
reflns_simple.dirpath().ensure(dir=1)
reflns_g_simple.dirpath().ensure(dir=1)
result = procrunner.run(
[
"dials.integrate",
"nproc=1",
experiments.strpath,
"profile.fitting=False",
"background.algorithm=simple",
"background.simple.outlier.algorithm=null",
"output.reflections=" + reflns_simple.strpath,
],
working_directory=tmpdir.strpath,
)
assert not result.returncode and not result.stderr
assert reflns_simple.check()
result = procrunner.run(
[
"dials.integrate",
"nproc=1",
experiments.strpath,
"profile.fitting=False",
"background.algorithm=gmodel",
"background.gmodel.robust.algorithm=False",
"background.gmodel.model=model.pickle",
"output.reflections=" + reflns_g_simple.strpath,
],
working_directory=tmpdir.strpath,
)
assert not result.returncode and not result.stderr
assert reflns_g_simple.check()
from dials.array_family import flex
reflections1 = flex.reflection_table.from_file(reflns_simple.strpath)
reflections3 = flex.reflection_table.from_file(reflns_g_simple.strpath)
assert len(reflections1) == len(reflections3)
flag = flex.reflection_table.flags.integrated_sum
integrated1 = reflections1.select(reflections1.get_flags(flag, all=True))
integrated3 = reflections3.select(reflections3.get_flags(flag, all=True))
assert len(integrated1) > 0
assert len(integrated1) == len(integrated3)
mean_bg1 = integrated1["background.mean"]
mean_bg3 = integrated3["background.mean"]
scale3 = integrated3["background.scale"]
diff1 = flex.abs(mean_bg1 - mean_bg3)
assert (scale3 > 0).count(False) == 0
assert (diff1 < 1e-5).count(False) == 0
def _filter_reflections(self, params, experiments, reflections):
''' Filter the reflections to integrate. '''
from dials.util.command_line import Command
from dials.algorithms import filtering
from dials.array_family import flex
# Set all reflections which overlap bad pixels to zero
Command.start('Filtering reflections by detector mask')
if experiments[0].scan == None:
array_range = 1
else:
array_range = experiments[0].scan.get_array_range()
mask = filtering.by_detector_mask(
reflections['bbox'],
experiments[0].imageset.get_raw_data(0)[0] >= 0,
array_range)
reflections.del_selected(not mask)
Command.end('Filtered %d reflections by detector mask' % len(reflections))
# Filter the reflections by zeta
min_zeta = params.integration.filter.by_zeta
if min_zeta > 0:
Command.start('Filtering reflections by zeta >= %f' % min_zeta)
zeta = reflections.compute_zeta(experiments[0])
reflections.del_selected(flex.abs(zeta) < min_zeta)
n = len(reflections)
Command.end('Filtered %d reflections by zeta >= %f' % (n, min_zeta))
return reflections
def _local_setup(self, reflections):
"""Setup additional attributes used in gradients calculation. These are
specific to scans-type prediction parameterisations"""
# Spindle rotation matrices for every reflection
# R = self._axis.axis_and_angle_as_r3_rotation_matrix(phi)
# R = flex.mat3_double(len(reflections))
# NB for now use flex.vec3_double.rotate_around_origin each time I need the
# rotation matrix R.
# r is the reciprocal lattice vector, in the lab frame
self._phi_calc = reflections["xyzcal.mm"].parts()[2]
q = self._fixed_rotation * (self._UB * self._h)
self._r = self._setting_rotation * q.rotate_around_origin(
self._axis, self._phi_calc
)
# All of the derivatives of phi have a common denominator, given by
# (e X r).s0, where e is the rotation axis. Calculate this once, here.
self._e_X_r = (self._setting_rotation * self._axis).cross(self._r)
self._e_r_s0 = (self._e_X_r).dot(self._s0)
# Note that e_r_s0 -> 0 when the rotation axis, beam vector and
# relp are coplanar. This occurs when a reflection just touches
# the Ewald sphere.
#
# There is a relationship between e_r_s0 and zeta_factor.
# Uncommenting the code below shows that
# s0.(e X r) = zeta * |s X s0|
# from dials.algorithms.profile_model.gaussian_rs import zeta_factor
# from libtbx.test_utils import approx_equal
# s = matrix.col(reflections['s1'][0])
# z = zeta_factor(axis[0], s0[0], s)
# ss0 = (s.cross(matrix.col(s0[0]))).length()
# assert approx_equal(e_r_s0[0], z * ss0)
# catch small values of e_r_s0
e_r_s0_mag = flex.abs(self._e_r_s0)
try:
assert flex.min(e_r_s0_mag) > 1.0e-6
except AssertionError as e:
imin = flex.min_index(e_r_s0_mag)
print("(e X r).s0 too small:")
print("for", (e_r_s0_mag <= 1.0e-6).count(True), "reflections")
print("out of", len(e_r_s0_mag), "total")
print("such as", reflections["miller_index"][imin])
print("with scattering vector", reflections["s1"][imin])
print("where r =", self._r[imin])
print("e =", self._axis[imin])
print("s0 =", self._s0[imin])
print("this reflection forms angle with the equatorial plane " "normal:")
vecn = (
matrix.col(self._s0[imin])
.cross(matrix.col(self._axis[imin]))
.normalize()
)
print(matrix.col(reflections["s1"][imin]).accute_angle(vecn))
raise e
def __init__(self, crystal, beam, detector, goniometer, scan, reflections):
"""Initialise the algorithm. Calculate the list of tau and zetas.
Params:
reflections The list of reflections
experiment The experiment object
"""
# Get the oscillation width
dphi2 = scan.get_oscillation(deg=False)[1] / 2.0
# Calculate a list of angles and zeta's
tau, zeta = self._calculate_tau_and_zeta(crystal, beam, detector,
goniometer, scan, reflections)
# Calculate zeta * (tau +- dphi / 2) / math.sqrt(2)
self.e1 = (tau + dphi2) * flex.abs(zeta) / math.sqrt(2.0)
self.e2 = (tau - dphi2) * flex.abs(zeta) / math.sqrt(2.0)
def _id_refs_to_keep(self, obs_data):
"""Create a selection of observations that pass certain conditions.
This step includes rejection of reflections too close to the spindle,
reflections measured outside the scan range, rejection of the (0,0,0)
Miller index and rejection of reflections with the overload flag set.
Outlier rejection is done later."""
# first exclude reflections with miller index set to 0,0,0
sel1 = obs_data['miller_index'] != (0, 0, 0)
# exclude reflections with overloads, as these have worse centroids
sel2 = ~obs_data.get_flags(obs_data.flags.overloaded)
# combine selections
sel = sel1 & sel2
inc = flex.size_t_range(len(obs_data)).select(sel)
obs_data = obs_data.select(sel)
# Default to True to pass the following test if there is no rotation axis
# for a particular experiment
to_keep = flex.bool(len(inc), True)
for iexp, exp in enumerate(self._experiments):
axis = self._axes[iexp]
if not axis or exp.scan is None: continue
if exp.scan.get_oscillation()[1] == 0.0: continue
sel = obs_data['id'] == iexp
s0 = self._s0vecs[iexp]
s1 = obs_data['s1'].select(sel)
phi = obs_data['xyzobs.mm.value'].parts()[2].select(sel)
# first test: reject reflections for which the parallelepiped formed
# between the gonio axis, s0 and s1 has a volume of less than the cutoff.
# Those reflections are by definition closer to the spindle-beam
# plane and for low values of the cutoff are troublesome to
# integrate anyway.
p_vol = flex.abs(
s1.cross(flex.vec3_double(s1.size(), s0)).dot(axis))
passed1 = p_vol > self._close_to_spindle_cutoff
# second test: reject reflections that lie outside the scan range
passed2 = exp.scan.is_angle_valid(phi, deg=False)
# sanity check to catch a mutilated scan that does not make sense
if passed2.count(True) == 0:
from libtbx.utils import Sorry
raise Sorry(
"Experiment id {0} contains no reflections with valid "
"scan angles".format(iexp))
# combine tests
to_update = passed1 & passed2
to_keep.set_selected(sel, to_update)
inc = inc.select(to_keep)
return inc
def chosen_weights(observation_set, params):
data = observation_set.data()
sigmas = observation_set.sigmas()
return {
"unit": flex.double(len(data), 1.),
"variance": 1. / (sigmas * sigmas),
"gentle": flex.pow(flex.sqrt(flex.abs(data)) / sigmas, 2),
"extreme": flex.pow(data / sigmas, 2)
}[params.postrefinement.target_weighting]
def select_strong(reflections):
from dials.array_family import flex
selection1 = reflections.get_flags(reflections.flags.indexed)
selection2 = reflections.get_flags(reflections.flags.centroid_outlier)
selection3 = flex.abs(reflections["zeta"]) > 0.05
selection4 = reflections["partiality"] > 0.99
selection = (selection1) & (~selection2) & (selection3) & (selection4)
return reflections.select(selection)
def _local_setup(self, reflections):
"""Setup additional attributes used in gradients calculation. These are
specific to scans-type prediction parameterisations"""
# Spindle rotation matrices for every reflection
#R = self._axis.axis_and_angle_as_r3_rotation_matrix(phi)
#R = flex.mat3_double(len(reflections))
# NB for now use flex.vec3_double.rotate_around_origin each time I need the
# rotation matrix R.
# r is the reciprocal lattice vector, in the lab frame
self._phi_calc = reflections['xyzcal.mm'].parts()[2]
q = self._fixed_rotation * (self._UB * self._h)
self._r = self._setting_rotation * q.rotate_around_origin(self._axis, self._phi_calc)
# All of the derivatives of phi have a common denominator, given by
# (e X r).s0, where e is the rotation axis. Calculate this once, here.
self._e_X_r = (self._setting_rotation * self._axis).cross(self._r)
self._e_r_s0 = (self._e_X_r).dot(self._s0)
# Note that e_r_s0 -> 0 when the rotation axis, beam vector and
# relp are coplanar. This occurs when a reflection just touches
# the Ewald sphere.
#
# There is a relationship between e_r_s0 and zeta_factor.
# Uncommenting the code below shows that
# s0.(e X r) = zeta * |s X s0|
#from dials.algorithms.profile_model.gaussian_rs import zeta_factor
#from libtbx.test_utils import approx_equal
#s = matrix.col(reflections['s1'][0])
#z = zeta_factor(axis[0], s0[0], s)
#ss0 = (s.cross(matrix.col(s0[0]))).length()
#assert approx_equal(e_r_s0[0], z * ss0)
# catch small values of e_r_s0
e_r_s0_mag = flex.abs(self._e_r_s0)
try:
assert flex.min(e_r_s0_mag) > 1.e-6
except AssertionError as e:
imin = flex.min_index(e_r_s0_mag)
print "(e X r).s0 too small:"
print "for", (e_r_s0_mag <= 1.e-6).count(True), "reflections"
print "out of", len(e_r_s0_mag), "total"
print "such as", reflections['miller_index'][imin]
print "with scattering vector", reflections['s1'][imin]
print "where r =", self._r[imin]
print "e =", self._axis[imin]
print "s0 =", self._s0[imin]
print ("this reflection forms angle with the equatorial plane "
"normal:")
vecn = matrix.col(self._s0[imin]).cross(matrix.col(self._axis[imin])).normalize()
print matrix.col(reflections['s1'][imin]).accute_angle(vecn)
raise e
return
def test_multi_sweep(dials_regression, tmpdir):
expts = os.path.join(
dials_regression, "integration_test_data", "multi_sweep", "experiments.json"
)
experiments = load.experiment_list(expts)
for i, expt in enumerate(experiments):
expt.identifier = str(100 + i)
experiments.as_json(tmpdir.join("modified_input.json").strpath)
refls = os.path.join(
dials_regression, "integration_test_data", "multi_sweep", "indexed.pickle"
)
result = procrunner.run(
[
"dials.integrate",
"nproc=1",
"modified_input.json",
refls,
"prediction.padding=0",
],
working_directory=tmpdir,
)
assert not result.returncode and not result.stderr
assert (tmpdir / "integrated.refl").check()
assert (tmpdir / "integrated.expt").check()
experiments = load.experiment_list(tmpdir.join("integrated.expt").strpath)
for i, expt in enumerate(experiments):
assert expt.identifier == str(100 + i)
table = flex.reflection_table.from_file(tmpdir / "integrated.refl")
assert len(table) == 4020
assert dict(table.experiment_identifiers()) == {0: "100", 1: "101"}
# Check the results
T1 = table[:2010]
T2 = table[2010:]
ID1 = list(set(T1["id"]))
ID2 = list(set(T2["id"]))
assert len(ID1) == 1
assert len(ID2) == 1
assert ID1[0] == 0
assert ID2[0] == 1
I1 = T1["intensity.prf.value"]
I2 = T2["intensity.prf.value"]
F1 = T1.get_flags(T1.flags.integrated_prf)
F2 = T2.get_flags(T2.flags.integrated_prf)
assert F1 == F2
I1 = I1.select(F1)
I2 = I2.select(F2)
assert flex.abs(I1 - I2) < 1e-6
def apply_iterative_weights(self):
e_i = (
self.Ih_table.intensities
- (self.Ih_table.inverse_scale_factors * self.Ih_table.Ih_values)
) / (self.Ih_table.inverse_scale_factors * self.Ih_table.Ih_values)
abs_ei = flex.abs(e_i)
self.Ih_table.weights = flex.double(self.Ih_table.size, 1.0)
sel = abs_ei > self.c
sel_abs_ei = abs_ei.select(sel)
sel_weight_fn = self.c / sel_abs_ei
self.Ih_table.weights.set_selected(sel, sel_weight_fn)
def __call__(self, d):
"""
True if within powder ring.
:param d: The resolution
:return: True/False in powder ring
"""
result = flex.bool(len(d), False)
d_star_sq = uctbx.d_as_d_star_sq(d)
for ds2 in self.d_star_sq:
result = result | (flex.abs(d_star_sq - ds2) < self.half_width)
return result
def r_split(self,
this,
other,
assume_index_matching=False,
use_binning=False):
# Used in Boutet et al. (2012), which credit it to Owen et al
# (2006). See also R_mrgd_I in Diederichs & Karplus (1997)?
# Barends cites Collaborative Computational Project Number 4. The
# CCP4 suite: programs for protein crystallography. Acta
# Crystallogr. Sect. D-Biol. Crystallogr. 50, 760-763 (1994) and
# White, T. A. et al. CrystFEL: a software suite for snapshot
# serial crystallography. J. Appl. Cryst. 45, 335-341 (2012).
if not use_binning:
assert other.indices().size() == this.indices().size()
if this.data().size() == 0:
return None
if assume_index_matching:
(o, c) = (this, other)
else:
(o, c) = this.common_sets(other=other, assert_no_singles=True)
# The case where the denominator is less or equal to zero is
# pathological and should never arise in practice.
den = flex.sum(flex.abs(o.data() + c.data()))
assert den > 0
return math.sqrt(2) * flex.sum(flex.abs(o.data() - c.data())) / den
assert this.binner is not None
results = []
for i_bin in this.binner().range_all():
sel = this.binner().selection(i_bin)
results.append(
self.r_split(this.select(sel),
other.select(sel),
assume_index_matching=assume_index_matching,
use_binning=False))
return binned_data(binner=this.binner(),
data=results,
data_fmt='%7.4f')
def r1_factor(self,
this,
other,
scale_factor=None,
assume_index_matching=False,
use_binning=False):
"""Get the R1 factor according to this formula
.. math::
R1 = \dfrac{\sum{||F| - k|F'||}}{\sum{|F|}}
where F is this.data() and F' is other.data() and
k is the factor to put F' on the same scale as F"""
assert not use_binning or this.binner() is not None
assert other.indices().size() == this.indices().size()
if not use_binning:
if this.data().size() == 0: return None
if (assume_index_matching):
o, c = this, other
else:
o, c = this.common_sets(other=other, assert_no_singles=True)
o = flex.abs(o.data())
c = flex.abs(c.data())
if (scale_factor is None):
den = flex.sum(c * c)
if (den != 0):
c *= (flex.sum(o * c) / den)
elif (scale_factor is not None):
c *= scale_factor
return flex.sum(flex.abs(o - c)) / flex.sum(o)
results = []
for i_bin in this.binner().range_all():
sel = this.binner().selection(i_bin)
results.append(
self.r1_factor(this.select(sel), other.select(sel),
scale_factor.data[i_bin],
assume_index_matching))
return binned_data(binner=this.binner(),
data=results,
data_fmt="%7.4f")
def __call__(self, d):
'''
True if within powder ring.
:param d: The resolution
:return: True/False in powder ring
'''
from dials.array_family import flex
result = flex.bool(len(d), False)
for d_spacing in self.d_spacings:
result = result | (flex.abs(d - d_spacing) < self.half_width)
return result
def __call__(self, d):
'''
True if within powder ring.
:param d: The resolution
:return: True/False in powder ring
'''
from dials.array_family import flex
result = flex.bool(len(d), False)
for d_spacing in self.d_spacings:
result = result | (flex.abs(d - d_spacing) < self.half_width)
return result
def _id_refs_to_keep(self, obs_data):
"""Create a selection of observations that pass certain conditions.
This step includes rejection of reflections too close to the spindle,
reflections measured outside the scan range, rejection of the (0,0,0)
Miller index and rejection of reflections with the overload flag set.
Outlier rejection is done later."""
# first exclude reflections with miller index set to 0,0,0
sel1 = obs_data['miller_index'] != (0,0,0)
# exclude reflections with overloads, as these have worse centroids
sel2 = ~obs_data.get_flags(obs_data.flags.overloaded)
# combine selections
sel = sel1 & sel2
inc = flex.size_t_range(len(obs_data)).select(sel)
obs_data = obs_data.select(sel)
# Default to True to pass the following test if there is no rotation axis
# for a particular experiment
to_keep = flex.bool(len(inc), True)
for iexp, exp in enumerate(self._experiments):
axis = self._axes[iexp]
if not axis or exp.scan is None: continue
if exp.scan.get_oscillation()[1] == 0.0: continue
sel = obs_data['id'] == iexp
s0 = self._s0vecs[iexp]
s1 = obs_data['s1'].select(sel)
phi = obs_data['xyzobs.mm.value'].parts()[2].select(sel)
# first test: reject reflections for which the parallelepiped formed
# between the gonio axis, s0 and s1 has a volume of less than the cutoff.
# Those reflections are by definition closer to the spindle-beam
# plane and for low values of the cutoff are troublesome to
# integrate anyway.
p_vol = flex.abs(s1.cross(flex.vec3_double(s1.size(), s0)).dot(axis))
passed1 = p_vol > self._close_to_spindle_cutoff
# second test: reject reflections that lie outside the scan range
phi_min, phi_max = exp.scan.get_oscillation_range(deg=False)
passed2 = (phi >= phi_min) & (phi <= phi_max)
# combine tests
to_update = passed1 & passed2
to_keep.set_selected(sel, to_update)
inc = inc.select(to_keep)
return inc
def __call__(self, d):
'''
True if within powder ring.
:param d: The resolution
:return: True/False in powder ring
'''
from dials.array_family import flex
from cctbx import uctbx
result = flex.bool(len(d), False)
d_star_sq = uctbx.d_as_d_star_sq(d)
for ds2 in self.d_star_sq:
result = result | (flex.abs(d_star_sq - ds2) < self.half_width)
return result
def filter_unsuitable_reflections(Ih_table, cutoff, min_Ih, min_partiality,
min_reflections_required):
"""Do a first pass to calculate delta_hl and filter out the largest
deviants, so that the error model is not misled by these and instead
operates on the central ~90% of the data. Also choose reflection groups
with n_h > 1, as these have deltas of zero by definition and will bias
the variance calculations. Also, only use groups where <Ih> > 25.0, as
the assumptions of normally distributed deltas will not hold for low
<Ih>."""
n_h = Ih_table.calc_nh()
sigmaprime = calc_sigmaprime([1.0, 0.0], Ih_table)
delta_hl = calc_deltahl(Ih_table, n_h, sigmaprime)
# make sure the fit isn't misled by extreme values
sel = flex.abs(delta_hl) < cutoff
if "partiality" in Ih_table.Ih_table:
sel &= Ih_table.Ih_table["partiality"] > min_partiality
Ih_table = Ih_table.select(sel)
n = Ih_table.size
sum_I_over_var = (Ih_table.intensities /
Ih_table.variances) * Ih_table.h_index_matrix
n_per_group = flex.double(n, 1) * Ih_table.h_index_matrix
avg_I_over_var = sum_I_over_var / n_per_group
sel = avg_I_over_var > 0.85
Ih_table = Ih_table.select_on_groups(sel)
n_h = Ih_table.calc_nh()
scaled_Ih = Ih_table.Ih_values * Ih_table.inverse_scale_factors
# need a scaled min_Ih, where can reasonably expect norm distribution
# (use min_Ih=25 by default, sigma ~ 5)
sel2 = scaled_Ih > min_Ih
# can't calculate a true deviation for groups of 1
sel3 = n_h > 1.0
sel4 = Ih_table.intensities > 0.001
# don't want to include weaker reflections where the background adds
# significantly to the variances, as these would no longer be normally
# distributed and skew the fit.
Ih_table = Ih_table.select(sel2 & sel3 & sel4)
n = Ih_table.size
if n < min_reflections_required:
raise ValueError(
"Insufficient reflections (%s < %s) to perform error modelling." %
(n, min_reflections_required))
n_h = Ih_table.calc_nh()
# now make sure any left also have n > 1
sel = n_h > 1.0
Ih_table = Ih_table.select(sel)
return Ih_table
def run(self):
from dials.algorithms.integration import CorrectionsMulti, Corrections
from dials.array_family import flex
corrector = CorrectionsMulti()
for experiment in self.exlist:
corrector.append(
Corrections(experiment.beam, experiment.goniometer,
experiment.detector))
lp1 = corrector.lp(self.rlist['id'], self.rlist['s1'])
lp2 = self.compute_expected()
diff = flex.abs(lp1 - lp2)
assert (diff.all_lt(1e-7))
print 'OK'
def __call__(self, reflections):
"""
Do some pre-processing.
"""
# Compute some reflection properties
reflections.compute_zeta_multi(self.experiments)
reflections.compute_d(self.experiments)
reflections.compute_bbox(self.experiments)
# Filter the reflections by zeta
mask = flex.abs(reflections["zeta"]) < self.params.filter.min_zeta
reflections.set_flags(mask, reflections.flags.dont_integrate)
# Filter the reflections by powder ring
if self.params.filter.powder_filter is not None:
mask = self.params.filter.powder_filter(reflections["d"])
reflections.set_flags(mask, reflections.flags.in_powder_ring)
def run(self):
from dials.algorithms.integration import CorrectionsMulti, Corrections
from dials.array_family import flex
corrector = CorrectionsMulti()
for experiment in self.exlist:
corrector.append(Corrections(
experiment.beam,
experiment.goniometer,
experiment.detector))
lp1 = corrector.lp(self.rlist['id'], self.rlist['s1'])
lp2 = self.compute_expected()
diff = flex.abs(lp1 - lp2)
assert(diff.all_lt(1e-7))
print 'OK'
def test_multi_sequence(dials_regression, run_in_tmpdir):
result = procrunner.run(
[
"dials.integrate",
os.path.join(
dials_regression,
"integration_test_data",
"multi_sweep",
"experiments.json",
),
os.path.join(
dials_regression,
"integration_test_data",
"multi_sweep",
"indexed.pickle",
),
"prediction.padding=0",
]
)
assert not result.returncode and not result.stderr
assert os.path.exists("integrated.refl")
with open("integrated.refl", "rb") as fh:
table = pickle.load(fh)
assert len(table) == 4020
# Check the results
T1 = table[:2010]
T2 = table[2010:]
ID1 = list(set(T1["id"]))
ID2 = list(set(T2["id"]))
assert len(ID1) == 1
assert len(ID2) == 1
assert ID1[0] == 0
assert ID2[0] == 1
I1 = T1["intensity.prf.value"]
I2 = T2["intensity.prf.value"]
F1 = T1.get_flags(T1.flags.integrated_prf)
F2 = T2.get_flags(T2.flags.integrated_prf)
assert F1 == F2
I1 = I1.select(F1)
I2 = I2.select(F2)
assert flex.abs(I1 - I2) < 1e-6
def test_multi_sweep(self):
from os.path import join
from libtbx import easy_run
import os
dirname = 'multi_sweep'
os.mkdir(dirname)
os.chdir(dirname)
# Call dials.integrate
easy_run.fully_buffered([
'dials.integrate',
join(self.integration_test_data, 'multi_sweep',
'experiments.json'),
join(self.integration_test_data, 'multi_sweep', 'indexed.pickle'),
'prediction.padding=0',
]).raise_if_errors()
import cPickle as pickle
table = pickle.load(open('integrated.pickle', 'rb'))
assert len(table) == 4020
# Check the results
T1 = table[:2010]
T2 = table[2010:]
ID1 = list(set(T1['id']))
ID2 = list(set(T2['id']))
assert len(ID1) == 1
assert len(ID2) == 1
assert ID1[0] == 0
assert ID2[0] == 1
I1 = T1['intensity.prf.value']
I2 = T2['intensity.prf.value']
F1 = T1.get_flags(T1.flags.integrated_prf)
F2 = T2.get_flags(T2.flags.integrated_prf)
assert F1 == F2
I1 = I1.select(F1)
I2 = I2.select(F2)
assert flex.abs(I1 - I2) < 1e-6
print 'OK'
def __call__(self, reflections):
'''
Do some pre-processing.
'''
from dials.array_family import flex
# Compute some reflection properties
reflections.compute_zeta_multi(self.experiments)
reflections.compute_d(self.experiments)
reflections.compute_bbox(self.experiments)
# Filter the reflections by zeta
mask = flex.abs(reflections['zeta']) < self.params.filter.min_zeta
num_ignore = mask.count(True)
reflections.set_flags(mask, reflections.flags.dont_integrate)
# Filter the reflections by powder ring
if self.params.filter.powder_filter is not None:
mask = self.params.filter.powder_filter(reflections['d'])
reflections.set_flags(mask, reflections.flags.in_powder_ring)
def test_multi_sweep(self):
from os.path import join
from libtbx import easy_run
import os
dirname ='multi_sweep'
os.mkdir(dirname)
os.chdir(dirname)
# Call dials.integrate
easy_run.fully_buffered([
'dials.integrate',
join(self.integration_test_data, 'multi_sweep', 'experiments.json'),
join(self.integration_test_data, 'multi_sweep', 'indexed.pickle')
]).raise_if_errors()
import cPickle as pickle
table = pickle.load(open('integrated.pickle', 'rb'))
assert len(table) == 4020
# Check the results
T1 = table[:2010]
T2 = table[2010:]
ID1 = list(set(T1['id']))
ID2 = list(set(T2['id']))
assert len(ID1) == 1
assert len(ID2) == 1
assert ID1[0] == 0
assert ID2[0] == 1
I1 = T1['intensity.prf.value']
I2 = T2['intensity.prf.value']
F1 = T1.get_flags(T1.flags.integrated_prf)
F2 = T2.get_flags(T2.flags.integrated_prf)
assert F1 == F2
I1 = I1.select(F1)
I2 = I2.select(F2)
assert flex.abs(I1 - I2) < 1e-6
print 'OK'
def predict_reflections(experiments):
from dials.array_family import flex
print("Predicting reflections")
# Predict
reflections = flex.reflection_table.from_predictions_multi(experiments)
# Compute some reflection properties
reflections.compute_zeta_multi(experiments)
reflections.compute_d(experiments)
reflections.compute_bbox(experiments)
# Filter the reflections by zeta
mask = flex.abs(reflections["zeta"]) < 0.05
num_ignore = mask.count(True)
reflections.set_flags(mask, reflections.flags.dont_integrate)
print("Predicted %d reflections" % len(reflections))
print("Ignoring %d reflections" % num_ignore)
return reflections
def test_run(dials_data):
filename = dials_data("centroid_test_data").join("experiments.json").strpath
exlist = ExperimentListFactory.from_json_file(filename)
assert len(exlist) == 1
rlist = flex.reflection_table.from_predictions_multi(exlist)
corrector = CorrectionsMulti()
for experiment in exlist:
corrector.append(
Corrections(experiment.beam, experiment.goniometer, experiment.detector)
)
lp1 = corrector.lp(rlist["id"], rlist["s1"])
lp2 = flex.double(
[LP_calculations(exlist[i], s1) for i, s1 in zip(rlist["id"], rlist["s1"])]
)
diff = flex.abs(lp1 - lp2)
assert diff.all_lt(1e-7)
def __call__(self, reflections):
'''
Do some pre-processing.
'''
from dials.array_family import flex
from scitbx.array_family import shared
# Compute some reflection properties
reflections.compute_zeta_multi(self.experiments)
reflections.compute_d(self.experiments)
reflections.compute_bbox(self.experiments)
# Filter the reflections by zeta
mask = flex.abs(reflections['zeta']) < self.params.filter.min_zeta
num_ignore = mask.count(True)
reflections.set_flags(mask, reflections.flags.dont_integrate)
# Filter the reflections by powder ring
if self.params.filter.powder_filter is not None:
mask = self.params.filter.powder_filter(reflections['d'])
reflections.set_flags(mask, reflections.flags.in_powder_ring)
def _write_pickle(self, batch):
pass
def _write_predictions(self, predictions):
pass
@contextmanager
def open_shoebox_writer(filename):
writer = ShoeboxWriter(filename)
yield writer
if __name__ == '__main__':
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
from dials.array_family import flex
experiments = ExperimentListFactory.from_json_file(
'/home/upc86896/Data/Data/i04-BAG-training/dials_processed/experiments.json')
predictions = flex.reflection_table.from_predictions(experiments[0])
predictions.compute_bbox(experiments[0], nsigma=3, sigma_d=0.024,
sigma_m=0.044)
zeta = predictions.compute_zeta(experiments[0])
mask = flex.abs(zeta) < 0.05
predictions.del_selected(mask)
with open_shoebox_writer("extracted.tar") as writer:
writer.write(predictions, experiments[0].imageset)
def onclick(event): import math ts = event.xdata diffs = flex.abs(t - ts) ts = t[flex.first_index(diffs, flex.min(diffs))] print get_paths_from_timestamps([ts], tag="shot")[0]
def test_for_reference(self):
from dials.array_family import flex
from math import sqrt, pi
# Integrate
integration = self.experiments[0].profile.fitting_class()(self.experiments[0])
# Integrate the reference profiles
integration(self.experiments, self.reference)
locator = integration.learner.locate()
# Check the reference profiles and spots are ok
#self.check_profiles(integration.learner)
# Make sure background is zero
profiles = self.reference['rs_shoebox']
eps = 1e-7
for p in profiles:
assert(abs(flex.sum(p.background) - 0) < eps)
print 'OK'
# Only select variances greater than zero
mask = self.reference.get_flags(self.reference.flags.integrated, all=False)
assert(mask.count(True) > 0)
I_cal = self.reference['intensity.prf.value']
I_var = self.reference['intensity.prf.variance']
B_sim = self.reference['background.sim.a'].as_double()
I_sim = self.reference['intensity.sim'].as_double()
I_exp = self.reference['intensity.exp']
P_cor = self.reference['profile.correlation']
X_pos, Y_pos, Z_pos = self.reference['xyzcal.px'].parts()
I_cal = I_cal.select(mask)
I_var = I_var.select(mask)
I_sim = I_sim.select(mask)
I_exp = I_exp.select(mask)
P_cor = P_cor.select(mask)
max_ind = flex.max_index(flex.abs(I_cal-I_sim))
max_I = I_cal[max_ind]
max_P = self.reference[max_ind]['rs_shoebox'].data
max_C = self.reference[max_ind]['xyzcal.px']
max_S = self.reference[max_ind]['shoebox'].data
ref_ind = locator.index(max_C)
ref_P = locator.profile(ref_ind)
ref_C = locator.coord(ref_ind)
#def f(I):
#mask = flex.bool(flex.grid(9,9,9), False)
#for k in range(9):
#for j in range(9):
#for i in range(9):
#dx = 5 * (i - 4.5) / 4.5
#dy = 5 * (j - 4.5) / 4.5
#dz = 5 * (k - 4.5) / 4.5
#dd = sqrt(dx**2 + dy**2 + dz**2)
#if dd <= 3:
#mask[k,j,i] = True
#mask = mask.as_1d() & (ref_P.as_1d() > 0)
#p = ref_P.as_1d().select(mask)
#c = max_P.as_1d().select(mask)
#return flex.sum((c - I * p)**2 / (I * p))
#ff = []
#for I in range(9500, 11500):
#ff.append(f(I))
#print 'Old I: ', sorted(range(len(ff)), key=lambda x: ff[x])[0] + 9500
#from matplotlib import pylab
#pylab.plot(range(9500, 11500), ff)
#pylab.show()
#def estI(I):
#mask = flex.bool(flex.grid(9,9,9), False)
#for k in range(9):
#for j in range(9):
#for i in range(9):
#dx = 5 * (i - 4.5) / 4.5
#dy = 5 * (j - 4.5) / 4.5
#dz = 5 * (k - 4.5) / 4.5
#dd = sqrt(dx**2 + dy**2 + dz**2)
#if dd <= 3:
#mask[k,j,i] = True
#mask = mask.as_1d() & (ref_P.as_1d() > 0)
#p = ref_P.as_1d().select(mask)
#c = max_P.as_1d().select(mask)
#v = I * p
#return flex.sum(c * p / v) / flex.sum(p*p/v)
#def iterI(I0):
#I = estI(I0)
#print I
#if abs(I - I0) < 1e-3:
#return I
#return iterI(I)
#newI = iterI(10703)#flex.sum(max_P))
#print "New I: ", newI
# Calculate the z score
perc = self.mv3n_tolerance_interval(3*3)
Z = (I_cal - I_sim) / flex.sqrt(I_var)
mv = flex.mean_and_variance(Z)
Z_mean = mv.mean()
Z_var = mv.unweighted_sample_variance()
print "Z: mean: %f, var: %f, sig: %f" % (Z_mean, Z_var, sqrt(Z_var))
def plot_obs_colored_by_transverse_deltas(self, reflections, panel = None, ax = None, bounds = None): return self.plot_obs_colored_by_data(flex.abs(reflections['transverse_displacements']), reflections, panel, ax, bounds)
def lorentz_callable(self,values): Rh = self.get_Rh_array(values) Rs = flex.double(len(self.MILLER),1./values.DEFF)+flex.double(len(self.MILLER),values.ETA/2.)/self.DVEC ratio = Rh / Rs ratio_abs = flex.abs(ratio) return ratio_abs
def _get_gradients_core(self, reflections, D, s0, U, B, axis, fixed_rotation, callback=None):
"""Calculate gradients of the prediction formula with respect to
each of the parameters of the contained models, for reflection h
that reflects at rotation angle phi with scattering vector s that
intersects panel panel_id. That is, calculate dX/dp, dY/dp and
dphi/dp"""
# Spindle rotation matrices for every reflection
#R = self._axis.axis_and_angle_as_r3_rotation_matrix(phi)
#R = flex.mat3_double(len(reflections))
# NB for now use flex.vec3_double.rotate_around_origin each time I need the
# rotation matrix R.
self._axis = axis
self._fixed_rotation = fixed_rotation
self._s0 = s0
# pv is the 'projection vector' for the ray along s1.
self._D = D
self._s1 = reflections['s1']
self._pv = D * self._s1
# also need quantities derived from pv, precalculated for efficiency
u, v, w = self._pv.parts()
self._w_inv = 1/w
self._u_w_inv = u * self._w_inv
self._v_w_inv = v * self._w_inv
self._UB = U * B
self._U = U
self._B = B
# r is the reciprocal lattice vector, in the lab frame
self._h = reflections['miller_index'].as_vec3_double()
self._phi_calc = reflections['xyzcal.mm'].parts()[2]
self._r = (self._fixed_rotation * (self._UB * self._h)).rotate_around_origin(self._axis, self._phi_calc)
# All of the derivatives of phi have a common denominator, given by
# (e X r).s0, where e is the rotation axis. Calculate this once, here.
self._e_X_r = self._axis.cross(self._r)
self._e_r_s0 = (self._e_X_r).dot(self._s0)
# Note that e_r_s0 -> 0 when the rotation axis, beam vector and
# relp are coplanar. This occurs when a reflection just touches
# the Ewald sphere.
#
# There is a relationship between e_r_s0 and zeta_factor.
# Uncommenting the code below shows that
# s0.(e X r) = zeta * |s X s0|
#from dials.algorithms.profile_model.gaussian_rs import zeta_factor
#from libtbx.test_utils import approx_equal
#s = matrix.col(reflections['s1'][0])
#z = zeta_factor(axis[0], s0[0], s)
#ss0 = (s.cross(matrix.col(s0[0]))).length()
#assert approx_equal(e_r_s0[0], z * ss0)
# catch small values of e_r_s0
e_r_s0_mag = flex.abs(self._e_r_s0)
try:
assert flex.min(e_r_s0_mag) > 1.e-6
except AssertionError as e:
imin = flex.min_index(e_r_s0_mag)
print "(e X r).s0 too small:"
print "for", (e_r_s0_mag <= 1.e-6).count(True), "reflections"
print "out of", len(e_r_s0_mag), "total"
print "such as", reflections['miller_index'][imin]
print "with scattering vector", reflections['s1'][imin]
print "where r =", self._r[imin]
print "e =", self._axis[imin]
print "s0 =", self._s0[imin]
print ("this reflection forms angle with the equatorial plane "
"normal:")
vecn = matrix.col(self._s0[imin]).cross(matrix.col(self._axis[imin])).normalize()
print matrix.col(reflections['s1'][imin]).accute_angle(vecn)
raise e
# Set up empty list in which to store gradients
m = len(reflections)
results = []
# determine experiment to indices mappings once, here
experiment_to_idx = []
for iexp, exp in enumerate(self._experiments):
sel = reflections['id'] == iexp
isel = sel.iselection()
experiment_to_idx.append(isel)
# reset a pointer to the parameter number
self._iparam = 0
### Work through the parameterisations, calculating their contributions
### to derivatives d[pv]/dp and d[phi]/dp
# loop over the detector parameterisations
for dp in self._detector_parameterisations:
# Determine (sub)set of reflections affected by this parameterisation
isel = flex.size_t()
for exp_id in dp.get_experiment_ids():
isel.extend(experiment_to_idx[exp_id])
# Access the detector model being parameterised
detector = dp.get_model()
# Get panel numbers of the affected reflections
panel = reflections['panel'].select(isel)
# Extend derivative vectors for this detector parameterisation
results = self._extend_gradient_vectors(results, m, dp.num_free(),
keys=self._grad_names)
# loop through the panels in this detector
for panel_id, _ in enumerate(exp.detector):
# get the right subset of array indices to set for this panel
sub_isel = isel.select(panel == panel_id)
if len(sub_isel) == 0:
# if no reflections intersect this panel, skip calculation
continue
sub_pv = self._pv.select(sub_isel)
sub_D = self._D.select(sub_isel)
dpv_ddet_p = self._detector_derivatives(dp, sub_pv, sub_D, panel_id)
# convert to dX/dp, dY/dp and assign the elements of the vectors
# corresponding to this experiment and panel
sub_w_inv = self._w_inv.select(sub_isel)
sub_u_w_inv = self._u_w_inv.select(sub_isel)
sub_v_w_inv = self._v_w_inv.select(sub_isel)
dX_ddet_p, dY_ddet_p = self._calc_dX_dp_and_dY_dp_from_dpv_dp(
sub_w_inv, sub_u_w_inv, sub_v_w_inv, dpv_ddet_p)
# use a local parameter index pointer because we set all derivatives
# for this panel before moving on to the next
iparam = self._iparam
for dX, dY in zip(dX_ddet_p, dY_ddet_p):
if dX is not None:
results[iparam]['dX_dp'].set_selected(sub_isel, dX)
if dY is not None:
results[iparam]['dY_dp'].set_selected(sub_isel, dY)
# increment the local parameter index pointer
iparam += 1
if callback is not None:
iparam = self._iparam
for i in range(dp.num_free()):
results[iparam] = callback(results[iparam])
iparam += 1
# increment the parameter index pointer to the last detector parameter
self._iparam += dp.num_free()
# loop over the beam parameterisations
for bp in self._beam_parameterisations:
# Determine (sub)set of reflections affected by this parameterisation
isel = flex.size_t()
for exp_id in bp.get_experiment_ids():
isel.extend(experiment_to_idx[exp_id])
# Extend derivative vectors for this beam parameterisation
results = self._extend_gradient_vectors(results, m, bp.num_free(),
keys=self._grad_names)
if len(isel) == 0:
# if no reflections are in this experiment, skip calculation
self._iparam += bp.num_free()
continue
# Get required data from those reflections
r = self._r.select(isel)
e_X_r = self._e_X_r.select(isel)
e_r_s0 = self._e_r_s0.select(isel)
D = self._D.select(isel)
w_inv = self._w_inv.select(isel)
u_w_inv = self._u_w_inv.select(isel)
v_w_inv = self._v_w_inv.select(isel)
dpv_dbeam_p, dphi_dbeam_p = self._beam_derivatives(bp, r, e_X_r, e_r_s0, D)
# convert to dX/dp, dY/dp and assign the elements of the vectors
# corresponding to this experiment
dX_dbeam_p, dY_dbeam_p = self._calc_dX_dp_and_dY_dp_from_dpv_dp(
w_inv, u_w_inv, v_w_inv, dpv_dbeam_p)
for dX, dY, dphi in zip(dX_dbeam_p, dY_dbeam_p, dphi_dbeam_p):
results[self._iparam][self._grad_names[0]].set_selected(isel, dX)
results[self._iparam][self._grad_names[1]].set_selected(isel, dY)
results[self._iparam][self._grad_names[2]].set_selected(isel, dphi)
if callback is not None:
results[self._iparam] = callback(results[self._iparam])
# increment the parameter index pointer
self._iparam += 1
# loop over the crystal orientation parameterisations
for xlop in self._xl_orientation_parameterisations:
# Determine (sub)set of reflections affected by this parameterisation
isel = flex.size_t()
for exp_id in xlop.get_experiment_ids():
isel.extend(experiment_to_idx[exp_id])
# Extend derivative vectors for this crystal orientation parameterisation
results = self._extend_gradient_vectors(results, m, xlop.num_free(),
keys=self._grad_names)
if len(isel) == 0:
# if no reflections are in this experiment, skip calculation
self._iparam += xlop.num_free()
continue
# Get required data from those reflections
axis = self._axis.select(isel)
fixed_rotation = self._fixed_rotation.select(isel)
phi_calc = self._phi_calc.select(isel)
h = self._h.select(isel)
s1 = self._s1.select(isel)
e_X_r = self._e_X_r.select(isel)
e_r_s0 = self._e_r_s0.select(isel)
B = self._B.select(isel)
D = self._D.select(isel)
w_inv = self._w_inv.select(isel)
u_w_inv = self._u_w_inv.select(isel)
v_w_inv = self._v_w_inv.select(isel)
# get derivatives of the U matrix wrt the parameters
dU_dxlo_p = [reflections["dU_dp{0}".format(i)].select(isel) \
for i in range(xlop.num_free())]
dpv_dxlo_p, dphi_dxlo_p = self._xl_orientation_derivatives(
dU_dxlo_p, axis, fixed_rotation, phi_calc, h, s1, e_X_r, e_r_s0, B, D)
# convert to dX/dp, dY/dp and assign the elements of the vectors
# corresponding to this experiment
dX_dxlo_p, dY_dxlo_p = self._calc_dX_dp_and_dY_dp_from_dpv_dp(
w_inv, u_w_inv, v_w_inv, dpv_dxlo_p)
for dX, dY, dphi in zip(dX_dxlo_p, dY_dxlo_p, dphi_dxlo_p):
results[self._iparam][self._grad_names[0]].set_selected(isel, dX)
results[self._iparam][self._grad_names[1]].set_selected(isel, dY)
results[self._iparam][self._grad_names[2]].set_selected(isel, dphi)
if callback is not None:
results[self._iparam] = callback(results[self._iparam])
# increment the parameter index pointer
self._iparam += 1
# loop over the crystal unit cell parameterisations
for xlucp in self._xl_unit_cell_parameterisations:
# Determine (sub)set of reflections affected by this parameterisation
isel = flex.size_t()
for exp_id in xlucp.get_experiment_ids():
isel.extend(experiment_to_idx[exp_id])
# Extend derivative vectors for this crystal unit cell parameterisation
results = self._extend_gradient_vectors(results, m, xlucp.num_free(),
keys=self._grad_names)
if len(isel) == 0:
# if no reflections are in this experiment, skip calculation
self._iparam += xlucp.num_free()
continue
# Get required data from those reflections
axis = self._axis.select(isel)
fixed_rotation = self._fixed_rotation.select(isel)
phi_calc = self._phi_calc.select(isel)
h = self._h.select(isel)
s1 = self._s1.select(isel)
e_X_r = self._e_X_r.select(isel)
e_r_s0 = self._e_r_s0.select(isel)
U = self._U.select(isel)
D = self._D.select(isel)
w_inv = self._w_inv.select(isel)
u_w_inv = self._u_w_inv.select(isel)
v_w_inv = self._v_w_inv.select(isel)
dB_dxluc_p = [reflections["dB_dp{0}".format(i)].select(isel) \
for i in range(xlucp.num_free())]
dpv_dxluc_p, dphi_dxluc_p = self._xl_unit_cell_derivatives(
dB_dxluc_p, axis, fixed_rotation, phi_calc, h, s1, e_X_r, e_r_s0, U, D)
# convert to dX/dp, dY/dp and assign the elements of the vectors
# corresponding to this experiment
dX_dxluc_p, dY_dxluc_p = self._calc_dX_dp_and_dY_dp_from_dpv_dp(
w_inv, u_w_inv, v_w_inv, dpv_dxluc_p)
for dX, dY, dphi in zip(dX_dxluc_p, dY_dxluc_p, dphi_dxluc_p):
results[self._iparam][self._grad_names[0]].set_selected(isel, dX)
results[self._iparam][self._grad_names[1]].set_selected(isel, dY)
results[self._iparam][self._grad_names[2]].set_selected(isel, dphi)
if callback is not None:
results[self._iparam] = callback(results[self._iparam])
# increment the parameter index pointer
self._iparam += 1
return results
def run_stills_pred_param(self, verbose = False):
if verbose:
print 'Testing derivatives for StillsPredictionParameterisation'
print '========================================================'
# Build a prediction parameterisation for the stills experiment
pred_param = StillsPredictionParameterisation(self.stills_experiments,
detector_parameterisations = [self.det_param],
beam_parameterisations = [self.s0_param],
xl_orientation_parameterisations = [self.xlo_param],
xl_unit_cell_parameterisations = [self.xluc_param])
# Predict the reflections in place. Must do this ahead of calculating
# the analytical gradients so quantities like s1 are correct
from dials.algorithms.refinement.prediction import ExperimentsPredictor
ref_predictor = ExperimentsPredictor(self.stills_experiments)
ref_predictor.update()
ref_predictor.predict(self.reflections)
# get analytical gradients
an_grads = pred_param.get_gradients(self.reflections)
fd_grads = self.get_fd_gradients(pred_param, ref_predictor)
for i, (an_grad, fd_grad) in enumerate(zip(an_grads, fd_grads)):
# compare FD with analytical calculations
if verbose: print "\nParameter {0}: {1}". format(i, fd_grad['name'])
for idx, name in enumerate(["dX_dp", "dY_dp", "dDeltaPsi_dp"]):
if verbose: print name
a = fd_grad[name]
b = an_grad[name]
abs_error = a - b
denom = a + b
fns = five_number_summary(abs_error)
if verbose: print (" summary of absolute errors: %9.6f %9.6f %9.6f " + \
"%9.6f %9.6f") % fns
assert flex.max(flex.abs(abs_error)) < 0.0003
# largest absolute error found to be about 0.00025 for dY/dp of
# Crystal0g_param_3. Reject outlying absolute errors and test again.
iqr = fns[3] - fns[1]
# skip further stats on errors with an iqr of near zero, e.g. dDeltaPsi_dp
# for detector parameters, which are all equal to zero
if iqr < 1.e-10:
continue
sel1 = abs_error < fns[3] + 1.5 * iqr
sel2 = abs_error > fns[1] - 1.5 * iqr
sel = sel1 & sel2
tst = flex.max_index(flex.abs(abs_error.select(sel)))
tst_val = abs_error.select(sel)[tst]
n_outliers = sel.count(False)
if verbose: print (" {0} outliers rejected, leaving greatest " + \
"absolute error: {1:9.6f}").format(n_outliers, tst_val)
# largest absolute error now 0.000086 for dX/dp of Beam0Mu2
assert abs(tst_val) < 0.00009
# Completely skip parameters with FD gradients all zero (e.g. gradients of
# DeltaPsi for detector parameters)
sel1 = flex.abs(a) < 1.e-10
if sel1.all_eq(True):
continue
# otherwise calculate normalised errors, by dividing absolute errors by
# the IQR (more stable than relative error calculation)
norm_error = abs_error / iqr
fns = five_number_summary(norm_error)
if verbose: print (" summary of normalised errors: %9.6f %9.6f %9.6f " + \
"%9.6f %9.6f") % fns
# largest normalised error found to be about 25.7 for dY/dp of
# Crystal0g_param_3.
try:
assert flex.max(flex.abs(norm_error)) < 30
except AssertionError as e:
e.args += ("extreme normalised error value: {0}".format(
flex.max(flex.abs(norm_error))),)
raise e
# Reject outlying normalised errors and test again
iqr = fns[3] - fns[1]
if iqr > 0.:
sel1 = norm_error < fns[3] + 1.5 * iqr
sel2 = norm_error > fns[1] - 1.5 * iqr
sel = sel1 & sel2
tst = flex.max_index(flex.abs(norm_error.select(sel)))
tst_val = norm_error.select(sel)[tst]
n_outliers = sel.count(False)
# most outliers found for for dY/dp of Crystal0g_param_3 (which had
# largest errors, so no surprise there).
try:
assert n_outliers < 250
except AssertionError as e:
e.args += ("too many outliers rejected: {0}".format(n_outliers),)
raise e
if verbose: print (" {0} outliers rejected, leaving greatest " + \
"normalised error: {1:9.6f}").format(n_outliers, tst_val)
# largest normalied error now about -4. for dX/dp of Detector0Tau1
assert abs(tst_val) < 4.5
if verbose: print
return
def test3(self):
from os.path import join
from libtbx import easy_run
import shutil
from os.path import join
shutil.copyfile(join(self.path, "experiments.json"), "experiments.json")
for i in range(1, 10):
shutil.copyfile(join(self.path, "centroid_000%d.cbf" % i),
"centroid_001%d.cbf" % i)
with open("experiments.json", "r") as infile:
lines = infile.readlines()
inscan = False
inimagerange = False
inoscillation = False
count = 0
done1 = False
done2 = False
for i, line in enumerate(lines):
if not inscan:
if line.strip().startswith('"scan": ['):
inscan = True
else:
if not inimagerange and not inoscillation:
if line.strip().startswith('"image_range": ['):
inimagerange = True
if line.strip().startswith('"oscillation": ['):
inoscillation = True
elif inimagerange:
if count == 0:
lines[i] = '11,'
count += 1
elif count == 1:
lines[i] = '19'
done1 = True
inimagerange = False
count = 0
elif inoscillation:
if count == 0:
lines[i] = '360.0,'
done2 = True
inoscillation = False
inscan = False
break
assert(done1)
assert(done2)
with open("experiments.json", "w") as outfile:
outfile.write('\n'.join(lines))
# Call dials.integrate
easy_run.fully_buffered([
'dials.integrate',
'experiments.json',
'profile.fitting=False',
'integration.integrator=3d',
]).raise_if_errors()
from math import pi
import cPickle as pickle
table = pickle.load(open('integrated.pickle', 'rb'))
mask1 = table.get_flags(table.flags.integrated,all=False)
assert(len(table) == 1996), "Table has %d entries instead of 1996" % len(table)
assert(mask1.count(True) == 1666)
mask2 = self.table.get_flags(table.flags.integrated,all=False)
assert(mask1.all_eq(mask2))
t1 = table.select(mask1)
t2 = self.table.select(mask1)
Cal_P1 = t1['xyzcal.mm'].parts()[2]
Cal_Z1 = t1['xyzcal.px'].parts()[2]
Obs_Z1 = t1['xyzobs.px.value'].parts()[2]
# Obs_P1 = t1['xyzobs.mm.value'].parts()[2]
Cal_Z2 = t2['xyzcal.px'].parts()[2]
Cal_P2 = t2['xyzcal.mm'].parts()[2]
Obs_Z2 = t2['xyzobs.px.value'].parts()[2]
# Obs_P2 = t2['xyzobs.mm.value'].parts()[2]
diff_I = t1['intensity.sum.value'] - t2['intensity.sum.value']
diff_Cal_Z = Cal_Z1 - (Cal_Z2 + 10)
diff_Obs_Z = Obs_Z1 - (Obs_Z2 + 10)
diff_Cal_P = Cal_P1 - (Cal_P2 + 2*pi)
# diff_Obs_P = Obs_P1 - (Obs_P2 + 2*pi)
assert(flex.abs(diff_I).all_lt(1e-7))
assert(flex.abs(diff_Cal_Z).all_lt(1e-7))
assert(flex.abs(diff_Cal_P).all_lt(1e-7))
assert(flex.abs(diff_Obs_Z).all_lt(1e-7))
# assert(flex.abs(diff_Obs_P).all_lt(1e-7))
print 'OK'
def run(self):
from os.path import join, exists
from libtbx import easy_run
import os
from uuid import uuid4
os.mkdir("simple")
os.mkdir("robust")
os.mkdir("gmodel_simple")
os.mkdir("gmodel_robust")
assert exists(join(self.path, "experiments.json"))
from dials.array_family import flex
from dials.algorithms.background.gmodel import StaticBackgroundModel
ysize = 2527
xsize = 2463
data = flex.double(flex.grid(ysize, xsize), 1)
model = StaticBackgroundModel()
model.add(data)
import cPickle as pickle
pickle.dump(model, open("model.pickle", "w"))
# Call dials.integrate
easy_run.fully_buffered(
[
"dials.integrate",
join(self.path, "experiments.json"),
"profile.fitting=False",
"background.algorithm=simple",
"background.simple.outlier.algorithm=null",
"output.reflections=./simple/reflections.pickle",
]
).raise_if_errors()
assert exists("simple/reflections.pickle")
# Call dials.integrate
easy_run.fully_buffered(
[
"dials.integrate",
join(self.path, "experiments.json"),
"profile.fitting=False",
"background.algorithm=glm",
"output.reflections=./robust/reflections.pickle",
]
).raise_if_errors()
assert exists("robust/reflections.pickle")
# Call dials.integrate
easy_run.fully_buffered(
[
"dials.integrate",
join(self.path, "experiments.json"),
"profile.fitting=False",
"background.algorithm=gmodel",
"background.gmodel.robust.algorithm=False",
"background.gmodel.model=model.pickle",
"output.reflections=./gmodel_simple/reflections.pickle",
]
).raise_if_errors()
assert exists("gmodel_simple/reflections.pickle")
# Call dials.integrate
easy_run.fully_buffered(
[
"dials.integrate",
join(self.path, "experiments.json"),
"profile.fitting=False",
"background.algorithm=gmodel",
"background.gmodel.robust.algorithm=True",
"background.gmodel.model=model.pickle",
"output.reflections=./gmodel_robust/reflections.pickle",
]
).raise_if_errors()
assert exists("gmodel_robust/reflections.pickle")
from dials.array_family import flex
reflections1 = flex.reflection_table.from_pickle("simple/reflections.pickle")
reflections2 = flex.reflection_table.from_pickle("robust/reflections.pickle")
reflections3 = flex.reflection_table.from_pickle("gmodel_simple/reflections.pickle")
reflections4 = flex.reflection_table.from_pickle("gmodel_robust/reflections.pickle")
assert len(reflections1) == len(reflections3)
assert len(reflections2) == len(reflections4)
flag = flex.reflection_table.flags.integrated_sum
integrated1 = reflections1.select(reflections1.get_flags(flag, all=True))
integrated2 = reflections2.select(reflections2.get_flags(flag, all=True))
integrated3 = reflections3.select(reflections3.get_flags(flag, all=True))
integrated4 = reflections4.select(reflections4.get_flags(flag, all=True))
assert len(integrated1) > 0
assert len(integrated2) > 0
assert len(integrated1) == len(integrated3)
assert len(integrated2) == len(integrated4)
mean_bg1 = integrated1["background.mean"]
mean_bg2 = integrated2["background.mean"]
mean_bg3 = integrated3["background.mean"]
mean_bg4 = integrated4["background.mean"]
scale3 = integrated3["background.scale"]
scale4 = integrated4["background.scale"]
diff1 = flex.abs(mean_bg1 - mean_bg3)
diff2 = flex.abs(mean_bg2 - mean_bg4)
assert (scale3 > 0).count(False) == 0
assert (scale4 > 0).count(False) == 0
assert (diff1 < 1e-5).count(False) == 0
print "OK"
def _id_refs_to_keep(self, obs_data):
"""Create a selection of observations that pass certain conditions.
This step includes rejection of reflections too close to the spindle,
reflections measured outside the scan range, rejection of the (0,0,0)
Miller index and rejection of reflections with the overload flag set.
Outlier rejection is done later."""
# first exclude reflections with miller index set to 0,0,0
sel1 = obs_data["miller_index"] != (0, 0, 0)
# exclude reflections with overloads, as these have worse centroids
sel2 = ~obs_data.get_flags(obs_data.flags.overloaded)
# combine selections
sel = sel1 & sel2
inc = flex.size_t_range(len(obs_data)).select(sel)
obs_data = obs_data.select(sel)
# Default to True to pass the following test if there is no rotation axis
# for a particular experiment
to_keep = flex.bool(len(inc), True)
for iexp, exp in enumerate(self._experiments):
axis = self._axes[iexp]
if not axis or exp.scan is None:
continue
if exp.scan.is_still():
continue
sel = obs_data["id"] == iexp
s0 = self._s0vecs[iexp]
s1 = obs_data["s1"].select(sel)
phi = obs_data["xyzobs.mm.value"].parts()[2].select(sel)
# first test: reject reflections for which the parallelepiped formed
# between the gonio axis, s0 and s1 has a volume of less than the cutoff.
# Those reflections are by definition closer to the spindle-beam
# plane and for low values of the cutoff are troublesome to
# integrate anyway.
p_vol = flex.abs(s1.cross(flex.vec3_double(s1.size(), s0)).dot(axis))
passed1 = p_vol > self._close_to_spindle_cutoff
# second test: reject reflections that lie outside the scan range
passed2 = exp.scan.is_angle_valid(phi, deg=False)
# sanity check to catch a mutilated scan that does not make sense
if passed2.count(True) == 0:
raise DialsRefineConfigError(
"Experiment id {} contains no reflections with valid "
"scan angles".format(iexp)
)
# combine tests so far
to_update = passed1 & passed2
# third test: reject reflections close to the centres of the first and
# last images in the scan
if self._scan_margin > 0.0:
edge1, edge2 = [e + 0.5 for e in exp.scan.get_image_range()]
edge1 = exp.scan.get_angle_from_image_index(edge1, deg=False)
edge1 += self._scan_margin
edge2 = exp.scan.get_angle_from_image_index(edge2, deg=False)
edge2 -= self._scan_margin
passed3 = (edge1 <= phi) & (phi <= edge2)
# combine the last test only if there would be a reasonable number of
# reflections left for refinement
tmp = to_update
to_update = to_update & passed3
if to_update.count(True) < 40:
logger.warning(
"Too few reflections to trim centroids from the scan "
"edges. Resetting scan_margin=0.0"
)
to_update = tmp
# make selection
to_keep.set_selected(sel, to_update)
inc = inc.select(to_keep)
return inc
def __call__(self):
"""Determine optimal mosaicity and domain size model (monochromatic)"""
RR = self.refinery.predict_for_reflection_table(self.reflections)
excursion_rad = RR["delpsical.rad"]
delta_psi_deg = excursion_rad * 180./math.pi
print
print flex.max(delta_psi_deg), flex.min(delta_psi_deg)
mean_excursion = flex.mean(delta_psi_deg)
print "The mean excursion is %7.3f degrees, r.m.s.d %7.3f"%(mean_excursion, math.sqrt(flex.mean(RR["delpsical2"])))
crystal = self.experiments[0].crystal
beam = self.experiments[0].beam
miller_indices = self.reflections["miller_index"]
# FIXME XXX revise this formula so as to use a different wavelength potentially for each reflection
two_thetas = crystal.get_unit_cell().two_theta(miller_indices,beam.get_wavelength(),deg=True)
dspacings = crystal.get_unit_cell().d(miller_indices)
dspace_sq = dspacings * dspacings
# First -- try to get a reasonable envelope for the observed excursions.
## minimum of three regions; maximum of 50 measurements in each bin
print "fitting parameters on %d spots"%len(excursion_rad)
n_bins = min(max(3, len(excursion_rad)//25),50)
bin_sz = len(excursion_rad)//n_bins
print "nbins",n_bins,"bin_sz",bin_sz
order = flex.sort_permutation(two_thetas)
two_thetas_env = flex.double()
dspacings_env = flex.double()
excursion_rads_env = flex.double()
for x in xrange(0,n_bins):
subset = order[x*bin_sz:(x+1)*bin_sz]
two_thetas_env.append(flex.mean(two_thetas.select(subset)))
dspacings_env.append(flex.mean(dspacings.select(subset)))
excursion_rads_env.append(flex.max(flex.abs(excursion_rad.select(subset))))
# Second -- parameter fit
## solve the normal equations
sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env)
sum_inv_u = flex.sum(dspacings_env)
sum_te_u = flex.sum(dspacings_env * excursion_rads_env)
sum_te = flex.sum(excursion_rads_env)
Normal_Mat = sqr((sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env)))
Vector = col((sum_te_u, sum_te))
solution = Normal_Mat.inverse() * Vector
s_ang = 1./(2*solution[0])
print "Best LSQ fit Scheerer domain size is %9.2f ang"%(
s_ang)
tan_phi_rad = dspacings / (2. * s_ang)
tan_phi_deg = tan_phi_rad * 180./math.pi
k_degrees = solution[1]* 180./math.pi
print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f"%(2*k_degrees, k_degrees)
tan_outer_deg = tan_phi_deg + k_degrees
from xfel.mono_simulation.max_like import minimizer
# coerce the estimates to be positive for max-likelihood
lower_limit_domain_size = math.pow(crystal.get_unit_cell().volume(),
1./3.)*3 # params.refinement.domain_size_lower_limit
d_estimate = max(s_ang, lower_limit_domain_size)
M = minimizer(d_i = dspacings, psi_i = excursion_rad, eta_rad = abs(2. * solution[1]),
Deff = d_estimate)
print "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas))
tan_phi_rad_ML = dspacings / (2. / M.x[0])
tan_phi_deg_ML = tan_phi_rad_ML * 180./math.pi
tan_outer_deg_ML = tan_phi_deg_ML + 0.5*M.x[1]*180./math.pi
self.nv_acceptance_flags = flex.abs(delta_psi_deg) < tan_outer_deg_ML
if self.graph_verbose: #params.refinement.mosaic.enable_AD14F7B: # Excursion vs resolution fit
AD1TF7B_MAX2T = 30.
AD1TF7B_MAXDP = 1.
from matplotlib import pyplot as plt
plt.plot(two_thetas, delta_psi_deg, "bo")
minplot = flex.min(two_thetas)
plt.plot([0,minplot],[mean_excursion,mean_excursion],"k-")
LR = flex.linear_regression(two_thetas, delta_psi_deg)
model_y = LR.slope()*two_thetas + LR.y_intercept()
plt.plot(two_thetas, model_y, "k-")
plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas)))
plt.plot(two_thetas, tan_phi_deg_ML, "r.")
plt.plot(two_thetas, -tan_phi_deg_ML, "r.")
plt.plot(two_thetas, tan_outer_deg_ML, "g.")
plt.plot(two_thetas, -tan_outer_deg_ML, "g.")
plt.xlim([0,AD1TF7B_MAX2T])
plt.ylim([-AD1TF7B_MAXDP,AD1TF7B_MAXDP])
plt.show()
plt.close()
from xfel.mono_simulation.util import green_curve_area
self.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML)
print "The green curve area is ", self.green_curve_area
crystal._ML_half_mosaicity_deg = M.x[1]*180./(2.*math.pi)
crystal._ML_domain_size_ang = 2./M.x[0]
self._ML_full_mosaicity_rad = M.x[1]
self._ML_domain_size_ang = 2./M.x[0]
#params.refinement.mosaic.model_expansion_factor
"""The expansion factor should be initially set to 1, then expanded so that the # reflections matched becomes
as close as possible to # of observed reflections input, in the last integration call. Determine this by
inspecting the output log file interactively. Do not exceed the bare minimum threshold needed.
The intention is to find an optimal value, global for a given dataset."""
model_expansion_factor = 1.4
crystal._ML_half_mosaicity_deg *= model_expansion_factor
crystal._ML_domain_size_ang /= model_expansion_factor
return crystal
