def cross_validate_to_determine_number_of_terms(
x_obs, y_obs, w_obs=None, min_terms=10, max_terms=25, n_free=100, n_goes=5
):
if n_goes == None:
if min_terms < 2:
min_terms = 2
free_residuals = []
free_flags = flex.bool(x_obs.size(), True)
free_permut = flex.random_permutation(x_obs.size())
for ii in range(n_free):
free_flags[free_permut[ii]] = False
for count in range(min_terms, max_terms):
fit = chebyshev_lsq_fit(count, x_obs, y_obs, w_obs, free_flags)
free_residuals.append(fit.free_f)
return flex.double(free_residuals)
else:
if w_obs is None:
w_obs = flex.double(x_obs.size(), 1)
free_resid = flex.double(max_terms - min_terms, 0)
for jj in range(n_goes):
free_resid += cross_validate_to_determine_number_of_terms(
x_obs, y_obs, w_obs, min_terms=min_terms, max_terms=max_terms, n_free=n_free, n_goes=None
)
return min_terms + flex.min_index(free_resid)
def split_into_groups(self, sample, ngroups):
"""Split each vector in the data sample into groups of approximately equal
size."""
# number of obs in the sample
sample_size = len(sample[0])
# random permutation
p = flex.random_permutation(sample_size)
permuted = [col.select(p) for col in sample]
# determine groups
blocksize = int(sample_size / ngroups)
rem = sample_size % ngroups
blocksizes = [blocksize] * (ngroups - rem) + [blocksize + 1] * rem
starts = [0]
ends = [blocksizes[0]]
for b in blocksizes[1:]:
starts.append(ends[-1])
ends.append(ends[-1] + b)
blocks = zip(starts, ends)
# split into groups
groups = []
for s,e in blocks:
groups.append([col[s:e] for col in permuted])
return groups
def exercise_densely_distributed_singular_values(show_progress, full_coverage,
klass):
n = 40
m = 2 * n
n_runs = 20
tol = 10 * scitbx.math.double_numeric_limits.epsilon
gen = scitbx.linalg.random_normal_matrix_generator(m, n)
sigmas = []
sigmas.append(flex.double([10**(-i / n) for i in range(n)]))
sigmas.append(sigmas[0].select(flex.random_permutation(n)))
sigmas.append(sigmas[0].reversed())
print("Densely distributed singular values:", end=' ')
n_tests = 0
for i in range(n_runs):
if not full_coverage and random.random() < 0.8: continue
n_tests += 1
for i_case, sigma in enumerate(sigmas):
a = gen.matrix_with_singular_values(sigma)
svd = klass(a, accumulate_u=False, accumulate_v=False)
if i_case > 0:
sigma = sigma.select(flex.sort_permutation(sigma,
reverse=True))
delta = (svd.sigma - sigma) / sigma / tol
assert delta.all_lt(5)
print("%i done." % n_tests)
def split_into_groups(self, sample, ngroups):
"""Split each vector in the data sample into groups of approximately equal
size."""
# number of obs in the sample
sample_size = len(sample[0])
# random permutation
p = flex.random_permutation(sample_size)
permuted = [col.select(p) for col in sample]
# determine groups
blocksize = int(sample_size / ngroups)
rem = sample_size % ngroups
blocksizes = [blocksize] * (ngroups - rem) + [blocksize + 1] * rem
starts = [0]
ends = [blocksizes[0]]
for b in blocksizes[1:]:
starts.append(ends[-1])
ends.append(ends[-1] + b)
blocks = zip(starts, ends)
# split into groups
groups = [[col[start:end] for col in permuted] for start, end in blocks]
return groups
def estimate_cc_sig_fac(self):
# A1.1. Estimation of sigma(CC) as a function of sample size.
binner = self.intensities.setup_binner_counting_sorted(reflections_per_bin=200)
a = flex.double()
b = flex.double()
for i in range(binner.n_bins_all()):
count = binner.counts()[i]
if count == 0:
continue
bin_isel = binner.array_indices(i)
p = flex.random_permutation(count)
p = p[:2 * (count // 2)] # ensure even count
a.extend(self.intensities.data().select(bin_isel.select(p[:count//2])))
b.extend(self.intensities.data().select(bin_isel.select(p[count//2:])))
perm = flex.random_selection(a.size(), min(20000, a.size()))
a = a.select(perm)
b = b.select(perm)
self.corr_unrelated = CorrelationCoefficientAccumulator(a, b)
n_pairs = a.size()
min_num_groups = 10 # minimum number of groups
max_n_group = int(min(n_pairs/min_num_groups, 200)) # maximum number in group
min_n_group = int(min(5, max_n_group)) # minimum number in group
mean_ccs = flex.double()
rms_ccs = flex.double()
ns = flex.double()
for n in range(min_n_group, max_n_group):
ns.append(n)
ccs = flex.double()
for i in range(200):
isel = flex.random_selection(a.size(), n)
corr = CorrelationCoefficientAccumulator(a.select(isel), b.select(isel))
ccs.append(corr.coefficient())
mean_ccs.append(flex.mean(ccs))
rms_ccs.append(flex.mean(flex.pow2(ccs))**0.5)
x = 1/flex.pow(ns, 0.5)
y = rms_ccs
fit = flex.linear_regression(x, y)
assert fit.is_well_defined()
self.cc_sig_fac = fit.slope()
if 0:
from matplotlib import pyplot as plt
plt.plot(x, y)
plt.plot(
plt.xlim(), [fit.slope() * x_ + fit.y_intercept() for x_ in plt.xlim()])
plt.show()
def test_median_and_iqr():
# Even number of values, one bin
vals1 = flex.double((1, 2, 3, 4))
bins = flex.size_t([0] * len(vals1))
binned_statistics = BinnedStatistics(vals1, bins, 1)
assert binned_statistics.get_medians().all_eq(flex.double((2.5,)))
assert binned_statistics.get_iqrs().all_eq(flex.double((1.5,)))
# NB this differs from the calculation performed via
# _,q1,_,q3,_=five_number_summary(vals1);q3-q1
# which gives 2.0. However, it matches the result of R's IQR function
# Odd number of values, one bin, and robustness test
vals2 = flex.double((1, 2, 3, 100, 1000))
bins = flex.size_t([0] * len(vals2))
binned_statistics = BinnedStatistics(vals2, bins, 1)
assert binned_statistics.get_medians().all_eq(flex.double((3.0,)))
assert binned_statistics.get_iqrs().all_eq(flex.double((98,)))
# NB this is the same as the calculation performed via
# _,q1,_,q3,_=five_number_summary(vals2);q3-q1, and matches R's IQR function
# Now combine the data and randomise order
vals3 = vals1.concatenate(vals2)
bins = flex.size_t([0] * len(vals1) + [1] * len(vals2))
perm = flex.random_permutation(len(bins))
vals3 = vals3.select(perm)
bins = bins.select(perm)
binned_statistics = BinnedStatistics(vals3, bins, 2)
assert binned_statistics.get_medians().all_eq(
flex.double(
(
2.5,
3.0,
)
)
)
assert binned_statistics.get_iqrs().all_eq(
flex.double(
(
1.5,
98,
)
)
)
def form_initial_subset(self, h, data):
"""Method 2 of subsection 3.1 of R&vD"""
# permutation of input data for sampling
p = flex.random_permutation(len(data[0]))
permuted = [col.select(p) for col in data]
# draw random p+1 subset J (or larger if required)
detS0 = 0.0
i = 0
while not detS0 > 0.0:
subset_size = self._p + 1 + i
J = [e[0:subset_size] for e in permuted]
i += 1
T0, S0 = self.means_and_covariance(J)
detS0 = S0.matrix_determinant_via_lu()
H1 = self.concentration_step(h, data, T0, S0)
return H1
def block(data_size, n_repeats):
data = flex.random_double(size=data_size)
permutation = flex.random_permutation(data.size())
for use_pointers, use_iterators_range in [(False, 2), (True, 3)]:
print " use_pointers =", use_pointers
for use_iterators in range(use_iterators_range):
t0 = time.time()
result = flex.integer_offsets_vs_pointers(data, permutation, 0,
use_pointers,
use_iterators)
t_overhead = time.time() - t0
t0 = time.time()
result = flex.integer_offsets_vs_pointers(data, permutation,
n_repeats, use_pointers,
use_iterators)
print " use_iterators =", use_iterators, \
"time = %.2f s" % (time.time() - t0 - t_overhead), \
"overhead = %.2f s" % t_overhead, \
"(%-7s %.6g)" % result
def form_initial_subset(self, h, data):
"""Method 2 of subsection 3.1 of R&vD"""
# permutation of input data for sampling
p = flex.random_permutation(len(data[0]))
permuted = [col.select(p) for col in data]
# draw random p+1 subset J (or larger if required)
detS0 = 0.0
i = 0
while not detS0 > 0.0:
subset_size = self._p + 1 + i
J = [e[0:subset_size] for e in permuted]
i += 1
T0, S0 = self.means_and_covariance(J)
detS0 = S0.matrix_determinant_via_lu()
H1 = self.concentration_step(h, data, T0, S0)
return H1
def block(data_size, n_repeats):
data = flex.random_double(size=data_size)
permutation = flex.random_permutation(data.size())
for use_pointers, use_iterators_range in [(False, 2), (True, 3)]:
print " use_pointers =", use_pointers
for use_iterators in range(use_iterators_range):
t0 = time.time()
result = flex.integer_offsets_vs_pointers(
data, permutation, 0, use_pointers, use_iterators)
t_overhead = time.time() - t0
t0 = time.time()
result = flex.integer_offsets_vs_pointers(
data,
permutation,
n_repeats,
use_pointers,
use_iterators)
print " use_iterators =", use_iterators, \
"time = %.2f s" % (time.time() - t0 - t_overhead), \
"overhead = %.2f s" % t_overhead, \
"(%-7s %.6g)" % result
def exercise_densely_distributed_singular_values(show_progress, full_coverage):
n = 40
m = 2*n
n_runs = 20
tol = 10*scitbx.math.double_numeric_limits.epsilon
gen = scitbx.linalg.random_normal_matrix_generator(m, n)
sigmas = []
sigmas.append( flex.double([ 10**(-i/n) for i in xrange(n) ]) )
sigmas.append( sigmas[0].select(flex.random_permutation(n)) )
sigmas.append( sigmas[0].reversed() )
print "Densely distributed singular values:",
n_tests = 0
for i in xrange(n_runs):
if not full_coverage and random.random() < 0.8: continue
n_tests += 1
for i_case, sigma in enumerate(sigmas):
a = gen.matrix_with_singular_values(sigma)
svd = scitbx.linalg.svd.real(a, accumulate_u=False, accumulate_v=False)
if i_case > 0: sigma = sigma.select(
flex.sort_permutation(sigma, reverse=True))
delta = (svd.sigma - sigma)/sigma/tol
assert delta.all_lt(5)
print "%i done." % n_tests
def cross_validate_to_determine_number_of_terms(x_obs,
y_obs,
w_obs=None,
min_terms=10,
max_terms=25,
n_free=100,
n_goes=5):
if (n_goes == None):
if (min_terms < 2):
min_terms = 2
free_residuals = []
free_flags = flex.bool(x_obs.size(), True)
free_permut = flex.random_permutation(x_obs.size())
for ii in range(n_free):
free_flags[free_permut[ii]] = False
for count in range(min_terms, max_terms):
fit = chebyshev_lsq_fit(count, x_obs, y_obs, w_obs, free_flags)
free_residuals.append(fit.free_f)
return (flex.double(free_residuals))
else:
if w_obs is None:
w_obs = flex.double(x_obs.size(), 1)
free_resid = flex.double(max_terms - min_terms, 0)
for jj in range(n_goes):
free_resid += cross_validate_to_determine_number_of_terms(
x_obs,
y_obs,
w_obs,
min_terms=min_terms,
max_terms=max_terms,
n_free=n_free,
n_goes=None)
return (min_terms + flex.min_index(free_resid))
def _estimate_cc_sig_fac(self):
"""Estimation of sigma(CC) as a function of sample size.
Estimate the error in the correlation coefficient, sigma(CC) by using
pairs of reflections at similar resolutions that are not related by
potential symmetry. Using pairs of unrelated reflections at similar
resolutions, calculate sigma(CC) == rms(CC) for groups of size N = 3..200.
The constant CCsigFac is obtained from a linear fit of
sigma(CC) to 1/N^(1/2), i.e.:
sigma(CC) = CCsigFac/N^(1/2)
"""
max_bins = 500
reflections_per_bin = max(
200, int(math.ceil(self.intensities.size() / max_bins)))
binner = self.intensities.setup_binner_counting_sorted(
reflections_per_bin=reflections_per_bin)
a = flex.double()
b = flex.double()
ma_tmp = self.intensities.customized_copy(
crystal_symmetry=crystal.symmetry(
space_group=self.lattice_group,
unit_cell=self.intensities.unit_cell(),
assert_is_compatible_unit_cell=False,
)).map_to_asu()
for i in range(binner.n_bins_all()):
count = binner.counts()[i]
if count == 0:
continue
bin_isel = binner.array_indices(i)
p = flex.random_permutation(count)
p = p[:2 * (count // 2)] # ensure even count
ma_a = ma_tmp.select(bin_isel.select(p[:count // 2]))
ma_b = ma_tmp.select(bin_isel.select(p[count // 2:]))
# only choose pairs of reflections that don't have the same indices
# in the asu of the lattice group
sel = ma_a.indices() != ma_b.indices()
a.extend(ma_a.data().select(sel))
b.extend(ma_b.data().select(sel))
perm = flex.random_selection(a.size(), min(20000, a.size()))
a = a.select(perm)
b = b.select(perm)
self.corr_unrelated = CorrelationCoefficientAccumulator(a, b)
n_pairs = a.size()
min_num_groups = 10 # minimum number of groups
max_n_group = int(min(n_pairs / min_num_groups,
200)) # maximum number in group
min_n_group = int(min(5, max_n_group)) # minimum number in group
if (max_n_group - min_n_group) < 4:
self.cc_sig_fac = 0
return
mean_ccs = flex.double()
rms_ccs = flex.double()
ns = flex.double()
for n in range(min_n_group, max_n_group + 1):
ns.append(n)
ccs = flex.double()
for i in range(200):
isel = flex.random_selection(a.size(), n)
corr = CorrelationCoefficientAccumulator(
a.select(isel), b.select(isel))
ccs.append(corr.coefficient())
mean_ccs.append(flex.mean(ccs))
rms_ccs.append(flex.mean(flex.pow2(ccs))**0.5)
x = 1 / flex.pow(ns, 0.5)
y = rms_ccs
fit = flex.linear_regression(x, y)
if fit.is_well_defined():
self.cc_sig_fac = fit.slope()
else:
self.cc_sig_fac = 0
def update_minimum_covering_sphere(self):
n_points = min(1000, self.points.size())
isel = flex.random_permutation(self.points.size())[:n_points]
self.minimum_covering_sphere = minimum_covering_sphere(
self.points.select(isel))
def discover_better_experimental_model(
experiments,
reflections,
params,
nproc=1,
d_min=None,
mm_search_scope=4.0,
wide_search_binning=1,
plot_search_scope=False,
):
assert len(experiments) == len(reflections)
assert len(experiments) > 0
refl_lists = []
max_cell_list = []
# The detector/beam of the first experiment is used to define the basis for the
# optimisation, so assert that the beam intersects with the detector
detector = experiments[0].detector
beam = experiments[0].beam
beam_panel = detector.get_panel_intersection(beam.get_s0())
if beam_panel == -1:
raise Sorry("input beam does not intersect detector")
for expt, refl in zip(experiments, reflections):
refl = copy.deepcopy(refl)
refl["imageset_id"] = flex.int(len(refl), 0)
refl.centroid_px_to_mm([expt])
refl.map_centroids_to_reciprocal_space([expt])
if d_min is not None:
d_spacings = 1 / refl["rlp"].norms()
sel = d_spacings > d_min
refl = refl.select(sel)
# derive a max_cell from mm spots
if params.max_cell is None:
max_cell = find_max_cell(refl,
max_cell_multiplier=1.3,
step_size=45).max_cell
max_cell_list.append(max_cell)
if params.max_reflections is not None and refl.size(
) > params.max_reflections:
logger.info("Selecting subset of %i reflections for analysis" %
params.max_reflections)
perm = flex.random_permutation(refl.size())
sel = perm[:params.max_reflections]
refl = refl.select(sel)
refl_lists.append(refl)
if params.max_cell is None:
max_cell = flex.median(flex.double(max_cell_list))
else:
max_cell = params.max_cell
with concurrent.futures.ProcessPoolExecutor(max_workers=nproc) as pool:
solution_lists = []
amax_list = []
for result in pool.map(run_dps, experiments, refl_lists,
itertools.repeat(max_cell)):
if result.get("solutions"):
solution_lists.append(result["solutions"])
amax_list.append(result["amax"])
if not solution_lists:
raise Sorry("No solutions found")
new_experiments = optimize_origin_offset_local_scope(
experiments,
refl_lists,
solution_lists,
amax_list,
mm_search_scope=mm_search_scope,
wide_search_binning=wide_search_binning,
plot_search_scope=plot_search_scope,
)
new_detector = new_experiments[0].detector
old_panel, old_beam_centre = detector.get_ray_intersection(beam.get_s0())
new_panel, new_beam_centre = new_detector.get_ray_intersection(
beam.get_s0())
old_beam_centre_px = detector[old_panel].millimeter_to_pixel(
old_beam_centre)
new_beam_centre_px = new_detector[new_panel].millimeter_to_pixel(
new_beam_centre)
logger.info("Old beam centre: %.2f, %.2f mm" % old_beam_centre +
" (%.1f, %.1f px)" % old_beam_centre_px)
logger.info("New beam centre: %.2f, %.2f mm" % new_beam_centre +
" (%.1f, %.1f px)" % new_beam_centre_px)
logger.info(
"Shift: %.2f, %.2f mm" %
(matrix.col(old_beam_centre) - matrix.col(new_beam_centre)).elems +
" (%.1f, %.1f px)" %
(matrix.col(old_beam_centre_px) - matrix.col(new_beam_centre_px)).elems
)
return new_experiments
def random_permutation(s): from scitbx.array_family import flex return flex.select(s, flex.random_permutation(size=len(s)))
def get_optimisation_test_set(
n_grp,
n_dst,
n_atm,
atomic_amplitude=1.0,
random_dataset_order=False,
):
"""Generate a test set of random uijs for optimisation"""
assert n_grp > 0
assert n_grp <= len(TLS_MATRICES)
assert n_dst > 0
assert n_atm > 3
target_uijs = numpy.zeros((n_dst, n_atm, 6))
# Default to equal weights
target_weights = flex.double(flex.grid((n_dst, n_atm)), 1.0)
# Create atomic base
atomic_values_numpy = rran(n_atm * 6).reshape((n_atm, 6))
real_atomic_amps = atomic_amplitude * rran(n_atm)
atomic_base = flex.sym_mat3_double(atomic_values_numpy)
# Create atomic total and add to target
atomic_uijs = real_atomic_amps.reshape((n_atm, 1)) * atomic_values_numpy
for i_dst in range(n_dst):
target_uijs[i_dst] = atomic_uijs
# Generate a random set of amplitudes
real_group_amps = rran(n_grp * n_dst).reshape((n_grp, n_dst))
# Output lists of base uijs and atoms they are associated with
base_uijs = []
base_sels = []
# Mapping of elements to datasets
if random_dataset_order is True:
dataset_hash = flex.size_t(
numpy.concatenate(
[flex.random_permutation(n_dst) for _ in range(n_grp)]))
else:
dataset_hash = flex.size_t(list(range(n_dst)) * n_grp)
# base counter
i_cnt = 0
for i_grp in range(n_grp):
# Number of atoms in this group -- at least 2
if n_grp == 1:
n_atm_this = n_atm
else:
n_atm_this = max(2, iran(n_atm))
# Which atoms are covered by this group
i_sel = iran(n_atm, size=n_atm_this, replace=False)
# Generate some uijs for this
tls_m = TLSMatrices(TLS_MATRICES[i_grp])
for _ in range(n_dst):
# Which dataset?
i_dst = dataset_hash[i_cnt]
i_cnt += 1
# Generate random coordinates
x_ = 50.0 * (-0.5 + rran(n_atm_this * 3).reshape((n_atm_this, 3)))
x_ = flex.vec3_double(x_)
# Generate the base elements
u_ = tls_m.uijs(x_, origin=(0., 0., 0.))
base_uijs.append(flex.sym_mat3_double(u_))
base_sels.append(flex.size_t(i_sel))
# Generate the amplitude-multiplied equivalent
u_m = (real_group_amps[i_grp, i_dst] * tls_m).uijs(x_,
origin=(0., 0.,
0.))
target_uijs[i_dst, i_sel, :] += numpy.array(u_m)
# Reshape
target_uijs = flex.sym_mat3_double(target_uijs.reshape((n_dst * n_atm, 6)))
target_uijs.reshape(flex.grid((n_dst, n_atm)))
return (
target_uijs, target_weights, \
base_uijs, base_sels, dataset_hash, \
atomic_base, \
real_group_amps.flatten(), real_atomic_amps,
)
def scale_map_coeffs(n_real=None,
randomized_map_coeffs=None,
map_coeffs=None,
high_resolution_noise_fraction=None,
low_resolution_noise_fraction=None,
random_selection_within_bins=False,
low_resolution_noise_cutoff=None,
log=sys.stdout):
'''
Scale map coefficients to target value vs resolution, optionally randomize
Scales map coefficients in resolution bins, scale factor adjusted
to yield high_resolution_noise_fraction at high-resolution limit
and low_resolution_noise_fraction at low_resolution_noise_cutoff and
linearly between in 1/resolution.
Optionally randomizes amplitudes and phases by shuffling within bins
'''
assert random_selection_within_bins or randomized_map_coeffs
if not hasattr(map_coeffs, 'binner') or not map_coeffs.binner():
map_coeffs.setup_binner(auto_binning=True)
d_max, d_min = map_coeffs.d_max_min()
if d_max < 0: d_max = 1.e+10
if random_selection_within_bins:
new_map_coeffs = map_coeffs.customized_copy(
data=flex.complex_double(map_coeffs.size(), (0 + 0.j)))
print("\nGenerating map randomized in Fourier space", file=log)
else:
new_map_coeffs = randomized_map_coeffs
print("Relative error added at high-resolution: %.3f" %
(high_resolution_noise_fraction),
file=log)
print("Relative error added at low-resolution: %.3f" %
(low_resolution_noise_fraction),
file=log)
print("Resolution Noise ratio RMS original RMS error ",
file=log)
for i_bin in map_coeffs.binner().range_used():
sel = map_coeffs.binner().selection(i_bin)
dd = map_coeffs.d_spacings().data().select(sel)
local_d_mean = dd.min_max_mean().mean
local_s_mean = 1 / local_d_mean
s_max = 1 / max(1.e-10, d_min)
if low_resolution_noise_cutoff:
s_min = 1 / max(1.e-10, min(d_max, low_resolution_noise_cutoff))
else:
s_min = 1 / max(1.e-10, d_max)
fraction_high = max(
0., min(1., (local_s_mean - s_min) / max(1.e-10, s_max - s_min)))
noise_ratio=low_resolution_noise_fraction+\
fraction_high * (
high_resolution_noise_fraction-
low_resolution_noise_fraction)
mc = map_coeffs.select(sel)
rms_original = norm(mc.data())
if random_selection_within_bins: # randomize, normalize, scale
fp, phi = map_coeffs_as_fp_phi(mc)
sel_fp = flex.random_permutation(fp.size())
new_fp = fp.select(sel_fp)
data = new_fp.data()
data *= noise_ratio
sel_phi = flex.random_permutation(phi.size())
new_phi = phi.select(sel_phi)
new_mc = fp_phi_as_map_coeffs(new_fp, new_phi)
else: # just normalize and scale
randomized_mc = randomized_map_coeffs.select(sel)
rms_new = norm(randomized_mc.data())
scale = rms_original / max(1.e-10, rms_new)
new_fp, new_phi = map_coeffs_as_fp_phi(randomized_mc)
data = new_fp.data()
data *= noise_ratio * scale
new_mc = fp_phi_as_map_coeffs(new_fp, new_phi)
rms_new = norm(new_mc.data())
new_map_coeffs.data().set_selected(sel, new_mc.data())
print(" %.3f %.3f %.3f %.3f " %
(local_d_mean, noise_ratio, rms_original, rms_new),
file=log)
# Convert to map
from cctbx import maptbx
return maptbx.map_coefficients_to_map(
map_coeffs=new_map_coeffs,
crystal_symmetry=new_map_coeffs.crystal_symmetry(),
n_real=n_real)
def update_minimum_covering_sphere(self):
n_points = min(1000, self.points.size())
isel = flex.random_permutation(self.points.size())[:n_points]
self.minimum_covering_sphere = minimum_covering_sphere(
self.points.select(isel))
def discover_better_experimental_model(imagesets,
spot_lists,
params,
dps_params,
nproc=1,
wide_search_binning=1):
assert len(imagesets) == len(spot_lists)
assert len(imagesets) > 0
# XXX should check that all the detector and beam objects are the same
spot_lists_mm = []
max_cell_list = []
detector = imagesets[0].get_detector()
beam = imagesets[0].get_beam()
beam_panel = detector.get_panel_intersection(beam.get_s0())
if beam_panel == -1:
raise Sorry("input beam does not intersect detector")
for imageset, spots in zip(imagesets, spot_lists):
if "imageset_id" not in spots:
spots["imageset_id"] = spots["id"]
spots_mm = copy.deepcopy(spots)
spots_mm.centroid_px_to_mm(imageset.get_detector(),
scan=imageset.get_scan())
spots_mm.map_centroids_to_reciprocal_space(
detector=imageset.get_detector(),
beam=imageset.get_beam(),
goniometer=imageset.get_goniometer(),
)
if dps_params.d_min is not None:
d_spacings = 1 / spots_mm["rlp"].norms()
sel = d_spacings > dps_params.d_min
spots_mm = spots_mm.select(sel)
# derive a max_cell from mm spots
if params.max_cell is None:
max_cell = find_max_cell(spots_mm,
max_cell_multiplier=1.3,
step_size=45).max_cell
max_cell_list.append(max_cell)
if (params.max_reflections is not None
and spots_mm.size() > params.max_reflections):
logger.info("Selecting subset of %i reflections for analysis" %
params.max_reflections)
perm = flex.random_permutation(spots_mm.size())
sel = perm[:params.max_reflections]
spots_mm = spots_mm.select(sel)
spot_lists_mm.append(spots_mm)
if params.max_cell is None:
max_cell = flex.median(flex.double(max_cell_list))
else:
max_cell = params.max_cell
args = [(imageset, spots, max_cell, dps_params)
for imageset, spots in zip(imagesets, spot_lists_mm)]
results = easy_mp.parallel_map(
func=run_dps,
iterable=args,
processes=nproc,
method="multiprocessing",
preserve_order=True,
asynchronous=True,
preserve_exception_message=True,
)
solution_lists = [r["solutions"] for r in results]
amax_list = [r["amax"] for r in results]
assert len(solution_lists) > 0
detector = imagesets[0].get_detector()
beam = imagesets[0].get_beam()
# perform calculation
if dps_params.indexing.improve_local_scope == "origin_offset":
discoverer = better_experimental_model_discovery(
imagesets,
spot_lists_mm,
solution_lists,
amax_list,
dps_params,
wide_search_binning=wide_search_binning,
)
new_detector = discoverer.optimize_origin_offset_local_scope()
old_panel, old_beam_centre = detector.get_ray_intersection(
beam.get_s0())
new_panel, new_beam_centre = new_detector.get_ray_intersection(
beam.get_s0())
old_beam_centre_px = detector[old_panel].millimeter_to_pixel(
old_beam_centre)
new_beam_centre_px = new_detector[new_panel].millimeter_to_pixel(
new_beam_centre)
logger.info("Old beam centre: %.2f, %.2f mm" % old_beam_centre +
" (%.1f, %.1f px)" % old_beam_centre_px)
logger.info("New beam centre: %.2f, %.2f mm" % new_beam_centre +
" (%.1f, %.1f px)" % new_beam_centre_px)
logger.info(
"Shift: %.2f, %.2f mm" %
(matrix.col(old_beam_centre) - matrix.col(new_beam_centre)).elems +
" (%.1f, %.1f px)" % (matrix.col(old_beam_centre_px) -
matrix.col(new_beam_centre_px)).elems)
return new_detector, beam
elif dps_params.indexing.improve_local_scope == "S0_vector":
raise NotImplementedError()
