def __init__(self,
hooft_analysis,
use_students_t_distribution=False,
students_t_nu=None,
probability_plot_slope=None):
self.delta_fo2, minus_fo2 =\
hooft_analysis.delta_fo2.generate_bijvoet_mates().hemispheres_acentrics()
self.delta_fc2, minus_fc2 =\
hooft_analysis.delta_fc2.generate_bijvoet_mates().hemispheres_acentrics()
# we want to plot both hemispheres
self.delta_fo2.indices().extend(minus_fo2.indices())
self.delta_fo2.data().extend(minus_fo2.data() * -1)
self.delta_fo2.sigmas().extend(minus_fo2.sigmas())
self.delta_fc2.indices().extend(minus_fc2.indices())
self.delta_fc2.data().extend(minus_fc2.data() * -1)
self.indices = self.delta_fo2.indices()
observed_deviations = (hooft_analysis.G * self.delta_fc2.data()
- self.delta_fo2.data())/self.delta_fo2.sigmas()
if probability_plot_slope is not None:
observed_deviations /= probability_plot_slope
selection = flex.sort_permutation(observed_deviations)
observed_deviations = observed_deviations.select(selection)
if use_students_t_distribution:
if students_t_nu is None:
students_t_nu = maximise_students_t_correlation_coefficient(
observed_deviations, 1, 200)
self.distribution = distributions.students_t_distribution(students_t_nu)
else:
self.distribution = distributions.normal_distribution()
self.x = self.distribution.quantiles(observed_deviations.size())
self.y = observed_deviations
self.fit = flex.linear_regression(self.x[5:-5], self.y[5:-5])
self.correlation = flex.linear_correlation(self.x[5:-5], self.y[5:-5])
assert self.fit.is_well_defined()
def exercise_sdb(verbose=0):
structure = random_structure.xray_structure(
space_group_info=sgtbx.space_group_info("P 31"),
elements=["N","C","C","O"]*2,
volume_per_atom=500,
min_distance=2.,
general_positions_only=False,
random_u_iso=True)
f_abs = abs(structure.structure_factors(
anomalous_flag=False, d_min=2, algorithm="direct").f_calc())
sdb_out = structure.as_cns_sdb_file(
file="foo.sdb",
description="random_structure",
comment=["any", "thing"],
group="best")
if (0 or verbose):
sys.stdout.write(sdb_out)
sdb_files = sdb_reader.multi_sdb_parser(StringIO(sdb_out))
assert len(sdb_files) == 1
structure_read = sdb_files[0].as_xray_structure(
crystal_symmetry=crystal.symmetry(
unit_cell=structure.unit_cell(),
space_group_info=None))
f_read = abs(f_abs.structure_factors_from_scatterers(
xray_structure=structure_read, algorithm="direct").f_calc())
regression = flex.linear_regression(f_abs.data(), f_read.data())
assert regression.is_well_defined()
if (0 or verbose):
regression.show_summary()
assert abs(regression.slope()-1) < 1.e-4
assert abs(regression.y_intercept()) < 1.e-3
def __init__(self,
hooft_analysis,
use_students_t_distribution=False,
students_t_nu=None,
probability_plot_slope=None):
self.delta_fo2, minus_fo2 =\
hooft_analysis.delta_fo2.generate_bijvoet_mates().hemispheres_acentrics()
self.delta_fc2, minus_fc2 =\
hooft_analysis.delta_fc2.generate_bijvoet_mates().hemispheres_acentrics()
# we want to plot both hemispheres
self.delta_fo2.indices().extend(minus_fo2.indices())
self.delta_fo2.data().extend(minus_fo2.data() * -1)
self.delta_fo2.sigmas().extend(minus_fo2.sigmas())
self.delta_fc2.indices().extend(minus_fc2.indices())
self.delta_fc2.data().extend(minus_fc2.data() * -1)
self.indices = self.delta_fo2.indices()
observed_deviations = (hooft_analysis.G * self.delta_fc2.data()
- self.delta_fo2.data())/self.delta_fo2.sigmas()
if probability_plot_slope is not None:
observed_deviations /= probability_plot_slope
selection = flex.sort_permutation(observed_deviations)
observed_deviations = observed_deviations.select(selection)
if use_students_t_distribution:
if students_t_nu is None:
students_t_nu = maximise_students_t_correlation_coefficient(
observed_deviations, 1, 200)
self.distribution = distributions.students_t_distribution(students_t_nu)
else:
self.distribution = distributions.normal_distribution()
self.x = self.distribution.quantiles(observed_deviations.size())
self.y = observed_deviations
self.fit = flex.linear_regression(self.x[5:-5], self.y[5:-5])
self.correlation = flex.linear_correlation(self.x[5:-5], self.y[5:-5])
assert self.fit.is_well_defined()
def exercise_xray_structure(use_u_aniso, verbose=0):
structure = random_structure.xray_structure(
space_group_info=sgtbx.space_group_info("P 31"),
elements=["N", "C", "C", "O", "Si"] * 2,
volume_per_atom=500,
min_distance=2.0,
general_positions_only=False,
random_u_iso=True,
use_u_aniso=use_u_aniso,
)
f_abs = abs(structure.structure_factors(anomalous_flag=False, d_min=2, algorithm="direct").f_calc())
for resname in (None, "res"):
for fractional_coordinates in (False, True):
pdb_file = structure.as_pdb_file(
remark="Title", remarks=["Any", "Thing"], fractional_coordinates=fractional_coordinates, resname=resname
)
if 0 or verbose:
sys.stdout.write(pdb_file)
structure_read = iotbx.pdb.input(
source_info=None, lines=flex.std_string(pdb_file.splitlines())
).xray_structure_simple(fractional_coordinates=fractional_coordinates, use_scale_matrix_if_available=False)
f_read = abs(
f_abs.structure_factors_from_scatterers(xray_structure=structure_read, algorithm="direct").f_calc()
)
regression = flex.linear_regression(f_abs.data(), f_read.data())
assert regression.is_well_defined()
if 0 or verbose:
regression.show_summary()
assert approx_equal(regression.slope(), 1, eps=1.0e-2)
assert approx_equal(regression.y_intercept(), 0, eps=flex.max(f_abs.data()) * 0.01)
def linear_regression_test(d_analytical,
d_numerical,
test_hard=True,
slope_tolerance=1.e-3,
correlation_min=0.999,
verbose=0):
if (type(d_analytical) != type(flex.double())):
d_analytical = flex_tuple_as_flex_double(d_analytical)
if (type(d_numerical) != type(flex.double())):
d_numerical = flex_tuple_as_flex_double(d_numerical)
if (0 or verbose):
print("analytical:", tuple(d_analytical))
print("numerical: ", tuple(d_numerical))
if (flex.max(flex.abs(d_analytical)) == 0
and flex.max(flex.abs(d_numerical)) == 0):
return
regr = flex.linear_regression(d_analytical, d_numerical)
corr = flex.linear_correlation(d_analytical, d_numerical).coefficient()
assert regr.is_well_defined()
if (abs(regr.slope() - 1) > slope_tolerance or corr < correlation_min):
print("Error: finite difference mismatch:")
print("slope:", regr.slope())
print("correlation:", corr)
if (0 or verbose):
for a, n in zip(d_analytical, d_numerical):
print(a, n)
assert not test_hard
def exercise_xray_structure(use_u_aniso, verbose=0):
structure = random_structure.xray_structure(
space_group_info=sgtbx.space_group_info("P 31"),
elements=["N","C","C","O","Si"]*2,
volume_per_atom=500,
min_distance=2.,
general_positions_only=False,
random_u_iso=True,
use_u_aniso=use_u_aniso)
f_abs = abs(structure.structure_factors(
anomalous_flag=False, d_min=2, algorithm="direct").f_calc())
for resname in (None, "res"):
for fractional_coordinates in (False, True):
pdb_file = structure.as_pdb_file(
remark="Title", remarks=["Any", "Thing"],
fractional_coordinates=fractional_coordinates,
resname=resname)
if (0 or verbose):
sys.stdout.write(pdb_file)
structure_read = iotbx.pdb.input(
source_info=None,
lines=flex.std_string(pdb_file.splitlines())).xray_structure_simple(
fractional_coordinates=fractional_coordinates,
use_scale_matrix_if_available=False)
f_read = abs(f_abs.structure_factors_from_scatterers(
xray_structure=structure_read, algorithm="direct").f_calc())
regression = flex.linear_regression(f_abs.data(), f_read.data())
assert regression.is_well_defined()
if (0 or verbose):
regression.show_summary()
assert approx_equal(regression.slope(), 1, eps=1.e-2)
assert approx_equal(
regression.y_intercept(), 0, eps=flex.max(f_abs.data())*0.01)
def linear_regression_test(d_analytical, d_numerical, test_hard=True,
slope_tolerance=1.e-3,
correlation_min=0.999,
verbose=0):
if (type(d_analytical) != type(flex.double())):
d_analytical = flex_tuple_as_flex_double(d_analytical)
if (type(d_numerical) != type(flex.double())):
d_numerical = flex_tuple_as_flex_double(d_numerical)
if (0 or verbose):
print "analytical:", tuple(d_analytical)
print "numerical: ", tuple(d_numerical)
if ( flex.max(flex.abs(d_analytical)) == 0
and flex.max(flex.abs(d_numerical)) == 0):
return
regr = flex.linear_regression(d_analytical, d_numerical)
corr = flex.linear_correlation(d_analytical, d_numerical).coefficient()
assert regr.is_well_defined()
if (abs(regr.slope() - 1) > slope_tolerance or corr < correlation_min):
print "Error: finite difference mismatch:"
print "slope:", regr.slope()
print "correlation:", corr
if (0 or verbose):
for a, n in zip(d_analytical, d_numerical):
print a, n
assert not test_hard
def anomalous_probability_plot(intensities, expected_delta=None):
from scitbx.math import distributions
from scitbx.array_family import flex
assert intensities.is_unique_set_under_symmetry()
assert intensities.anomalous_flag()
dI = intensities.anomalous_differences()
y = dI.data()/dI.sigmas()
perm = flex.sort_permutation(y)
y = y.select(perm)
distribution = distributions.normal_distribution()
x = distribution.quantiles(y.size())
if expected_delta is not None:
sel = flex.abs(x) < expected_delta
x = x.select(sel)
y = y.select(sel)
fit = flex.linear_regression(x, y)
correlation = flex.linear_correlation(x, y)
assert fit.is_well_defined()
if 0:
from matplotlib import pyplot
pyplot.scatter(x, y)
m = fit.slope()
c = fit.y_intercept()
pyplot.plot(pyplot.xlim(), [m * x_ + c for x_ in pyplot.xlim()])
pyplot.show()
return fit.slope(), fit.y_intercept(), x.size()
def absolute_structure_analysis(xs, fo2, fc, scale, nu=None, log=None,
outlier_cutoff_factor=None):
if log is None:
log = sys.stdout
hooft_analysis = absolute_structure.hooft_analysis(
fo2, fc, scale_factor=scale, outlier_cutoff_factor=outlier_cutoff_factor)
print >> log, "Gaussian analysis:"
hooft_analysis.show(out=log)
NPP = absolute_structure.bijvoet_differences_probability_plot(
hooft_analysis)
print >> log, "Probability plot:"
NPP.show(out=log)
print >> log
if nu is None:
nu = absolute_structure.maximise_students_t_correlation_coefficient(
NPP.y, min_nu=1, max_nu=200)
distribution = distributions.students_t_distribution(nu)
observed_deviations = NPP.y
expected_deviations = distribution.quantiles(observed_deviations.size())
fit = flex.linear_regression(
expected_deviations[5:-5], observed_deviations[5:-5])
t_analysis = absolute_structure.students_t_hooft_analysis(
fo2, fc, nu, scale_factor=scale, probability_plot_slope=fit.slope(),
outlier_cutoff_factor=outlier_cutoff_factor)
tPP = absolute_structure.bijvoet_differences_probability_plot(
t_analysis, use_students_t_distribution=True, students_t_nu=nu)
print >> log, "Student's t analysis:"
print >> log, "nu: %.2f" %nu
t_analysis.show(out=log)
print >> log, "Probability plot:"
tPP.show(out=log)
print >> log
if xs is not None:
flack = absolute_structure.flack_analysis(xs, fo2.as_xray_observations())
flack.show(out=log)
def anomalous_probability_plot(intensities, expected_delta=None):
from scitbx.math import distributions
from scitbx.array_family import flex
assert intensities.is_unique_set_under_symmetry()
assert intensities.anomalous_flag()
dI = intensities.anomalous_differences()
y = dI.data() / dI.sigmas()
perm = flex.sort_permutation(y)
y = y.select(perm)
distribution = distributions.normal_distribution()
x = distribution.quantiles(y.size())
if expected_delta is not None:
sel = flex.abs(x) < expected_delta
x = x.select(sel)
y = y.select(sel)
fit = flex.linear_regression(x, y)
correlation = flex.linear_correlation(x, y)
assert fit.is_well_defined()
if 0:
from matplotlib import pyplot
pyplot.scatter(x, y)
m = fit.slope()
c = fit.y_intercept()
pyplot.plot(pyplot.xlim(), [m * x_ + c for x_ in pyplot.xlim()])
pyplot.show()
return fit.slope(), fit.y_intercept(), x.size()
def exercise_sdb(verbose=0):
structure = random_structure.xray_structure(
space_group_info=sgtbx.space_group_info("P 31"),
elements=["N", "C", "C", "O"] * 2,
volume_per_atom=500,
min_distance=2.,
general_positions_only=False,
random_u_iso=True)
f_abs = abs(
structure.structure_factors(anomalous_flag=False,
d_min=2,
algorithm="direct").f_calc())
sdb_out = structure.as_cns_sdb_file(file="foo.sdb",
description="random_structure",
comment=["any", "thing"],
group="best")
if (0 or verbose):
sys.stdout.write(sdb_out)
sdb_files = sdb_reader.multi_sdb_parser(StringIO(sdb_out))
assert len(sdb_files) == 1
structure_read = sdb_files[0].as_xray_structure(
crystal_symmetry=crystal.symmetry(unit_cell=structure.unit_cell(),
space_group_info=None))
f_read = abs(
f_abs.structure_factors_from_scatterers(xray_structure=structure_read,
algorithm="direct").f_calc())
regression = flex.linear_regression(f_abs.data(), f_read.data())
assert regression.is_well_defined()
if (0 or verbose):
regression.show_summary()
assert abs(regression.slope() - 1) < 1.e-4
assert abs(regression.y_intercept()) < 1.e-3
def __init__(self, f_obs, asu_contents, e_statistics=False):
assert f_obs.is_real_array()
self.info = f_obs.info()
f_obs_selected = f_obs.select(f_obs.data() > 0)
f_obs_selected.use_binning_of(f_obs)
# compute <fobs^2> in resolution shells
self.mean_fobs_sq = f_obs_selected.mean_sq(
use_binning=True,
use_multiplicities=True).data[1:-1]
n_none = self.mean_fobs_sq.count(None)
if (n_none > 0):
error_message = "wilson_plot error: %d empty bin%s:" % plural_s(n_none)
if (self.info is not None):
error_message += "\n Info: " + str(self.info)
error_message += "\n Number of bins: %d" % len(self.mean_fobs_sq)
error_message += "\n Number of f_obs > 0: %d" % (
f_obs_selected.indices().size())
error_message += "\n Number of f_obs <= 0: %d" % (
f_obs.indices().size() - f_obs_selected.indices().size())
raise RuntimeError(error_message)
self.mean_fobs_sq = flex.double(self.mean_fobs_sq)
# compute <s^2> = <(sin(theta)/lambda)^2> in resolution shells
stol_sq = f_obs_selected.sin_theta_over_lambda_sq()
stol_sq.use_binner_of(f_obs_selected)
self.mean_stol_sq = flex.double(stol_sq.mean(
use_binning=True,
use_multiplicities=True).data[1:-1])
# cache scattering factor info
gaussians = {}
for chemical_type in asu_contents.keys():
gaussians[chemical_type] = eltbx.xray_scattering.wk1995(
chemical_type).fetch()
# compute expected f_calc^2 in resolution shells
self.expected_f_sq = flex.double()
for stol_sq in self.mean_stol_sq:
sum_fj_sq = 0
for chemical_type, n_atoms in asu_contents.items():
f0 = gaussians[chemical_type].at_stol_sq(stol_sq)
sum_fj_sq += f0 * f0 * n_atoms
self.expected_f_sq.append(sum_fj_sq)
self.expected_f_sq *= f_obs_selected.space_group().order_z() \
* f_obs_selected.space_group().n_ltr()
# fit to straight line
self.x = self.mean_stol_sq
self.y = flex.log(self.mean_fobs_sq / self.expected_f_sq)
fit = flex.linear_regression(self.x, self.y)
assert fit.is_well_defined()
self.fit_y_intercept = fit.y_intercept()
self.fit_slope = fit.slope()
self.wilson_intensity_scale_factor = math.exp(self.fit_y_intercept) # intensity scale factor
self.wilson_k = math.sqrt(self.wilson_intensity_scale_factor) # conversion to amplitude scale factor
self.wilson_b = -self.fit_slope / 2
self.fit_correlation = flex.linear_correlation(self.x,self.y).coefficient()
if e_statistics:
normalised = f_obs_selected.normalised_amplitudes(asu_contents, self)
self.normalised_f_obs = normalised.array()
self.mean_e_sq_minus_1 = normalised.mean_e_sq_minus_1()
self.percent_e_sq_gt_2 = normalised.percent_e_sq_gt_2()
def check_regression(x, y, label, min_correlation=0, verbose=0):
xy_regr = flex.linear_regression(x, y)
xy_corr = flex.linear_correlation(x, y)
assert xy_regr.is_well_defined()
if (0 or verbose):
print label, "correlation: %.4f slope: %.3f" % (xy_corr.coefficient(),
xy_regr.slope())
assert min_correlation == 0 or xy_corr.coefficient() >= min_correlation
def check_regression(x, y, label, min_correlation=0, verbose=0):
xy_regr = flex.linear_regression(x, y)
xy_corr = flex.linear_correlation(x, y)
assert xy_regr.is_well_defined()
if (0 or verbose):
print label, "correlation: %.4f slope: %.3f" % (
xy_corr.coefficient(), xy_regr.slope())
assert min_correlation == 0 or xy_corr.coefficient() >= min_correlation
def exercise(space_group_info, n_scatterers=8, d_min=2, verbose=0, e_min=1.5):
structure = random_structure.xray_structure(
space_group_info,
elements=["const"] * n_scatterers,
volume_per_atom=200,
min_distance=3.0,
general_positions_only=True,
u_iso=0.0,
)
if 0 or verbose:
structure.show_summary().show_scatterers()
f_calc = structure.structure_factors(d_min=d_min, anomalous_flag=False).f_calc()
f_obs = abs(f_calc)
q_obs = miller.array(
miller_set=f_obs,
data=f_obs.data() / math.sqrt(f_obs.space_group().order_p() * n_scatterers) / f_obs.space_group().n_ltr(),
)
q_obs = q_obs.sort(by_value="abs")
q_obs.setup_binner(auto_binning=True)
n_obs = q_obs.quasi_normalize_structure_factors()
r = flex.linear_regression(q_obs.data(), n_obs.data())
if 0 or verbose:
r.show_summary()
assert r.is_well_defined()
assert abs(r.y_intercept()) < 0.1
assert abs(r.slope() - 1) < 0.2
q_large = q_obs.select(q_obs.quasi_normalized_as_normalized().data() > e_min)
if 0 or verbose:
print "Number of e-values > %.6g: %d" % (e_min, q_large.size())
other_structure = random_structure.xray_structure(
space_group_info,
elements=["const"] * n_scatterers,
volume_per_atom=200,
min_distance=3.0,
general_positions_only=True,
u_iso=0.0,
)
assert other_structure.unit_cell().is_similar_to(structure.unit_cell())
q_calc = q_large.structure_factors_from_scatterers(other_structure, algorithm="direct").f_calc()
start = q_large.phase_transfer(q_calc.data())
for selection_fixed in (None, flex.double([random.random() for i in xrange(start.size())]) < 0.4):
from_map_data = direct_space_squaring(start, selection_fixed)
direct_space_result = start.phase_transfer(phase_source=from_map_data)
new_phases = reciprocal_space_squaring(start, selection_fixed, verbose)
reciprocal_space_result = start.phase_transfer(phase_source=flex.polar(1, new_phases))
mwpe = direct_space_result.mean_weighted_phase_error(reciprocal_space_result)
if 0 or verbose:
print "mwpe: %.2f" % mwpe, start.space_group_info()
for i, h in enumerate(direct_space_result.indices()):
amp_d, phi_d = complex_math.abs_arg(direct_space_result.data()[i], deg=True)
amp_r, phi_r = complex_math.abs_arg(reciprocal_space_result.data()[i], deg=True)
phase_err = scitbx.math.phase_error(phi_d, phi_r, deg=True)
assert phase_err < 1.0 or abs(from_map_data[i]) < 1.0e-6
exercise_truncate(q_large)
def recycle(miller_array):
merge.write(file_name="tmp.sca", miller_array=miller_array)
read_back_file = merge.reader(file_handle=open("tmp.sca"))
read_back_arrays = read_back_file.as_miller_arrays()
assert len(read_back_arrays) == 1
read_back_array = read_back_arrays[0]
read_back_input_indexing = read_back_array.adopt_set(miller_array)
if (miller_array.is_xray_amplitude_array()):
read_back_input_indexing = read_back_input_indexing.f_sq_as_f()
regression = flex.linear_regression(miller_array.data(),
read_back_input_indexing.data())
assert approx_equal(regression.slope(), 1, eps=1.e-3)
assert abs(regression.y_intercept()) < 1
regression = flex.linear_regression(miller_array.sigmas(),
read_back_input_indexing.sigmas())
if (miller_array.is_xray_intensity_array()):
assert approx_equal(regression.slope(), 1, eps=1.e-3)
else:
assert approx_equal(regression.slope(), 1, eps=1.e-1)
assert abs(regression.y_intercept()) < 1
def sigmaa_model_error(self):
x = 0.25*flex.pow( self.h_array, 2.0/3.0 ) # h was in d*^-3 !!!
y = flex.log( self.sigmaa_fitted )
#compute the slope please
result = flex.linear_regression( x, y )
result = -(result.slope()/math.pi*3)
if result < 0:
result = None
else:
result = math.sqrt( result )
return result
def sigmaa_model_error(self):
x = 0.25*flex.pow( self.h_array, 2.0/3.0 ) # h was in d*^-3 !!!
y = flex.log( self.sigmaa_fitted )
#compute the slope please
result = flex.linear_regression( x, y )
result = -(result.slope()/math.pi*3)
if result < 0:
result = None
else:
result = math.sqrt( result )
return result
def recycle(miller_array):
merge.write(file_name="tmp.sca", miller_array=miller_array)
read_back_file = merge.reader(file_handle=open("tmp.sca"))
read_back_arrays = read_back_file.as_miller_arrays()
assert len(read_back_arrays) == 1
read_back_array = read_back_arrays[0]
read_back_input_indexing = read_back_array.adopt_set(miller_array)
if (miller_array.is_xray_amplitude_array()):
read_back_input_indexing = read_back_input_indexing.f_sq_as_f()
regression = flex.linear_regression(
miller_array.data(),
read_back_input_indexing.data())
assert approx_equal(regression.slope(), 1, eps=1.e-3)
assert abs(regression.y_intercept()) < 1
regression = flex.linear_regression(
miller_array.sigmas(),
read_back_input_indexing.sigmas())
if (miller_array.is_xray_intensity_array()):
assert approx_equal(regression.slope(), 1, eps=1.e-3)
else:
assert approx_equal(regression.slope(), 1, eps=1.e-1)
assert abs(regression.y_intercept()) < 1
def __init__(self, f_obs, asu_contents, e_statistics=False):
assert f_obs.is_real_array()
self.info = f_obs.info()
f_obs_selected = f_obs.select(f_obs.data() > 0)
f_obs_selected.use_binning_of(f_obs)
# compute <fobs^2> in resolution shells
self.mean_fobs_sq = f_obs_selected.mean_sq(use_binning=True, use_multiplicities=True).data[1:-1]
n_none = self.mean_fobs_sq.count(None)
if n_none > 0:
error_message = "wilson_plot error: %d empty bin%s:" % plural_s(n_none)
if self.info is not None:
error_message += "\n Info: " + str(self.info)
error_message += "\n Number of bins: %d" % len(self.mean_fobs_sq)
error_message += "\n Number of f_obs > 0: %d" % (f_obs_selected.indices().size())
error_message += "\n Number of f_obs <= 0: %d" % (f_obs.indices().size() - f_obs_selected.indices().size())
raise RuntimeError(error_message)
self.mean_fobs_sq = flex.double(self.mean_fobs_sq)
# compute <s^2> = <(sin(theta)/lambda)^2> in resolution shells
stol_sq = f_obs_selected.sin_theta_over_lambda_sq()
stol_sq.use_binner_of(f_obs_selected)
self.mean_stol_sq = flex.double(stol_sq.mean(use_binning=True, use_multiplicities=True).data[1:-1])
# cache scattering factor info
gaussians = {}
for chemical_type in asu_contents.keys():
gaussians[chemical_type] = eltbx.xray_scattering.wk1995(chemical_type).fetch()
# compute expected f_calc^2 in resolution shells
self.expected_f_sq = flex.double()
for stol_sq in self.mean_stol_sq:
sum_fj_sq = 0
for chemical_type, n_atoms in asu_contents.items():
f0 = gaussians[chemical_type].at_stol_sq(stol_sq)
sum_fj_sq += f0 * f0 * n_atoms
self.expected_f_sq.append(sum_fj_sq)
self.expected_f_sq *= f_obs_selected.space_group().order_z() * f_obs_selected.space_group().n_ltr()
# fit to straight line
self.x = self.mean_stol_sq
self.y = flex.log(self.mean_fobs_sq / self.expected_f_sq)
fit = flex.linear_regression(self.x, self.y)
assert fit.is_well_defined()
self.fit_y_intercept = fit.y_intercept()
self.fit_slope = fit.slope()
self.wilson_intensity_scale_factor = math.exp(self.fit_y_intercept) # intensity scale factor
self.wilson_k = math.sqrt(self.wilson_intensity_scale_factor) # conversion to amplitude scale factor
self.wilson_b = -self.fit_slope / 2
self.fit_correlation = flex.linear_correlation(self.x, self.y).coefficient()
if e_statistics:
normalised = f_obs_selected.normalised_amplitudes(asu_contents, self)
self.normalised_f_obs = normalised.array()
self.mean_e_sq_minus_1 = normalised.mean_e_sq_minus_1()
self.percent_e_sq_gt_2 = normalised.percent_e_sq_gt_2()
def absolute_structure_analysis(xs,
fo2,
fc,
scale,
nu=None,
log=None,
outlier_cutoff_factor=None):
if log is None:
log = sys.stdout
hooft_analysis = absolute_structure.hooft_analysis(
fo2,
fc,
scale_factor=scale,
outlier_cutoff_factor=outlier_cutoff_factor)
print >> log, "Gaussian analysis:"
hooft_analysis.show(out=log)
NPP = absolute_structure.bijvoet_differences_probability_plot(
hooft_analysis)
print >> log, "Probability plot:"
NPP.show(out=log)
print >> log
if nu is None:
nu = absolute_structure.maximise_students_t_correlation_coefficient(
NPP.y, min_nu=1, max_nu=200)
distribution = distributions.students_t_distribution(nu)
observed_deviations = NPP.y
expected_deviations = distribution.quantiles(observed_deviations.size())
fit = flex.linear_regression(expected_deviations[5:-5],
observed_deviations[5:-5])
t_analysis = absolute_structure.students_t_hooft_analysis(
fo2,
fc,
nu,
scale_factor=scale,
probability_plot_slope=fit.slope(),
outlier_cutoff_factor=outlier_cutoff_factor)
tPP = absolute_structure.bijvoet_differences_probability_plot(
t_analysis, use_students_t_distribution=True, students_t_nu=nu)
print >> log, "Student's t analysis:"
print >> log, "nu: %.2f" % nu
t_analysis.show(out=log)
print >> log, "Probability plot:"
tPP.show(out=log)
print >> log
if xs is not None:
flack = absolute_structure.flack_analysis(xs,
fo2.as_xray_observations())
flack.show(out=log)
def run(args):
import libtbx.load_env
usage = "%s [options]" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
check_format=False,
epilog=help_message)
params, options, args = parser.parse_args(show_diff_phil=True,
return_unhandled=True)
assert len(args) == 1
from iotbx.reflection_file_reader import any_reflection_file
intensities = None
f = args[0]
arrays = any_reflection_file(f).as_miller_arrays(merge_equivalents=False)
for ma in arrays:
print ma.info().labels
if ma.info().labels == ['I', 'SIGI']:
intensities = ma
elif ma.info().labels == ['IMEAN', 'SIGIMEAN']:
intensities = ma
elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']:
intensities = ma
assert intensities is not None
if params.d_min is not None:
intensities = intensities.resolution_filter(d_min=params.d_min)
from cctbx.array_family import flex
# see also:
# cctbx/miller/merge_equivalents.h
# cctbx/miller/equivalent_reflection_merging.tex
# this should calculate the external variance, i.e. V(y) = sum(v_i)
merging_external = intensities.merge_equivalents(use_internal_variance=False)
multiplicities = merging_external.redundancies().data()
external_sigmas = merging_external.array().sigmas()
# sigmas should be bigger not smaller
external_sigmas *= flex.sqrt(multiplicities.as_double())
# set the sigmas to 1, and calculate the mean intensities and internal variances
intensities_copy = intensities.customized_copy(
sigmas=flex.double(intensities.size(), 1))
merging_internal = intensities_copy.merge_equivalents()
merged_intensities = merging_internal.array()
internal_sigmas = merging_internal.array().sigmas()
# sigmas should be bigger not smaller
internal_sigmas *= flex.sqrt(multiplicities.as_double())
# select only those reflections with sufficient repeat observations
sel = (multiplicities > 3)
external_sigmas = external_sigmas.select(sel)
internal_sigmas = internal_sigmas.select(sel)
merged_intensities = merged_intensities.select(sel)
# what we want to plot/do linear regression with
y = flex.pow2(internal_sigmas/merged_intensities.data())
x = flex.pow2(external_sigmas/merged_intensities.data())
sel = (x < 1) & (y < 1)
x = x.select(sel)
y = y.select(sel)
# set backend before importing pyplot
import matplotlib
#matplotlib.use('Agg')
linreg = flex.linear_regression(x, y)
linreg.show_summary()
import math
print 1/math.sqrt(linreg.slope() * linreg.y_intercept())
#x = -flex.log10(x)
#y = -flex.log10(y)
x = 1/x
y = 1/y
from matplotlib import pyplot
pyplot.scatter(x, y, marker='+', s=20, alpha=1, c='black')
pyplot.show()
pyplot.clf()
# chi^2 plot vs resolution
# i.e. <var(int)>/<var(ext)>
# where var(ext) and var(int) are as defined in equations 4 & 5 respectively
# in Blessing (1997)
internal_var = merged_intensities.customized_copy(
data=flex.pow2(internal_sigmas))
external_var = merged_intensities.customized_copy(
data=flex.pow2(external_sigmas))
n_bins = 10
internal_var.setup_binner(n_bins=n_bins)
external_var.use_binning_of(internal_var)
mean_internal = internal_var.mean(use_binning=True)
mean_external = external_var.mean(use_binning=True)
y = [mean_internal.data[i+1]/mean_external.data[i+1] for i in range(n_bins)]
x = [mean_internal.binner.bin_centers(2)]
pyplot.scatter(x, y)
pyplot.xlabel('1/d^2')
pyplot.ylabel('<var(int)>/<var(ext)>')
pyplot.show()
pyplot.clf()
return
def exercise(space_group_info, n_scatterers=8, d_min=2, verbose=0, e_min=1.5):
structure = random_structure.xray_structure(space_group_info,
elements=["const"] *
n_scatterers,
volume_per_atom=200,
min_distance=3.,
general_positions_only=True,
u_iso=0.0)
if (0 or verbose):
structure.show_summary().show_scatterers()
f_calc = structure.structure_factors(d_min=d_min,
anomalous_flag=False).f_calc()
f_obs = abs(f_calc)
q_obs = miller.array(
miller_set=f_obs,
data=f_obs.data() /
math.sqrt(f_obs.space_group().order_p() * n_scatterers) /
f_obs.space_group().n_ltr())
q_obs = q_obs.sort(by_value="abs")
q_obs.setup_binner(auto_binning=True)
n_obs = q_obs.quasi_normalize_structure_factors()
r = flex.linear_regression(q_obs.data(), n_obs.data())
if (0 or verbose):
r.show_summary()
assert r.is_well_defined()
assert abs(r.y_intercept()) < 0.1
assert abs(r.slope() - 1) < 0.2
q_large = q_obs.select(
q_obs.quasi_normalized_as_normalized().data() > e_min)
if (0 or verbose):
print("Number of e-values > %.6g: %d" % (e_min, q_large.size()))
other_structure = random_structure.xray_structure(
space_group_info,
elements=["const"] * n_scatterers,
volume_per_atom=200,
min_distance=3.,
general_positions_only=True,
u_iso=0.0)
assert other_structure.unit_cell().is_similar_to(structure.unit_cell())
q_calc = q_large.structure_factors_from_scatterers(
other_structure, algorithm="direct").f_calc()
start = q_large.phase_transfer(q_calc.data())
for selection_fixed in (None,
flex.double(
[random.random()
for i in range(start.size())]) < 0.4):
from_map_data = direct_space_squaring(start, selection_fixed)
direct_space_result = start.phase_transfer(phase_source=from_map_data)
new_phases = reciprocal_space_squaring(start, selection_fixed, verbose)
reciprocal_space_result = start.phase_transfer(
phase_source=flex.polar(1, new_phases))
mwpe = direct_space_result.mean_weighted_phase_error(
reciprocal_space_result)
if (0 or verbose):
print("mwpe: %.2f" % mwpe, start.space_group_info())
for i, h in enumerate(direct_space_result.indices()):
amp_d, phi_d = complex_math.abs_arg(direct_space_result.data()[i],
deg=True)
amp_r, phi_r = complex_math.abs_arg(
reciprocal_space_result.data()[i], deg=True)
phase_err = scitbx.math.phase_error(phi_d, phi_r, deg=True)
assert phase_err < 1.0 or abs(from_map_data[i]) < 1.e-6
exercise_truncate(q_large)
def run(args):
import libtbx.load_env
usage = "%s [options]" % libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage, phil=phil_scope, check_format=False, epilog=help_message
)
params, options, args = parser.parse_args(
show_diff_phil=True, return_unhandled=True
)
assert len(args) == 1
from iotbx.reflection_file_reader import any_reflection_file
intensities = None
f = args[0]
arrays = any_reflection_file(f).as_miller_arrays(merge_equivalents=False)
for ma in arrays:
print(ma.info().labels)
if ma.info().labels == ["I", "SIGI"]:
intensities = ma
elif ma.info().labels == ["IMEAN", "SIGIMEAN"]:
intensities = ma
elif ma.info().labels == ["I(+)", "SIGI(+)", "I(-)", "SIGI(-)"]:
intensities = ma
assert intensities is not None
if params.d_min is not None:
intensities = intensities.resolution_filter(d_min=params.d_min)
from cctbx.array_family import flex
# see also:
# cctbx/miller/merge_equivalents.h
# cctbx/miller/equivalent_reflection_merging.tex
# this should calculate the external variance, i.e. V(y) = sum(v_i)
merging_external = intensities.merge_equivalents(use_internal_variance=False)
multiplicities = merging_external.redundancies().data()
external_sigmas = merging_external.array().sigmas()
# sigmas should be bigger not smaller
external_sigmas *= flex.sqrt(multiplicities.as_double())
# set the sigmas to 1, and calculate the mean intensities and internal variances
intensities_copy = intensities.customized_copy(
sigmas=flex.double(intensities.size(), 1)
)
merging_internal = intensities_copy.merge_equivalents()
merged_intensities = merging_internal.array()
internal_sigmas = merging_internal.array().sigmas()
# sigmas should be bigger not smaller
internal_sigmas *= flex.sqrt(multiplicities.as_double())
# select only those reflections with sufficient repeat observations
sel = multiplicities > 3
external_sigmas = external_sigmas.select(sel)
internal_sigmas = internal_sigmas.select(sel)
merged_intensities = merged_intensities.select(sel)
# what we want to plot/do linear regression with
y = flex.pow2(internal_sigmas / merged_intensities.data())
x = flex.pow2(external_sigmas / merged_intensities.data())
sel = (x < 1) & (y < 1)
x = x.select(sel)
y = y.select(sel)
# set backend before importing pyplot
import matplotlib
# matplotlib.use('Agg')
linreg = flex.linear_regression(x, y)
linreg.show_summary()
import math
print(1 / math.sqrt(linreg.slope() * linreg.y_intercept()))
# x = -flex.log10(x)
# y = -flex.log10(y)
x = 1 / x
y = 1 / y
from matplotlib import pyplot
pyplot.scatter(x, y, marker="+", s=20, alpha=1, c="black")
pyplot.show()
pyplot.clf()
# chi^2 plot vs resolution
# i.e. <var(int)>/<var(ext)>
# where var(ext) and var(int) are as defined in equations 4 & 5 respectively
# in Blessing (1997)
internal_var = merged_intensities.customized_copy(data=flex.pow2(internal_sigmas))
external_var = merged_intensities.customized_copy(data=flex.pow2(external_sigmas))
n_bins = 10
internal_var.setup_binner(n_bins=n_bins)
external_var.use_binning_of(internal_var)
mean_internal = internal_var.mean(use_binning=True)
mean_external = external_var.mean(use_binning=True)
y = [mean_internal.data[i + 1] / mean_external.data[i + 1] for i in range(n_bins)]
x = [mean_internal.binner.bin_centers(2)]
pyplot.scatter(x, y)
pyplot.xlabel("1/d^2")
pyplot.ylabel("<var(int)>/<var(ext)>")
pyplot.show()
pyplot.clf()
return
def refine_rotx_roty2(OO,enable_rotational_target=True):
helper = OO.per_frame_helper_factory()
helper.restart()
if enable_rotational_target:
print "Trying least squares minimization of excursions",
from scitbx.lstbx import normal_eqns_solving
iterations = normal_eqns_solving.naive_iterations(
non_linear_ls = helper,
gradient_threshold = 1.E-10)
results = helper.x
print "with %d reflections"%len(OO.parent.indexed_pairs),
print "result %6.2f degrees"%(results[1]*180./math.pi),
print "result %6.2f degrees"%(results[0]*180./math.pi)
if False: # Excursion histogram
print "The input mosaicity is %7.3f deg full width"%OO.parent.inputai.getMosaicity()
# final histogram
if OO.pvr_fix:
final = 360.* helper.fvec_callable_pvr(results)
else:
final = 360.* helper.fvec_callable_NOT_USED_AFTER_BUGFIX(results)
rmsdexc = math.sqrt(flex.mean(final*final))
from matplotlib import pyplot as plt
nbins = len(final)//20
n,bins,patches = plt.hist(final,
nbins, normed=0, facecolor="orange", alpha=0.75)
plt.xlabel("Rotation on e1 axis, rmsd %7.3f deg"%rmsdexc)
plt.title("Histogram of cctbx.xfel misorientation")
plt.axis([-0.5,0.5,0,100])
plt.plot([rmsdexc],[18],"b|")
plt.show()
# Determine optimal mosaicity and domain size model (monochromatic)
if OO.pvr_fix:
final = 360.* helper.fvec_callable_pvr(results)
else:
final = 360.* helper.fvec_callable_NOT_USED_AFTER_BUGFIX(results)
#Guard against misindexing -- seen in simulated data, with zone nearly perfectly aligned
guard_stats = flex.max(final), flex.min(final)
if False and REMOVETEST_KILLING_LEGITIMATE_EXCURSIONS (guard_stats[0] > 2.0 or guard_stats[1] < -2.0):
raise Exception("Misindexing diagnosed by meaningless excursion angle (bandpass_gaussian model)");
print "The mean excursion is %7.3f degrees"%(flex.mean(final))
two_thetas = helper.last_set_orientation.unit_cell().two_theta(OO.reserve_indices,OO.central_wavelength_ang,deg=True)
dspacings = helper.last_set_orientation.unit_cell().d(OO.reserve_indices)
dspace_sq = dspacings * dspacings
excursion_rad = final * math.pi/ 180.
# First -- try to get a reasonable envelope for the observed excursions.
## minimum of three regions; maximum of 50 measurements in each bin
print "fitting parameters on %d spots"%len(excursion_rad)
n_bins = min(max(3, len(excursion_rad)//25),50)
bin_sz = len(excursion_rad)//n_bins
print "nbins",n_bins,"bin_sz",bin_sz
order = flex.sort_permutation(two_thetas)
two_thetas_env = flex.double()
dspacings_env = flex.double()
excursion_rads_env = flex.double()
for x in xrange(0,n_bins):
subset = order[x*bin_sz:(x+1)*bin_sz]
two_thetas_env.append( flex.mean(two_thetas.select(subset)) )
dspacings_env.append( flex.mean(dspacings.select(subset)))
excursion_rads_env.append( flex.max( flex.abs( excursion_rad.select(subset))))
# Second -- parameter fit
## solve the normal equations
sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env)
sum_inv_u = flex.sum(dspacings_env)
sum_te_u = flex.sum(dspacings_env * excursion_rads_env)
sum_te = flex.sum(excursion_rads_env)
Normal_Mat = sqr((sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env)))
Vector = col((sum_te_u, sum_te))
solution = Normal_Mat.inverse() * Vector
s_ang = 1./(2*solution[0])
print "Best LSQ fit Scheerer domain size is %9.2f ang"%(
s_ang)
tan_phi_rad = helper.last_set_orientation.unit_cell().d(OO.reserve_indices) / (2. * s_ang)
tan_phi_deg = tan_phi_rad * 180./math.pi
k_degrees = solution[1]* 180./math.pi
print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f"%(2*k_degrees, k_degrees)
tan_outer_deg = tan_phi_deg + k_degrees
if OO.mosaic_refinement_target=="ML":
from xfel.mono_simulation.max_like import minimizer
print "input", s_ang,2. * solution[1]*180/math.pi
# coerce the estimates to be positive for max-likelihood
lower_limit_domain_size = math.pow(
helper.last_set_orientation.unit_cell().volume(),
1./3.)*20 # 10-unit cell block size minimum reasonable domain
d_estimate = max(s_ang, lower_limit_domain_size)
M = minimizer(d_i = dspacings, psi_i = excursion_rad, eta_rad = abs(2. * solution[1]),
Deff = d_estimate)
print "output",1./M.x[0], M.x[1]*180./math.pi
tan_phi_rad_ML = helper.last_set_orientation.unit_cell().d(OO.reserve_indices) / (2. / M.x[0])
tan_phi_deg_ML = tan_phi_rad_ML * 180./math.pi
# bugfix: Need factor of 0.5 because the plot shows half mosaicity (displacement from the center point defined as zero)
tan_outer_deg_ML = tan_phi_deg_ML + 0.5*M.x[1]*180./math.pi
if OO.parent.horizons_phil.integration.mosaic.enable_polychromatic:
# add code here to perform polychromatic modeling.
"""
get miller indices DONE
get model-predicted mono-wavelength centroid S1 vectors
back-convert S1vec, with mono-wavelength, to detector-plane position, factoring in subpixel correction
compare with spot centroid measured position
compare with locus of bodypixels
"""
print list(OO.reserve_indices)
print len(OO.reserve_indices), len(two_thetas)
positions = [
OO.ucbp3.simple_forward_calculation_spot_position(
wavelength = OO.central_wavelength_ang,
observation_no = obsno).position
for obsno in xrange(len(OO.parent.indexed_pairs))]
print len(positions)
print positions # model-predicted positions
print len(OO.parent.spots)
print OO.parent.indexed_pairs
print OO.parent.spots
print len(OO.parent.spots)
meas_spots = [OO.parent.spots[pair["spot"]] for pair in OO.parent.indexed_pairs]
# for xspot in meas_spots:
# xspot.ctr_mass_x(),xspot.ctr_mass_y()
# xspot.max_pxl_x()
# xspot.bodypixels
# xspot.ctr_mass_x()
# Do some work to calculate an rmsd
diff_vecs = flex.vec3_double()
for p,xspot in zip(positions, meas_spots):
diff_vecs.append((p[0]-xspot.ctr_mass_y(), p[1]-xspot.ctr_mass_x(), 0.0))
# could use diff_vecs.rms_length()
diff_vecs_sq = diff_vecs.dot(diff_vecs)
mean_diff_vec_sq = flex.mean(diff_vecs_sq)
rmsd = math.sqrt(mean_diff_vec_sq)
print "mean obs-pred diff vec on %d spots is %6.2f pixels"%(len(positions),rmsd)
positions_to_fictitious = [
OO.ucbp3.simple_forward_calculation_spot_position(
wavelength = OO.central_wavelength_ang,
observation_no = obsno).position_to_fictitious
for obsno in xrange(len(OO.parent.indexed_pairs))]
# Do some work to calculate an rmsd
diff_vecs = flex.vec3_double()
for p,xspot in zip(positions_to_fictitious, meas_spots):
diff_vecs.append((p[0]-xspot.ctr_mass_y(), p[1]-xspot.ctr_mass_x(), 0.0))
rmsd = diff_vecs.rms_length()
print "mean obs-pred_to_fictitious diff vec on %d spots is %6.2f pixels"%(len(positions),rmsd)
"""
actually, it might be better if the entire set of experimental observations
is transformed into the ideal detector plane, for the purposes of poly_treatment.
start here. Now it would be good to actually implement probability of observing a body pixel given the model.
We have everything needed right here.
"""
if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B:
# Image plot: obs and predicted positions + bodypixels
from matplotlib import pyplot as plt
plt.plot( [p[0] for p in positions_to_fictitious], [p[1] for p in positions_to_fictitious], "r.")
plt.plot( [xspot.ctr_mass_y() for xspot in meas_spots],
[xspot.ctr_mass_x() for xspot in meas_spots], "g.")
bodypx = []
for xspot in meas_spots:
for body in xspot.bodypixels:
bodypx.append(body)
plt.plot( [b.y for b in bodypx], [b.x for b in bodypx], "b.")
plt.axes().set_aspect("equal")
plt.show()
print "MEAN excursion",flex.mean(final),
if OO.mosaic_refinement_target=="ML":
print "mosaicity deg FW=",M.x[1]*180./math.pi
else:
print
if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B: # Excursion vs resolution fit
AD1TF7B_MAX2T = 30.
AD1TF7B_MAXDP = 1.
from matplotlib import pyplot as plt
fig = plt.figure()
plt.plot(two_thetas, final, "bo")
mean = flex.mean(final)
minplot = flex.min(two_thetas)
plt.plot([0,minplot],[mean,mean],"k-")
LR = flex.linear_regression(two_thetas, final)
#LR.show_summary()
model_y = LR.slope()*two_thetas + LR.y_intercept()
plt.plot(two_thetas, model_y, "k-")
print helper.last_set_orientation.unit_cell()
#for sdp,tw in zip (dspacings,two_thetas):
#print sdp,tw
if OO.mosaic_refinement_target=="ML":
plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas)))
plt.plot(two_thetas, tan_phi_deg_ML, "r.")
plt.plot(two_thetas, -tan_phi_deg_ML, "r.")
plt.plot(two_thetas, tan_outer_deg_ML, "g.")
plt.plot(two_thetas, -tan_outer_deg_ML, "g.")
else:
plt.plot(two_thetas_env, excursion_rads_env *180./math.pi, "r|")
plt.plot(two_thetas_env, -excursion_rads_env *180./math.pi, "r|")
plt.plot(two_thetas_env, excursion_rads_env *180./math.pi, "r-")
plt.plot(two_thetas_env, -excursion_rads_env *180./math.pi, "r-")
plt.plot(two_thetas, tan_phi_deg, "r.")
plt.plot(two_thetas, -tan_phi_deg, "r.")
plt.plot(two_thetas, tan_outer_deg, "g.")
plt.plot(two_thetas, -tan_outer_deg, "g.")
plt.xlim([0,AD1TF7B_MAX2T])
plt.ylim([-AD1TF7B_MAXDP,AD1TF7B_MAXDP])
OO.parent.show_figure(plt,fig,"psi")
plt.close()
from xfel.mono_simulation.util import green_curve_area,ewald_proximal_volume
if OO.mosaic_refinement_target=="ML":
OO.parent.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML)
OO.parent.inputai.setMosaicity(M.x[1]*180./math.pi) # full width, degrees
OO.parent.ML_half_mosaicity_deg = M.x[1]*180./(2.*math.pi)
OO.parent.ML_domain_size_ang = 1./M.x[0]
OO.parent.ewald_proximal_volume = ewald_proximal_volume(
wavelength_ang = OO.central_wavelength_ang,
resolution_cutoff_ang = OO.parent.horizons_phil.integration.mosaic.ewald_proximal_volume_resolution_cutoff,
domain_size_ang = 1./M.x[0],
full_mosaicity_rad = M.x[1])
return results, helper.last_set_orientation,1./M.x[0] # full width domain size, angstroms
else:
assert OO.mosaic_refinement_target=="LSQ"
OO.parent.green_curve_area = green_curve_area(two_thetas, tan_outer_deg)
OO.parent.inputai.setMosaicity(2*k_degrees) # full width
OO.parent.ML_half_mosaicity_deg = k_degrees
OO.parent.ML_domain_size_ang = s_ang
OO.parent.ewald_proximal_volume = ewald_proximal_volume(
wavelength_ang = OO.central_wavelength_ang,
resolution_cutoff_ang = OO.parent.horizons_phil.integration.mosaic.ewald_proximal_volume_resolution_cutoff,
domain_size_ang = s_ang,
full_mosaicity_rad = 2*k_degrees*math.pi/180.)
return results, helper.last_set_orientation,s_ang # full width domain size, angstroms
def refine_rotx_roty2(OO, enable_rotational_target=True):
helper = OO.per_frame_helper_factory()
helper.restart()
if enable_rotational_target:
print "Trying least squares minimization of excursions",
from scitbx.lstbx import normal_eqns_solving
iterations = normal_eqns_solving.naive_iterations(
non_linear_ls=helper, gradient_threshold=1.E-10)
results = helper.x
print "with %d reflections" % len(OO.parent.indexed_pairs),
print "result %6.2f degrees" % (results[1] * 180. / math.pi),
print "result %6.2f degrees" % (results[0] * 180. / math.pi)
if False: # Excursion histogram
print "The input mosaicity is %7.3f deg full width" % OO.parent.inputai.getMosaicity(
)
# final histogram
if OO.pvr_fix:
final = 360. * helper.fvec_callable_pvr(results)
else:
final = 360. * helper.fvec_callable_NOT_USED_AFTER_BUGFIX(
results)
rmsdexc = math.sqrt(flex.mean(final * final))
from matplotlib import pyplot as plt
nbins = len(final) // 20
n, bins, patches = plt.hist(final,
nbins,
normed=0,
facecolor="orange",
alpha=0.75)
plt.xlabel("Rotation on e1 axis, rmsd %7.3f deg" % rmsdexc)
plt.title("Histogram of cctbx.xfel misorientation")
plt.axis([-0.5, 0.5, 0, 100])
plt.plot([rmsdexc], [18], "b|")
plt.show()
# Determine optimal mosaicity and domain size model (monochromatic)
if OO.pvr_fix:
final = 360. * helper.fvec_callable_pvr(results)
else:
final = 360. * helper.fvec_callable_NOT_USED_AFTER_BUGFIX(results)
#Guard against misindexing -- seen in simulated data, with zone nearly perfectly aligned
guard_stats = flex.max(final), flex.min(final)
if False and REMOVETEST_KILLING_LEGITIMATE_EXCURSIONS(
guard_stats[0] > 2.0 or guard_stats[1] < -2.0):
raise Exception(
"Misindexing diagnosed by meaningless excursion angle (bandpass_gaussian model)"
)
print "The mean excursion is %7.3f degrees" % (flex.mean(final))
two_thetas = helper.last_set_orientation.unit_cell().two_theta(
OO.reserve_indices, OO.central_wavelength_ang, deg=True)
dspacings = helper.last_set_orientation.unit_cell().d(
OO.reserve_indices)
dspace_sq = dspacings * dspacings
excursion_rad = final * math.pi / 180.
# First -- try to get a reasonable envelope for the observed excursions.
## minimum of three regions; maximum of 50 measurements in each bin
print "fitting parameters on %d spots" % len(excursion_rad)
n_bins = min(max(3, len(excursion_rad) // 25), 50)
bin_sz = len(excursion_rad) // n_bins
print "nbins", n_bins, "bin_sz", bin_sz
order = flex.sort_permutation(two_thetas)
two_thetas_env = flex.double()
dspacings_env = flex.double()
excursion_rads_env = flex.double()
for x in range(0, n_bins):
subset = order[x * bin_sz:(x + 1) * bin_sz]
two_thetas_env.append(flex.mean(two_thetas.select(subset)))
dspacings_env.append(flex.mean(dspacings.select(subset)))
excursion_rads_env.append(
flex.max(flex.abs(excursion_rad.select(subset))))
# Second -- parameter fit
## solve the normal equations
sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env)
sum_inv_u = flex.sum(dspacings_env)
sum_te_u = flex.sum(dspacings_env * excursion_rads_env)
sum_te = flex.sum(excursion_rads_env)
Normal_Mat = sqr(
(sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env)))
Vector = col((sum_te_u, sum_te))
solution = Normal_Mat.inverse() * Vector
s_ang = 1. / (2 * solution[0])
print "Best LSQ fit Scheerer domain size is %9.2f ang" % (s_ang)
tan_phi_rad = helper.last_set_orientation.unit_cell().d(
OO.reserve_indices) / (2. * s_ang)
tan_phi_deg = tan_phi_rad * 180. / math.pi
k_degrees = solution[1] * 180. / math.pi
print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f" % (
2 * k_degrees, k_degrees)
tan_outer_deg = tan_phi_deg + k_degrees
if OO.mosaic_refinement_target == "ML":
from xfel.mono_simulation.max_like import minimizer
print "input", s_ang, 2. * solution[1] * 180 / math.pi
# coerce the estimates to be positive for max-likelihood
lower_limit_domain_size = math.pow(
helper.last_set_orientation.unit_cell().volume(), 1. /
3.) * 20 # 10-unit cell block size minimum reasonable domain
d_estimate = max(s_ang, lower_limit_domain_size)
M = minimizer(d_i=dspacings,
psi_i=excursion_rad,
eta_rad=abs(2. * solution[1]),
Deff=d_estimate)
print "output", 1. / M.x[0], M.x[1] * 180. / math.pi
tan_phi_rad_ML = helper.last_set_orientation.unit_cell().d(
OO.reserve_indices) / (2. / M.x[0])
tan_phi_deg_ML = tan_phi_rad_ML * 180. / math.pi
# bugfix: Need factor of 0.5 because the plot shows half mosaicity (displacement from the center point defined as zero)
tan_outer_deg_ML = tan_phi_deg_ML + 0.5 * M.x[1] * 180. / math.pi
if OO.parent.horizons_phil.integration.mosaic.enable_polychromatic:
# add code here to perform polychromatic modeling.
"""
get miller indices DONE
get model-predicted mono-wavelength centroid S1 vectors
back-convert S1vec, with mono-wavelength, to detector-plane position, factoring in subpixel correction
compare with spot centroid measured position
compare with locus of bodypixels
"""
print list(OO.reserve_indices)
print len(OO.reserve_indices), len(two_thetas)
positions = [
OO.ucbp3.simple_forward_calculation_spot_position(
wavelength=OO.central_wavelength_ang,
observation_no=obsno).position
for obsno in range(len(OO.parent.indexed_pairs))
]
print len(positions)
print positions # model-predicted positions
print len(OO.parent.spots)
print OO.parent.indexed_pairs
print OO.parent.spots
print len(OO.parent.spots)
meas_spots = [
OO.parent.spots[pair["spot"]]
for pair in OO.parent.indexed_pairs
]
# for xspot in meas_spots:
# xspot.ctr_mass_x(),xspot.ctr_mass_y()
# xspot.max_pxl_x()
# xspot.bodypixels
# xspot.ctr_mass_x()
# Do some work to calculate an rmsd
diff_vecs = flex.vec3_double()
for p, xspot in zip(positions, meas_spots):
diff_vecs.append((p[0] - xspot.ctr_mass_y(),
p[1] - xspot.ctr_mass_x(), 0.0))
# could use diff_vecs.rms_length()
diff_vecs_sq = diff_vecs.dot(diff_vecs)
mean_diff_vec_sq = flex.mean(diff_vecs_sq)
rmsd = math.sqrt(mean_diff_vec_sq)
print "mean obs-pred diff vec on %d spots is %6.2f pixels" % (
len(positions), rmsd)
positions_to_fictitious = [
OO.ucbp3.simple_forward_calculation_spot_position(
wavelength=OO.central_wavelength_ang,
observation_no=obsno).position_to_fictitious
for obsno in range(len(OO.parent.indexed_pairs))
]
# Do some work to calculate an rmsd
diff_vecs = flex.vec3_double()
for p, xspot in zip(positions_to_fictitious, meas_spots):
diff_vecs.append((p[0] - xspot.ctr_mass_y(),
p[1] - xspot.ctr_mass_x(), 0.0))
rmsd = diff_vecs.rms_length()
print "mean obs-pred_to_fictitious diff vec on %d spots is %6.2f pixels" % (
len(positions), rmsd)
"""
actually, it might be better if the entire set of experimental observations
is transformed into the ideal detector plane, for the purposes of poly_treatment.
start here. Now it would be good to actually implement probability of observing a body pixel given the model.
We have everything needed right here.
"""
if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B:
# Image plot: obs and predicted positions + bodypixels
from matplotlib import pyplot as plt
plt.plot([p[0] for p in positions_to_fictitious],
[p[1] for p in positions_to_fictitious], "r.")
plt.plot([xspot.ctr_mass_y() for xspot in meas_spots],
[xspot.ctr_mass_x() for xspot in meas_spots], "g.")
bodypx = []
for xspot in meas_spots:
for body in xspot.bodypixels:
bodypx.append(body)
plt.plot([b.y for b in bodypx], [b.x for b in bodypx], "b.")
plt.axes().set_aspect("equal")
plt.show()
print "MEAN excursion", flex.mean(final),
if OO.mosaic_refinement_target == "ML":
print "mosaicity deg FW=", M.x[1] * 180. / math.pi
else:
print
if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B: # Excursion vs resolution fit
AD1TF7B_MAX2T = 30.
AD1TF7B_MAXDP = 1.
from matplotlib import pyplot as plt
fig = plt.figure()
plt.plot(two_thetas, final, "bo")
mean = flex.mean(final)
minplot = flex.min(two_thetas)
plt.plot([0, minplot], [mean, mean], "k-")
LR = flex.linear_regression(two_thetas, final)
#LR.show_summary()
model_y = LR.slope() * two_thetas + LR.y_intercept()
plt.plot(two_thetas, model_y, "k-")
print helper.last_set_orientation.unit_cell()
#for sdp,tw in zip (dspacings,two_thetas):
#print sdp,tw
if OO.mosaic_refinement_target == "ML":
plt.title(
"ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" %
(M.x[1] * 180. / math.pi, 2. / M.x[0], len(two_thetas)))
plt.plot(two_thetas, tan_phi_deg_ML, "r.")
plt.plot(two_thetas, -tan_phi_deg_ML, "r.")
plt.plot(two_thetas, tan_outer_deg_ML, "g.")
plt.plot(two_thetas, -tan_outer_deg_ML, "g.")
else:
plt.plot(two_thetas_env, excursion_rads_env * 180. / math.pi,
"r|")
plt.plot(two_thetas_env, -excursion_rads_env * 180. / math.pi,
"r|")
plt.plot(two_thetas_env, excursion_rads_env * 180. / math.pi,
"r-")
plt.plot(two_thetas_env, -excursion_rads_env * 180. / math.pi,
"r-")
plt.plot(two_thetas, tan_phi_deg, "r.")
plt.plot(two_thetas, -tan_phi_deg, "r.")
plt.plot(two_thetas, tan_outer_deg, "g.")
plt.plot(two_thetas, -tan_outer_deg, "g.")
plt.xlim([0, AD1TF7B_MAX2T])
plt.ylim([-AD1TF7B_MAXDP, AD1TF7B_MAXDP])
OO.parent.show_figure(plt, fig, "psi")
plt.close()
from xfel.mono_simulation.util import green_curve_area, ewald_proximal_volume
if OO.mosaic_refinement_target == "ML":
OO.parent.green_curve_area = green_curve_area(
two_thetas, tan_outer_deg_ML)
OO.parent.inputai.setMosaicity(M.x[1] * 180. /
math.pi) # full width, degrees
OO.parent.ML_half_mosaicity_deg = M.x[1] * 180. / (2. * math.pi)
OO.parent.ML_domain_size_ang = 1. / M.x[0]
OO.parent.ewald_proximal_volume = ewald_proximal_volume(
wavelength_ang=OO.central_wavelength_ang,
resolution_cutoff_ang=OO.parent.horizons_phil.integration.
mosaic.ewald_proximal_volume_resolution_cutoff,
domain_size_ang=1. / M.x[0],
full_mosaicity_rad=M.x[1])
return results, helper.last_set_orientation, 1. / M.x[
0] # full width domain size, angstroms
else:
assert OO.mosaic_refinement_target == "LSQ"
OO.parent.green_curve_area = green_curve_area(
two_thetas, tan_outer_deg)
OO.parent.inputai.setMosaicity(2 * k_degrees) # full width
OO.parent.ML_half_mosaicity_deg = k_degrees
OO.parent.ML_domain_size_ang = s_ang
OO.parent.ewald_proximal_volume = ewald_proximal_volume(
wavelength_ang=OO.central_wavelength_ang,
resolution_cutoff_ang=OO.parent.horizons_phil.integration.
mosaic.ewald_proximal_volume_resolution_cutoff,
domain_size_ang=s_ang,
full_mosaicity_rad=2 * k_degrees * math.pi / 180.)
return results, helper.last_set_orientation, s_ang # full width domain size, angstroms
