def result_for_cxi_merge(self, file_name):
values = self.get_parameter_values()
self.rs2_parameter_range_assertions(values)
scaler = self.nave1_refinery.scaler_callable(
self.get_parameter_values())
partiality_array = self.refinery.get_partiality_array(values)
p_scaler = flex.pow(
partiality_array,
0.5 * self.params.postrefinement.merge_partiality_exponent)
fat_selection = (
self.nave1_refinery.lorentz_callable(self.get_parameter_values()) >
self.params.postrefinement.rs_hybrid.partiality_threshold
) # was 0.2 for rs2
fat_count = fat_selection.count(True)
scaler_s = scaler.select(fat_selection)
p_scaler_s = p_scaler.select(fat_selection)
#avoid empty database INSERT, if insufficient centrally-located Bragg spots:
# in samosa, handle this at a higher level, but handle it somehow.
if fat_count < 3:
raise ValueError("< 3 near-fulls after refinement")
print >> self.out, "On total %5d the fat selection is %5d" % (len(
self.observations_pair1_selected.indices()), fat_count)
observations_original_index = \
self.observations_original_index_pair1_selected.select(fat_selection)
observations = self.observations_pair1_selected.customized_copy(
indices=self.observations_pair1_selected.indices().select(
fat_selection),
data=(
self.observations_pair1_selected.data().select(fat_selection) /
scaler_s),
sigmas=(
self.observations_pair1_selected.sigmas().select(fat_selection)
/ (scaler_s * p_scaler_s)))
matches = miller.match_multi_indices(
miller_indices_unique=self.miller_set.indices(),
miller_indices=observations.indices())
I_weight = flex.double(len(observations.sigmas()), 1.)
I_reference = flex.double(
[self.i_model.data()[pair[0]] for pair in matches.pairs()])
I_invalid = flex.bool(
[self.i_model.sigmas()[pair[0]] < 0. for pair in matches.pairs()])
I_weight.set_selected(I_invalid, 0.)
SWC = simple_weighted_correlation(I_weight, I_reference,
observations.data())
print >> self.out, "CORR: NEW correlation is", SWC.corr
print >> self.out, "ASTAR_FILE", file_name, tuple(
self.nave1_refinery.get_eff_Astar(values))
self.final_corr = SWC.corr
#another range assertion
assert self.final_corr > 0.1, "correlation coefficient out of range (<= 0.1) after LevMar refinement"
# XXX Specific to the hybrid_rs method, and likely these limits are problem-specific (especially G-max) so look for another approach
# or expose the limits as phil parameters.
assert values.G < 0.5, "G-scale value out of range ( > 0.5 XXX may be too strict ) after LevMar refinement"
return observations_original_index, observations, matches
def result_for_cxi_merge(self, file_name):
values = self.get_parameter_values()
self.rs2_parameter_range_assertions(values)
scaler = self.refinery.scaler_callable(
self.parameterization_class(self.MINI.x))
partiality_array = self.refinery.get_partiality_array(values)
p_scaler = flex.pow(
partiality_array,
0.5 * self.params.postrefinement.merge_partiality_exponent)
fat_selection = (partiality_array > 0.2)
fat_count = fat_selection.count(True)
scaler_s = scaler.select(fat_selection)
p_scaler_s = p_scaler.select(fat_selection)
#avoid empty database INSERT, if insufficient centrally-located Bragg spots:
# in samosa, handle this at a higher level, but handle it somehow.
if fat_count < 3:
raise ValueError("< 3 near-fulls after refinement")
print("On total %5d the fat selection is %5d" %
(len(self.observations_pair1_selected.indices()), fat_count),
file=self.out)
observations_original_index = \
self.observations_original_index_pair1_selected.select(fat_selection)
observations = self.observations_pair1_selected.customized_copy(
indices=self.observations_pair1_selected.indices().select(
fat_selection),
data=(
self.observations_pair1_selected.data().select(fat_selection) /
scaler_s),
sigmas=(
self.observations_pair1_selected.sigmas().select(fat_selection)
/ (scaler_s * p_scaler_s)))
matches = miller.match_multi_indices(
miller_indices_unique=self.miller_set.indices(),
miller_indices=observations.indices())
I_weight = flex.double(len(observations.sigmas()), 1.)
I_reference = flex.double(
[self.i_model.data()[pair[0]] for pair in matches.pairs()])
I_invalid = flex.bool(
[self.i_model.sigmas()[pair[0]] < 0. for pair in matches.pairs()])
I_weight.set_selected(I_invalid, 0.)
SWC = simple_weighted_correlation(I_weight, I_reference,
observations.data())
print("CORR: NEW correlation is", SWC.corr, file=self.out)
print("ASTAR_FILE",
file_name,
tuple(self.refinery.get_eff_Astar(values)),
file=self.out)
self.final_corr = SWC.corr
self.refined_mini = self.MINI
#another range assertion
assert self.final_corr > 0.1, "correlation coefficient out of range (<= 0.1) after rs2 refinement"
return observations_original_index, observations, matches
def result_for_cxi_merge(self):
values = self.get_parameter_values()
self.rs2_parameter_range_assertions(values)
scaler = self.refinery.scaler_callable(self.parameterization_class(self.MINI.x))
partiality_array = self.refinery.get_partiality_array(values)
p_scaler = flex.pow(partiality_array,
0.5*self.params.postrefinement.merge_partiality_exponent)
fat_selection = (partiality_array > 0.2)
fat_count = fat_selection.count(True)
scaler_s = scaler.select(fat_selection)
p_scaler_s = p_scaler.select(fat_selection)
# reject an experiment with insufficient number of near-full reflections
if fat_count < 3:
if self.params.output.log_level == 0:
self.logger.log("Rejected experiment, because: On total %5d the fat selection is %5d"%(len(self.observations_pair1_selected.indices()), fat_count))
raise ValueError("< 3 near-fulls after refinement")
if self.params.output.log_level == 0:
self.logger.log("On total %5d the fat selection is %5d"%(len(self.observations_pair1_selected.indices()), fat_count))
observations_original_index = self.observations_original_index_pair1_selected.select(fat_selection)
observations = self.observations_pair1_selected.customized_copy(
indices = self.observations_pair1_selected.indices().select(fat_selection),
data = (self.observations_pair1_selected.data().select(fat_selection)/scaler_s),
sigmas = (self.observations_pair1_selected.sigmas().select(fat_selection)/(scaler_s * p_scaler_s))
)
matches = miller.match_multi_indices(
miller_indices_unique=self.params.scaling.miller_set.indices(),
miller_indices=observations.indices())
I_weight = flex.double(len(observations.sigmas()), 1.)
I_reference = flex.double([self.params.scaling.i_model.data()[pair[0]] for pair in matches.pairs()])
I_invalid = flex.bool([self.params.scaling.i_model.sigmas()[pair[0]] < 0. for pair in matches.pairs()])
I_weight.set_selected(I_invalid,0.)
SWC = simple_weighted_correlation(I_weight, I_reference, observations.data())
if self.params.output.log_level == 0:
self.logger.log("CORR: NEW correlation is: %f"%SWC.corr)
self.logger.log("ASTAR: ")
self.logger.log(tuple(self.refinery.get_eff_Astar(values)))
self.final_corr = SWC.corr
self.refined_mini = self.MINI
#another range assertion
assert self.final_corr > 0.1,"correlation coefficient out of range (<= 0.1) after rs2 refinement"
return observations_original_index, observations, matches
def result_for_cxi_merge(self, file_name):
values = self.get_parameter_values()
self.rs2_parameter_range_assertions(values)
scaler = self.nave1_refinery.scaler_callable(self.get_parameter_values())
partiality_array = self.refinery.get_partiality_array(values)
p_scaler = flex.pow(partiality_array,
0.5*self.params.postrefinement.merge_partiality_exponent)
fat_selection = (self.nave1_refinery.lorentz_callable(self.get_parameter_values()) >
self.params.postrefinement.rs_hybrid.partiality_threshold) # was 0.2 for rs2
fat_count = fat_selection.count(True)
scaler_s = scaler.select(fat_selection)
p_scaler_s = p_scaler.select(fat_selection)
#avoid empty database INSERT, if insufficient centrally-located Bragg spots:
# in samosa, handle this at a higher level, but handle it somehow.
if fat_count < 3:
raise ValueError, "< 3 near-fulls after refinement"
print >> self.out, "On total %5d the fat selection is %5d"%(
len(self.observations_pair1_selected.indices()), fat_count)
observations_original_index = \
self.observations_original_index_pair1_selected.select(fat_selection)
observations = self.observations_pair1_selected.customized_copy(
indices = self.observations_pair1_selected.indices().select(fat_selection),
data = (self.observations_pair1_selected.data().select(fat_selection)/scaler_s),
sigmas = (self.observations_pair1_selected.sigmas().select(fat_selection)/(scaler_s * p_scaler_s))
)
matches = miller.match_multi_indices(
miller_indices_unique=self.miller_set.indices(),
miller_indices=observations.indices())
I_weight = flex.double(len(observations.sigmas()), 1.)
I_reference = flex.double([self.i_model.data()[pair[0]] for pair in matches.pairs()])
SWC = simple_weighted_correlation(I_weight, I_reference, observations.data())
print >> self.out, "CORR: NEW correlation is", SWC.corr
self.final_corr = SWC.corr
#another range assertion
assert self.final_corr > 0.1,"correlation coefficient out of range (<= 0.1) after LevMar refinement"
# XXX Specific to the hybrid_rs method, and likely these limits are problem-specific (especially G-max) so look for another approach
# or expose the limits as phil parameters.
assert values.G < 0.5 , "G-scale value out of range ( > 0.5 XXX may be too strict ) after LevMar refinement"
return observations_original_index,observations,matches
def __init__(self, measurements_orig, params, i_model, miller_set, result,
out):
measurements = measurements_orig.deep_copy()
# Now manipulate the data to conform to unit cell, asu, and space group
# of reference. The resolution will be cut later.
# Only works if there is NOT an indexing ambiguity!
observations = measurements.customized_copy(
anomalous_flag=not params.merge_anomalous,
crystal_symmetry=miller_set.crystal_symmetry()).map_to_asu()
observations_original_index = measurements.customized_copy(
anomalous_flag=not params.merge_anomalous,
crystal_symmetry=miller_set.crystal_symmetry())
# Ensure that match_multi_indices() will return identical results
# when a frame's observations are matched against the
# pre-generated Miller set, self.miller_set, and the reference
# data set, self.i_model. The implication is that the same match
# can be used to map Miller indices to array indices for intensity
# accumulation, and for determination of the correlation
# coefficient in the presence of a scaling reference.
assert len(i_model.indices()) == len(miller_set.indices()) \
and (i_model.indices() ==
miller_set.indices()).count(False) == 0
matches = miller.match_multi_indices(
miller_indices_unique=miller_set.indices(),
miller_indices=observations.indices())
pair1 = flex.int([pair[1] for pair in matches.pairs()])
pair0 = flex.int([pair[0] for pair in matches.pairs()])
# narrow things down to the set that matches, only
observations_pair1_selected = observations.customized_copy(
indices=flex.miller_index(
[observations.indices()[p] for p in pair1]),
data=flex.double([observations.data()[p] for p in pair1]),
sigmas=flex.double([observations.sigmas()[p] for p in pair1]),
)
observations_original_index_pair1_selected = observations_original_index.customized_copy(
indices=flex.miller_index(
[observations_original_index.indices()[p] for p in pair1]),
data=flex.double(
[observations_original_index.data()[p] for p in pair1]),
sigmas=flex.double(
[observations_original_index.sigmas()[p] for p in pair1]),
)
###################
I_observed = observations_pair1_selected.data()
MILLER = observations_original_index_pair1_selected.indices()
ORI = result["current_orientation"][0]
Astar = matrix.sqr(ORI.reciprocal_matrix())
WAVE = result["wavelength"]
BEAM = matrix.col((0.0, 0.0, -1. / WAVE))
BFACTOR = 0.
#calculation of correlation here
I_reference = flex.double(
[i_model.data()[pair[0]] for pair in matches.pairs()])
I_invalid = flex.bool(
[i_model.sigmas()[pair[0]] < 0. for pair in matches.pairs()])
use_weights = False # New facility for getting variance-weighted correlation
if use_weights:
#variance weighting
I_weight = flex.double([
1. / (observations_pair1_selected.sigmas()[pair[1]])**2
for pair in matches.pairs()
])
else:
I_weight = flex.double(len(observations_pair1_selected.sigmas()),
1.)
I_weight.set_selected(I_invalid, 0.)
"""Explanation of 'include_negatives' semantics as originally implemented in cxi.merge postrefinement:
include_negatives = True
+ and - reflections both used for Rh distribution for initial estimate of RS parameter
+ and - reflections both used for calc/obs correlation slope for initial estimate of G parameter
+ and - reflections both passed to the refinery and used in the target function (makes sense if
you look at it from a certain point of view)
include_negatives = False
+ and - reflections both used for Rh distribution for initial estimate of RS parameter
+ reflections only used for calc/obs correlation slope for initial estimate of G parameter
+ and - reflections both passed to the refinery and used in the target function (makes sense if
you look at it from a certain point of view)
"""
if params.include_negatives:
SWC = simple_weighted_correlation(I_weight, I_reference,
I_observed)
else:
non_positive = (observations_pair1_selected.data() <= 0)
SWC = simple_weighted_correlation(
I_weight.select(~non_positive),
I_reference.select(~non_positive),
I_observed.select(~non_positive))
print >> out, "Old correlation is", SWC.corr
if params.postrefinement.algorithm == "rs":
Rhall = flex.double()
for mill in MILLER:
H = matrix.col(mill)
Xhkl = Astar * H
Rh = (Xhkl + BEAM).length() - (1. / WAVE)
Rhall.append(Rh)
Rs = math.sqrt(flex.mean(Rhall * Rhall))
RS = 1. / 10000. # reciprocal effective domain size of 1 micron
RS = Rs # try this empirically determined approximate, monochrome, a-mosaic value
current = flex.double([SWC.slope, BFACTOR, RS, 0., 0.])
parameterization_class = rs_parameterization
refinery = rs_refinery(ORI=ORI,
MILLER=MILLER,
BEAM=BEAM,
WAVE=WAVE,
ICALCVEC=I_reference,
IOBSVEC=I_observed)
elif params.postrefinement.algorithm == "eta_deff":
eta_init = 2. * result["ML_half_mosaicity_deg"][0] * math.pi / 180.
D_eff_init = 2. * result["ML_domain_size_ang"][0]
current = flex.double([
SWC.slope,
BFACTOR,
eta_init,
0.,
0.,
D_eff_init,
])
parameterization_class = eta_deff_parameterization
refinery = eta_deff_refinery(ORI=ORI,
MILLER=MILLER,
BEAM=BEAM,
WAVE=WAVE,
ICALCVEC=I_reference,
IOBSVEC=I_observed)
func = refinery.fvec_callable(parameterization_class(current))
functional = flex.sum(func * func)
print >> out, "functional", functional
self.current = current
self.parameterization_class = parameterization_class
self.refinery = refinery
self.out = out
self.params = params
self.miller_set = miller_set
self.observations_pair1_selected = observations_pair1_selected
self.observations_original_index_pair1_selected = observations_original_index_pair1_selected
def run(self, experiments, reflections):
self.logger.log_step_time("POSTREFINEMENT")
if not self.params.postrefinement.enable:
self.logger.log("Postrefinement was not done")
if self.mpi_helper.rank == 0:
self.logger.main_log("Postrefinement was not done")
return experiments, reflections
target_symm = symmetry(
unit_cell=self.params.scaling.unit_cell,
space_group_info=self.params.scaling.space_group)
i_model = self.params.scaling.i_model
miller_set = self.params.scaling.miller_set
# Ensure that match_multi_indices() will return identical results
# when a frame's observations are matched against the
# pre-generated Miller set, self.miller_set, and the reference
# data set, self.i_model. The implication is that the same match
# can be used to map Miller indices to array indices for intensity
# accumulation, and for determination of the correlation
# coefficient in the presence of a scaling reference.
assert len(i_model.indices()) == len(miller_set.indices())
assert (i_model.indices() == miller_set.indices()).count(False) == 0
new_experiments = ExperimentList()
new_reflections = flex.reflection_table()
experiments_rejected_by_reason = {} # reason:how_many_rejected
for experiment in experiments:
exp_reflections = reflections.select(
reflections['exp_id'] == experiment.identifier)
# Build a miller array for the experiment reflections with original miller indexes
exp_miller_indices_original = miller.set(
target_symm, exp_reflections['miller_index'], True)
observations_original_index = miller.array(
exp_miller_indices_original,
exp_reflections['intensity.sum.value'],
flex.double(
flex.sqrt(exp_reflections['intensity.sum.variance'])))
assert exp_reflections.size() == exp_miller_indices_original.size()
assert observations_original_index.size(
) == exp_miller_indices_original.size()
# Build a miller array for the experiment reflections with asu miller indexes
exp_miller_indices_asu = miller.set(
target_symm, exp_reflections['miller_index_asymmetric'], True)
observations = miller.array(
exp_miller_indices_asu, exp_reflections['intensity.sum.value'],
flex.double(
flex.sqrt(exp_reflections['intensity.sum.variance'])))
matches = miller.match_multi_indices(
miller_indices_unique=miller_set.indices(),
miller_indices=observations.indices())
pair1 = flex.int([pair[1] for pair in matches.pairs()
]) # refers to the observations
pair0 = flex.int([pair[0] for pair in matches.pairs()
]) # refers to the model
assert exp_reflections.size() == exp_miller_indices_original.size()
assert observations_original_index.size(
) == exp_miller_indices_original.size()
# narrow things down to the set that matches, only
observations_pair1_selected = observations.customized_copy(
indices=flex.miller_index(
[observations.indices()[p] for p in pair1]),
data=flex.double([observations.data()[p] for p in pair1]),
sigmas=flex.double([observations.sigmas()[p] for p in pair1]))
observations_original_index_pair1_selected = observations_original_index.customized_copy(
indices=flex.miller_index(
[observations_original_index.indices()[p] for p in pair1]),
data=flex.double(
[observations_original_index.data()[p] for p in pair1]),
sigmas=flex.double(
[observations_original_index.sigmas()[p] for p in pair1]))
I_observed = observations_pair1_selected.data()
MILLER = observations_original_index_pair1_selected.indices()
ORI = crystal_orientation(experiment.crystal.get_A(),
basis_type.reciprocal)
Astar = matrix.sqr(ORI.reciprocal_matrix())
Astar_from_experiment = matrix.sqr(experiment.crystal.get_A())
assert Astar == Astar_from_experiment
WAVE = experiment.beam.get_wavelength()
BEAM = matrix.col((0.0, 0.0, -1. / WAVE))
BFACTOR = 0.
MOSAICITY_DEG = experiment.crystal.get_half_mosaicity_deg()
DOMAIN_SIZE_A = experiment.crystal.get_domain_size_ang()
# calculation of correlation here
I_reference = flex.double(
[i_model.data()[pair[0]] for pair in matches.pairs()])
I_invalid = flex.bool(
[i_model.sigmas()[pair[0]] < 0. for pair in matches.pairs()])
use_weights = False # New facility for getting variance-weighted correlation
if use_weights:
# variance weighting
I_weight = flex.double([
1. / (observations_pair1_selected.sigmas()[pair[1]])**2
for pair in matches.pairs()
])
else:
I_weight = flex.double(
len(observations_pair1_selected.sigmas()), 1.)
I_weight.set_selected(I_invalid, 0.)
"""Explanation of 'include_negatives' semantics as originally implemented in cxi.merge postrefinement:
include_negatives = True
+ and - reflections both used for Rh distribution for initial estimate of RS parameter
+ and - reflections both used for calc/obs correlation slope for initial estimate of G parameter
+ and - reflections both passed to the refinery and used in the target function (makes sense if
you look at it from a certain point of view)
include_negatives = False
+ and - reflections both used for Rh distribution for initial estimate of RS parameter
+ reflections only used for calc/obs correlation slope for initial estimate of G parameter
+ and - reflections both passed to the refinery and used in the target function (makes sense if
you look at it from a certain point of view)
"""
# RB: By design, for MPI-Merge "include negatives" is implicitly True
SWC = simple_weighted_correlation(I_weight, I_reference,
I_observed)
if self.params.output.log_level == 0:
self.logger.log("Old correlation is: %f" % SWC.corr)
if self.params.postrefinement.algorithm == "rs":
Rhall = flex.double()
for mill in MILLER:
H = matrix.col(mill)
Xhkl = Astar * H
Rh = (Xhkl + BEAM).length() - (1. / WAVE)
Rhall.append(Rh)
Rs = math.sqrt(flex.mean(Rhall * Rhall))
RS = 1. / 10000. # reciprocal effective domain size of 1 micron
RS = Rs # try this empirically determined approximate, monochrome, a-mosaic value
current = flex.double([SWC.slope, BFACTOR, RS, 0., 0.])
parameterization_class = rs_parameterization
refinery = rs_refinery(ORI=ORI,
MILLER=MILLER,
BEAM=BEAM,
WAVE=WAVE,
ICALCVEC=I_reference,
IOBSVEC=I_observed)
elif self.params.postrefinement.algorithm == "eta_deff":
eta_init = 2. * MOSAICITY_DEG * math.pi / 180.
D_eff_init = 2. * DOMAIN_SIZE_A
current = flex.double(
[SWC.slope, BFACTOR, eta_init, 0., 0., D_eff_init])
parameterization_class = eta_deff_parameterization
refinery = eta_deff_refinery(ORI=ORI,
MILLER=MILLER,
BEAM=BEAM,
WAVE=WAVE,
ICALCVEC=I_reference,
IOBSVEC=I_observed)
func = refinery.fvec_callable(parameterization_class(current))
functional = flex.sum(func * func)
if self.params.output.log_level == 0:
self.logger.log("functional: %f" % functional)
self.current = current
self.parameterization_class = parameterization_class
self.refinery = refinery
self.observations_pair1_selected = observations_pair1_selected
self.observations_original_index_pair1_selected = observations_original_index_pair1_selected
error_detected = False
try:
self.run_plain()
result_observations_original_index, result_observations, result_matches = self.result_for_cxi_merge(
)
assert result_observations_original_index.size(
) == result_observations.size()
assert result_matches.pairs().size(
) == result_observations_original_index.size()
except (AssertionError, ValueError, RuntimeError) as e:
error_detected = True
reason = repr(e)
if not reason:
reason = "Unknown error"
if not reason in experiments_rejected_by_reason:
experiments_rejected_by_reason[reason] = 1
else:
experiments_rejected_by_reason[reason] += 1
if not error_detected:
new_experiments.append(experiment)
new_exp_reflections = flex.reflection_table()
new_exp_reflections[
'miller_index_asymmetric'] = flex.miller_index(
result_observations.indices())
new_exp_reflections['intensity.sum.value'] = flex.double(
result_observations.data())
new_exp_reflections['intensity.sum.variance'] = flex.double(
flex.pow(result_observations.sigmas(), 2))
new_exp_reflections['exp_id'] = flex.std_string(
len(new_exp_reflections), experiment.identifier)
new_reflections.extend(new_exp_reflections)
'''
# debugging
elif reason.startswith("ValueError"):
self.logger.log("Rejected b/c of value error exp id: %s; unit cell: %s"%(exp_id, str(experiment.crystal.get_unit_cell())) )
'''
# report rejected experiments, reflections
experiments_rejected_by_postrefinement = len(experiments) - len(
new_experiments)
reflections_rejected_by_postrefinement = reflections.size(
) - new_reflections.size()
self.logger.log("Experiments rejected by post-refinement: %d" %
experiments_rejected_by_postrefinement)
self.logger.log("Reflections rejected by post-refinement: %d" %
reflections_rejected_by_postrefinement)
all_reasons = []
for reason, count in experiments_rejected_by_reason.iteritems():
self.logger.log("Experiments rejected due to %s: %d" %
(reason, count))
all_reasons.append(reason)
comm = self.mpi_helper.comm
MPI = self.mpi_helper.MPI
# Collect all rejection reasons from all ranks. Use allreduce to let each rank have all reasons.
all_reasons = comm.allreduce(all_reasons, MPI.SUM)
all_reasons = set(all_reasons)
# Now that each rank has all reasons from all ranks, we can treat the reasons in a uniform way.
total_experiments_rejected_by_reason = {}
for reason in all_reasons:
rejected_experiment_count = 0
if reason in experiments_rejected_by_reason:
rejected_experiment_count = experiments_rejected_by_reason[
reason]
total_experiments_rejected_by_reason[reason] = comm.reduce(
rejected_experiment_count, MPI.SUM, 0)
total_accepted_experiment_count = comm.reduce(len(new_experiments),
MPI.SUM, 0)
# how many reflections have we rejected due to post-refinement?
rejected_reflections = len(reflections) - len(new_reflections)
total_rejected_reflections = self.mpi_helper.sum(rejected_reflections)
if self.mpi_helper.rank == 0:
for reason, count in total_experiments_rejected_by_reason.iteritems(
):
self.logger.main_log(
"Total experiments rejected due to %s: %d" %
(reason, count))
self.logger.main_log("Total experiments accepted: %d" %
total_accepted_experiment_count)
self.logger.main_log(
"Total reflections rejected due to post-refinement: %d" %
total_rejected_reflections)
self.logger.log_step_time("POSTREFINEMENT", True)
return new_experiments, new_reflections
def __init__(self,measurements_orig, params, i_model, miller_set, result, out):
measurements = measurements_orig.deep_copy()
# Now manipulate the data to conform to unit cell, asu, and space group
# of reference. The resolution will be cut later.
# Only works if there is NOT an indexing ambiguity!
observations = measurements.customized_copy(
anomalous_flag=not params.merge_anomalous,
crystal_symmetry=miller_set.crystal_symmetry()
).map_to_asu()
observations_original_index = measurements.customized_copy(
anomalous_flag=not params.merge_anomalous,
crystal_symmetry=miller_set.crystal_symmetry()
)
# Ensure that match_multi_indices() will return identical results
# when a frame's observations are matched against the
# pre-generated Miller set, self.miller_set, and the reference
# data set, self.i_model. The implication is that the same match
# can be used to map Miller indices to array indices for intensity
# accumulation, and for determination of the correlation
# coefficient in the presence of a scaling reference.
assert len(i_model.indices()) == len(miller_set.indices()) \
and (i_model.indices() ==
miller_set.indices()).count(False) == 0
matches = miller.match_multi_indices(
miller_indices_unique=miller_set.indices(),
miller_indices=observations.indices())
pair1 = flex.int([pair[1] for pair in matches.pairs()])
pair0 = flex.int([pair[0] for pair in matches.pairs()])
# narrow things down to the set that matches, only
observations_pair1_selected = observations.customized_copy(
indices = flex.miller_index([observations.indices()[p] for p in pair1]),
data = flex.double([observations.data()[p] for p in pair1]),
sigmas = flex.double([observations.sigmas()[p] for p in pair1]),
)
observations_original_index_pair1_selected = observations_original_index.customized_copy(
indices = flex.miller_index([observations_original_index.indices()[p] for p in pair1]),
data = flex.double([observations_original_index.data()[p] for p in pair1]),
sigmas = flex.double([observations_original_index.sigmas()[p] for p in pair1]),
)
###################
I_observed = observations_pair1_selected.data()
chosen = chosen_weights(observations_pair1_selected, params)
MILLER = observations_original_index_pair1_selected.indices()
ORI = result["current_orientation"][0]
Astar = matrix.sqr(ORI.reciprocal_matrix())
WAVE = result["wavelength"]
BEAM = matrix.col((0.0,0.0,-1./WAVE))
BFACTOR = 0.
#calculation of correlation here
I_reference = flex.double([i_model.data()[pair[0]] for pair in matches.pairs()])
use_weights = False # New facility for getting variance-weighted correlation
if use_weights:
#variance weighting
I_weight = flex.double(
[1./(observations_pair1_selected.sigmas()[pair[1]])**2 for pair in matches.pairs()])
else:
I_weight = flex.double(len(observations_pair1_selected.sigmas()), 1.)
"""Explanation of 'include_negatives' semantics as originally implemented in cxi.merge postrefinement:
include_negatives = True
+ and - reflections both used for Rh distribution for initial estimate of RS parameter
+ and - reflections both used for calc/obs correlation slope for initial estimate of G parameter
+ and - reflections both passed to the refinery and used in the target function (makes sense if
you look at it from a certain point of view)
include_negatives = False
+ and - reflections both used for Rh distribution for initial estimate of RS parameter
+ reflections only used for calc/obs correlation slope for initial estimate of G parameter
+ and - reflections both passed to the refinery and used in the target function (makes sense if
you look at it from a certain point of view)
"""
if params.include_negatives:
SWC = simple_weighted_correlation(I_weight, I_reference, I_observed)
else:
non_positive = ( observations_pair1_selected.data() <= 0 )
SWC = simple_weighted_correlation(I_weight.select(~non_positive),
I_reference.select(~non_positive), I_observed.select(~non_positive))
print >> out, "Old correlation is", SWC.corr
assert params.postrefinement.algorithm=="rs_hybrid"
Rhall = flex.double()
for mill in MILLER:
H = matrix.col(mill)
Xhkl = Astar*H
Rh = ( Xhkl + BEAM ).length() - (1./WAVE)
Rhall.append(Rh)
Rs = math.sqrt(flex.mean(Rhall*Rhall))
RS = 1./10000. # reciprocal effective domain size of 1 micron
RS = Rs # try this empirically determined approximate, monochrome, a-mosaic value
self.rs2_current = flex.double([SWC.slope, BFACTOR, RS, 0., 0.])
self.rs2_parameterization_class = rs_parameterization
self.rs2_refinery = rs2_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE,
ICALCVEC = I_reference, IOBSVEC = I_observed, WEIGHTS = chosen)
self.rs2_refinery.set_profile_shape(params.postrefinement.lineshape)
self.nave1_refinery = nave1_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE,
ICALCVEC = I_reference, IOBSVEC = I_observed, WEIGHTS = chosen)
self.nave1_refinery.set_profile_shape(params.postrefinement.lineshape)
self.out=out; self.params = params;
self.miller_set = miller_set
self.observations_pair1_selected = observations_pair1_selected;
self.observations_original_index_pair1_selected = observations_original_index_pair1_selected
self.i_model = i_model
