def scale_frame_detail(self, result, file_name, db_mgr, out):
# If the pickled integration file does not contain a wavelength,
# fall back on the value given on the command line. XXX The
# wavelength parameter should probably be removed from master_phil
# once all pickled integration files contain it.
if ("wavelength" in result):
wavelength = result["wavelength"]
elif (self.params.wavelength is not None):
wavelength = self.params.wavelength
else:
# XXX Give error, or raise exception?
return None
assert (wavelength > 0)
observations = result["observations"][0]
cos_two_polar_angle = result["cos_two_polar_angle"]
assert observations.size() == cos_two_polar_angle.size()
tt_vec = observations.two_theta(wavelength)
#print "mean tt degrees",180.*flex.mean(tt_vec.data())/math.pi
cos_tt_vec = flex.cos(tt_vec.data())
sin_tt_vec = flex.sin(tt_vec.data())
cos_sq_tt_vec = cos_tt_vec * cos_tt_vec
sin_sq_tt_vec = sin_tt_vec * sin_tt_vec
P_nought_vec = 0.5 * (1. + cos_sq_tt_vec)
F_prime = -1.0 # Hard-coded value defines the incident polarization axis
P_prime = 0.5 * F_prime * cos_two_polar_angle * sin_sq_tt_vec
# XXX added as a diagnostic
prange = P_nought_vec - P_prime
other_F_prime = 1.0
otherP_prime = 0.5 * other_F_prime * cos_two_polar_angle * sin_sq_tt_vec
otherprange = P_nought_vec - otherP_prime
diff2 = flex.abs(prange - otherprange)
print "mean diff is", flex.mean(diff2), "range", flex.min(
diff2), flex.max(diff2)
# XXX done
observations = observations / (P_nought_vec - P_prime)
# This corrects observations for polarization assuming 100% polarization on
# one axis (thus the F_prime = -1.0 rather than the perpendicular axis, 1.0)
# Polarization model as described by Kahn, Fourme, Gadet, Janin, Dumas & Andre
# (1982) J. Appl. Cryst. 15, 330-337, equations 13 - 15.
print "Step 3. Correct for polarization."
indexed_cell = observations.unit_cell()
observations_original_index = observations.deep_copy()
if result.get(
"model_partialities", None
) is not None and result["model_partialities"][0] is not None:
# some recordkeeping useful for simulations
partialities_original_index = observations.customized_copy(
crystal_symmetry=self.miller_set.crystal_symmetry(),
data=result["model_partialities"][0]["data"],
sigmas=flex.double(result["model_partialities"][0]
["data"].size()), #dummy value for sigmas
indices=result["model_partialities"][0]["indices"],
).resolution_filter(d_min=self.params.d_min)
assert len(observations_original_index.indices()) == len(
observations.indices())
# Now manipulate the data to conform to unit cell, asu, and space group
# of reference. The resolution will be cut later.
# Only works if there is NOT an indexing ambiguity!
observations = observations.customized_copy(
anomalous_flag=not self.params.merge_anomalous,
crystal_symmetry=self.miller_set.crystal_symmetry()).map_to_asu()
observations_original_index = observations_original_index.customized_copy(
anomalous_flag=not self.params.merge_anomalous,
crystal_symmetry=self.miller_set.crystal_symmetry())
print "Step 4. Filter on global resolution and map to asu"
print >> out, "Data in reference setting:"
#observations.show_summary(f=out, prefix=" ")
show_observations(observations, out=out)
#if self.params.significance_filter.apply is True:
# raise Exception("significance filter not implemented in samosa")
if self.params.significance_filter.apply is True: #------------------------------------
# Apply an I/sigma filter ... accept resolution bins only if they
# have significant signal; tends to screen out higher resolution observations
# if the integration model doesn't quite fit
N_obs_pre_filter = observations.size()
N_bins_small_set = N_obs_pre_filter // self.params.significance_filter.min_ct
N_bins_large_set = N_obs_pre_filter // self.params.significance_filter.max_ct
# Ensure there is at least one bin.
N_bins = max([
min([self.params.significance_filter.n_bins,
N_bins_small_set]), N_bins_large_set, 1
])
print "Total obs %d Choose n bins = %d" % (N_obs_pre_filter,
N_bins)
bin_results = show_observations(observations,
out=out,
n_bins=N_bins)
#show_observations(observations, out=sys.stdout, n_bins=N_bins)
acceptable_resolution_bins = [
bin.mean_I_sigI > self.params.significance_filter.sigma
for bin in bin_results
]
acceptable_nested_bin_sequences = [
i for i in xrange(len(acceptable_resolution_bins))
if False not in acceptable_resolution_bins[:i + 1]
]
if len(acceptable_nested_bin_sequences) == 0:
return null_data(file_name=file_name,
log_out=out.getvalue(),
low_signal=True)
else:
N_acceptable_bins = max(acceptable_nested_bin_sequences) + 1
imposed_res_filter = float(bin_results[N_acceptable_bins -
1].d_range.split()[2])
imposed_res_sel = observations.resolution_filter_selection(
d_min=imposed_res_filter)
observations = observations.select(imposed_res_sel)
observations_original_index = observations_original_index.select(
imposed_res_sel)
print "New resolution filter at %7.2f" % imposed_res_filter, file_name
print "N acceptable bins", N_acceptable_bins
print "Old n_obs: %d, new n_obs: %d" % (N_obs_pre_filter,
observations.size())
print "Step 5. Frame by frame resolution filter"
# Finished applying the binwise I/sigma filter---------------------------------------
if self.params.raw_data.sdfac_auto is True:
raise Exception("sdfac auto not implemented in samosa.")
print "Step 6. Match to reference intensities, filter by correlation, filter out negative intensities."
assert len(observations_original_index.indices()) \
== len(observations.indices())
data = frame_data(self.n_refl, file_name)
data.set_indexed_cell(indexed_cell)
data.d_min = observations.d_min()
# Ensure that match_multi_indices() will return identical results
# when a frame's observations are matched against the
# pre-generated Miller set, self.miller_set, and the reference
# data set, self.i_model. The implication is that the same match
# can be used to map Miller indices to array indices for intensity
# accumulation, and for determination of the correlation
# coefficient in the presence of a scaling reference.
if self.i_model is not None:
assert len(self.i_model.indices()) == len(self.miller_set.indices()) \
and (self.i_model.indices() ==
self.miller_set.indices()).count(False) == 0
matches = miller.match_multi_indices(
miller_indices_unique=self.miller_set.indices(),
miller_indices=observations.indices())
use_weights = False # New facility for getting variance-weighted correlation
if self.params.scaling.algorithm in ['mark1', 'levmar']:
# Because no correlation is computed, the correlation
# coefficient is fixed at zero. Setting slope = 1 means
# intensities are added without applying a scale factor.
sum_x = 0
sum_y = 0
for pair in matches.pairs():
data.n_obs += 1
if not self.params.include_negatives and observations.data()[
pair[1]] <= 0:
data.n_rejected += 1
else:
sum_y += observations.data()[pair[1]]
N = data.n_obs - data.n_rejected
# Early return if there are no positive reflections on the frame.
if data.n_obs <= data.n_rejected:
return null_data(file_name=file_name,
log_out=out.getvalue(),
low_signal=True)
# Update the count for each matched reflection. This counts
# reflections with non-positive intensities, too.
data.completeness += matches.number_of_matches(0).as_int()
data.wavelength = wavelength
if not self.params.scaling.enable: # Do not scale anything
print "Scale factor to an isomorphous reference PDB will NOT be applied."
slope = 1.0
offset = 0.0
observations_original_index_indices = observations_original_index.indices(
)
if db_mgr is None:
return unpack(MINI.x) # special exit for two-color indexing
kwargs = {
'wavelength': wavelength,
'beam_x': result['xbeam'],
'beam_y': result['ybeam'],
'distance': result['distance'],
'unique_file_name': data.file_name
}
ORI = result["current_orientation"][0]
Astar = matrix.sqr(ORI.reciprocal_matrix())
kwargs['res_ori_1'] = Astar[0]
kwargs['res_ori_2'] = Astar[1]
kwargs['res_ori_3'] = Astar[2]
kwargs['res_ori_4'] = Astar[3]
kwargs['res_ori_5'] = Astar[4]
kwargs['res_ori_6'] = Astar[5]
kwargs['res_ori_7'] = Astar[6]
kwargs['res_ori_8'] = Astar[7]
kwargs['res_ori_9'] = Astar[8]
assert self.params.scaling.report_ML is True
kwargs['half_mosaicity_deg'] = result["ML_half_mosaicity_deg"][0]
kwargs['domain_size_ang'] = result["ML_domain_size_ang"][0]
frame_id_0_base = db_mgr.insert_frame(**kwargs)
xypred = result["mapped_predictions"][0]
indices = flex.size_t([pair[1] for pair in matches.pairs()])
sel_observations = flex.intersection(size=observations.data().size(),
iselections=[indices])
set_original_hkl = observations_original_index_indices.select(
flex.intersection(size=observations_original_index_indices.size(),
iselections=[indices]))
set_xypred = xypred.select(
flex.intersection(size=xypred.size(), iselections=[indices]))
kwargs = {
'hkl_id_0_base': [pair[0] for pair in matches.pairs()],
'i': observations.data().select(sel_observations),
'sigi': observations.sigmas().select(sel_observations),
'detector_x': [xy[0] for xy in set_xypred],
'detector_y': [xy[1] for xy in set_xypred],
'frame_id_0_base': [frame_id_0_base] * len(matches.pairs()),
'overload_flag': [0] * len(matches.pairs()),
'original_h': [hkl[0] for hkl in set_original_hkl],
'original_k': [hkl[1] for hkl in set_original_hkl],
'original_l': [hkl[2] for hkl in set_original_hkl]
}
db_mgr.insert_observation(**kwargs)
print >> out, "Lattice: %d reflections" % (data.n_obs -
data.n_rejected)
print >> out, "average obs", sum_y / (data.n_obs - data.n_rejected), \
"average calc", sum_x / (data.n_obs - data.n_rejected)
print >> out, "Rejected %d reflections with negative intensities" % \
data.n_rejected
data.accept = True
for pair in matches.pairs():
if not self.params.include_negatives and (
observations.data()[pair[1]] <= 0):
continue
Intensity = observations.data()[pair[1]]
# Super-rare exception. If saved sigmas instead of I/sigmas in the ISIGI dict, this wouldn't be needed.
if Intensity == 0:
continue
# Add the reflection as a two-tuple of intensity and I/sig(I)
# to the dictionary of observations.
index = self.miller_set.indices()[pair[0]]
isigi = (Intensity, observations.data()[pair[1]] /
observations.sigmas()[pair[1]], 1.0)
if index in data.ISIGI:
data.ISIGI[index].append(isigi)
else:
data.ISIGI[index] = [isigi]
sigma = observations.sigmas()[pair[1]]
variance = sigma * sigma
data.summed_N[pair[0]] += 1
data.summed_wt_I[pair[0]] += Intensity / variance
data.summed_weight[pair[0]] += 1 / variance
data.set_log_out(out.getvalue())
return data
def scale_frame_detail(self,timestamp,cursor,do_inserts=True,result=None):#, file_name, db_mgr, out):
if result is None:
result = self.params
# If the pickled integration file does not contain a wavelength,
# fall back on the value given on the command line. XXX The
# wavelength parameter should probably be removed from master_phil
# once all pickled integration files contain it.
wavelength = result["wavelength"]
assert (wavelength > 0)
# Do not apply polarization correction here, as this requires knowledge of
# pixel size at minimum, and full detector geometry in general. The optimal
# redesign would be to apply the polarization correction just after the integration
# step in the integration code.
print "Step 3. Correct for polarization."
observations = result["observations"][0]
indexed_cell = observations.unit_cell()
observations_original_index = observations.deep_copy()
assert len(observations_original_index.indices()) == len(observations.indices())
# Now manipulate the data to conform to unit cell, asu, and space group
# of reference. The resolution will be cut later.
# Only works if there is NOT an indexing ambiguity!
#observations = observations.customized_copy(
# anomalous_flag=not self.params.merge_anomalous,
# crystal_symmetry=self.miller_set.crystal_symmetry()
# ).map_to_asu()
#observations_original_index = observations_original_index.customized_copy(
# anomalous_flag=not self.params.merge_anomalous,
# crystal_symmetry=self.miller_set.crystal_symmetry()
# )
observations = observations.customized_copy(anomalous_flag=False).map_to_asu()
print "Step 4. Filter on global resolution and map to asu"
#observations.show_summary(f=out, prefix=" ")
from rstbx.dials_core.integration_core import show_observations
show_observations(observations)
print "Step 6. Match to reference intensities, filter by correlation, filter out negative intensities."
assert len(observations_original_index.indices()) \
== len(observations.indices())
# Ensure that match_multi_indices() will return identical results
# when a frame's observations are matched against the
# pre-generated Miller set, self.miller_set, and the reference
# data set, self.i_model. The implication is that the same match
# can be used to map Miller indices to array indices for intensity
# accumulation, and for determination of the correlation
# coefficient in the presence of a scaling reference.
self.miller_set.show_summary(prefix="mset ")
matches = match_multi_indices(
miller_indices_unique=self.miller_set.indices(),
miller_indices=observations.indices())
slope = 1.0
offset = 0.0
print result.get("sa_parameters")[0]
have_sa_params = ( type(result.get("sa_parameters")[0]) == type(dict()) )
observations_original_index_indices = observations_original_index.indices()
print result.keys()
kwargs = {'wavelength': wavelength,
'beam_x': result['xbeam'],
'beam_y': result['ybeam'],
'distance': result['distance'],
'slope': slope,
'offset': offset,
'unique_file_name': timestamp,
'eventstamp':timestamp,
'sifoil': 0.0}
trial_id = self.get_trial_id(cursor)
run_id = self.get_run_id(cursor)
kwargs["trials_id"] = trial_id
kwargs["rungroups_id"] = self.rungroup_id
kwargs["runs_run_id"] = run_id
kwargs["isoforms_isoform_id"] = self.isoform_id
res_ori_direct = matrix.sqr(
observations.unit_cell().orthogonalization_matrix()).transpose().elems
kwargs['res_ori_1'] = res_ori_direct[0]
kwargs['res_ori_2'] = res_ori_direct[1]
kwargs['res_ori_3'] = res_ori_direct[2]
kwargs['res_ori_4'] = res_ori_direct[3]
kwargs['res_ori_5'] = res_ori_direct[4]
kwargs['res_ori_6'] = res_ori_direct[5]
kwargs['res_ori_7'] = res_ori_direct[6]
kwargs['res_ori_8'] = res_ori_direct[7]
kwargs['res_ori_9'] = res_ori_direct[8]
kwargs['mosaic_block_rotation'] = result.get("ML_half_mosaicity_deg",[float("NaN")])[0]
kwargs['mosaic_block_size'] = result.get("ML_domain_size_ang",[float("NaN")])[0]
kwargs['ewald_proximal_volume'] = result.get("ewald_proximal_volume",[float("NaN")])[0]
sql, parameters = self._insert(
table='`%s_frames`' % self.db_experiment_tag,
**kwargs)
print sql
print parameters
results = {'frame':[sql, parameters, kwargs]}
if do_inserts:
cursor.execute(sql, parameters[0])
frame_id = cursor.lastrowid
else:
frame_id = None
xypred = result["mapped_predictions"][0]
indices = flex.size_t([pair[1] for pair in matches.pairs()])
sel_observations = flex.intersection(
size=observations.data().size(),
iselections=[indices])
set_original_hkl = observations_original_index_indices.select(
flex.intersection(
size=observations_original_index_indices.size(),
iselections=[indices]))
set_xypred = xypred.select(
flex.intersection(
size=xypred.size(),
iselections=[indices]))
''' debugging printout
print len(observations.data())
print len(indices)
print len(sel_observations)
for x in xrange(len(observations.data())):
print x,observations.indices().select(sel_observations)[x],
print set_original_hkl[x],
index_into_hkl_id = matches.pairs()[x][0]
print index_into_hkl_id,
print self.miller_set.indices()[index_into_hkl_id],
cursor.execute('SELECT H,K,L FROM %s_hkls WHERE hkl_id = %d'%(
self.db_experiment_tag, self.miller_set_id[index_into_hkl_id]))
print cursor.fetchall()[0]
'''
print "Adding %d observations for this frame"%(len(sel_observations))
kwargs = {'hkls_id': self.miller_set_id.select(flex.size_t([pair[0] for pair in matches.pairs()])),
'i': observations.data().select(sel_observations),
'sigi': observations.sigmas().select(sel_observations),
'detector_x_px': [xy[0] for xy in set_xypred],
'detector_y_px': [xy[1] for xy in set_xypred],
'frames_id': [frame_id] * len(matches.pairs()),
'overload_flag': [0] * len(matches.pairs()),
'original_h': [hkl[0] for hkl in set_original_hkl],
'original_k': [hkl[1] for hkl in set_original_hkl],
'original_l': [hkl[2] for hkl in set_original_hkl],
'frames_rungroups_id': [self.rungroup_id] * len(matches.pairs()),
'frames_trials_id': [trial_id] * len(matches.pairs()),
'panel': [0] * len(matches.pairs())
}
if do_inserts:
# For MySQLdb executemany() is six times slower than a single big
# execute() unless the "values" keyword is given in lowercase
# (http://sourceforge.net/p/mysql-python/bugs/305).
#
# See also merging_database_sqlite3._insert()
query = ("INSERT INTO `%s_observations` (" % self.db_experiment_tag) \
+ ", ".join(kwargs.keys()) + ") values (" \
+ ", ".join(["%s"] * len(kwargs.keys())) + ")"
try:
parameters = zip(*kwargs.values())
except TypeError:
parameters = [kwargs.values()]
cursor.executemany(query, parameters)
#print "done execute many"
#print cursor._last_executed
results['observations'] = [query, parameters, kwargs]
else:
# since frame_id isn't valid in the query here, don't include a sql statement or parameters array in the results
results['observations'] = [None, None, kwargs]
return results
def scale_frame_detail(self,
timestamp,
cursor,
do_inserts=True,
result=None): #, file_name, db_mgr, out):
if result is None:
result = self.params
# If the pickled integration file does not contain a wavelength,
# fall back on the value given on the command line. XXX The
# wavelength parameter should probably be removed from master_phil
# once all pickled integration files contain it.
wavelength = result["wavelength"]
assert (wavelength > 0)
# Do not apply polarization correction here, as this requires knowledge of
# pixel size at minimum, and full detector geometry in general. The optimal
# redesign would be to apply the polarization correction just after the integration
# step in the integration code.
print("Step 3. Correct for polarization.")
observations = result["observations"][0]
indexed_cell = observations.unit_cell()
observations_original_index = observations.deep_copy()
assert len(observations_original_index.indices()) == len(
observations.indices())
# Now manipulate the data to conform to unit cell, asu, and space group
# of reference. The resolution will be cut later.
# Only works if there is NOT an indexing ambiguity!
#observations = observations.customized_copy(
# anomalous_flag=not self.params.merge_anomalous,
# crystal_symmetry=self.miller_set.crystal_symmetry()
# ).map_to_asu()
#observations_original_index = observations_original_index.customized_copy(
# anomalous_flag=not self.params.merge_anomalous,
# crystal_symmetry=self.miller_set.crystal_symmetry()
# )
observations = observations.customized_copy(
anomalous_flag=False).map_to_asu()
print("Step 4. Filter on global resolution and map to asu")
#observations.show_summary(f=out, prefix=" ")
from rstbx.dials_core.integration_core import show_observations
show_observations(observations)
print(
"Step 6. Match to reference intensities, filter by correlation, filter out negative intensities."
)
assert len(observations_original_index.indices()) \
== len(observations.indices())
# Ensure that match_multi_indices() will return identical results
# when a frame's observations are matched against the
# pre-generated Miller set, self.miller_set, and the reference
# data set, self.i_model. The implication is that the same match
# can be used to map Miller indices to array indices for intensity
# accumulation, and for determination of the correlation
# coefficient in the presence of a scaling reference.
self.miller_set.show_summary(prefix="mset ")
matches = match_multi_indices(
miller_indices_unique=self.miller_set.indices(),
miller_indices=observations.indices())
slope = 1.0
offset = 0.0
print(result.get("sa_parameters")[0])
have_sa_params = (type(result.get("sa_parameters")[0]) == type(dict()))
observations_original_index_indices = observations_original_index.indices(
)
print(list(result.keys()))
kwargs = {
'wavelength': wavelength,
'beam_x': result['xbeam'],
'beam_y': result['ybeam'],
'distance': result['distance'],
'slope': slope,
'offset': offset,
'unique_file_name': timestamp,
'eventstamp': timestamp,
'sifoil': 0.0
}
trial_id = self.get_trial_id(cursor)
run_id = self.get_run_id(cursor)
kwargs["trials_id"] = trial_id
kwargs["rungroups_id"] = self.rungroup_id
kwargs["runs_run_id"] = run_id
kwargs["isoforms_isoform_id"] = self.isoform_id
res_ori_direct = matrix.sqr(observations.unit_cell(
).orthogonalization_matrix()).transpose().elems
kwargs['res_ori_1'] = res_ori_direct[0]
kwargs['res_ori_2'] = res_ori_direct[1]
kwargs['res_ori_3'] = res_ori_direct[2]
kwargs['res_ori_4'] = res_ori_direct[3]
kwargs['res_ori_5'] = res_ori_direct[4]
kwargs['res_ori_6'] = res_ori_direct[5]
kwargs['res_ori_7'] = res_ori_direct[6]
kwargs['res_ori_8'] = res_ori_direct[7]
kwargs['res_ori_9'] = res_ori_direct[8]
kwargs['mosaic_block_rotation'] = result.get("ML_half_mosaicity_deg",
[float("NaN")])[0]
kwargs['mosaic_block_size'] = result.get("ML_domain_size_ang",
[float("NaN")])[0]
kwargs['ewald_proximal_volume'] = result.get("ewald_proximal_volume",
[float("NaN")])[0]
sql, parameters = self._insert(table='`%s_frames`' %
self.db_experiment_tag,
**kwargs)
print(sql)
print(parameters)
results = {'frame': [sql, parameters, kwargs]}
if do_inserts:
cursor.execute(sql, parameters[0])
frame_id = cursor.lastrowid
else:
frame_id = None
xypred = result["mapped_predictions"][0]
indices = flex.size_t([pair[1] for pair in matches.pairs()])
sel_observations = flex.intersection(size=observations.data().size(),
iselections=[indices])
set_original_hkl = observations_original_index_indices.select(
flex.intersection(size=observations_original_index_indices.size(),
iselections=[indices]))
set_xypred = xypred.select(
flex.intersection(size=xypred.size(), iselections=[indices]))
''' debugging printout
print len(observations.data())
print len(indices)
print len(sel_observations)
for x in range(len(observations.data())):
print x,observations.indices().select(sel_observations)[x],
print set_original_hkl[x],
index_into_hkl_id = matches.pairs()[x][0]
print index_into_hkl_id,
print self.miller_set.indices()[index_into_hkl_id],
cursor.execute('SELECT H,K,L FROM %s_hkls WHERE hkl_id = %d'%(
self.db_experiment_tag, self.miller_set_id[index_into_hkl_id]))
print cursor.fetchall()[0]
'''
print("Adding %d observations for this frame" %
(len(sel_observations)))
kwargs = {
'hkls_id':
self.miller_set_id.select(
flex.size_t([pair[0] for pair in matches.pairs()])),
'i':
observations.data().select(sel_observations),
'sigi':
observations.sigmas().select(sel_observations),
'detector_x_px': [xy[0] for xy in set_xypred],
'detector_y_px': [xy[1] for xy in set_xypred],
'frames_id': [frame_id] * len(matches.pairs()),
'overload_flag': [0] * len(matches.pairs()),
'original_h': [hkl[0] for hkl in set_original_hkl],
'original_k': [hkl[1] for hkl in set_original_hkl],
'original_l': [hkl[2] for hkl in set_original_hkl],
'frames_rungroups_id': [self.rungroup_id] * len(matches.pairs()),
'frames_trials_id': [trial_id] * len(matches.pairs()),
'panel': [0] * len(matches.pairs())
}
if do_inserts:
# For MySQLdb executemany() is six times slower than a single big
# execute() unless the "values" keyword is given in lowercase
# (http://sourceforge.net/p/mysql-python/bugs/305).
#
# See also merging_database_sqlite3._insert()
query = ("INSERT INTO `%s_observations` (" % self.db_experiment_tag) \
+ ", ".join(kwargs) + ") values (" \
+ ", ".join(["%s"] * len(kwargs)) + ")"
try:
parameters = list(zip(*list(kwargs.values())))
except TypeError:
parameters = [list(kwargs.values())]
cursor.executemany(query, parameters)
#print "done execute many"
#print cursor._last_executed
results['observations'] = [query, parameters, kwargs]
else:
# since frame_id isn't valid in the query here, don't include a sql statement or parameters array in the results
results['observations'] = [None, None, kwargs]
return results
def scale_frame_detail(self, result, file_name, db_mgr, out):
# If the pickled integration file does not contain a wavelength,
# fall back on the value given on the command line. XXX The
# wavelength parameter should probably be removed from master_phil
# once all pickled integration files contain it.
if (result.has_key("wavelength")):
wavelength = result["wavelength"]
elif (self.params.wavelength is not None):
wavelength = self.params.wavelength
else:
# XXX Give error, or raise exception?
return None
assert (wavelength > 0)
observations = result["observations"][0]
cos_two_polar_angle = result["cos_two_polar_angle"]
assert observations.size() == cos_two_polar_angle.size()
tt_vec = observations.two_theta(wavelength)
#print "mean tt degrees",180.*flex.mean(tt_vec.data())/math.pi
cos_tt_vec = flex.cos( tt_vec.data() )
sin_tt_vec = flex.sin( tt_vec.data() )
cos_sq_tt_vec = cos_tt_vec * cos_tt_vec
sin_sq_tt_vec = sin_tt_vec * sin_tt_vec
P_nought_vec = 0.5 * (1. + cos_sq_tt_vec)
F_prime = -1.0 # Hard-coded value defines the incident polarization axis
P_prime = 0.5 * F_prime * cos_two_polar_angle * sin_sq_tt_vec
# XXX added as a diagnostic
prange=P_nought_vec - P_prime
other_F_prime = 1.0
otherP_prime = 0.5 * other_F_prime * cos_two_polar_angle * sin_sq_tt_vec
otherprange=P_nought_vec - otherP_prime
diff2 = flex.abs(prange - otherprange)
print "mean diff is",flex.mean(diff2), "range",flex.min(diff2), flex.max(diff2)
# XXX done
observations = observations / ( P_nought_vec - P_prime )
# This corrects observations for polarization assuming 100% polarization on
# one axis (thus the F_prime = -1.0 rather than the perpendicular axis, 1.0)
# Polarization model as described by Kahn, Fourme, Gadet, Janin, Dumas & Andre
# (1982) J. Appl. Cryst. 15, 330-337, equations 13 - 15.
print "Step 3. Correct for polarization."
indexed_cell = observations.unit_cell()
observations_original_index = observations.deep_copy()
if result.get("model_partialities",None) is not None and result["model_partialities"][0] is not None:
# some recordkeeping useful for simulations
partialities_original_index = observations.customized_copy(
crystal_symmetry=self.miller_set.crystal_symmetry(),
data = result["model_partialities"][0]["data"],
sigmas = flex.double(result["model_partialities"][0]["data"].size()), #dummy value for sigmas
indices = result["model_partialities"][0]["indices"],
).resolution_filter(d_min=self.params.d_min)
assert len(observations_original_index.indices()) == len(observations.indices())
# Now manipulate the data to conform to unit cell, asu, and space group
# of reference. The resolution will be cut later.
# Only works if there is NOT an indexing ambiguity!
observations = observations.customized_copy(
anomalous_flag=not self.params.merge_anomalous,
crystal_symmetry=self.miller_set.crystal_symmetry()
).map_to_asu()
observations_original_index = observations_original_index.customized_copy(
anomalous_flag=not self.params.merge_anomalous,
crystal_symmetry=self.miller_set.crystal_symmetry()
)
print "Step 4. Filter on global resolution and map to asu"
print >> out, "Data in reference setting:"
#observations.show_summary(f=out, prefix=" ")
show_observations(observations, out=out)
#if self.params.significance_filter.apply is True:
# raise Exception("significance filter not implemented in samosa")
if self.params.significance_filter.apply is True: #------------------------------------
# Apply an I/sigma filter ... accept resolution bins only if they
# have significant signal; tends to screen out higher resolution observations
# if the integration model doesn't quite fit
N_obs_pre_filter = observations.size()
N_bins_small_set = N_obs_pre_filter // self.params.significance_filter.min_ct
N_bins_large_set = N_obs_pre_filter // self.params.significance_filter.max_ct
# Ensure there is at least one bin.
N_bins = max(
[min([self.params.significance_filter.n_bins,N_bins_small_set]),
N_bins_large_set, 1]
)
print "Total obs %d Choose n bins = %d"%(N_obs_pre_filter,N_bins)
bin_results = show_observations(observations, out=out, n_bins=N_bins)
#show_observations(observations, out=sys.stdout, n_bins=N_bins)
acceptable_resolution_bins = [
bin.mean_I_sigI > self.params.significance_filter.sigma for bin in bin_results]
acceptable_nested_bin_sequences = [i for i in xrange(len(acceptable_resolution_bins))
if False not in acceptable_resolution_bins[:i+1]]
if len(acceptable_nested_bin_sequences)==0:
return null_data(
file_name=file_name, log_out=out.getvalue(), low_signal=True)
else:
N_acceptable_bins = max(acceptable_nested_bin_sequences) + 1
imposed_res_filter = float(bin_results[N_acceptable_bins-1].d_range.split()[2])
imposed_res_sel = observations.resolution_filter_selection(
d_min=imposed_res_filter)
observations = observations.select(
imposed_res_sel)
observations_original_index = observations_original_index.select(
imposed_res_sel)
print "New resolution filter at %7.2f"%imposed_res_filter,file_name
print "N acceptable bins",N_acceptable_bins
print "Old n_obs: %d, new n_obs: %d"%(N_obs_pre_filter,observations.size())
print "Step 5. Frame by frame resolution filter"
# Finished applying the binwise I/sigma filter---------------------------------------
if self.params.raw_data.sdfac_auto is True:
raise Exception("sdfac auto not implemented in samosa.")
print "Step 6. Match to reference intensities, filter by correlation, filter out negative intensities."
assert len(observations_original_index.indices()) \
== len(observations.indices())
data = frame_data(self.n_refl, file_name)
data.set_indexed_cell(indexed_cell)
data.d_min = observations.d_min()
# Ensure that match_multi_indices() will return identical results
# when a frame's observations are matched against the
# pre-generated Miller set, self.miller_set, and the reference
# data set, self.i_model. The implication is that the same match
# can be used to map Miller indices to array indices for intensity
# accumulation, and for determination of the correlation
# coefficient in the presence of a scaling reference.
if self.i_model is not None:
assert len(self.i_model.indices()) == len(self.miller_set.indices()) \
and (self.i_model.indices() ==
self.miller_set.indices()).count(False) == 0
matches = miller.match_multi_indices(
miller_indices_unique=self.miller_set.indices(),
miller_indices=observations.indices())
use_weights = False # New facility for getting variance-weighted correlation
if self.params.scaling.algorithm in ['mark1','levmar']:
# Because no correlation is computed, the correlation
# coefficient is fixed at zero. Setting slope = 1 means
# intensities are added without applying a scale factor.
sum_x = 0
sum_y = 0
for pair in matches.pairs():
data.n_obs += 1
if not self.params.include_negatives and observations.data()[pair[1]] <= 0:
data.n_rejected += 1
else:
sum_y += observations.data()[pair[1]]
N = data.n_obs - data.n_rejected
# Early return if there are no positive reflections on the frame.
if data.n_obs <= data.n_rejected:
return null_data(
file_name=file_name, log_out=out.getvalue(), low_signal=True)
# Update the count for each matched reflection. This counts
# reflections with non-positive intensities, too.
data.completeness += matches.number_of_matches(0).as_int()
data.wavelength = wavelength
if not self.params.scaling.enable: # Do not scale anything
print "Scale factor to an isomorphous reference PDB will NOT be applied."
slope = 1.0
offset = 0.0
observations_original_index_indices = observations_original_index.indices()
if db_mgr is None: return unpack(MINI.x) # special exit for two-color indexing
kwargs = {'wavelength': wavelength,
'beam_x': result['xbeam'],
'beam_y': result['ybeam'],
'distance': result['distance'],
'unique_file_name': data.file_name}
ORI = result["current_orientation"][0]
Astar = matrix.sqr(ORI.reciprocal_matrix())
kwargs['res_ori_1'] = Astar[0]
kwargs['res_ori_2'] = Astar[1]
kwargs['res_ori_3'] = Astar[2]
kwargs['res_ori_4'] = Astar[3]
kwargs['res_ori_5'] = Astar[4]
kwargs['res_ori_6'] = Astar[5]
kwargs['res_ori_7'] = Astar[6]
kwargs['res_ori_8'] = Astar[7]
kwargs['res_ori_9'] = Astar[8]
assert self.params.scaling.report_ML is True
kwargs['half_mosaicity_deg'] = result["ML_half_mosaicity_deg"][0]
kwargs['domain_size_ang'] = result["ML_domain_size_ang"][0]
frame_id_0_base = db_mgr.insert_frame(**kwargs)
xypred = result["mapped_predictions"][0]
indices = flex.size_t([pair[1] for pair in matches.pairs()])
sel_observations = flex.intersection(
size=observations.data().size(),
iselections=[indices])
set_original_hkl = observations_original_index_indices.select(
flex.intersection(
size=observations_original_index_indices.size(),
iselections=[indices]))
set_xypred = xypred.select(
flex.intersection(
size=xypred.size(),
iselections=[indices]))
kwargs = {'hkl_id_0_base': [pair[0] for pair in matches.pairs()],
'i': observations.data().select(sel_observations),
'sigi': observations.sigmas().select(sel_observations),
'detector_x': [xy[0] for xy in set_xypred],
'detector_y': [xy[1] for xy in set_xypred],
'frame_id_0_base': [frame_id_0_base] * len(matches.pairs()),
'overload_flag': [0] * len(matches.pairs()),
'original_h': [hkl[0] for hkl in set_original_hkl],
'original_k': [hkl[1] for hkl in set_original_hkl],
'original_l': [hkl[2] for hkl in set_original_hkl]}
db_mgr.insert_observation(**kwargs)
print >> out, "Lattice: %d reflections" % (data.n_obs - data.n_rejected)
print >> out, "average obs", sum_y / (data.n_obs - data.n_rejected), \
"average calc", sum_x / (data.n_obs - data.n_rejected)
print >> out, "Rejected %d reflections with negative intensities" % \
data.n_rejected
data.accept = True
for pair in matches.pairs():
if not self.params.include_negatives and (observations.data()[pair[1]] <= 0) :
continue
Intensity = observations.data()[pair[1]]
# Super-rare exception. If saved sigmas instead of I/sigmas in the ISIGI dict, this wouldn't be needed.
if Intensity == 0:
continue
# Add the reflection as a two-tuple of intensity and I/sig(I)
# to the dictionary of observations.
index = self.miller_set.indices()[pair[0]]
isigi = (Intensity,
observations.data()[pair[1]] / observations.sigmas()[pair[1]],
1.0)
if index in data.ISIGI:
data.ISIGI[index].append(isigi)
else:
data.ISIGI[index] = [isigi]
sigma = observations.sigmas()[pair[1]]
variance = sigma * sigma
data.summed_N[pair[0]] += 1
data.summed_wt_I[pair[0]] += Intensity / variance
data.summed_weight[pair[0]] += 1 / variance
data.set_log_out(out.getvalue())
return data
