def __init__(self,integration_dict,horizons_phil,verbose=False):
self.integration_dict = integration_dict
self.horizons_phil = horizons_phil
results = self.integration_dict["results"]
obs = [item.get_obs(self.integration_dict["spacegroup"]) for item in results]
if verbose:
for item in results:
show_observations(item.get_obs(self.integration_dict["spacegroup"]))
self.stats_mtz(self.integration_dict,obs)
self.integration_dict["resolution"] = self.value
def __init__(self,integration_dict,horizons_phil,verbose=False):
self.integration_dict = integration_dict
self.horizons_phil = horizons_phil
results = self.integration_dict["results"]
obs = [item.get_obs(self.integration_dict["spacegroup"]) for item in results]
if verbose:
for item in results:
show_observations(item.get_obs(self.integration_dict["spacegroup"]))
self.stats_mtz(self.integration_dict,obs)
self.integration_dict["resolution"] = self.value
def get_imposed_res_filter(self, out):
if self.stash_res_filter is not None: return self.stash_res_filter
if self.params.significance_filter.apply is True: #------------------------------------
print >> out, "Step 5. Frame by frame resolution filter"
# Apply an I/sigma filter ... accept resolution bins only if they
# have significant signal; tends to screen out higher resolution observations
# if the integration model doesn't quite fit
N_obs_pre_filter = self.i_sigi.size()
N_bins_small_set = N_obs_pre_filter // self.params.significance_filter.min_ct
N_bins_large_set = N_obs_pre_filter // self.params.significance_filter.max_ct
# Ensure there is at least one bin.
N_bins = max([
min([self.params.significance_filter.n_bins,
N_bins_small_set]), N_bins_large_set, 1
])
print >> out, "Total obs %d Choose n bins = %d" % (
N_obs_pre_filter, N_bins)
bin_results = show_observations(self.measurements,
out=sys.stdout,
n_bins=N_bins)
if True: # no fuller kapton -- not implemented here,
# but code can and should be borrowed from cxi.merge
acceptable_resolution_bins = [
bin.mean_I_sigI > self.params.significance_filter.sigma
for bin in bin_results
]
acceptable_nested_bin_sequences = [
i for i in range(len(acceptable_resolution_bins))
if False not in acceptable_resolution_bins[:i + 1]
]
N_acceptable_bins = max(acceptable_nested_bin_sequences) + 1
imposed_res_filter = float(bin_results[N_acceptable_bins -
1].d_range.split()[2])
print >> out, "New resolution filter at %7.2f" % imposed_res_filter, self.filename
print >> out, "N acceptable bins", N_acceptable_bins
print >> out, "Old n_obs: %d, new n_obs: %d" % (
N_obs_pre_filter, self.measurements.size())
# Finished applying the binwise I/sigma filter---------------------------------------
else:
imposed_res_filter = None
self.stash_res_filter = imposed_res_filter
return imposed_res_filter
def apply_significance_filter(self, experiments, reflections):
self.logger.log_step_time("SIGNIFICANCE_FILTER")
# Apply an I/sigma filter ... accept resolution bins only if they
# have significant signal; tends to screen out higher resolution observations
# if the integration model doesn't quite fit
unit_cell = self.params.scaling.unit_cell
if unit_cell is None:
try:
unit_cell = self.params.statistics.average_unit_cell
except AttributeError:
pass
target_symm = symmetry(
unit_cell=unit_cell,
space_group_info=self.params.scaling.space_group)
new_experiments = ExperimentList()
new_reflections = flex.reflection_table()
for expt_id, experiment in enumerate(experiments):
exp_reflections = reflections.select(
reflections['exp_id'] == experiment.identifier)
if not len(exp_reflections): continue
N_obs_pre_filter = exp_reflections.size()
N_bins_small_set = N_obs_pre_filter // self.params.select.significance_filter.min_ct
N_bins_large_set = N_obs_pre_filter // self.params.select.significance_filter.max_ct
# Ensure there is at least one bin.
N_bins = max([
min([
self.params.select.significance_filter.n_bins,
N_bins_small_set
]), N_bins_large_set, 1
])
#print ("\nN_obs_pre_filter %d"%N_obs_pre_filter)
#print >> out, "Total obs %d Choose n bins = %d"%(N_obs_pre_filter,N_bins)
#if indices_to_edge is not None:
# print >> out, "Total preds %d to edge of detector"%indices_to_edge.size()
# Build a miller array for the experiment reflections
exp_miller_indices = miller.set(target_symm,
exp_reflections['miller_index'],
True)
exp_observations = miller.array(
exp_miller_indices, exp_reflections['intensity.sum.value'],
flex.sqrt(exp_reflections['intensity.sum.variance']))
assert exp_observations.size() == exp_reflections.size()
out = StringIO()
bin_results = show_observations(exp_observations,
out=out,
n_bins=N_bins)
if self.params.output.log_level == 0:
self.logger.log(out.getvalue())
acceptable_resolution_bins = [
bin.mean_I_sigI > self.params.select.significance_filter.sigma
for bin in bin_results
]
acceptable_nested_bin_sequences = [
i for i in range(len(acceptable_resolution_bins))
if False not in acceptable_resolution_bins[:i + 1]
]
if len(acceptable_nested_bin_sequences) == 0:
continue
else:
N_acceptable_bins = max(acceptable_nested_bin_sequences) + 1
imposed_res_filter = float(bin_results[N_acceptable_bins -
1].d_range.split()[2])
self.logger.log(
"Experiment id %d, image index %d, resolution cutoff %f\n"
% (expt_id, experiment.imageset.indices()[0],
imposed_res_filter))
imposed_res_sel = exp_observations.resolution_filter_selection(
d_min=imposed_res_filter)
assert imposed_res_sel.size() == exp_reflections.size()
new_exp_reflections = exp_reflections.select(imposed_res_sel)
if new_exp_reflections.size() > 0:
new_experiments.append(experiment)
new_reflections.extend(new_exp_reflections)
#self.logger.log("N acceptable bins %d"%N_acceptable_bins)
#self.logger.log("Old n_obs: %d, new n_obs: %d"%(N_obs_pre_filter, exp_observations.size()))
#if indices_to_edge is not None:
# print >> out, "Total preds %d to edge of detector"%indices_to_edge.size()
removed_reflections = len(reflections) - len(new_reflections)
removed_experiments = len(experiments) - len(new_experiments)
self.logger.log(
"Reflections rejected because of significance filter: %d" %
removed_reflections)
self.logger.log(
"Experiments rejected because of significance filter: %d" %
removed_experiments)
# MPI-reduce total counts
comm = self.mpi_helper.comm
MPI = self.mpi_helper.MPI
total_removed_reflections = comm.reduce(removed_reflections, MPI.SUM,
0)
total_removed_experiments = comm.reduce(removed_experiments, MPI.SUM,
0)
# rank 0: log total counts
if self.mpi_helper.rank == 0:
self.logger.main_log(
"Total reflections rejected because of significance filter: %d"
% total_removed_reflections)
self.logger.main_log(
"Total experiments rejected because of significance filter: %d"
% total_removed_experiments)
self.logger.log_step_time("SIGNIFICANCE_FILTER", True)
return new_experiments, new_reflections
def scale_frame_detail(self, result, file_name, db_mgr, out):
# If the pickled integration file does not contain a wavelength,
# fall back on the value given on the command line. XXX The
# wavelength parameter should probably be removed from master_phil
# once all pickled integration files contain it.
if ("wavelength" in result):
wavelength = result["wavelength"]
elif (self.params.wavelength is not None):
wavelength = self.params.wavelength
else:
# XXX Give error, or raise exception?
return None
assert (wavelength > 0)
observations = result["observations"][0]
cos_two_polar_angle = result["cos_two_polar_angle"]
assert observations.size() == cos_two_polar_angle.size()
tt_vec = observations.two_theta(wavelength)
#print "mean tt degrees",180.*flex.mean(tt_vec.data())/math.pi
cos_tt_vec = flex.cos(tt_vec.data())
sin_tt_vec = flex.sin(tt_vec.data())
cos_sq_tt_vec = cos_tt_vec * cos_tt_vec
sin_sq_tt_vec = sin_tt_vec * sin_tt_vec
P_nought_vec = 0.5 * (1. + cos_sq_tt_vec)
F_prime = -1.0 # Hard-coded value defines the incident polarization axis
P_prime = 0.5 * F_prime * cos_two_polar_angle * sin_sq_tt_vec
# XXX added as a diagnostic
prange = P_nought_vec - P_prime
other_F_prime = 1.0
otherP_prime = 0.5 * other_F_prime * cos_two_polar_angle * sin_sq_tt_vec
otherprange = P_nought_vec - otherP_prime
diff2 = flex.abs(prange - otherprange)
print "mean diff is", flex.mean(diff2), "range", flex.min(
diff2), flex.max(diff2)
# XXX done
observations = observations / (P_nought_vec - P_prime)
# This corrects observations for polarization assuming 100% polarization on
# one axis (thus the F_prime = -1.0 rather than the perpendicular axis, 1.0)
# Polarization model as described by Kahn, Fourme, Gadet, Janin, Dumas & Andre
# (1982) J. Appl. Cryst. 15, 330-337, equations 13 - 15.
print "Step 3. Correct for polarization."
indexed_cell = observations.unit_cell()
observations_original_index = observations.deep_copy()
if result.get(
"model_partialities", None
) is not None and result["model_partialities"][0] is not None:
# some recordkeeping useful for simulations
partialities_original_index = observations.customized_copy(
crystal_symmetry=self.miller_set.crystal_symmetry(),
data=result["model_partialities"][0]["data"],
sigmas=flex.double(result["model_partialities"][0]
["data"].size()), #dummy value for sigmas
indices=result["model_partialities"][0]["indices"],
).resolution_filter(d_min=self.params.d_min)
assert len(observations_original_index.indices()) == len(
observations.indices())
# Now manipulate the data to conform to unit cell, asu, and space group
# of reference. The resolution will be cut later.
# Only works if there is NOT an indexing ambiguity!
observations = observations.customized_copy(
anomalous_flag=not self.params.merge_anomalous,
crystal_symmetry=self.miller_set.crystal_symmetry()).map_to_asu()
observations_original_index = observations_original_index.customized_copy(
anomalous_flag=not self.params.merge_anomalous,
crystal_symmetry=self.miller_set.crystal_symmetry())
print "Step 4. Filter on global resolution and map to asu"
print >> out, "Data in reference setting:"
#observations.show_summary(f=out, prefix=" ")
show_observations(observations, out=out)
#if self.params.significance_filter.apply is True:
# raise Exception("significance filter not implemented in samosa")
if self.params.significance_filter.apply is True: #------------------------------------
# Apply an I/sigma filter ... accept resolution bins only if they
# have significant signal; tends to screen out higher resolution observations
# if the integration model doesn't quite fit
N_obs_pre_filter = observations.size()
N_bins_small_set = N_obs_pre_filter // self.params.significance_filter.min_ct
N_bins_large_set = N_obs_pre_filter // self.params.significance_filter.max_ct
# Ensure there is at least one bin.
N_bins = max([
min([self.params.significance_filter.n_bins,
N_bins_small_set]), N_bins_large_set, 1
])
print "Total obs %d Choose n bins = %d" % (N_obs_pre_filter,
N_bins)
bin_results = show_observations(observations,
out=out,
n_bins=N_bins)
#show_observations(observations, out=sys.stdout, n_bins=N_bins)
acceptable_resolution_bins = [
bin.mean_I_sigI > self.params.significance_filter.sigma
for bin in bin_results
]
acceptable_nested_bin_sequences = [
i for i in xrange(len(acceptable_resolution_bins))
if False not in acceptable_resolution_bins[:i + 1]
]
if len(acceptable_nested_bin_sequences) == 0:
return null_data(file_name=file_name,
log_out=out.getvalue(),
low_signal=True)
else:
N_acceptable_bins = max(acceptable_nested_bin_sequences) + 1
imposed_res_filter = float(bin_results[N_acceptable_bins -
1].d_range.split()[2])
imposed_res_sel = observations.resolution_filter_selection(
d_min=imposed_res_filter)
observations = observations.select(imposed_res_sel)
observations_original_index = observations_original_index.select(
imposed_res_sel)
print "New resolution filter at %7.2f" % imposed_res_filter, file_name
print "N acceptable bins", N_acceptable_bins
print "Old n_obs: %d, new n_obs: %d" % (N_obs_pre_filter,
observations.size())
print "Step 5. Frame by frame resolution filter"
# Finished applying the binwise I/sigma filter---------------------------------------
if self.params.raw_data.sdfac_auto is True:
raise Exception("sdfac auto not implemented in samosa.")
print "Step 6. Match to reference intensities, filter by correlation, filter out negative intensities."
assert len(observations_original_index.indices()) \
== len(observations.indices())
data = frame_data(self.n_refl, file_name)
data.set_indexed_cell(indexed_cell)
data.d_min = observations.d_min()
# Ensure that match_multi_indices() will return identical results
# when a frame's observations are matched against the
# pre-generated Miller set, self.miller_set, and the reference
# data set, self.i_model. The implication is that the same match
# can be used to map Miller indices to array indices for intensity
# accumulation, and for determination of the correlation
# coefficient in the presence of a scaling reference.
if self.i_model is not None:
assert len(self.i_model.indices()) == len(self.miller_set.indices()) \
and (self.i_model.indices() ==
self.miller_set.indices()).count(False) == 0
matches = miller.match_multi_indices(
miller_indices_unique=self.miller_set.indices(),
miller_indices=observations.indices())
use_weights = False # New facility for getting variance-weighted correlation
if self.params.scaling.algorithm in ['mark1', 'levmar']:
# Because no correlation is computed, the correlation
# coefficient is fixed at zero. Setting slope = 1 means
# intensities are added without applying a scale factor.
sum_x = 0
sum_y = 0
for pair in matches.pairs():
data.n_obs += 1
if not self.params.include_negatives and observations.data()[
pair[1]] <= 0:
data.n_rejected += 1
else:
sum_y += observations.data()[pair[1]]
N = data.n_obs - data.n_rejected
# Early return if there are no positive reflections on the frame.
if data.n_obs <= data.n_rejected:
return null_data(file_name=file_name,
log_out=out.getvalue(),
low_signal=True)
# Update the count for each matched reflection. This counts
# reflections with non-positive intensities, too.
data.completeness += matches.number_of_matches(0).as_int()
data.wavelength = wavelength
if not self.params.scaling.enable: # Do not scale anything
print "Scale factor to an isomorphous reference PDB will NOT be applied."
slope = 1.0
offset = 0.0
observations_original_index_indices = observations_original_index.indices(
)
if db_mgr is None:
return unpack(MINI.x) # special exit for two-color indexing
kwargs = {
'wavelength': wavelength,
'beam_x': result['xbeam'],
'beam_y': result['ybeam'],
'distance': result['distance'],
'unique_file_name': data.file_name
}
ORI = result["current_orientation"][0]
Astar = matrix.sqr(ORI.reciprocal_matrix())
kwargs['res_ori_1'] = Astar[0]
kwargs['res_ori_2'] = Astar[1]
kwargs['res_ori_3'] = Astar[2]
kwargs['res_ori_4'] = Astar[3]
kwargs['res_ori_5'] = Astar[4]
kwargs['res_ori_6'] = Astar[5]
kwargs['res_ori_7'] = Astar[6]
kwargs['res_ori_8'] = Astar[7]
kwargs['res_ori_9'] = Astar[8]
assert self.params.scaling.report_ML is True
kwargs['half_mosaicity_deg'] = result["ML_half_mosaicity_deg"][0]
kwargs['domain_size_ang'] = result["ML_domain_size_ang"][0]
frame_id_0_base = db_mgr.insert_frame(**kwargs)
xypred = result["mapped_predictions"][0]
indices = flex.size_t([pair[1] for pair in matches.pairs()])
sel_observations = flex.intersection(size=observations.data().size(),
iselections=[indices])
set_original_hkl = observations_original_index_indices.select(
flex.intersection(size=observations_original_index_indices.size(),
iselections=[indices]))
set_xypred = xypred.select(
flex.intersection(size=xypred.size(), iselections=[indices]))
kwargs = {
'hkl_id_0_base': [pair[0] for pair in matches.pairs()],
'i': observations.data().select(sel_observations),
'sigi': observations.sigmas().select(sel_observations),
'detector_x': [xy[0] for xy in set_xypred],
'detector_y': [xy[1] for xy in set_xypred],
'frame_id_0_base': [frame_id_0_base] * len(matches.pairs()),
'overload_flag': [0] * len(matches.pairs()),
'original_h': [hkl[0] for hkl in set_original_hkl],
'original_k': [hkl[1] for hkl in set_original_hkl],
'original_l': [hkl[2] for hkl in set_original_hkl]
}
db_mgr.insert_observation(**kwargs)
print >> out, "Lattice: %d reflections" % (data.n_obs -
data.n_rejected)
print >> out, "average obs", sum_y / (data.n_obs - data.n_rejected), \
"average calc", sum_x / (data.n_obs - data.n_rejected)
print >> out, "Rejected %d reflections with negative intensities" % \
data.n_rejected
data.accept = True
for pair in matches.pairs():
if not self.params.include_negatives and (
observations.data()[pair[1]] <= 0):
continue
Intensity = observations.data()[pair[1]]
# Super-rare exception. If saved sigmas instead of I/sigmas in the ISIGI dict, this wouldn't be needed.
if Intensity == 0:
continue
# Add the reflection as a two-tuple of intensity and I/sig(I)
# to the dictionary of observations.
index = self.miller_set.indices()[pair[0]]
isigi = (Intensity, observations.data()[pair[1]] /
observations.sigmas()[pair[1]], 1.0)
if index in data.ISIGI:
data.ISIGI[index].append(isigi)
else:
data.ISIGI[index] = [isigi]
sigma = observations.sigmas()[pair[1]]
variance = sigma * sigma
data.summed_N[pair[0]] += 1
data.summed_wt_I[pair[0]] += Intensity / variance
data.summed_weight[pair[0]] += 1 / variance
data.set_log_out(out.getvalue())
return data
def integrate_one_character(self,setting,integration_limit):
#from libtbx.development.timers import Profiler
#P = Profiler("Preliminary")
import copy
local = copy.deepcopy(self.process_dictionary)
local['cell']=cellstr(setting)
print("Cell in setting",setting["counter"],local["cell"])
frames = list(sorted(self.spotfinder_results.pd['osc_start'].keys()))
local['maxcel']='0'
local['xbeam']="%f"%setting['minimizer'].new['xbeam']
local['ybeam']="%f"%setting['minimizer'].new['ybeam']
local['distance']="%f"%setting['minimizer'].new['distance']
local["resolution"]= "%f"%integration_limit
from labelit.steps import primaries
local['spacegroup'] = primaries[setting['bravais']]
local['procstart'] = local['procend'] = "%d"%frames[0]
self.pixel_size = float(local['pixel_size'])
from labelit.dptbx import AutoIndexEngine, Parameters
ai = AutoIndexEngine(local['endstation'])
P = Parameters(xbeam=setting["refined x beam"],ybeam=setting["refined y beam"],
distance=setting["refined distance"],twotheta=float(local["twotheta"]))
ai.setBase(P)
ai.setWavelength(float(local['wavelength']))
ai.setMaxcell(float(local['ref_maxcel']))
print("Deltaphi is",float(local['deltaphi']))
ai.setDeltaphi(float(local['deltaphi'])*math.pi/180.)
ai.setMosaicity(setting["mosaicity"])
ai.setOrientation(setting["orient"])
refimage = self.files.images[0]
ai.set_active_areas(self.horizons_phil,
beam=(int(refimage.beamx/refimage.pixel_size),
int(refimage.beamy/refimage.pixel_size)))
image_centers = [(math.pi/180.)*float(x) for x in local["osc_start"].values()]
print("Limiting resolution",integration_limit)
local["results"] = []
for i in range(len(frames)):
print("---------BEGIN Integrate one frame %d %s" % \
(frames[i], os.path.split(self.files.filenames()[i])[-1]))
#P = Profiler("worker")
if self.horizons_phil.integration.combine_sym_constraints_and_3D_target and setting["counter"]>1:
from rstbx.apps.stills.dials_refinement_preceding_integration import integrate_one_frame
integrate_worker = integrate_one_frame(self.triclinic["integration"]["results"][0])
else:
from rstbx.apps.stills.deltapsi_refinement_preceding_integration import integrate_one_frame
integrate_worker = integrate_one_frame()
integrate_worker.inputai = ai
integrate_worker.inputpd = dict(masks=local["masks"],
size1=local["size1"],
size2=local["size2"],
symmetry=setting["best_subsym"])
# carefully select only the data items needed for integrate_worker
# avoid giving the whole process dictionary; reference to "local"
# is a circular reference creating memory leak, while copying the
# whole thing is a big performance hit.
integrate_worker.frame_numbers = frames
integrate_worker.imagefiles = self.files
integrate_worker.spotfinder = self.spotfinder_results
integrate_worker.image_centers = image_centers
integrate_worker.limiting_resolution = integration_limit
integrate_worker.setting_id = setting["counter"]
integrate_worker.pixel_size = self.pixel_size
integrate_worker.set_pixel_size(self.pixel_size)
integrate_worker.set_detector_size(int(local["size1"]),int(local["size2"]))
integrate_worker.set_detector_saturation(refimage.saturation)
integrate_worker.set_up_mask_focus()
integrate_worker.initialize_increments(i)
integrate_worker.horizons_phil = self.horizons_phil
if self.horizons_phil.indexing.verbose_cv:
print("EFFECTIVE TILING"," ".join(
["%d"%z for z in refimage.get_tile_manager(self.horizons_phil).effective_tiling_as_flex_int()]))
integrate_worker.integration_concept(image_number = i,
cb_op_to_primitive = setting["cb_op_inp_best"].inverse(),
verbose_cv = self.horizons_phil.indexing.verbose_cv,
background_factor = self.horizons_phil.integration.background_factor,
)
#P = Profiler("proper")
integrate_worker.integration_proper()
local["results"].append(integrate_worker)
local["r_xbeam"]=ai.xbeam()
local["r_ybeam"]=ai.ybeam()
local["r_distance"]=ai.distance()
local["r_wavelength"]=ai.wavelength
local["r_residual"]=integrate_worker.r_residual
local["r_mosaicity"]=setting["mosaicity"]
try:
local["ewald_proximal_volume"]=integrate_worker.ewald_proximal_volume
except Exception as e:
local["ewald_proximal_volume"]=None
if (self.horizons_phil.indexing.open_wx_viewer):
if True: #use updated slip viewer
try:
import wx
from rstbx.slip_viewer.frame import XrayFrame as SlipXrayFrame
from rstbx.command_line.slip_viewer import master_str as slip_params
from iotbx import phil
from spotfinder import phil_str
from spotfinder.command_line.signal_strength import additional_spotfinder_phil_defs
work_phil = phil.process_command_line("",master_string=slip_params + phil_str + additional_spotfinder_phil_defs)
work_params = work_phil.work.extract()
app = wx.App(0)
wx.SystemOptions.SetOption("osx.openfiledialog.always-show-types", "1")
frame = SlipXrayFrame(None, -1, "X-ray image display", size=(800,720))
frame.Show()
# Update initial settings with values from the command line. Needs
# to be done before image is loaded (but after the frame is
# instantiated).
frame.inherited_params = integrate_worker.horizons_phil
frame.params = work_params
if (frame.pyslip is None):
frame.init_pyslip()
if (frame.settings_frame is None):
frame.OnShowSettings(None)
frame.Layout()
frame.pyslip.tiles.user_requests_antialiasing = work_params.anti_aliasing
frame.settings_frame.panel.center_ctrl.SetValue(True)
frame.settings_frame.panel.integ_ctrl.SetValue(True)
frame.settings_frame.panel.spots_ctrl.SetValue(False)
frame.settings.show_effective_tiling = work_params.show_effective_tiling
frame.settings_frame.panel.collect_values()
paths = work_phil.remaining_args
frame.user_callback = integrate_worker.slip_callback
frame.load_image(self.files.filenames()[i])
app.MainLoop()
del app
except Exception:
pass # must use phenix.wxpython for wx display
elif False : #original wx viewer
try:
from rstbx.viewer.frame import XrayFrame
import wx
from rstbx.viewer import display
display.user_callback = integrate_worker.user_callback
app = wx.App(0)
frame = XrayFrame(None, -1, "X-ray image display", size=(1200,1080))
frame.settings.show_spotfinder_spots = False
frame.settings.show_integration = False
#frame.settings.enable_collect_values = False
frame.SetSize((1024,780))
frame.load_image(self.files.filenames()[i])
frame.Show()
app.MainLoop()
del app
except Exception:
pass # must use phenix.wxpython for wx display
# for the wx image viewer
filename = self.horizons_phil.indexing.indexing_pickle
if filename != None:
filename = "%s_%d_%d.pkl"%(filename,setting["counter"],keys[i])
SIO = StringIO()
table_raw = show_observations(integrate_worker.get_obs(
local["spacegroup"]),out=SIO)
limitobject = ResolutionAnalysisMetaClass(local, self.horizons_phil)
info = dict(table = SIO.getvalue(),
table_raw = table_raw,
xbeam = setting["refined x beam"],
ybeam = setting["refined y beam"],
distance = setting["refined distance"],
residual = integrate_worker.r_residual,
resolution = limitobject.value, # FIXME not reliable?
mosaicity = setting["mosaicity"],
pointgroup = local["spacegroup"],
hkllist = integrate_worker.hkllist,
predictions = (1./integrate_worker.pixel_size)*integrate_worker.predicted,
mapped_predictions = integrate_worker.detector_xy,
integration_masks_xy = integrate_worker.integration_masks_as_xy_tuples(),
background_masks_xy = integrate_worker.background_masks_as_xy_tuples()
)
assert info["predictions"].size() >= info["mapped_predictions"].size()
assert info["predictions"].size() == info["hkllist"].size()
G = open(filename,"wb")
pickle.dump(info,G,pickle.HIGHEST_PROTOCOL)
print("---------END Integrate one frame",frames[i])
return local
def scale_frame_detail(self,
timestamp,
cursor,
do_inserts=True,
result=None): #, file_name, db_mgr, out):
if result is None:
result = self.params
# If the pickled integration file does not contain a wavelength,
# fall back on the value given on the command line. XXX The
# wavelength parameter should probably be removed from master_phil
# once all pickled integration files contain it.
wavelength = result["wavelength"]
assert (wavelength > 0)
# Do not apply polarization correction here, as this requires knowledge of
# pixel size at minimum, and full detector geometry in general. The optimal
# redesign would be to apply the polarization correction just after the integration
# step in the integration code.
print("Step 3. Correct for polarization.")
observations = result["observations"][0]
indexed_cell = observations.unit_cell()
observations_original_index = observations.deep_copy()
assert len(observations_original_index.indices()) == len(
observations.indices())
# Now manipulate the data to conform to unit cell, asu, and space group
# of reference. The resolution will be cut later.
# Only works if there is NOT an indexing ambiguity!
#observations = observations.customized_copy(
# anomalous_flag=not self.params.merge_anomalous,
# crystal_symmetry=self.miller_set.crystal_symmetry()
# ).map_to_asu()
#observations_original_index = observations_original_index.customized_copy(
# anomalous_flag=not self.params.merge_anomalous,
# crystal_symmetry=self.miller_set.crystal_symmetry()
# )
observations = observations.customized_copy(
anomalous_flag=False).map_to_asu()
print("Step 4. Filter on global resolution and map to asu")
#observations.show_summary(f=out, prefix=" ")
from rstbx.dials_core.integration_core import show_observations
show_observations(observations)
print(
"Step 6. Match to reference intensities, filter by correlation, filter out negative intensities."
)
assert len(observations_original_index.indices()) \
== len(observations.indices())
# Ensure that match_multi_indices() will return identical results
# when a frame's observations are matched against the
# pre-generated Miller set, self.miller_set, and the reference
# data set, self.i_model. The implication is that the same match
# can be used to map Miller indices to array indices for intensity
# accumulation, and for determination of the correlation
# coefficient in the presence of a scaling reference.
self.miller_set.show_summary(prefix="mset ")
matches = match_multi_indices(
miller_indices_unique=self.miller_set.indices(),
miller_indices=observations.indices())
slope = 1.0
offset = 0.0
print(result.get("sa_parameters")[0])
have_sa_params = (type(result.get("sa_parameters")[0]) == type(dict()))
observations_original_index_indices = observations_original_index.indices(
)
print(list(result.keys()))
kwargs = {
'wavelength': wavelength,
'beam_x': result['xbeam'],
'beam_y': result['ybeam'],
'distance': result['distance'],
'slope': slope,
'offset': offset,
'unique_file_name': timestamp,
'eventstamp': timestamp,
'sifoil': 0.0
}
trial_id = self.get_trial_id(cursor)
run_id = self.get_run_id(cursor)
kwargs["trials_id"] = trial_id
kwargs["rungroups_id"] = self.rungroup_id
kwargs["runs_run_id"] = run_id
kwargs["isoforms_isoform_id"] = self.isoform_id
res_ori_direct = matrix.sqr(observations.unit_cell(
).orthogonalization_matrix()).transpose().elems
kwargs['res_ori_1'] = res_ori_direct[0]
kwargs['res_ori_2'] = res_ori_direct[1]
kwargs['res_ori_3'] = res_ori_direct[2]
kwargs['res_ori_4'] = res_ori_direct[3]
kwargs['res_ori_5'] = res_ori_direct[4]
kwargs['res_ori_6'] = res_ori_direct[5]
kwargs['res_ori_7'] = res_ori_direct[6]
kwargs['res_ori_8'] = res_ori_direct[7]
kwargs['res_ori_9'] = res_ori_direct[8]
kwargs['mosaic_block_rotation'] = result.get("ML_half_mosaicity_deg",
[float("NaN")])[0]
kwargs['mosaic_block_size'] = result.get("ML_domain_size_ang",
[float("NaN")])[0]
kwargs['ewald_proximal_volume'] = result.get("ewald_proximal_volume",
[float("NaN")])[0]
sql, parameters = self._insert(table='`%s_frames`' %
self.db_experiment_tag,
**kwargs)
print(sql)
print(parameters)
results = {'frame': [sql, parameters, kwargs]}
if do_inserts:
cursor.execute(sql, parameters[0])
frame_id = cursor.lastrowid
else:
frame_id = None
xypred = result["mapped_predictions"][0]
indices = flex.size_t([pair[1] for pair in matches.pairs()])
sel_observations = flex.intersection(size=observations.data().size(),
iselections=[indices])
set_original_hkl = observations_original_index_indices.select(
flex.intersection(size=observations_original_index_indices.size(),
iselections=[indices]))
set_xypred = xypred.select(
flex.intersection(size=xypred.size(), iselections=[indices]))
''' debugging printout
print len(observations.data())
print len(indices)
print len(sel_observations)
for x in range(len(observations.data())):
print x,observations.indices().select(sel_observations)[x],
print set_original_hkl[x],
index_into_hkl_id = matches.pairs()[x][0]
print index_into_hkl_id,
print self.miller_set.indices()[index_into_hkl_id],
cursor.execute('SELECT H,K,L FROM %s_hkls WHERE hkl_id = %d'%(
self.db_experiment_tag, self.miller_set_id[index_into_hkl_id]))
print cursor.fetchall()[0]
'''
print("Adding %d observations for this frame" %
(len(sel_observations)))
kwargs = {
'hkls_id':
self.miller_set_id.select(
flex.size_t([pair[0] for pair in matches.pairs()])),
'i':
observations.data().select(sel_observations),
'sigi':
observations.sigmas().select(sel_observations),
'detector_x_px': [xy[0] for xy in set_xypred],
'detector_y_px': [xy[1] for xy in set_xypred],
'frames_id': [frame_id] * len(matches.pairs()),
'overload_flag': [0] * len(matches.pairs()),
'original_h': [hkl[0] for hkl in set_original_hkl],
'original_k': [hkl[1] for hkl in set_original_hkl],
'original_l': [hkl[2] for hkl in set_original_hkl],
'frames_rungroups_id': [self.rungroup_id] * len(matches.pairs()),
'frames_trials_id': [trial_id] * len(matches.pairs()),
'panel': [0] * len(matches.pairs())
}
if do_inserts:
# For MySQLdb executemany() is six times slower than a single big
# execute() unless the "values" keyword is given in lowercase
# (http://sourceforge.net/p/mysql-python/bugs/305).
#
# See also merging_database_sqlite3._insert()
query = ("INSERT INTO `%s_observations` (" % self.db_experiment_tag) \
+ ", ".join(kwargs) + ") values (" \
+ ", ".join(["%s"] * len(kwargs)) + ")"
try:
parameters = list(zip(*list(kwargs.values())))
except TypeError:
parameters = [list(kwargs.values())]
cursor.executemany(query, parameters)
#print "done execute many"
#print cursor._last_executed
results['observations'] = [query, parameters, kwargs]
else:
# since frame_id isn't valid in the query here, don't include a sql statement or parameters array in the results
results['observations'] = [None, None, kwargs]
return results
def integrate_one_character(self,setting,integration_limit):
#from libtbx.development.timers import Profiler
#P = Profiler("Preliminary")
import copy
local = copy.deepcopy(self.process_dictionary)
local['cell']=cellstr(setting)
print "Cell in setting",setting["counter"],local["cell"]
frames = self.spotfinder_results.pd['osc_start'].keys()
frames.sort()
local['maxcel']='0'
local['xbeam']="%f"%setting['minimizer'].new['xbeam']
local['ybeam']="%f"%setting['minimizer'].new['ybeam']
local['distance']="%f"%setting['minimizer'].new['distance']
local["resolution"]= "%f"%integration_limit
from labelit.steps import primaries
local['spacegroup'] = primaries[setting['bravais']]
local['procstart'] = local['procend'] = "%d"%frames[0]
self.pixel_size = float(local['pixel_size'])
from labelit.dptbx import AutoIndexEngine, Parameters
ai = AutoIndexEngine(local['endstation'])
P = Parameters(xbeam=setting["refined x beam"],ybeam=setting["refined y beam"],
distance=setting["refined distance"],twotheta=float(local["twotheta"]))
ai.setBase(P)
ai.setWavelength(float(local['wavelength']))
ai.setMaxcell(float(local['ref_maxcel']))
print "Deltaphi is",float(local['deltaphi'])
ai.setDeltaphi(float(local['deltaphi'])*math.pi/180.)
ai.setMosaicity(setting["mosaicity"])
ai.setOrientation(setting["orient"])
refimage = self.files.images[0]
ai.set_active_areas(self.horizons_phil,
beam=(int(refimage.beamx/refimage.pixel_size),
int(refimage.beamy/refimage.pixel_size)))
image_centers = [(math.pi/180.)*float(x) for x in local["osc_start"].values()]
print "Limiting resolution",integration_limit
local["results"] = []
for i in xrange(len(frames)):
print "---------BEGIN Integrate one frame %d %s" % \
(frames[i], os.path.split(self.files.filenames()[i])[-1])
#P = Profiler("worker")
if self.horizons_phil.integration.combine_sym_constraints_and_3D_target and setting["counter"]>1:
from rstbx.apps.stills.dials_refinement_preceding_integration import integrate_one_frame
integrate_worker = integrate_one_frame(self.triclinic["integration"]["results"][0])
else:
from rstbx.apps.stills.deltapsi_refinement_preceding_integration import integrate_one_frame
integrate_worker = integrate_one_frame()
integrate_worker.inputai = ai
integrate_worker.inputpd = dict(masks=local["masks"],
size1=local["size1"],
size2=local["size2"],
symmetry=setting["best_subsym"])
# carefully select only the data items needed for integrate_worker
# avoid giving the whole process dictionary; reference to "local"
# is a circular reference creating memory leak, while copying the
# whole thing is a big performance hit.
integrate_worker.frame_numbers = frames
integrate_worker.imagefiles = self.files
integrate_worker.spotfinder = self.spotfinder_results
integrate_worker.image_centers = image_centers
integrate_worker.limiting_resolution = integration_limit
integrate_worker.setting_id = setting["counter"]
integrate_worker.pixel_size = self.pixel_size
integrate_worker.set_pixel_size(self.pixel_size)
integrate_worker.set_detector_size(int(local["size1"]),int(local["size2"]))
integrate_worker.set_detector_saturation(refimage.saturation)
integrate_worker.set_up_mask_focus()
integrate_worker.initialize_increments(i)
integrate_worker.horizons_phil = self.horizons_phil
if self.horizons_phil.indexing.verbose_cv:
print "EFFECTIVE TILING"," ".join(
["%d"%z for z in refimage.get_tile_manager(self.horizons_phil).effective_tiling_as_flex_int()])
integrate_worker.integration_concept(image_number = i,
cb_op_to_primitive = setting["cb_op_inp_best"].inverse(),
verbose_cv = self.horizons_phil.indexing.verbose_cv,
background_factor = self.horizons_phil.integration.background_factor,
)
#P = Profiler("proper")
integrate_worker.integration_proper()
local["results"].append(integrate_worker)
local["r_xbeam"]=ai.xbeam()
local["r_ybeam"]=ai.ybeam()
local["r_distance"]=ai.distance()
local["r_wavelength"]=ai.wavelength
local["r_residual"]=integrate_worker.r_residual
local["r_mosaicity"]=setting["mosaicity"]
try:
local["ewald_proximal_volume"]=integrate_worker.ewald_proximal_volume
except Exception, e:
local["ewald_proximal_volume"]=None
if (self.horizons_phil.indexing.open_wx_viewer) :
if True: #use updated slip viewer
try:
import wx
from rstbx.slip_viewer.frame import XrayFrame as SlipXrayFrame
from rstbx.command_line.slip_viewer import master_str as slip_params
from iotbx import phil
from spotfinder import phil_str
from spotfinder.command_line.signal_strength import additional_spotfinder_phil_defs
work_phil = phil.process_command_line("",master_string=slip_params + phil_str + additional_spotfinder_phil_defs)
work_params = work_phil.work.extract()
app = wx.App(0)
wx.SystemOptions.SetOptionInt("osx.openfiledialog.always-show-types", 1)
frame = SlipXrayFrame(None, -1, "X-ray image display", size=(800,720))
frame.Show()
# Update initial settings with values from the command line. Needs
# to be done before image is loaded (but after the frame is
# instantiated).
frame.inherited_params = integrate_worker.horizons_phil
frame.params = work_params
if (frame.pyslip is None):
frame.init_pyslip()
if (frame.settings_frame is None):
frame.OnShowSettings(None)
frame.Layout()
frame.pyslip.tiles.user_requests_antialiasing = work_params.anti_aliasing
frame.settings_frame.panel.center_ctrl.SetValue(True)
frame.settings_frame.panel.integ_ctrl.SetValue(True)
frame.settings_frame.panel.spots_ctrl.SetValue(False)
frame.settings.show_effective_tiling = work_params.show_effective_tiling
frame.settings_frame.panel.collect_values()
paths = work_phil.remaining_args
frame.user_callback = integrate_worker.slip_callback
frame.load_image(self.files.filenames()[i])
app.MainLoop()
del app
except Exception:
pass # must use phenix.wxpython for wx display
elif False : #original wx viewer
try:
from rstbx.viewer.frame import XrayFrame
import wx
from rstbx.viewer import display
display.user_callback = integrate_worker.user_callback
app = wx.App(0)
frame = XrayFrame(None, -1, "X-ray image display", size=(1200,1080))
frame.settings.show_spotfinder_spots = False
frame.settings.show_integration = False
#frame.settings.enable_collect_values = False
frame.SetSize((1024,780))
frame.load_image(self.files.filenames()[i])
frame.Show()
app.MainLoop()
del app
except Exception:
pass # must use phenix.wxpython for wx display
# for the wx image viewer
filename = self.horizons_phil.indexing.indexing_pickle
if filename != None:
filename = "%s_%d_%d.pkl"%(filename,setting["counter"],keys[i])
SIO = StringIO.StringIO()
table_raw = show_observations(integrate_worker.get_obs(
local["spacegroup"]),out=SIO)
limitobject = ResolutionAnalysisMetaClass(local, self.horizons_phil)
info = dict(table = SIO.getvalue(),
table_raw = table_raw,
xbeam = setting["refined x beam"],
ybeam = setting["refined y beam"],
distance = setting["refined distance"],
residual = integrate_worker.r_residual,
resolution = limitobject.value, # FIXME not reliable?
mosaicity = setting["mosaicity"],
pointgroup = local["spacegroup"],
hkllist = integrate_worker.hkllist,
predictions = (1./integrate_worker.pixel_size)*integrate_worker.predicted,
mapped_predictions = integrate_worker.detector_xy,
integration_masks_xy = integrate_worker.integration_masks_as_xy_tuples(),
background_masks_xy = integrate_worker.background_masks_as_xy_tuples()
)
assert info["predictions"].size() >= info["mapped_predictions"].size()
assert info["predictions"].size() == info["hkllist"].size()
G = open(filename,"wb")
pickle.dump(info,G,pickle.HIGHEST_PROTOCOL)
print "---------END Integrate one frame",frames[i]
def scale_frame_detail(self,timestamp,cursor,do_inserts=True,result=None):#, file_name, db_mgr, out):
if result is None:
result = self.params
# If the pickled integration file does not contain a wavelength,
# fall back on the value given on the command line. XXX The
# wavelength parameter should probably be removed from master_phil
# once all pickled integration files contain it.
wavelength = result["wavelength"]
assert (wavelength > 0)
# Do not apply polarization correction here, as this requires knowledge of
# pixel size at minimum, and full detector geometry in general. The optimal
# redesign would be to apply the polarization correction just after the integration
# step in the integration code.
print "Step 3. Correct for polarization."
observations = result["observations"][0]
indexed_cell = observations.unit_cell()
observations_original_index = observations.deep_copy()
assert len(observations_original_index.indices()) == len(observations.indices())
# Now manipulate the data to conform to unit cell, asu, and space group
# of reference. The resolution will be cut later.
# Only works if there is NOT an indexing ambiguity!
#observations = observations.customized_copy(
# anomalous_flag=not self.params.merge_anomalous,
# crystal_symmetry=self.miller_set.crystal_symmetry()
# ).map_to_asu()
#observations_original_index = observations_original_index.customized_copy(
# anomalous_flag=not self.params.merge_anomalous,
# crystal_symmetry=self.miller_set.crystal_symmetry()
# )
observations = observations.customized_copy(anomalous_flag=False).map_to_asu()
print "Step 4. Filter on global resolution and map to asu"
#observations.show_summary(f=out, prefix=" ")
from rstbx.dials_core.integration_core import show_observations
show_observations(observations)
print "Step 6. Match to reference intensities, filter by correlation, filter out negative intensities."
assert len(observations_original_index.indices()) \
== len(observations.indices())
# Ensure that match_multi_indices() will return identical results
# when a frame's observations are matched against the
# pre-generated Miller set, self.miller_set, and the reference
# data set, self.i_model. The implication is that the same match
# can be used to map Miller indices to array indices for intensity
# accumulation, and for determination of the correlation
# coefficient in the presence of a scaling reference.
self.miller_set.show_summary(prefix="mset ")
matches = match_multi_indices(
miller_indices_unique=self.miller_set.indices(),
miller_indices=observations.indices())
slope = 1.0
offset = 0.0
print result.get("sa_parameters")[0]
have_sa_params = ( type(result.get("sa_parameters")[0]) == type(dict()) )
observations_original_index_indices = observations_original_index.indices()
print result.keys()
kwargs = {'wavelength': wavelength,
'beam_x': result['xbeam'],
'beam_y': result['ybeam'],
'distance': result['distance'],
'slope': slope,
'offset': offset,
'unique_file_name': timestamp,
'eventstamp':timestamp,
'sifoil': 0.0}
trial_id = self.get_trial_id(cursor)
run_id = self.get_run_id(cursor)
kwargs["trials_id"] = trial_id
kwargs["rungroups_id"] = self.rungroup_id
kwargs["runs_run_id"] = run_id
kwargs["isoforms_isoform_id"] = self.isoform_id
res_ori_direct = matrix.sqr(
observations.unit_cell().orthogonalization_matrix()).transpose().elems
kwargs['res_ori_1'] = res_ori_direct[0]
kwargs['res_ori_2'] = res_ori_direct[1]
kwargs['res_ori_3'] = res_ori_direct[2]
kwargs['res_ori_4'] = res_ori_direct[3]
kwargs['res_ori_5'] = res_ori_direct[4]
kwargs['res_ori_6'] = res_ori_direct[5]
kwargs['res_ori_7'] = res_ori_direct[6]
kwargs['res_ori_8'] = res_ori_direct[7]
kwargs['res_ori_9'] = res_ori_direct[8]
kwargs['mosaic_block_rotation'] = result.get("ML_half_mosaicity_deg",[float("NaN")])[0]
kwargs['mosaic_block_size'] = result.get("ML_domain_size_ang",[float("NaN")])[0]
kwargs['ewald_proximal_volume'] = result.get("ewald_proximal_volume",[float("NaN")])[0]
sql, parameters = self._insert(
table='`%s_frames`' % self.db_experiment_tag,
**kwargs)
print sql
print parameters
results = {'frame':[sql, parameters, kwargs]}
if do_inserts:
cursor.execute(sql, parameters[0])
frame_id = cursor.lastrowid
else:
frame_id = None
xypred = result["mapped_predictions"][0]
indices = flex.size_t([pair[1] for pair in matches.pairs()])
sel_observations = flex.intersection(
size=observations.data().size(),
iselections=[indices])
set_original_hkl = observations_original_index_indices.select(
flex.intersection(
size=observations_original_index_indices.size(),
iselections=[indices]))
set_xypred = xypred.select(
flex.intersection(
size=xypred.size(),
iselections=[indices]))
''' debugging printout
print len(observations.data())
print len(indices)
print len(sel_observations)
for x in xrange(len(observations.data())):
print x,observations.indices().select(sel_observations)[x],
print set_original_hkl[x],
index_into_hkl_id = matches.pairs()[x][0]
print index_into_hkl_id,
print self.miller_set.indices()[index_into_hkl_id],
cursor.execute('SELECT H,K,L FROM %s_hkls WHERE hkl_id = %d'%(
self.db_experiment_tag, self.miller_set_id[index_into_hkl_id]))
print cursor.fetchall()[0]
'''
print "Adding %d observations for this frame"%(len(sel_observations))
kwargs = {'hkls_id': self.miller_set_id.select(flex.size_t([pair[0] for pair in matches.pairs()])),
'i': observations.data().select(sel_observations),
'sigi': observations.sigmas().select(sel_observations),
'detector_x_px': [xy[0] for xy in set_xypred],
'detector_y_px': [xy[1] for xy in set_xypred],
'frames_id': [frame_id] * len(matches.pairs()),
'overload_flag': [0] * len(matches.pairs()),
'original_h': [hkl[0] for hkl in set_original_hkl],
'original_k': [hkl[1] for hkl in set_original_hkl],
'original_l': [hkl[2] for hkl in set_original_hkl],
'frames_rungroups_id': [self.rungroup_id] * len(matches.pairs()),
'frames_trials_id': [trial_id] * len(matches.pairs()),
'panel': [0] * len(matches.pairs())
}
if do_inserts:
# For MySQLdb executemany() is six times slower than a single big
# execute() unless the "values" keyword is given in lowercase
# (http://sourceforge.net/p/mysql-python/bugs/305).
#
# See also merging_database_sqlite3._insert()
query = ("INSERT INTO `%s_observations` (" % self.db_experiment_tag) \
+ ", ".join(kwargs.keys()) + ") values (" \
+ ", ".join(["%s"] * len(kwargs.keys())) + ")"
try:
parameters = zip(*kwargs.values())
except TypeError:
parameters = [kwargs.values()]
cursor.executemany(query, parameters)
#print "done execute many"
#print cursor._last_executed
results['observations'] = [query, parameters, kwargs]
else:
# since frame_id isn't valid in the query here, don't include a sql statement or parameters array in the results
results['observations'] = [None, None, kwargs]
return results
