def scale_frame_detail(self, result, file_name, db_mgr, out): # If the pickled integration file does not contain a wavelength, # fall back on the value given on the command line. XXX The # wavelength parameter should probably be removed from master_phil # once all pickled integration files contain it. if ("wavelength" in result): wavelength = result["wavelength"] elif (self.params.wavelength is not None): wavelength = self.params.wavelength else: # XXX Give error, or raise exception? return None assert (wavelength > 0) observations = result["observations"][0] cos_two_polar_angle = result["cos_two_polar_angle"] assert observations.size() == cos_two_polar_angle.size() tt_vec = observations.two_theta(wavelength) #print "mean tt degrees",180.*flex.mean(tt_vec.data())/math.pi cos_tt_vec = flex.cos(tt_vec.data()) sin_tt_vec = flex.sin(tt_vec.data()) cos_sq_tt_vec = cos_tt_vec * cos_tt_vec sin_sq_tt_vec = sin_tt_vec * sin_tt_vec P_nought_vec = 0.5 * (1. + cos_sq_tt_vec) F_prime = -1.0 # Hard-coded value defines the incident polarization axis P_prime = 0.5 * F_prime * cos_two_polar_angle * sin_sq_tt_vec # XXX added as a diagnostic prange = P_nought_vec - P_prime other_F_prime = 1.0 otherP_prime = 0.5 * other_F_prime * cos_two_polar_angle * sin_sq_tt_vec otherprange = P_nought_vec - otherP_prime diff2 = flex.abs(prange - otherprange) print "mean diff is", flex.mean(diff2), "range", flex.min( diff2), flex.max(diff2) # XXX done observations = observations / (P_nought_vec - P_prime) # This corrects observations for polarization assuming 100% polarization on # one axis (thus the F_prime = -1.0 rather than the perpendicular axis, 1.0) # Polarization model as described by Kahn, Fourme, Gadet, Janin, Dumas & Andre # (1982) J. Appl. Cryst. 15, 330-337, equations 13 - 15. print "Step 3. Correct for polarization." indexed_cell = observations.unit_cell() observations_original_index = observations.deep_copy() if result.get( "model_partialities", None ) is not None and result["model_partialities"][0] is not None: # some recordkeeping useful for simulations partialities_original_index = observations.customized_copy( crystal_symmetry=self.miller_set.crystal_symmetry(), data=result["model_partialities"][0]["data"], sigmas=flex.double(result["model_partialities"][0] ["data"].size()), #dummy value for sigmas indices=result["model_partialities"][0]["indices"], ).resolution_filter(d_min=self.params.d_min) assert len(observations_original_index.indices()) == len( observations.indices()) # Now manipulate the data to conform to unit cell, asu, and space group # of reference. The resolution will be cut later. # Only works if there is NOT an indexing ambiguity! observations = observations.customized_copy( anomalous_flag=not self.params.merge_anomalous, crystal_symmetry=self.miller_set.crystal_symmetry()).map_to_asu() observations_original_index = observations_original_index.customized_copy( anomalous_flag=not self.params.merge_anomalous, crystal_symmetry=self.miller_set.crystal_symmetry()) print "Step 4. Filter on global resolution and map to asu" print >> out, "Data in reference setting:" #observations.show_summary(f=out, prefix=" ") show_observations(observations, out=out) #if self.params.significance_filter.apply is True: # raise Exception("significance filter not implemented in samosa") if self.params.significance_filter.apply is True: #------------------------------------ # Apply an I/sigma filter ... accept resolution bins only if they # have significant signal; tends to screen out higher resolution observations # if the integration model doesn't quite fit N_obs_pre_filter = observations.size() N_bins_small_set = N_obs_pre_filter // self.params.significance_filter.min_ct N_bins_large_set = N_obs_pre_filter // self.params.significance_filter.max_ct # Ensure there is at least one bin. N_bins = max([ min([self.params.significance_filter.n_bins, N_bins_small_set]), N_bins_large_set, 1 ]) print "Total obs %d Choose n bins = %d" % (N_obs_pre_filter, N_bins) bin_results = show_observations(observations, out=out, n_bins=N_bins) #show_observations(observations, out=sys.stdout, n_bins=N_bins) acceptable_resolution_bins = [ bin.mean_I_sigI > self.params.significance_filter.sigma for bin in bin_results ] acceptable_nested_bin_sequences = [ i for i in xrange(len(acceptable_resolution_bins)) if False not in acceptable_resolution_bins[:i + 1] ] if len(acceptable_nested_bin_sequences) == 0: return null_data(file_name=file_name, log_out=out.getvalue(), low_signal=True) else: N_acceptable_bins = max(acceptable_nested_bin_sequences) + 1 imposed_res_filter = float(bin_results[N_acceptable_bins - 1].d_range.split()[2]) imposed_res_sel = observations.resolution_filter_selection( d_min=imposed_res_filter) observations = observations.select(imposed_res_sel) observations_original_index = observations_original_index.select( imposed_res_sel) print "New resolution filter at %7.2f" % imposed_res_filter, file_name print "N acceptable bins", N_acceptable_bins print "Old n_obs: %d, new n_obs: %d" % (N_obs_pre_filter, observations.size()) print "Step 5. Frame by frame resolution filter" # Finished applying the binwise I/sigma filter--------------------------------------- if self.params.raw_data.sdfac_auto is True: raise Exception("sdfac auto not implemented in samosa.") print "Step 6. Match to reference intensities, filter by correlation, filter out negative intensities." assert len(observations_original_index.indices()) \ == len(observations.indices()) data = frame_data(self.n_refl, file_name) data.set_indexed_cell(indexed_cell) data.d_min = observations.d_min() # Ensure that match_multi_indices() will return identical results # when a frame's observations are matched against the # pre-generated Miller set, self.miller_set, and the reference # data set, self.i_model. The implication is that the same match # can be used to map Miller indices to array indices for intensity # accumulation, and for determination of the correlation # coefficient in the presence of a scaling reference. if self.i_model is not None: assert len(self.i_model.indices()) == len(self.miller_set.indices()) \ and (self.i_model.indices() == self.miller_set.indices()).count(False) == 0 matches = miller.match_multi_indices( miller_indices_unique=self.miller_set.indices(), miller_indices=observations.indices()) use_weights = False # New facility for getting variance-weighted correlation if self.params.scaling.algorithm in ['mark1', 'levmar']: # Because no correlation is computed, the correlation # coefficient is fixed at zero. Setting slope = 1 means # intensities are added without applying a scale factor. sum_x = 0 sum_y = 0 for pair in matches.pairs(): data.n_obs += 1 if not self.params.include_negatives and observations.data()[ pair[1]] <= 0: data.n_rejected += 1 else: sum_y += observations.data()[pair[1]] N = data.n_obs - data.n_rejected # Early return if there are no positive reflections on the frame. if data.n_obs <= data.n_rejected: return null_data(file_name=file_name, log_out=out.getvalue(), low_signal=True) # Update the count for each matched reflection. This counts # reflections with non-positive intensities, too. data.completeness += matches.number_of_matches(0).as_int() data.wavelength = wavelength if not self.params.scaling.enable: # Do not scale anything print "Scale factor to an isomorphous reference PDB will NOT be applied." slope = 1.0 offset = 0.0 observations_original_index_indices = observations_original_index.indices( ) if db_mgr is None: return unpack(MINI.x) # special exit for two-color indexing kwargs = { 'wavelength': wavelength, 'beam_x': result['xbeam'], 'beam_y': result['ybeam'], 'distance': result['distance'], 'unique_file_name': data.file_name } ORI = result["current_orientation"][0] Astar = matrix.sqr(ORI.reciprocal_matrix()) kwargs['res_ori_1'] = Astar[0] kwargs['res_ori_2'] = Astar[1] kwargs['res_ori_3'] = Astar[2] kwargs['res_ori_4'] = Astar[3] kwargs['res_ori_5'] = Astar[4] kwargs['res_ori_6'] = Astar[5] kwargs['res_ori_7'] = Astar[6] kwargs['res_ori_8'] = Astar[7] kwargs['res_ori_9'] = Astar[8] assert self.params.scaling.report_ML is True kwargs['half_mosaicity_deg'] = result["ML_half_mosaicity_deg"][0] kwargs['domain_size_ang'] = result["ML_domain_size_ang"][0] frame_id_0_base = db_mgr.insert_frame(**kwargs) xypred = result["mapped_predictions"][0] indices = flex.size_t([pair[1] for pair in matches.pairs()]) sel_observations = flex.intersection(size=observations.data().size(), iselections=[indices]) set_original_hkl = observations_original_index_indices.select( flex.intersection(size=observations_original_index_indices.size(), iselections=[indices])) set_xypred = xypred.select( flex.intersection(size=xypred.size(), iselections=[indices])) kwargs = { 'hkl_id_0_base': [pair[0] for pair in matches.pairs()], 'i': observations.data().select(sel_observations), 'sigi': observations.sigmas().select(sel_observations), 'detector_x': [xy[0] for xy in set_xypred], 'detector_y': [xy[1] for xy in set_xypred], 'frame_id_0_base': [frame_id_0_base] * len(matches.pairs()), 'overload_flag': [0] * len(matches.pairs()), 'original_h': [hkl[0] for hkl in set_original_hkl], 'original_k': [hkl[1] for hkl in set_original_hkl], 'original_l': [hkl[2] for hkl in set_original_hkl] } db_mgr.insert_observation(**kwargs) print >> out, "Lattice: %d reflections" % (data.n_obs - data.n_rejected) print >> out, "average obs", sum_y / (data.n_obs - data.n_rejected), \ "average calc", sum_x / (data.n_obs - data.n_rejected) print >> out, "Rejected %d reflections with negative intensities" % \ data.n_rejected data.accept = True for pair in matches.pairs(): if not self.params.include_negatives and ( observations.data()[pair[1]] <= 0): continue Intensity = observations.data()[pair[1]] # Super-rare exception. If saved sigmas instead of I/sigmas in the ISIGI dict, this wouldn't be needed. if Intensity == 0: continue # Add the reflection as a two-tuple of intensity and I/sig(I) # to the dictionary of observations. index = self.miller_set.indices()[pair[0]] isigi = (Intensity, observations.data()[pair[1]] / observations.sigmas()[pair[1]], 1.0) if index in data.ISIGI: data.ISIGI[index].append(isigi) else: data.ISIGI[index] = [isigi] sigma = observations.sigmas()[pair[1]] variance = sigma * sigma data.summed_N[pair[0]] += 1 data.summed_wt_I[pair[0]] += Intensity / variance data.summed_weight[pair[0]] += 1 / variance data.set_log_out(out.getvalue()) return data
def scale_frame_detail(self,timestamp,cursor,do_inserts=True,result=None):#, file_name, db_mgr, out): if result is None: result = self.params # If the pickled integration file does not contain a wavelength, # fall back on the value given on the command line. XXX The # wavelength parameter should probably be removed from master_phil # once all pickled integration files contain it. wavelength = result["wavelength"] assert (wavelength > 0) # Do not apply polarization correction here, as this requires knowledge of # pixel size at minimum, and full detector geometry in general. The optimal # redesign would be to apply the polarization correction just after the integration # step in the integration code. print "Step 3. Correct for polarization." observations = result["observations"][0] indexed_cell = observations.unit_cell() observations_original_index = observations.deep_copy() assert len(observations_original_index.indices()) == len(observations.indices()) # Now manipulate the data to conform to unit cell, asu, and space group # of reference. The resolution will be cut later. # Only works if there is NOT an indexing ambiguity! #observations = observations.customized_copy( # anomalous_flag=not self.params.merge_anomalous, # crystal_symmetry=self.miller_set.crystal_symmetry() # ).map_to_asu() #observations_original_index = observations_original_index.customized_copy( # anomalous_flag=not self.params.merge_anomalous, # crystal_symmetry=self.miller_set.crystal_symmetry() # ) observations = observations.customized_copy(anomalous_flag=False).map_to_asu() print "Step 4. Filter on global resolution and map to asu" #observations.show_summary(f=out, prefix=" ") from rstbx.dials_core.integration_core import show_observations show_observations(observations) print "Step 6. Match to reference intensities, filter by correlation, filter out negative intensities." assert len(observations_original_index.indices()) \ == len(observations.indices()) # Ensure that match_multi_indices() will return identical results # when a frame's observations are matched against the # pre-generated Miller set, self.miller_set, and the reference # data set, self.i_model. The implication is that the same match # can be used to map Miller indices to array indices for intensity # accumulation, and for determination of the correlation # coefficient in the presence of a scaling reference. self.miller_set.show_summary(prefix="mset ") matches = match_multi_indices( miller_indices_unique=self.miller_set.indices(), miller_indices=observations.indices()) slope = 1.0 offset = 0.0 print result.get("sa_parameters")[0] have_sa_params = ( type(result.get("sa_parameters")[0]) == type(dict()) ) observations_original_index_indices = observations_original_index.indices() print result.keys() kwargs = {'wavelength': wavelength, 'beam_x': result['xbeam'], 'beam_y': result['ybeam'], 'distance': result['distance'], 'slope': slope, 'offset': offset, 'unique_file_name': timestamp, 'eventstamp':timestamp, 'sifoil': 0.0} trial_id = self.get_trial_id(cursor) run_id = self.get_run_id(cursor) kwargs["trials_id"] = trial_id kwargs["rungroups_id"] = self.rungroup_id kwargs["runs_run_id"] = run_id kwargs["isoforms_isoform_id"] = self.isoform_id res_ori_direct = matrix.sqr( observations.unit_cell().orthogonalization_matrix()).transpose().elems kwargs['res_ori_1'] = res_ori_direct[0] kwargs['res_ori_2'] = res_ori_direct[1] kwargs['res_ori_3'] = res_ori_direct[2] kwargs['res_ori_4'] = res_ori_direct[3] kwargs['res_ori_5'] = res_ori_direct[4] kwargs['res_ori_6'] = res_ori_direct[5] kwargs['res_ori_7'] = res_ori_direct[6] kwargs['res_ori_8'] = res_ori_direct[7] kwargs['res_ori_9'] = res_ori_direct[8] kwargs['mosaic_block_rotation'] = result.get("ML_half_mosaicity_deg",[float("NaN")])[0] kwargs['mosaic_block_size'] = result.get("ML_domain_size_ang",[float("NaN")])[0] kwargs['ewald_proximal_volume'] = result.get("ewald_proximal_volume",[float("NaN")])[0] sql, parameters = self._insert( table='`%s_frames`' % self.db_experiment_tag, **kwargs) print sql print parameters results = {'frame':[sql, parameters, kwargs]} if do_inserts: cursor.execute(sql, parameters[0]) frame_id = cursor.lastrowid else: frame_id = None xypred = result["mapped_predictions"][0] indices = flex.size_t([pair[1] for pair in matches.pairs()]) sel_observations = flex.intersection( size=observations.data().size(), iselections=[indices]) set_original_hkl = observations_original_index_indices.select( flex.intersection( size=observations_original_index_indices.size(), iselections=[indices])) set_xypred = xypred.select( flex.intersection( size=xypred.size(), iselections=[indices])) ''' debugging printout print len(observations.data()) print len(indices) print len(sel_observations) for x in xrange(len(observations.data())): print x,observations.indices().select(sel_observations)[x], print set_original_hkl[x], index_into_hkl_id = matches.pairs()[x][0] print index_into_hkl_id, print self.miller_set.indices()[index_into_hkl_id], cursor.execute('SELECT H,K,L FROM %s_hkls WHERE hkl_id = %d'%( self.db_experiment_tag, self.miller_set_id[index_into_hkl_id])) print cursor.fetchall()[0] ''' print "Adding %d observations for this frame"%(len(sel_observations)) kwargs = {'hkls_id': self.miller_set_id.select(flex.size_t([pair[0] for pair in matches.pairs()])), 'i': observations.data().select(sel_observations), 'sigi': observations.sigmas().select(sel_observations), 'detector_x_px': [xy[0] for xy in set_xypred], 'detector_y_px': [xy[1] for xy in set_xypred], 'frames_id': [frame_id] * len(matches.pairs()), 'overload_flag': [0] * len(matches.pairs()), 'original_h': [hkl[0] for hkl in set_original_hkl], 'original_k': [hkl[1] for hkl in set_original_hkl], 'original_l': [hkl[2] for hkl in set_original_hkl], 'frames_rungroups_id': [self.rungroup_id] * len(matches.pairs()), 'frames_trials_id': [trial_id] * len(matches.pairs()), 'panel': [0] * len(matches.pairs()) } if do_inserts: # For MySQLdb executemany() is six times slower than a single big # execute() unless the "values" keyword is given in lowercase # (http://sourceforge.net/p/mysql-python/bugs/305). # # See also merging_database_sqlite3._insert() query = ("INSERT INTO `%s_observations` (" % self.db_experiment_tag) \ + ", ".join(kwargs.keys()) + ") values (" \ + ", ".join(["%s"] * len(kwargs.keys())) + ")" try: parameters = zip(*kwargs.values()) except TypeError: parameters = [kwargs.values()] cursor.executemany(query, parameters) #print "done execute many" #print cursor._last_executed results['observations'] = [query, parameters, kwargs] else: # since frame_id isn't valid in the query here, don't include a sql statement or parameters array in the results results['observations'] = [None, None, kwargs] return results
def scale_frame_detail(self, timestamp, cursor, do_inserts=True, result=None): #, file_name, db_mgr, out): if result is None: result = self.params # If the pickled integration file does not contain a wavelength, # fall back on the value given on the command line. XXX The # wavelength parameter should probably be removed from master_phil # once all pickled integration files contain it. wavelength = result["wavelength"] assert (wavelength > 0) # Do not apply polarization correction here, as this requires knowledge of # pixel size at minimum, and full detector geometry in general. The optimal # redesign would be to apply the polarization correction just after the integration # step in the integration code. print("Step 3. Correct for polarization.") observations = result["observations"][0] indexed_cell = observations.unit_cell() observations_original_index = observations.deep_copy() assert len(observations_original_index.indices()) == len( observations.indices()) # Now manipulate the data to conform to unit cell, asu, and space group # of reference. The resolution will be cut later. # Only works if there is NOT an indexing ambiguity! #observations = observations.customized_copy( # anomalous_flag=not self.params.merge_anomalous, # crystal_symmetry=self.miller_set.crystal_symmetry() # ).map_to_asu() #observations_original_index = observations_original_index.customized_copy( # anomalous_flag=not self.params.merge_anomalous, # crystal_symmetry=self.miller_set.crystal_symmetry() # ) observations = observations.customized_copy( anomalous_flag=False).map_to_asu() print("Step 4. Filter on global resolution and map to asu") #observations.show_summary(f=out, prefix=" ") from rstbx.dials_core.integration_core import show_observations show_observations(observations) print( "Step 6. Match to reference intensities, filter by correlation, filter out negative intensities." ) assert len(observations_original_index.indices()) \ == len(observations.indices()) # Ensure that match_multi_indices() will return identical results # when a frame's observations are matched against the # pre-generated Miller set, self.miller_set, and the reference # data set, self.i_model. The implication is that the same match # can be used to map Miller indices to array indices for intensity # accumulation, and for determination of the correlation # coefficient in the presence of a scaling reference. self.miller_set.show_summary(prefix="mset ") matches = match_multi_indices( miller_indices_unique=self.miller_set.indices(), miller_indices=observations.indices()) slope = 1.0 offset = 0.0 print(result.get("sa_parameters")[0]) have_sa_params = (type(result.get("sa_parameters")[0]) == type(dict())) observations_original_index_indices = observations_original_index.indices( ) print(list(result.keys())) kwargs = { 'wavelength': wavelength, 'beam_x': result['xbeam'], 'beam_y': result['ybeam'], 'distance': result['distance'], 'slope': slope, 'offset': offset, 'unique_file_name': timestamp, 'eventstamp': timestamp, 'sifoil': 0.0 } trial_id = self.get_trial_id(cursor) run_id = self.get_run_id(cursor) kwargs["trials_id"] = trial_id kwargs["rungroups_id"] = self.rungroup_id kwargs["runs_run_id"] = run_id kwargs["isoforms_isoform_id"] = self.isoform_id res_ori_direct = matrix.sqr(observations.unit_cell( ).orthogonalization_matrix()).transpose().elems kwargs['res_ori_1'] = res_ori_direct[0] kwargs['res_ori_2'] = res_ori_direct[1] kwargs['res_ori_3'] = res_ori_direct[2] kwargs['res_ori_4'] = res_ori_direct[3] kwargs['res_ori_5'] = res_ori_direct[4] kwargs['res_ori_6'] = res_ori_direct[5] kwargs['res_ori_7'] = res_ori_direct[6] kwargs['res_ori_8'] = res_ori_direct[7] kwargs['res_ori_9'] = res_ori_direct[8] kwargs['mosaic_block_rotation'] = result.get("ML_half_mosaicity_deg", [float("NaN")])[0] kwargs['mosaic_block_size'] = result.get("ML_domain_size_ang", [float("NaN")])[0] kwargs['ewald_proximal_volume'] = result.get("ewald_proximal_volume", [float("NaN")])[0] sql, parameters = self._insert(table='`%s_frames`' % self.db_experiment_tag, **kwargs) print(sql) print(parameters) results = {'frame': [sql, parameters, kwargs]} if do_inserts: cursor.execute(sql, parameters[0]) frame_id = cursor.lastrowid else: frame_id = None xypred = result["mapped_predictions"][0] indices = flex.size_t([pair[1] for pair in matches.pairs()]) sel_observations = flex.intersection(size=observations.data().size(), iselections=[indices]) set_original_hkl = observations_original_index_indices.select( flex.intersection(size=observations_original_index_indices.size(), iselections=[indices])) set_xypred = xypred.select( flex.intersection(size=xypred.size(), iselections=[indices])) ''' debugging printout print len(observations.data()) print len(indices) print len(sel_observations) for x in range(len(observations.data())): print x,observations.indices().select(sel_observations)[x], print set_original_hkl[x], index_into_hkl_id = matches.pairs()[x][0] print index_into_hkl_id, print self.miller_set.indices()[index_into_hkl_id], cursor.execute('SELECT H,K,L FROM %s_hkls WHERE hkl_id = %d'%( self.db_experiment_tag, self.miller_set_id[index_into_hkl_id])) print cursor.fetchall()[0] ''' print("Adding %d observations for this frame" % (len(sel_observations))) kwargs = { 'hkls_id': self.miller_set_id.select( flex.size_t([pair[0] for pair in matches.pairs()])), 'i': observations.data().select(sel_observations), 'sigi': observations.sigmas().select(sel_observations), 'detector_x_px': [xy[0] for xy in set_xypred], 'detector_y_px': [xy[1] for xy in set_xypred], 'frames_id': [frame_id] * len(matches.pairs()), 'overload_flag': [0] * len(matches.pairs()), 'original_h': [hkl[0] for hkl in set_original_hkl], 'original_k': [hkl[1] for hkl in set_original_hkl], 'original_l': [hkl[2] for hkl in set_original_hkl], 'frames_rungroups_id': [self.rungroup_id] * len(matches.pairs()), 'frames_trials_id': [trial_id] * len(matches.pairs()), 'panel': [0] * len(matches.pairs()) } if do_inserts: # For MySQLdb executemany() is six times slower than a single big # execute() unless the "values" keyword is given in lowercase # (http://sourceforge.net/p/mysql-python/bugs/305). # # See also merging_database_sqlite3._insert() query = ("INSERT INTO `%s_observations` (" % self.db_experiment_tag) \ + ", ".join(kwargs) + ") values (" \ + ", ".join(["%s"] * len(kwargs)) + ")" try: parameters = list(zip(*list(kwargs.values()))) except TypeError: parameters = [list(kwargs.values())] cursor.executemany(query, parameters) #print "done execute many" #print cursor._last_executed results['observations'] = [query, parameters, kwargs] else: # since frame_id isn't valid in the query here, don't include a sql statement or parameters array in the results results['observations'] = [None, None, kwargs] return results
def scale_frame_detail(self, result, file_name, db_mgr, out): # If the pickled integration file does not contain a wavelength, # fall back on the value given on the command line. XXX The # wavelength parameter should probably be removed from master_phil # once all pickled integration files contain it. if (result.has_key("wavelength")): wavelength = result["wavelength"] elif (self.params.wavelength is not None): wavelength = self.params.wavelength else: # XXX Give error, or raise exception? return None assert (wavelength > 0) observations = result["observations"][0] cos_two_polar_angle = result["cos_two_polar_angle"] assert observations.size() == cos_two_polar_angle.size() tt_vec = observations.two_theta(wavelength) #print "mean tt degrees",180.*flex.mean(tt_vec.data())/math.pi cos_tt_vec = flex.cos( tt_vec.data() ) sin_tt_vec = flex.sin( tt_vec.data() ) cos_sq_tt_vec = cos_tt_vec * cos_tt_vec sin_sq_tt_vec = sin_tt_vec * sin_tt_vec P_nought_vec = 0.5 * (1. + cos_sq_tt_vec) F_prime = -1.0 # Hard-coded value defines the incident polarization axis P_prime = 0.5 * F_prime * cos_two_polar_angle * sin_sq_tt_vec # XXX added as a diagnostic prange=P_nought_vec - P_prime other_F_prime = 1.0 otherP_prime = 0.5 * other_F_prime * cos_two_polar_angle * sin_sq_tt_vec otherprange=P_nought_vec - otherP_prime diff2 = flex.abs(prange - otherprange) print "mean diff is",flex.mean(diff2), "range",flex.min(diff2), flex.max(diff2) # XXX done observations = observations / ( P_nought_vec - P_prime ) # This corrects observations for polarization assuming 100% polarization on # one axis (thus the F_prime = -1.0 rather than the perpendicular axis, 1.0) # Polarization model as described by Kahn, Fourme, Gadet, Janin, Dumas & Andre # (1982) J. Appl. Cryst. 15, 330-337, equations 13 - 15. print "Step 3. Correct for polarization." indexed_cell = observations.unit_cell() observations_original_index = observations.deep_copy() if result.get("model_partialities",None) is not None and result["model_partialities"][0] is not None: # some recordkeeping useful for simulations partialities_original_index = observations.customized_copy( crystal_symmetry=self.miller_set.crystal_symmetry(), data = result["model_partialities"][0]["data"], sigmas = flex.double(result["model_partialities"][0]["data"].size()), #dummy value for sigmas indices = result["model_partialities"][0]["indices"], ).resolution_filter(d_min=self.params.d_min) assert len(observations_original_index.indices()) == len(observations.indices()) # Now manipulate the data to conform to unit cell, asu, and space group # of reference. The resolution will be cut later. # Only works if there is NOT an indexing ambiguity! observations = observations.customized_copy( anomalous_flag=not self.params.merge_anomalous, crystal_symmetry=self.miller_set.crystal_symmetry() ).map_to_asu() observations_original_index = observations_original_index.customized_copy( anomalous_flag=not self.params.merge_anomalous, crystal_symmetry=self.miller_set.crystal_symmetry() ) print "Step 4. Filter on global resolution and map to asu" print >> out, "Data in reference setting:" #observations.show_summary(f=out, prefix=" ") show_observations(observations, out=out) #if self.params.significance_filter.apply is True: # raise Exception("significance filter not implemented in samosa") if self.params.significance_filter.apply is True: #------------------------------------ # Apply an I/sigma filter ... accept resolution bins only if they # have significant signal; tends to screen out higher resolution observations # if the integration model doesn't quite fit N_obs_pre_filter = observations.size() N_bins_small_set = N_obs_pre_filter // self.params.significance_filter.min_ct N_bins_large_set = N_obs_pre_filter // self.params.significance_filter.max_ct # Ensure there is at least one bin. N_bins = max( [min([self.params.significance_filter.n_bins,N_bins_small_set]), N_bins_large_set, 1] ) print "Total obs %d Choose n bins = %d"%(N_obs_pre_filter,N_bins) bin_results = show_observations(observations, out=out, n_bins=N_bins) #show_observations(observations, out=sys.stdout, n_bins=N_bins) acceptable_resolution_bins = [ bin.mean_I_sigI > self.params.significance_filter.sigma for bin in bin_results] acceptable_nested_bin_sequences = [i for i in xrange(len(acceptable_resolution_bins)) if False not in acceptable_resolution_bins[:i+1]] if len(acceptable_nested_bin_sequences)==0: return null_data( file_name=file_name, log_out=out.getvalue(), low_signal=True) else: N_acceptable_bins = max(acceptable_nested_bin_sequences) + 1 imposed_res_filter = float(bin_results[N_acceptable_bins-1].d_range.split()[2]) imposed_res_sel = observations.resolution_filter_selection( d_min=imposed_res_filter) observations = observations.select( imposed_res_sel) observations_original_index = observations_original_index.select( imposed_res_sel) print "New resolution filter at %7.2f"%imposed_res_filter,file_name print "N acceptable bins",N_acceptable_bins print "Old n_obs: %d, new n_obs: %d"%(N_obs_pre_filter,observations.size()) print "Step 5. Frame by frame resolution filter" # Finished applying the binwise I/sigma filter--------------------------------------- if self.params.raw_data.sdfac_auto is True: raise Exception("sdfac auto not implemented in samosa.") print "Step 6. Match to reference intensities, filter by correlation, filter out negative intensities." assert len(observations_original_index.indices()) \ == len(observations.indices()) data = frame_data(self.n_refl, file_name) data.set_indexed_cell(indexed_cell) data.d_min = observations.d_min() # Ensure that match_multi_indices() will return identical results # when a frame's observations are matched against the # pre-generated Miller set, self.miller_set, and the reference # data set, self.i_model. The implication is that the same match # can be used to map Miller indices to array indices for intensity # accumulation, and for determination of the correlation # coefficient in the presence of a scaling reference. if self.i_model is not None: assert len(self.i_model.indices()) == len(self.miller_set.indices()) \ and (self.i_model.indices() == self.miller_set.indices()).count(False) == 0 matches = miller.match_multi_indices( miller_indices_unique=self.miller_set.indices(), miller_indices=observations.indices()) use_weights = False # New facility for getting variance-weighted correlation if self.params.scaling.algorithm in ['mark1','levmar']: # Because no correlation is computed, the correlation # coefficient is fixed at zero. Setting slope = 1 means # intensities are added without applying a scale factor. sum_x = 0 sum_y = 0 for pair in matches.pairs(): data.n_obs += 1 if not self.params.include_negatives and observations.data()[pair[1]] <= 0: data.n_rejected += 1 else: sum_y += observations.data()[pair[1]] N = data.n_obs - data.n_rejected # Early return if there are no positive reflections on the frame. if data.n_obs <= data.n_rejected: return null_data( file_name=file_name, log_out=out.getvalue(), low_signal=True) # Update the count for each matched reflection. This counts # reflections with non-positive intensities, too. data.completeness += matches.number_of_matches(0).as_int() data.wavelength = wavelength if not self.params.scaling.enable: # Do not scale anything print "Scale factor to an isomorphous reference PDB will NOT be applied." slope = 1.0 offset = 0.0 observations_original_index_indices = observations_original_index.indices() if db_mgr is None: return unpack(MINI.x) # special exit for two-color indexing kwargs = {'wavelength': wavelength, 'beam_x': result['xbeam'], 'beam_y': result['ybeam'], 'distance': result['distance'], 'unique_file_name': data.file_name} ORI = result["current_orientation"][0] Astar = matrix.sqr(ORI.reciprocal_matrix()) kwargs['res_ori_1'] = Astar[0] kwargs['res_ori_2'] = Astar[1] kwargs['res_ori_3'] = Astar[2] kwargs['res_ori_4'] = Astar[3] kwargs['res_ori_5'] = Astar[4] kwargs['res_ori_6'] = Astar[5] kwargs['res_ori_7'] = Astar[6] kwargs['res_ori_8'] = Astar[7] kwargs['res_ori_9'] = Astar[8] assert self.params.scaling.report_ML is True kwargs['half_mosaicity_deg'] = result["ML_half_mosaicity_deg"][0] kwargs['domain_size_ang'] = result["ML_domain_size_ang"][0] frame_id_0_base = db_mgr.insert_frame(**kwargs) xypred = result["mapped_predictions"][0] indices = flex.size_t([pair[1] for pair in matches.pairs()]) sel_observations = flex.intersection( size=observations.data().size(), iselections=[indices]) set_original_hkl = observations_original_index_indices.select( flex.intersection( size=observations_original_index_indices.size(), iselections=[indices])) set_xypred = xypred.select( flex.intersection( size=xypred.size(), iselections=[indices])) kwargs = {'hkl_id_0_base': [pair[0] for pair in matches.pairs()], 'i': observations.data().select(sel_observations), 'sigi': observations.sigmas().select(sel_observations), 'detector_x': [xy[0] for xy in set_xypred], 'detector_y': [xy[1] for xy in set_xypred], 'frame_id_0_base': [frame_id_0_base] * len(matches.pairs()), 'overload_flag': [0] * len(matches.pairs()), 'original_h': [hkl[0] for hkl in set_original_hkl], 'original_k': [hkl[1] for hkl in set_original_hkl], 'original_l': [hkl[2] for hkl in set_original_hkl]} db_mgr.insert_observation(**kwargs) print >> out, "Lattice: %d reflections" % (data.n_obs - data.n_rejected) print >> out, "average obs", sum_y / (data.n_obs - data.n_rejected), \ "average calc", sum_x / (data.n_obs - data.n_rejected) print >> out, "Rejected %d reflections with negative intensities" % \ data.n_rejected data.accept = True for pair in matches.pairs(): if not self.params.include_negatives and (observations.data()[pair[1]] <= 0) : continue Intensity = observations.data()[pair[1]] # Super-rare exception. If saved sigmas instead of I/sigmas in the ISIGI dict, this wouldn't be needed. if Intensity == 0: continue # Add the reflection as a two-tuple of intensity and I/sig(I) # to the dictionary of observations. index = self.miller_set.indices()[pair[0]] isigi = (Intensity, observations.data()[pair[1]] / observations.sigmas()[pair[1]], 1.0) if index in data.ISIGI: data.ISIGI[index].append(isigi) else: data.ISIGI[index] = [isigi] sigma = observations.sigmas()[pair[1]] variance = sigma * sigma data.summed_N[pair[0]] += 1 data.summed_wt_I[pair[0]] += Intensity / variance data.summed_weight[pair[0]] += 1 / variance data.set_log_out(out.getvalue()) return data