def result_for_cxi_merge(self, file_name): values = self.get_parameter_values() self.rs2_parameter_range_assertions(values) scaler = self.nave1_refinery.scaler_callable( self.get_parameter_values()) partiality_array = self.refinery.get_partiality_array(values) p_scaler = flex.pow( partiality_array, 0.5 * self.params.postrefinement.merge_partiality_exponent) fat_selection = ( self.nave1_refinery.lorentz_callable(self.get_parameter_values()) > self.params.postrefinement.rs_hybrid.partiality_threshold ) # was 0.2 for rs2 fat_count = fat_selection.count(True) scaler_s = scaler.select(fat_selection) p_scaler_s = p_scaler.select(fat_selection) #avoid empty database INSERT, if insufficient centrally-located Bragg spots: # in samosa, handle this at a higher level, but handle it somehow. if fat_count < 3: raise ValueError("< 3 near-fulls after refinement") print >> self.out, "On total %5d the fat selection is %5d" % (len( self.observations_pair1_selected.indices()), fat_count) observations_original_index = \ self.observations_original_index_pair1_selected.select(fat_selection) observations = self.observations_pair1_selected.customized_copy( indices=self.observations_pair1_selected.indices().select( fat_selection), data=( self.observations_pair1_selected.data().select(fat_selection) / scaler_s), sigmas=( self.observations_pair1_selected.sigmas().select(fat_selection) / (scaler_s * p_scaler_s))) matches = miller.match_multi_indices( miller_indices_unique=self.miller_set.indices(), miller_indices=observations.indices()) I_weight = flex.double(len(observations.sigmas()), 1.) I_reference = flex.double( [self.i_model.data()[pair[0]] for pair in matches.pairs()]) I_invalid = flex.bool( [self.i_model.sigmas()[pair[0]] < 0. for pair in matches.pairs()]) I_weight.set_selected(I_invalid, 0.) SWC = simple_weighted_correlation(I_weight, I_reference, observations.data()) print >> self.out, "CORR: NEW correlation is", SWC.corr print >> self.out, "ASTAR_FILE", file_name, tuple( self.nave1_refinery.get_eff_Astar(values)) self.final_corr = SWC.corr #another range assertion assert self.final_corr > 0.1, "correlation coefficient out of range (<= 0.1) after LevMar refinement" # XXX Specific to the hybrid_rs method, and likely these limits are problem-specific (especially G-max) so look for another approach # or expose the limits as phil parameters. assert values.G < 0.5, "G-scale value out of range ( > 0.5 XXX may be too strict ) after LevMar refinement" return observations_original_index, observations, matches
def result_for_cxi_merge(self, file_name): values = self.get_parameter_values() self.rs2_parameter_range_assertions(values) scaler = self.refinery.scaler_callable( self.parameterization_class(self.MINI.x)) partiality_array = self.refinery.get_partiality_array(values) p_scaler = flex.pow( partiality_array, 0.5 * self.params.postrefinement.merge_partiality_exponent) fat_selection = (partiality_array > 0.2) fat_count = fat_selection.count(True) scaler_s = scaler.select(fat_selection) p_scaler_s = p_scaler.select(fat_selection) #avoid empty database INSERT, if insufficient centrally-located Bragg spots: # in samosa, handle this at a higher level, but handle it somehow. if fat_count < 3: raise ValueError("< 3 near-fulls after refinement") print("On total %5d the fat selection is %5d" % (len(self.observations_pair1_selected.indices()), fat_count), file=self.out) observations_original_index = \ self.observations_original_index_pair1_selected.select(fat_selection) observations = self.observations_pair1_selected.customized_copy( indices=self.observations_pair1_selected.indices().select( fat_selection), data=( self.observations_pair1_selected.data().select(fat_selection) / scaler_s), sigmas=( self.observations_pair1_selected.sigmas().select(fat_selection) / (scaler_s * p_scaler_s))) matches = miller.match_multi_indices( miller_indices_unique=self.miller_set.indices(), miller_indices=observations.indices()) I_weight = flex.double(len(observations.sigmas()), 1.) I_reference = flex.double( [self.i_model.data()[pair[0]] for pair in matches.pairs()]) I_invalid = flex.bool( [self.i_model.sigmas()[pair[0]] < 0. for pair in matches.pairs()]) I_weight.set_selected(I_invalid, 0.) SWC = simple_weighted_correlation(I_weight, I_reference, observations.data()) print("CORR: NEW correlation is", SWC.corr, file=self.out) print("ASTAR_FILE", file_name, tuple(self.refinery.get_eff_Astar(values)), file=self.out) self.final_corr = SWC.corr self.refined_mini = self.MINI #another range assertion assert self.final_corr > 0.1, "correlation coefficient out of range (<= 0.1) after rs2 refinement" return observations_original_index, observations, matches
def result_for_cxi_merge(self): values = self.get_parameter_values() self.rs2_parameter_range_assertions(values) scaler = self.refinery.scaler_callable(self.parameterization_class(self.MINI.x)) partiality_array = self.refinery.get_partiality_array(values) p_scaler = flex.pow(partiality_array, 0.5*self.params.postrefinement.merge_partiality_exponent) fat_selection = (partiality_array > 0.2) fat_count = fat_selection.count(True) scaler_s = scaler.select(fat_selection) p_scaler_s = p_scaler.select(fat_selection) # reject an experiment with insufficient number of near-full reflections if fat_count < 3: if self.params.output.log_level == 0: self.logger.log("Rejected experiment, because: On total %5d the fat selection is %5d"%(len(self.observations_pair1_selected.indices()), fat_count)) raise ValueError("< 3 near-fulls after refinement") if self.params.output.log_level == 0: self.logger.log("On total %5d the fat selection is %5d"%(len(self.observations_pair1_selected.indices()), fat_count)) observations_original_index = self.observations_original_index_pair1_selected.select(fat_selection) observations = self.observations_pair1_selected.customized_copy( indices = self.observations_pair1_selected.indices().select(fat_selection), data = (self.observations_pair1_selected.data().select(fat_selection)/scaler_s), sigmas = (self.observations_pair1_selected.sigmas().select(fat_selection)/(scaler_s * p_scaler_s)) ) matches = miller.match_multi_indices( miller_indices_unique=self.params.scaling.miller_set.indices(), miller_indices=observations.indices()) I_weight = flex.double(len(observations.sigmas()), 1.) I_reference = flex.double([self.params.scaling.i_model.data()[pair[0]] for pair in matches.pairs()]) I_invalid = flex.bool([self.params.scaling.i_model.sigmas()[pair[0]] < 0. for pair in matches.pairs()]) I_weight.set_selected(I_invalid,0.) SWC = simple_weighted_correlation(I_weight, I_reference, observations.data()) if self.params.output.log_level == 0: self.logger.log("CORR: NEW correlation is: %f"%SWC.corr) self.logger.log("ASTAR: ") self.logger.log(tuple(self.refinery.get_eff_Astar(values))) self.final_corr = SWC.corr self.refined_mini = self.MINI #another range assertion assert self.final_corr > 0.1,"correlation coefficient out of range (<= 0.1) after rs2 refinement" return observations_original_index, observations, matches
def result_for_cxi_merge(self, file_name): values = self.get_parameter_values() self.rs2_parameter_range_assertions(values) scaler = self.nave1_refinery.scaler_callable(self.get_parameter_values()) partiality_array = self.refinery.get_partiality_array(values) p_scaler = flex.pow(partiality_array, 0.5*self.params.postrefinement.merge_partiality_exponent) fat_selection = (self.nave1_refinery.lorentz_callable(self.get_parameter_values()) > self.params.postrefinement.rs_hybrid.partiality_threshold) # was 0.2 for rs2 fat_count = fat_selection.count(True) scaler_s = scaler.select(fat_selection) p_scaler_s = p_scaler.select(fat_selection) #avoid empty database INSERT, if insufficient centrally-located Bragg spots: # in samosa, handle this at a higher level, but handle it somehow. if fat_count < 3: raise ValueError, "< 3 near-fulls after refinement" print >> self.out, "On total %5d the fat selection is %5d"%( len(self.observations_pair1_selected.indices()), fat_count) observations_original_index = \ self.observations_original_index_pair1_selected.select(fat_selection) observations = self.observations_pair1_selected.customized_copy( indices = self.observations_pair1_selected.indices().select(fat_selection), data = (self.observations_pair1_selected.data().select(fat_selection)/scaler_s), sigmas = (self.observations_pair1_selected.sigmas().select(fat_selection)/(scaler_s * p_scaler_s)) ) matches = miller.match_multi_indices( miller_indices_unique=self.miller_set.indices(), miller_indices=observations.indices()) I_weight = flex.double(len(observations.sigmas()), 1.) I_reference = flex.double([self.i_model.data()[pair[0]] for pair in matches.pairs()]) SWC = simple_weighted_correlation(I_weight, I_reference, observations.data()) print >> self.out, "CORR: NEW correlation is", SWC.corr self.final_corr = SWC.corr #another range assertion assert self.final_corr > 0.1,"correlation coefficient out of range (<= 0.1) after LevMar refinement" # XXX Specific to the hybrid_rs method, and likely these limits are problem-specific (especially G-max) so look for another approach # or expose the limits as phil parameters. assert values.G < 0.5 , "G-scale value out of range ( > 0.5 XXX may be too strict ) after LevMar refinement" return observations_original_index,observations,matches
def __init__(self, measurements_orig, params, i_model, miller_set, result, out): measurements = measurements_orig.deep_copy() # Now manipulate the data to conform to unit cell, asu, and space group # of reference. The resolution will be cut later. # Only works if there is NOT an indexing ambiguity! observations = measurements.customized_copy( anomalous_flag=not params.merge_anomalous, crystal_symmetry=miller_set.crystal_symmetry()).map_to_asu() observations_original_index = measurements.customized_copy( anomalous_flag=not params.merge_anomalous, crystal_symmetry=miller_set.crystal_symmetry()) # Ensure that match_multi_indices() will return identical results # when a frame's observations are matched against the # pre-generated Miller set, self.miller_set, and the reference # data set, self.i_model. The implication is that the same match # can be used to map Miller indices to array indices for intensity # accumulation, and for determination of the correlation # coefficient in the presence of a scaling reference. assert len(i_model.indices()) == len(miller_set.indices()) \ and (i_model.indices() == miller_set.indices()).count(False) == 0 matches = miller.match_multi_indices( miller_indices_unique=miller_set.indices(), miller_indices=observations.indices()) pair1 = flex.int([pair[1] for pair in matches.pairs()]) pair0 = flex.int([pair[0] for pair in matches.pairs()]) # narrow things down to the set that matches, only observations_pair1_selected = observations.customized_copy( indices=flex.miller_index( [observations.indices()[p] for p in pair1]), data=flex.double([observations.data()[p] for p in pair1]), sigmas=flex.double([observations.sigmas()[p] for p in pair1]), ) observations_original_index_pair1_selected = observations_original_index.customized_copy( indices=flex.miller_index( [observations_original_index.indices()[p] for p in pair1]), data=flex.double( [observations_original_index.data()[p] for p in pair1]), sigmas=flex.double( [observations_original_index.sigmas()[p] for p in pair1]), ) ################### I_observed = observations_pair1_selected.data() MILLER = observations_original_index_pair1_selected.indices() ORI = result["current_orientation"][0] Astar = matrix.sqr(ORI.reciprocal_matrix()) WAVE = result["wavelength"] BEAM = matrix.col((0.0, 0.0, -1. / WAVE)) BFACTOR = 0. #calculation of correlation here I_reference = flex.double( [i_model.data()[pair[0]] for pair in matches.pairs()]) I_invalid = flex.bool( [i_model.sigmas()[pair[0]] < 0. for pair in matches.pairs()]) use_weights = False # New facility for getting variance-weighted correlation if use_weights: #variance weighting I_weight = flex.double([ 1. / (observations_pair1_selected.sigmas()[pair[1]])**2 for pair in matches.pairs() ]) else: I_weight = flex.double(len(observations_pair1_selected.sigmas()), 1.) I_weight.set_selected(I_invalid, 0.) """Explanation of 'include_negatives' semantics as originally implemented in cxi.merge postrefinement: include_negatives = True + and - reflections both used for Rh distribution for initial estimate of RS parameter + and - reflections both used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) include_negatives = False + and - reflections both used for Rh distribution for initial estimate of RS parameter + reflections only used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) """ if params.include_negatives: SWC = simple_weighted_correlation(I_weight, I_reference, I_observed) else: non_positive = (observations_pair1_selected.data() <= 0) SWC = simple_weighted_correlation( I_weight.select(~non_positive), I_reference.select(~non_positive), I_observed.select(~non_positive)) print >> out, "Old correlation is", SWC.corr if params.postrefinement.algorithm == "rs": Rhall = flex.double() for mill in MILLER: H = matrix.col(mill) Xhkl = Astar * H Rh = (Xhkl + BEAM).length() - (1. / WAVE) Rhall.append(Rh) Rs = math.sqrt(flex.mean(Rhall * Rhall)) RS = 1. / 10000. # reciprocal effective domain size of 1 micron RS = Rs # try this empirically determined approximate, monochrome, a-mosaic value current = flex.double([SWC.slope, BFACTOR, RS, 0., 0.]) parameterization_class = rs_parameterization refinery = rs_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC=I_reference, IOBSVEC=I_observed) elif params.postrefinement.algorithm == "eta_deff": eta_init = 2. * result["ML_half_mosaicity_deg"][0] * math.pi / 180. D_eff_init = 2. * result["ML_domain_size_ang"][0] current = flex.double([ SWC.slope, BFACTOR, eta_init, 0., 0., D_eff_init, ]) parameterization_class = eta_deff_parameterization refinery = eta_deff_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC=I_reference, IOBSVEC=I_observed) func = refinery.fvec_callable(parameterization_class(current)) functional = flex.sum(func * func) print >> out, "functional", functional self.current = current self.parameterization_class = parameterization_class self.refinery = refinery self.out = out self.params = params self.miller_set = miller_set self.observations_pair1_selected = observations_pair1_selected self.observations_original_index_pair1_selected = observations_original_index_pair1_selected
def run(self, experiments, reflections): self.logger.log_step_time("POSTREFINEMENT") if not self.params.postrefinement.enable: self.logger.log("Postrefinement was not done") if self.mpi_helper.rank == 0: self.logger.main_log("Postrefinement was not done") return experiments, reflections target_symm = symmetry( unit_cell=self.params.scaling.unit_cell, space_group_info=self.params.scaling.space_group) i_model = self.params.scaling.i_model miller_set = self.params.scaling.miller_set # Ensure that match_multi_indices() will return identical results # when a frame's observations are matched against the # pre-generated Miller set, self.miller_set, and the reference # data set, self.i_model. The implication is that the same match # can be used to map Miller indices to array indices for intensity # accumulation, and for determination of the correlation # coefficient in the presence of a scaling reference. assert len(i_model.indices()) == len(miller_set.indices()) assert (i_model.indices() == miller_set.indices()).count(False) == 0 new_experiments = ExperimentList() new_reflections = flex.reflection_table() experiments_rejected_by_reason = {} # reason:how_many_rejected for experiment in experiments: exp_reflections = reflections.select( reflections['exp_id'] == experiment.identifier) # Build a miller array for the experiment reflections with original miller indexes exp_miller_indices_original = miller.set( target_symm, exp_reflections['miller_index'], True) observations_original_index = miller.array( exp_miller_indices_original, exp_reflections['intensity.sum.value'], flex.double( flex.sqrt(exp_reflections['intensity.sum.variance']))) assert exp_reflections.size() == exp_miller_indices_original.size() assert observations_original_index.size( ) == exp_miller_indices_original.size() # Build a miller array for the experiment reflections with asu miller indexes exp_miller_indices_asu = miller.set( target_symm, exp_reflections['miller_index_asymmetric'], True) observations = miller.array( exp_miller_indices_asu, exp_reflections['intensity.sum.value'], flex.double( flex.sqrt(exp_reflections['intensity.sum.variance']))) matches = miller.match_multi_indices( miller_indices_unique=miller_set.indices(), miller_indices=observations.indices()) pair1 = flex.int([pair[1] for pair in matches.pairs() ]) # refers to the observations pair0 = flex.int([pair[0] for pair in matches.pairs() ]) # refers to the model assert exp_reflections.size() == exp_miller_indices_original.size() assert observations_original_index.size( ) == exp_miller_indices_original.size() # narrow things down to the set that matches, only observations_pair1_selected = observations.customized_copy( indices=flex.miller_index( [observations.indices()[p] for p in pair1]), data=flex.double([observations.data()[p] for p in pair1]), sigmas=flex.double([observations.sigmas()[p] for p in pair1])) observations_original_index_pair1_selected = observations_original_index.customized_copy( indices=flex.miller_index( [observations_original_index.indices()[p] for p in pair1]), data=flex.double( [observations_original_index.data()[p] for p in pair1]), sigmas=flex.double( [observations_original_index.sigmas()[p] for p in pair1])) I_observed = observations_pair1_selected.data() MILLER = observations_original_index_pair1_selected.indices() ORI = crystal_orientation(experiment.crystal.get_A(), basis_type.reciprocal) Astar = matrix.sqr(ORI.reciprocal_matrix()) Astar_from_experiment = matrix.sqr(experiment.crystal.get_A()) assert Astar == Astar_from_experiment WAVE = experiment.beam.get_wavelength() BEAM = matrix.col((0.0, 0.0, -1. / WAVE)) BFACTOR = 0. MOSAICITY_DEG = experiment.crystal.get_half_mosaicity_deg() DOMAIN_SIZE_A = experiment.crystal.get_domain_size_ang() # calculation of correlation here I_reference = flex.double( [i_model.data()[pair[0]] for pair in matches.pairs()]) I_invalid = flex.bool( [i_model.sigmas()[pair[0]] < 0. for pair in matches.pairs()]) use_weights = False # New facility for getting variance-weighted correlation if use_weights: # variance weighting I_weight = flex.double([ 1. / (observations_pair1_selected.sigmas()[pair[1]])**2 for pair in matches.pairs() ]) else: I_weight = flex.double( len(observations_pair1_selected.sigmas()), 1.) I_weight.set_selected(I_invalid, 0.) """Explanation of 'include_negatives' semantics as originally implemented in cxi.merge postrefinement: include_negatives = True + and - reflections both used for Rh distribution for initial estimate of RS parameter + and - reflections both used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) include_negatives = False + and - reflections both used for Rh distribution for initial estimate of RS parameter + reflections only used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) """ # RB: By design, for MPI-Merge "include negatives" is implicitly True SWC = simple_weighted_correlation(I_weight, I_reference, I_observed) if self.params.output.log_level == 0: self.logger.log("Old correlation is: %f" % SWC.corr) if self.params.postrefinement.algorithm == "rs": Rhall = flex.double() for mill in MILLER: H = matrix.col(mill) Xhkl = Astar * H Rh = (Xhkl + BEAM).length() - (1. / WAVE) Rhall.append(Rh) Rs = math.sqrt(flex.mean(Rhall * Rhall)) RS = 1. / 10000. # reciprocal effective domain size of 1 micron RS = Rs # try this empirically determined approximate, monochrome, a-mosaic value current = flex.double([SWC.slope, BFACTOR, RS, 0., 0.]) parameterization_class = rs_parameterization refinery = rs_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC=I_reference, IOBSVEC=I_observed) elif self.params.postrefinement.algorithm == "eta_deff": eta_init = 2. * MOSAICITY_DEG * math.pi / 180. D_eff_init = 2. * DOMAIN_SIZE_A current = flex.double( [SWC.slope, BFACTOR, eta_init, 0., 0., D_eff_init]) parameterization_class = eta_deff_parameterization refinery = eta_deff_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC=I_reference, IOBSVEC=I_observed) func = refinery.fvec_callable(parameterization_class(current)) functional = flex.sum(func * func) if self.params.output.log_level == 0: self.logger.log("functional: %f" % functional) self.current = current self.parameterization_class = parameterization_class self.refinery = refinery self.observations_pair1_selected = observations_pair1_selected self.observations_original_index_pair1_selected = observations_original_index_pair1_selected error_detected = False try: self.run_plain() result_observations_original_index, result_observations, result_matches = self.result_for_cxi_merge( ) assert result_observations_original_index.size( ) == result_observations.size() assert result_matches.pairs().size( ) == result_observations_original_index.size() except (AssertionError, ValueError, RuntimeError) as e: error_detected = True reason = repr(e) if not reason: reason = "Unknown error" if not reason in experiments_rejected_by_reason: experiments_rejected_by_reason[reason] = 1 else: experiments_rejected_by_reason[reason] += 1 if not error_detected: new_experiments.append(experiment) new_exp_reflections = flex.reflection_table() new_exp_reflections[ 'miller_index_asymmetric'] = flex.miller_index( result_observations.indices()) new_exp_reflections['intensity.sum.value'] = flex.double( result_observations.data()) new_exp_reflections['intensity.sum.variance'] = flex.double( flex.pow(result_observations.sigmas(), 2)) new_exp_reflections['exp_id'] = flex.std_string( len(new_exp_reflections), experiment.identifier) new_reflections.extend(new_exp_reflections) ''' # debugging elif reason.startswith("ValueError"): self.logger.log("Rejected b/c of value error exp id: %s; unit cell: %s"%(exp_id, str(experiment.crystal.get_unit_cell())) ) ''' # report rejected experiments, reflections experiments_rejected_by_postrefinement = len(experiments) - len( new_experiments) reflections_rejected_by_postrefinement = reflections.size( ) - new_reflections.size() self.logger.log("Experiments rejected by post-refinement: %d" % experiments_rejected_by_postrefinement) self.logger.log("Reflections rejected by post-refinement: %d" % reflections_rejected_by_postrefinement) all_reasons = [] for reason, count in experiments_rejected_by_reason.iteritems(): self.logger.log("Experiments rejected due to %s: %d" % (reason, count)) all_reasons.append(reason) comm = self.mpi_helper.comm MPI = self.mpi_helper.MPI # Collect all rejection reasons from all ranks. Use allreduce to let each rank have all reasons. all_reasons = comm.allreduce(all_reasons, MPI.SUM) all_reasons = set(all_reasons) # Now that each rank has all reasons from all ranks, we can treat the reasons in a uniform way. total_experiments_rejected_by_reason = {} for reason in all_reasons: rejected_experiment_count = 0 if reason in experiments_rejected_by_reason: rejected_experiment_count = experiments_rejected_by_reason[ reason] total_experiments_rejected_by_reason[reason] = comm.reduce( rejected_experiment_count, MPI.SUM, 0) total_accepted_experiment_count = comm.reduce(len(new_experiments), MPI.SUM, 0) # how many reflections have we rejected due to post-refinement? rejected_reflections = len(reflections) - len(new_reflections) total_rejected_reflections = self.mpi_helper.sum(rejected_reflections) if self.mpi_helper.rank == 0: for reason, count in total_experiments_rejected_by_reason.iteritems( ): self.logger.main_log( "Total experiments rejected due to %s: %d" % (reason, count)) self.logger.main_log("Total experiments accepted: %d" % total_accepted_experiment_count) self.logger.main_log( "Total reflections rejected due to post-refinement: %d" % total_rejected_reflections) self.logger.log_step_time("POSTREFINEMENT", True) return new_experiments, new_reflections
def __init__(self,measurements_orig, params, i_model, miller_set, result, out): measurements = measurements_orig.deep_copy() # Now manipulate the data to conform to unit cell, asu, and space group # of reference. The resolution will be cut later. # Only works if there is NOT an indexing ambiguity! observations = measurements.customized_copy( anomalous_flag=not params.merge_anomalous, crystal_symmetry=miller_set.crystal_symmetry() ).map_to_asu() observations_original_index = measurements.customized_copy( anomalous_flag=not params.merge_anomalous, crystal_symmetry=miller_set.crystal_symmetry() ) # Ensure that match_multi_indices() will return identical results # when a frame's observations are matched against the # pre-generated Miller set, self.miller_set, and the reference # data set, self.i_model. The implication is that the same match # can be used to map Miller indices to array indices for intensity # accumulation, and for determination of the correlation # coefficient in the presence of a scaling reference. assert len(i_model.indices()) == len(miller_set.indices()) \ and (i_model.indices() == miller_set.indices()).count(False) == 0 matches = miller.match_multi_indices( miller_indices_unique=miller_set.indices(), miller_indices=observations.indices()) pair1 = flex.int([pair[1] for pair in matches.pairs()]) pair0 = flex.int([pair[0] for pair in matches.pairs()]) # narrow things down to the set that matches, only observations_pair1_selected = observations.customized_copy( indices = flex.miller_index([observations.indices()[p] for p in pair1]), data = flex.double([observations.data()[p] for p in pair1]), sigmas = flex.double([observations.sigmas()[p] for p in pair1]), ) observations_original_index_pair1_selected = observations_original_index.customized_copy( indices = flex.miller_index([observations_original_index.indices()[p] for p in pair1]), data = flex.double([observations_original_index.data()[p] for p in pair1]), sigmas = flex.double([observations_original_index.sigmas()[p] for p in pair1]), ) ################### I_observed = observations_pair1_selected.data() chosen = chosen_weights(observations_pair1_selected, params) MILLER = observations_original_index_pair1_selected.indices() ORI = result["current_orientation"][0] Astar = matrix.sqr(ORI.reciprocal_matrix()) WAVE = result["wavelength"] BEAM = matrix.col((0.0,0.0,-1./WAVE)) BFACTOR = 0. #calculation of correlation here I_reference = flex.double([i_model.data()[pair[0]] for pair in matches.pairs()]) use_weights = False # New facility for getting variance-weighted correlation if use_weights: #variance weighting I_weight = flex.double( [1./(observations_pair1_selected.sigmas()[pair[1]])**2 for pair in matches.pairs()]) else: I_weight = flex.double(len(observations_pair1_selected.sigmas()), 1.) """Explanation of 'include_negatives' semantics as originally implemented in cxi.merge postrefinement: include_negatives = True + and - reflections both used for Rh distribution for initial estimate of RS parameter + and - reflections both used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) include_negatives = False + and - reflections both used for Rh distribution for initial estimate of RS parameter + reflections only used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) """ if params.include_negatives: SWC = simple_weighted_correlation(I_weight, I_reference, I_observed) else: non_positive = ( observations_pair1_selected.data() <= 0 ) SWC = simple_weighted_correlation(I_weight.select(~non_positive), I_reference.select(~non_positive), I_observed.select(~non_positive)) print >> out, "Old correlation is", SWC.corr assert params.postrefinement.algorithm=="rs_hybrid" Rhall = flex.double() for mill in MILLER: H = matrix.col(mill) Xhkl = Astar*H Rh = ( Xhkl + BEAM ).length() - (1./WAVE) Rhall.append(Rh) Rs = math.sqrt(flex.mean(Rhall*Rhall)) RS = 1./10000. # reciprocal effective domain size of 1 micron RS = Rs # try this empirically determined approximate, monochrome, a-mosaic value self.rs2_current = flex.double([SWC.slope, BFACTOR, RS, 0., 0.]) self.rs2_parameterization_class = rs_parameterization self.rs2_refinery = rs2_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC = I_reference, IOBSVEC = I_observed, WEIGHTS = chosen) self.rs2_refinery.set_profile_shape(params.postrefinement.lineshape) self.nave1_refinery = nave1_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC = I_reference, IOBSVEC = I_observed, WEIGHTS = chosen) self.nave1_refinery.set_profile_shape(params.postrefinement.lineshape) self.out=out; self.params = params; self.miller_set = miller_set self.observations_pair1_selected = observations_pair1_selected; self.observations_original_index_pair1_selected = observations_original_index_pair1_selected self.i_model = i_model