def refine_error_model(params, experiments, reflection_tables): """Do error model refinement.""" # prepare relevant data for datastructures for i, table in enumerate(reflection_tables): # First get the good data table = table.select(~table.get_flags(table.flags.bad_for_scaling, all=False)) # Now chose intensities, ideally these two options could be combined # with a smart refactor if params.intensity_choice == "combine": if not params.combine.Imid: sys.exit("Imid value must be provided if intensity_choice=combine") table = calculate_prescaling_correction(table) # needed for below. I, V = combine_intensities(table, params.combine.Imid) table["intensity"] = I table["variance"] = V else: table = choose_initial_scaling_intensities( table, intensity_choice=params.intensity_choice ) reflection_tables[i] = table space_group = experiments[0].crystal.get_space_group() Ih_table = IhTable( reflection_tables, space_group, additional_cols=["partiality"], anomalous=True ) # now do the error model refinement model = BasicErrorModel(basic_params=params.basic) try: model = run_error_model_refinement(model, Ih_table) except (ValueError, RuntimeError) as e: logger.info(e) else: return model
def choose_scaling_intensities(reflection_table, intensity_choice="profile"): """Choose which intensities to use for scaling. The LP, QE and partiality corrections are also applied. Two new columns are added to the reflection table 'intensity' and 'variance', which have all corrections applied except an inverse scale factor.""" if intensity_choice == "profile": intensity_choice = "prf" # rename to allow string matching with refl table if intensity_choice == "prf": if (reflection_table.get_flags( reflection_table.flags.integrated_prf).count(True) == 0): logger.warning( "No profile fitted reflections in this dataset, using summation intensities" ) intensity_choice = "sum" reflection_table = calculate_prescaling_correction(reflection_table) conv = reflection_table["prescaling_correction"] intstr = "intensity." + intensity_choice + ".value" if intstr not in reflection_table: # Can't find selection, try to choose prf, if not then sum (also catches combine # which should not be used at this point) if "intensity.prf.value" in reflection_table: intstr = "intensity.prf.value" else: assert ("intensity.sum.value" in reflection_table), """No recognised intensity values found.""" intstr = "intensity.sum.value" varstr = intstr.rstrip("value") + "variance" else: varstr = intstr.rstrip("value") + "variance" logger.info( """%s intensities will be used for scaling (and mtz output if applicable). \n""", intstr, ) # prf partial intensities are the 'full' intensity values but sum are not if "partiality" in reflection_table and intstr == "intensity.sum.value": inverse_partiality = flex.double(reflection_table.size(), 1.0) nonzero_partiality_sel = reflection_table["partiality"] > 0.0 good_refl = reflection_table.select( reflection_table["partiality"] > 0.0) inverse_partiality.set_selected(nonzero_partiality_sel.iselection(), 1.0 / good_refl["partiality"]) conv *= inverse_partiality reflection_table["intensity"] = reflection_table[intstr] * conv reflection_table["variance"] = reflection_table[varstr] * conv * conv if ("partiality.inv.variance" in reflection_table and intstr == "intensity.sum.value"): reflection_table["variance"] += ( reflection_table[intstr] * reflection_table["partiality.inv.variance"]) variance_mask = reflection_table["variance"] <= 0.0 reflection_table.set_flags(variance_mask, reflection_table.flags.excluded_for_scaling) return reflection_table
def generated_refl_for_comb(): """Create a reflection table suitable for splitting into blocks.""" reflections = flex.reflection_table() reflections["intensity"] = flex.double( [1.0, 2.0, 3.0, 4.0, 500.0, 6.0, 2.0, 2.0]) reflections["variance"] = flex.double(8, 1.0) reflections["intensity.prf.value"] = flex.double( [1.0, 3.0, 3.0, 4.0, 50.0, 6.0, 3.0, 2.0]) reflections["intensity.prf.variance"] = flex.double( [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0]) reflections["intensity.sum.value"] = flex.double( [1.0, 4.0, 3.0, 4.0, 500.0, 6.0, 6.0, 2.0]) reflections["intensity.sum.variance"] = flex.double(8, 1.0) reflections["miller_index"] = flex.miller_index([ (1, 0, 0), (2, 0, 0), (0, 0, 1), (2, 2, 2), (1, 0, 0), (2, 0, 0), (1, 0, 0), (1, 0, 0), ]) reflections["d"] = flex.double([0.8, 2.1, 2.0, 1.4, 1.6, 2.5, 2.5, 2.5]) reflections["partiality"] = flex.double(8, 1.0) reflections["Esq"] = flex.double(8, 1.0) reflections["inverse_scale_factor"] = flex.double(8, 1.0) reflections["xyzobs.px.value"] = flex.vec3_double([ (0.0, 0.0, 0.0), (0.0, 0.0, 5.0), (0.0, 0.0, 8.0), (0.0, 0.0, 10.0), (0.0, 0.0, 12.0), (0.0, 0.0, 15.0), (0.0, 0.0, 15.0), (0.0, 0.0, 15.0), ]) reflections["s1"] = flex.vec3_double([ (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), (0.0, 0.1, 1.0), ]) reflections.set_flags(flex.bool(8, True), reflections.flags.integrated) reflections.set_flags(flex.bool([False] * 5 + [True] + [False] * 2), reflections.flags.bad_for_scaling) reflections["id"] = flex.int(8, 0) reflections.experiment_identifiers()[0] = "0" reflections = calculate_prescaling_correction(reflections) return reflections
def test_calculate_prescaling_correction(): """Test the helper function that applies the lp, dqe and partiality corr.""" reflection_table = flex.reflection_table() reflection_table["lp"] = flex.double([1.0, 0.9, 0.8]) reflection_table["qe"] = flex.double([0.6, 0.5, 0.4]) reflection_table = calculate_prescaling_correction(reflection_table) assert list(reflection_table["prescaling_correction"]) == [ 1.0 / 0.6, 0.9 / 0.5, 0.8 / 0.4, ] # Test compatibilty for old datasets del reflection_table["qe"] reflection_table["dqe"] = flex.double([0.6, 0.5, 0.4]) reflection_table = calculate_prescaling_correction(reflection_table) assert list(reflection_table["prescaling_correction"]) == [ 1.0 / 0.6, 0.9 / 0.5, 0.8 / 0.4, ]
def generate_simple_table(prf=True): """Generate a reflection table for testing intensity combination. The numbers are contrived to make sum intensities agree well at high intensity but terribly at low and vice versa for profile intensities.""" reflections = flex.reflection_table() reflections["miller_index"] = flex.miller_index( [ (0, 0, 1), (0, 0, 1), (0, 0, 1), (0, 0, 1), (0, 0, 1), (0, 0, 2), (0, 0, 2), (0, 0, 2), (0, 0, 2), (0, 0, 2), (0, 0, 3), (0, 0, 3), (0, 0, 3), (0, 0, 3), (0, 0, 3), (0, 0, 4), (0, 0, 4), (0, 0, 4), (0, 0, 4), (0, 0, 4), (0, 0, 5), (0, 0, 5), (0, 0, 5), (0, 0, 5), (0, 0, 5), ] ) reflections["inverse_scale_factor"] = flex.double(25, 1.0) # Contrive an example that should give the best cc12 when combined. # make sum intensities agree well at high intensity but terribly at low # and vice versa for profile intensities. # profile less consistent at high intensity here # sumless consistent at low intensity here reflections["intensity.sum.value"] = flex.double( [ 10000.0, 11000.0, 9000.0, 8000.0, 12000.0, 500.0, 5600.0, 5500.0, 2000.0, 6000.0, 100.0, 50.0, 150.0, 75.0, 125.0, 30.0, 10.0, 2.0, 35.0, 79.0, 1.0, 10.0, 20.0, 10.0, 5.0, ] ) reflections["intensity.sum.variance"] = flex.double( [10000] * 5 + [5000] * 5 + [100] * 5 + [30] * 5 + [10] * 5 ) reflections.set_flags(flex.bool(25, False), reflections.flags.outlier_in_scaling) reflections.set_flags(flex.bool(25, True), reflections.flags.integrated) reflections["lp"] = flex.double(25, 0.5) if prf: reflections["intensity.prf.value"] = flex.double( [ 10000.0, 16000.0, 12000.0, 6000.0, 9000.0, 5000.0, 2000.0, 1500.0, 1300.0, 9000.0, 100.0, 80.0, 120.0, 90.0, 100.0, 30.0, 40.0, 50.0, 30.0, 30.0, 10.0, 12.0, 9.0, 8.0, 10.0, ] ) reflections["intensity.prf.variance"] = flex.double( [1000] * 5 + [500] * 5 + [10] * 5 + [3] * 5 + [1] * 5 ) reflections = calculate_prescaling_correction(reflections) return reflections
def choose_initial_scaling_intensities(reflection_table, intensity_choice="profile"): """Choose which intensities to initially use for scaling. The LP, QE and partiality corrections are also applied. Two new columns are added to the reflection table 'intensity' and 'variance', which have all corrections applied except an inverse scale factor.""" if intensity_choice == "profile": intensity_choice = "prf" # rename to allow string matching with refl table if "intensity.prf.value" not in reflection_table: intensity_choice = "sum" elif intensity_choice == "prf": if ( reflection_table.get_flags(reflection_table.flags.integrated_prf).count( True ) == 0 ): logger.warning( "No profile fitted reflections in this dataset, using summation intensities" ) intensity_choice = "sum" reflection_table = calculate_prescaling_correction(reflection_table) conv = reflection_table["prescaling_correction"] # if prf/sum, use those. If combine, use prf else sum for each refl. if intensity_choice == "prf": reflection_table["intensity"] = reflection_table["intensity.prf.value"] * conv reflection_table["variance"] = ( reflection_table["intensity.prf.variance"] * conv * conv ) else: # first fill in summation intensities. if "partiality" in reflection_table: inverse_partiality = flex.double(reflection_table.size(), 1.0) nonzero_partiality_sel = reflection_table["partiality"] > 0.0 good_refl = reflection_table.select(reflection_table["partiality"] > 0.0) inverse_partiality.set_selected( nonzero_partiality_sel.iselection(), 1.0 / good_refl["partiality"] ) reflection_table["intensity"] = ( reflection_table["intensity.sum.value"] * conv * inverse_partiality ) reflection_table["variance"] = reflection_table[ "intensity.sum.variance" ] * flex.pow2(conv * inverse_partiality) if "partiality.inv.variance" in reflection_table: reflection_table["variance"] += ( reflection_table["intensity.sum.value"] * reflection_table["partiality.inv.variance"] ) else: reflection_table["intensity"] = ( reflection_table["intensity.sum.value"] * conv ) reflection_table["variance"] = ( reflection_table["intensity.sum.variance"] * conv * conv ) if intensity_choice == "combine": # now overwrite prf if we have it. sel = reflection_table.get_flags(reflection_table.flags.integrated_prf) isel = sel.iselection() Iprf = (reflection_table["intensity.prf.value"] * conv).select(sel) Vprf = (reflection_table["intensity.prf.variance"] * conv * conv).select( sel ) reflection_table["intensity"].set_selected(isel, Iprf) reflection_table["variance"].set_selected(isel, Vprf) variance_mask = reflection_table["variance"] <= 0.0 reflection_table.set_flags( variance_mask, reflection_table.flags.excluded_for_scaling ) return reflection_table