def exercise (debug=False) : if (not libtbx.env.has_module("phenix_regression")) : print "phenix_regression not configured, skipping." return hkl_file = libtbx.env.find_in_repositories( relative_path="phenix_regression/wizards/p9_se_w2.sca", test=os.path.isfile) args = [ hkl_file, "space_group=I4", "unit_cell=113.949,113.949,32.474,90,90,90", "loggraph=True", ] if (debug) : args.append("debug=True") print " ".join(args) out = StringIO() result = merging_statistics.run(args, out=out) if (debug) : print out.getvalue() assert ("R-merge: 0.073" in out.getvalue()) assert ("R-meas: 0.079" in out.getvalue()) assert (""" 1.81 1.74 12528 2073 6.04 97.05 1449.2 5.2 0.252 0.275 0.110 0.9""" in out.getvalue()), out.getvalue() cif_block = result.as_cif_block() assert "_reflns_shell" in cif_block assert approx_equal(float(cif_block["_reflns.pdbx_Rpim_I_obs"]), result.overall.r_pim) assert approx_equal(float(cif_block["_reflns.phenix_cc_star"]), result.overall.cc_star) assert approx_equal(float(cif_block["_reflns.phenix_cc_1/2"]), result.overall.cc_one_half) assert approx_equal( flex.int(cif_block["_reflns_shell.number_measured_obs"]), [15737, 15728, 15668, 15371, 14996, 14771, 13899, 13549, 13206, 12528]) assert "_reflns_shell.phenix_cc_star" in cif_block assert "_reflns_shell.phenix_cc_1/2" in cif_block remark_200 = result.as_remark_200(wavelength=0.9792).splitlines() assert ("REMARK 200 <I/SIGMA(I)> FOR SHELL : 5.1536" in remark_200),"\n".join(remark_200) assert ("REMARK 200 WAVELENGTH OR RANGE (A) : 0.9792" in remark_200) # test resolution cutoffs args2 = list(args[:-1]) + ["high_resolution=2.5", "low_resolution=15"] out = StringIO() result = merging_statistics.run(args2, out=out) if (debug) : print out.getvalue() assert ("Resolution: 14.96 - 2.50" in out.getvalue()) # extend binning args2 = list(args[:-1]) + ["high_resolution=1.5", "low_resolution=100", "--extend_d_max_min"] out = StringIO() result = merging_statistics.run(args2, out=out) if (debug) : print out.getvalue() assert ("Resolution: 100.00 - 1.50" in out.getvalue()) assert (" 1.55 1.50 0 0 0.00 0.00 0.0 0.0 None None None 0.000 0.000""" in out.getvalue()) # these should crash args2 = list(args[:-1]) + ["high_resolution=15", "low_resolution=2.5"] try : result = merging_statistics.run(args2, out=out) except Sorry, s : pass
def run(args): if len(args) == 0 or "-h" in args or "--help" in args: master_params.show() interp = master_params.command_line_argument_interpreter(args) phil_scope, unhandled = interp.process_and_fetch( args, custom_processor="collect_remaining") params = phil_scope.extract() merging_stats = [] for arg in unhandled: assert os.path.isfile(arg) merging_stats.append( merging_statistics.run([arg], master_params=phil_scope)) if params.latex: table1_tex(merging_stats)
def run(args=sys.argv[1:]): if len(args) == 0 or "-h" in args or "--help" in args: master_params.show() sys.exit() interp = master_params.command_line_argument_interpreter(args) phil_scope, unhandled = interp.process_and_fetch( args, custom_processor="collect_remaining" ) params = phil_scope.extract() input_files = [] for arg in unhandled: if not os.path.isfile(arg): sys.exit(f"{arg} is neither a file nor an interpretable parameter") input_files.append(arg) merging_stats = [ (merging_statistics.run([f], master_params=phil_scope)) for f in input_files ] if params.latex: table1_tex(merging_stats)
def exercise(debug=False): if not libtbx.env.has_module("phenix_regression"): print("phenix_regression not configured, skipping.") return hkl_file = libtbx.env.find_in_repositories( relative_path= "phenix_regression/wizards/help_tests/test_help/p9_se_w2.sca", test=os.path.isfile) assert hkl_file is not None args = [ hkl_file, "space_group=I4", "unit_cell=113.949,113.949,32.474,90,90,90", "loggraph=True", ] if debug: args.append("debug=True") print(" ".join(args)) out = StringIO() result = merging_statistics.run(args, out=out) if debug: print(out.getvalue()) assert ("R-merge: 0.073" in out.getvalue()) assert ("R-meas: 0.079" in out.getvalue()) assert ( """ 1.81 1.74 12528 2073 6.04 97.05 1449.2 5.2 0.252 0.275 0.110 0.967 0.281""" in out.getvalue()), out.getvalue() cif_block = result.as_cif_block() assert "_reflns_shell" in cif_block assert approx_equal(float(cif_block["_reflns.pdbx_Rpim_I_obs"]), result.overall.r_pim) assert approx_equal(float(cif_block["_reflns.pdbx_CC_half"]), result.overall.cc_one_half) assert approx_equal( flex.int(cif_block["_reflns_shell.number_measured_obs"]), [15737, 15728, 15668, 15371, 14996, 14771, 13899, 13549, 13206, 12528]) assert "_reflns_shell.pdbx_CC_half" in cif_block remark_200 = result.as_remark_200(wavelength=0.9792).splitlines() assert ("REMARK 200 <I/SIGMA(I)> FOR SHELL : 5.1536" in remark_200), "\n".join(remark_200) assert ("REMARK 200 WAVELENGTH OR RANGE (A) : 0.9792" in remark_200) # test resolution cutoffs args2 = list(args[:-1]) + ["high_resolution=2.5", "low_resolution=15"] out = StringIO() merging_statistics.run(args2, out=out) if debug: print(out.getvalue()) assert ("Resolution: 14.96 - 2.50" in out.getvalue()) # extend binning args2 = list(args[:-1]) + [ "high_resolution=1.5", "low_resolution=100", "--extend_d_max_min" ] out = StringIO() merging_statistics.run(args2, out=out) if debug: print(out.getvalue()) assert ("Resolution: 100.00 - 1.50" in out.getvalue()) assert ( " 1.55 1.50 0 0 0.00 0.00 0.0 0.0 None None None 0.000 0.000" "" in out.getvalue()) args2 = args + [ "json.file_name=merging_stats.json", "json.indent=2", "mmcif.file_name=merging_stats.mmcif", "mmcif.data_name=test" ] out = StringIO() merging_statistics.run(args2, out=out) assert os.path.exists("merging_stats.json") with open("merging_stats.json", "rb") as f: import json d = json.load(f) list(d.keys()) #FIXME why am I here? expected_keys = [ 'n_obs', 'd_star_sq_max', 'i_over_sigma_mean', 'completeness', 'cc_one_half', 'r_meas', 'd_star_sq_min', 'cc_anom', 'r_pim', 'i_mean', 'cc_one_half_critical_value', 'r_merge', 'multiplicity', 'cc_one_half_significance', 'n_uniq', 'anom_completeness', 'anom_signal', 'delta_i_mean_over_sig_delta_i_mean', ] for k in expected_keys: assert k in d assert approx_equal(d['i_over_sigma_mean'], [ 20.66560600258035, 20.345307828448572, 18.349521604862186, 16.032340432560876, 15.111031030727927, 12.289599369298855, 10.059003053756124, 8.042714696129208, 6.297114205813105, 5.153613598686054 ]) assert approx_equal(d['n_obs'], [ 15737, 15728, 15668, 15371, 14996, 14771, 13899, 13549, 13206, 12528 ]) assert 'overall' in d assert approx_equal(d['overall']['i_mean'], 18672.03367141032) assert approx_equal(d['overall']['multiplicity'], 6.747367444449599) import iotbx.cif assert os.path.exists("merging_stats.mmcif") cif = iotbx.cif.reader("merging_stats.mmcif").model() assert approx_equal(float(cif["test"]["_reflns.pdbx_CC_half"]), 0.997601585888) assert list(cif["test"]["_reflns_shell.number_measured_obs"]) == [ '15737', '15728', '15668', '15371', '14996', '14771', '13899', '13549', '13206', '12528' ] # args3 = args + ["anomalous=True"] result = merging_statistics.run(args3, out=out) assert approx_equal(result.overall.anom_signal, 0.0835294960933567) assert approx_equal(result.overall.anom_completeness, 0.9836098441180893) assert approx_equal(result.overall.delta_i_mean_over_sig_delta_i_mean, 1.1764137800824204) # these should crash args2 = list(args[:-1]) + ["high_resolution=15", "low_resolution=2.5"] try: merging_statistics.run(args2, out=out) except Sorry: pass else: raise Exception_expected args2 = list(args[:-1]) + ["high_resolution=1.5", "low_resolution=1.6"] try: merging_statistics.run(args2, out=out) except Sorry: pass else: raise Exception_expected # change space group args = [ hkl_file, "space_group=I422", "unit_cell=113.949,113.949,32.474,90,90,90", "loggraph=True", ] if debug: args.append("debug=True") print(" ".join(args)) out = StringIO() merging_statistics.run(args, out=out) if debug: print(out.getvalue()) assert ( " 28.49 3.76 15737 1224 12.86 99.84 47967.0 11.6 0.482 0.500 0.135 0.973 -0.513" in out.getvalue()), out.getvalue() # exercise 2: estimate resolution cutoffs (and symmetry_file argument) hkl_file = libtbx.env.find_in_repositories( relative_path="phenix_regression/harvesting/unmerged.sca", test=os.path.isfile) assert hkl_file is not None symm_file = libtbx.env.find_in_repositories( relative_path="phenix_regression/harvesting/refine.mtz", test=os.path.isfile) assert symm_file is not None args = [ hkl_file, "symmetry_file=\"%s\"" % symm_file, "--estimate_cutoffs", ] out = StringIO() merging_statistics.run(args, out=out) if debug: print(out.getvalue()) for line in """\ resolution of all data : 2.000 based on CC(1/2) >= 0.33 : 2.000 based on mean(I/sigma) >= 2.0 : 2.253 based on R-merge < 0.5 : 2.372 based on R-meas < 0.5 : 2.520 based on completeness >= 90% : 2.000 based on completeness >= 50% : 2.000""".splitlines(): assert line in out.getvalue(), out.getvalue() # check suitable error emitted given merged input mtz (not containing M_ISYM column) out = StringIO() hkl_file = libtbx.env.find_in_repositories( relative_path="phenix_regression/reflection_files/i_anomalous.mtz", test=os.path.isfile) assert hkl_file is not None # Test that merged anomalous data are already merged. try: merging_statistics.run([hkl_file, "anomalous=True"], out=out) except Sorry as e: assert str( e ) == 'The data in i_anomalous(+),SIGi_anomalous(+),i_anomalous(-),SIGi_anomalous(-) are already merged. Only unmerged (but scaled) data may be used in this program.' else: raise Exception_expected # Test that merged anomalous data can still be merged as non-anomalous data. merging_statistics.run([hkl_file], out=out) # test use_internal_variance option out = StringIO() hkl_file = libtbx.env.find_in_repositories( relative_path= "phenix_regression/wizards/help_tests/test_help/unmerged.mtz", test=os.path.isfile) assert hkl_file is not None args = [hkl_file, "use_internal_variance=False"] result = merging_statistics.run(args, out=out) assert approx_equal(result.overall.i_over_sigma_mean, 4.4867598237199) assert approx_equal(result.overall.unmerged_i_over_sigma_mean, 2.559049577429115) args = [ hkl_file, #"use_internal_variance=True" # this is the default behaviour ] result = merging_statistics.run(args, out=out) assert approx_equal(result.overall.i_over_sigma_mean, 4.063574245292925) assert approx_equal(result.overall.unmerged_i_over_sigma_mean, 2.559049577429115) # test eliminate_sys_absent option out = StringIO() hkl_file = libtbx.env.find_in_repositories( relative_path= "phenix_regression/reflection_files/AUTOMATIC_DEFAULT_scaled_unmerged_WAVE1.mtz", test=os.path.isfile) assert hkl_file is not None args = [ hkl_file, #"eliminate_sys_absent=True" # default behaviour ] result = merging_statistics.run(args, out=out) assert approx_equal(result.overall.d_max, 43.069972142418365) assert result.overall.n_obs == 118981, result.overall.n_obs args = [ hkl_file, "binning_method=counting_sorted", "eliminate_sys_absent=False", "n_bins=20", "cc_one_half_significance_level=0.01" ] result = merging_statistics.run(args, out=out) assert len(result.bins) == 20 assert approx_equal(result.overall.d_max, 52.445602416992195) assert result.overall.n_obs == 119045, result.overall.n_obs assert approx_equal(result.bins[0].cc_anom, 0.879550520045) assert result.bins[0].cc_anom_significance is True assert approx_equal(result.bins[0].cc_anom_critical_value, 0.0873548986308) assert approx_equal(result.cc_one_half_overall, 0.931122967496) assert approx_equal(result.cc_one_half_sigma_tau_overall, 0.9280192675969664) assert approx_equal(result.bins[0].cc_one_half, 0.9969293192434535) assert approx_equal(result.bins[0].cc_one_half_sigma_tau, 0.9968045160775104) assert result.bins[0].cc_one_half_significance is True assert result.bins[0].cc_one_half_sigma_tau_significance is True assert approx_equal(result.bins[-1].cc_one_half, 0.675340867481686) assert approx_equal(result.bins[-1].cc_one_half_sigma_tau, 0.6711734115834956) assert result.bins[-1].cc_one_half_significance is True assert result.bins[-1].cc_one_half_sigma_tau_significance is True # args2 = args + ["anomalous=True"] result = merging_statistics.run(args2, out=out) app = result.overall.anom_probability_plot_all_data assert approx_equal(app.slope, 1.6747371892218779) assert approx_equal(app.intercept, 0.03443233016003127) assert app.n_pairs == 24317 assert app.expected_delta is None app = result.overall.anom_probability_plot_expected_delta assert approx_equal(app.slope, 1.2190614967160294) assert approx_equal(app.intercept, 0.031151653693628906) assert app.n_pairs == 15365 assert app.expected_delta == 0.9 d = result.overall.as_dict() for k in ("anom_probability_plot_all_data", "anom_probability_plot_expected_delta"): assert list(d[k].keys()) == [ "slope", "intercept", "n_pairs", "expected_delta" ], list(d[k].keys()) out = StringIO() result.overall.show_anomalous_probability_plot(out) assert not show_diff( out.getvalue(), """\ Anomalous probability plot (all data): slope: 1.675 intercept: 0.034 n_pairs: 24317 Anomalous probability plot (expected delta = 0.9): slope: 1.219 intercept: 0.031 n_pairs: 15365 """) args = [ hkl_file, "binning_method=counting_sorted", "eliminate_sys_absent=False", "n_bins=20", "reflections_per_bin=2000", "cc_one_half_significance_level=0.01" ] result = merging_statistics.run(args, out=out) assert len(result.bins) == 12
result = merging_statistics.run(args2, out=out) if (debug) : print out.getvalue() assert ("Resolution: 100.00 - 1.50" in out.getvalue()) assert (" 1.55 1.50 0 0 0.00 0.00 0.0 0.0 None None None 0.000 0.000""" in out.getvalue()) # these should crash args2 = list(args[:-1]) + ["high_resolution=15", "low_resolution=2.5"] try : result = merging_statistics.run(args2, out=out) except Sorry, s : pass else : raise Exception_expected args2 = list(args[:-1]) + ["high_resolution=1.5", "low_resolution=1.6"] try : result = merging_statistics.run(args2, out=out) except Sorry, s : pass else : raise Exception_expected # change space group args = [ hkl_file, "space_group=I422", "unit_cell=113.949,113.949,32.474,90,90,90", "loggraph=True", ] if (debug) : args.append("debug=True") print " ".join(args) out = StringIO()
def run(args): merging_statistics.run(args, master_params=master_params)
def exercise (debug=False) : if (not libtbx.env.has_module("phenix_regression")) : print "phenix_regression not configured, skipping." return hkl_file = libtbx.env.find_in_repositories( relative_path="phenix_regression/wizards/help_tests/test_help/p9_se_w2.sca", test=os.path.isfile) assert hkl_file is not None args = [ hkl_file, "space_group=I4", "unit_cell=113.949,113.949,32.474,90,90,90", "loggraph=True", ] if (debug) : args.append("debug=True") print " ".join(args) out = StringIO() result = merging_statistics.run(args, out=out) if (debug) : print out.getvalue() assert ("R-merge: 0.073" in out.getvalue()) assert ("R-meas: 0.079" in out.getvalue()) assert (""" 1.81 1.74 12528 2073 6.04 97.05 1449.2 5.2 0.252 0.275 0.110 0.967 0.281""" in out.getvalue()), out.getvalue() cif_block = result.as_cif_block() assert "_reflns_shell" in cif_block assert approx_equal(float(cif_block["_reflns.pdbx_Rpim_I_obs"]), result.overall.r_pim) assert approx_equal(float(cif_block["_reflns.phenix_cc_star"]), result.overall.cc_star) assert approx_equal(float(cif_block["_reflns.phenix_cc_1/2"]), result.overall.cc_one_half) assert approx_equal( flex.int(cif_block["_reflns_shell.number_measured_obs"]), [15737, 15728, 15668, 15371, 14996, 14771, 13899, 13549, 13206, 12528]) assert "_reflns_shell.phenix_cc_star" in cif_block assert "_reflns_shell.phenix_cc_1/2" in cif_block remark_200 = result.as_remark_200(wavelength=0.9792).splitlines() assert ("REMARK 200 <I/SIGMA(I)> FOR SHELL : 5.1536" in remark_200),"\n".join(remark_200) assert ("REMARK 200 WAVELENGTH OR RANGE (A) : 0.9792" in remark_200) # test resolution cutoffs args2 = list(args[:-1]) + ["high_resolution=2.5", "low_resolution=15"] out = StringIO() result = merging_statistics.run(args2, out=out) if (debug) : print out.getvalue() assert ("Resolution: 14.96 - 2.50" in out.getvalue()) # extend binning args2 = list(args[:-1]) + ["high_resolution=1.5", "low_resolution=100", "--extend_d_max_min"] out = StringIO() result = merging_statistics.run(args2, out=out) if (debug) : print out.getvalue() assert ("Resolution: 100.00 - 1.50" in out.getvalue()) assert (" 1.55 1.50 0 0 0.00 0.00 0.0 0.0 None None None 0.000 0.000""" in out.getvalue()) args2 = args + ["json.file_name=merging_stats.json", "json.indent=2"] out = StringIO() result = merging_statistics.run(args2, out=out) assert os.path.exists("merging_stats.json") with open("merging_stats.json", "rb") as f: import json d = json.load(f) d.keys() expected_keys = [ 'n_obs', 'd_star_sq_max', 'i_over_sigma_mean', 'completeness', 'cc_one_half', 'r_meas', 'd_star_sq_min', 'cc_anom', 'r_pim', 'i_mean', 'cc_one_half_critical_value', 'r_merge', 'multiplicity', 'cc_one_half_significance', 'n_uniq'] for k in expected_keys: assert k in d assert approx_equal( d['i_over_sigma_mean'], [20.66560600258035, 20.345307828448572, 18.349521604862186, 16.032340432560876, 15.111031030727927, 12.289599369298855, 10.059003053756124, 8.042714696129208, 6.297114205813105, 5.153613598686054] ) assert approx_equal( d['n_obs'], [15737, 15728, 15668, 15371, 14996, 14771, 13899, 13549, 13206, 12528] ) assert 'overall' in d assert approx_equal(d['overall']['i_mean'], 18672.03367141032) assert approx_equal(d['overall']['multiplicity'], 6.747367444449599) # these should crash args2 = list(args[:-1]) + ["high_resolution=15", "low_resolution=2.5"] try : result = merging_statistics.run(args2, out=out) except Sorry, s : pass
[15737, 15728, 15668, 15371, 14996, 14771, 13899, 13549, 13206, 12528] ) assert 'overall' in d assert approx_equal(d['overall']['i_mean'], 18672.03367141032) assert approx_equal(d['overall']['multiplicity'], 6.747367444449599) # these should crash args2 = list(args[:-1]) + ["high_resolution=15", "low_resolution=2.5"] try : result = merging_statistics.run(args2, out=out) except Sorry, s : pass else : raise Exception_expected args2 = list(args[:-1]) + ["high_resolution=1.5", "low_resolution=1.6"] try : result = merging_statistics.run(args2, out=out) except Sorry, s : pass else : raise Exception_expected # change space group args = [ hkl_file, "space_group=I422", "unit_cell=113.949,113.949,32.474,90,90,90", "loggraph=True", ] if (debug) : args.append("debug=True") print " ".join(args) out = StringIO()
def exercise(debug=False): if (not libtbx.env.has_module("phenix_regression")): print "phenix_regression not configured, skipping." return hkl_file = libtbx.env.find_in_repositories( relative_path="phenix_regression/wizards/p9_se_w2.sca", test=os.path.isfile) args = [ hkl_file, "space_group=I4", "unit_cell=113.949,113.949,32.474,90,90,90", "loggraph=True", ] if (debug): args.append("debug=True") print " ".join(args) out = StringIO() result = merging_statistics.run(args, out=out) if (debug): print out.getvalue() assert ("R-merge: 0.073" in out.getvalue()) assert ("R-meas: 0.079" in out.getvalue()) assert ( """ 1.81 1.74 12528 2073 6.04 97.05 1449.2 5.2 0.252 0.275 0.110 0.967 0.281""" in out.getvalue()), out.getvalue() cif_block = result.as_cif_block() assert "_reflns_shell" in cif_block assert approx_equal(float(cif_block["_reflns.pdbx_Rpim_I_obs"]), result.overall.r_pim) assert approx_equal(float(cif_block["_reflns.phenix_cc_star"]), result.overall.cc_star) assert approx_equal(float(cif_block["_reflns.phenix_cc_1/2"]), result.overall.cc_one_half) assert approx_equal( flex.int(cif_block["_reflns_shell.number_measured_obs"]), [15737, 15728, 15668, 15371, 14996, 14771, 13899, 13549, 13206, 12528]) assert "_reflns_shell.phenix_cc_star" in cif_block assert "_reflns_shell.phenix_cc_1/2" in cif_block remark_200 = result.as_remark_200(wavelength=0.9792).splitlines() assert ("REMARK 200 <I/SIGMA(I)> FOR SHELL : 5.1536" in remark_200), "\n".join(remark_200) assert ("REMARK 200 WAVELENGTH OR RANGE (A) : 0.9792" in remark_200) # test resolution cutoffs args2 = list(args[:-1]) + ["high_resolution=2.5", "low_resolution=15"] out = StringIO() result = merging_statistics.run(args2, out=out) if (debug): print out.getvalue() assert ("Resolution: 14.96 - 2.50" in out.getvalue()) # extend binning args2 = list(args[:-1]) + [ "high_resolution=1.5", "low_resolution=100", "--extend_d_max_min" ] out = StringIO() result = merging_statistics.run(args2, out=out) if (debug): print out.getvalue() assert ("Resolution: 100.00 - 1.50" in out.getvalue()) assert ( " 1.55 1.50 0 0 0.00 0.00 0.0 0.0 None None None 0.000 0.000" "" in out.getvalue()) # these should crash args2 = list(args[:-1]) + ["high_resolution=15", "low_resolution=2.5"] try: result = merging_statistics.run(args2, out=out) except Sorry, s: pass