def load_log(self, file_name=None, processed_lines=None, update=True):
from iotbx import data_plots
if processed_lines :
self.tables = data_plots.import_ccp4i_logfile(log_lines=processed_lines)
else :
self.tables = data_plots.import_ccp4i_logfile(file_name=file_name)
if update :
self.update_interface()
def exercise_logfile():
log_file = libtbx.env.find_in_repositories(
relative_path="phenix_regression/tracking/scala.log",
test=os.path.isfile)
if log_file is not None:
tables = data_plots.import_ccp4i_logfile(log_file)
assert ([t.title for t in tables] == [
'>>> Scales v rotation range, Unspecified ',
'Analysis against Batch, Unspecified ',
'Analysis against resolution , Unspecified ',
'Analysis against intensity, Unspecified ',
'Completeness, multiplicity, Rmeas v. resolution, Unspecified ',
'Correlations within dataset, Unspecified ',
'Axial reflections, axis h, Unspecified ',
'Axial reflections, axis k, Unspecified ',
'Axial reflections, axis l, Unspecified ',
'Run 1, standard deviation v. Intensity, Unspecified '
])
loggraph3 = """\
$TABLE: Matthews coefficients:
$GRAPHS: Protein crystal computed at resolution of 2.450 :A:1,2,3,4
$$ Nmol/asym Matthews_Coeff sovlent_frac P(2.450) $$
$$
1 3.13 0.61 0.99
2 1.56 0.21 0.01
$$
"""
t3 = data_plots.table_data(None)
t3.import_loggraph(loggraph3)
g3 = t3.get_graph("Protein crystal computed at resolution of 2.450")
p3 = g3.get_plots()
def exercise_logfile () :
log_file = libtbx.env.find_in_repositories(
relative_path="phenix_regression/tracking/scala.log",
test=os.path.isfile)
if log_file is not None :
tables = data_plots.import_ccp4i_logfile(log_file)
assert ([ t.title for t in tables ] ==
['>>> Scales v rotation range, Unspecified ',
'Analysis against Batch, Unspecified ',
'Analysis against resolution , Unspecified ',
'Analysis against intensity, Unspecified ',
'Completeness, multiplicity, Rmeas v. resolution, Unspecified ',
'Correlations within dataset, Unspecified ',
'Axial reflections, axis h, Unspecified ',
'Axial reflections, axis k, Unspecified ',
'Axial reflections, axis l, Unspecified ',
'Run 1, standard deviation v. Intensity, Unspecified '])
loggraph3 = """\
$TABLE: Matthews coefficients:
$GRAPHS: Protein crystal computed at resolution of 2.450 :A:1,2,3,4
$$ Nmol/asym Matthews_Coeff sovlent_frac P(2.450) $$
$$
1 3.13 0.61 0.99
2 1.56 0.21 0.01
$$
"""
t3 = data_plots.table_data(None)
t3.import_loggraph(loggraph3)
g3 = t3.get_graph("Protein crystal computed at resolution of 2.450")
p3 = g3.get_plots()
def exercise_inline():
loggraph1 = """\
$TABLE: Resolution shell statistics
$GRAPHS
:R-free vs. resolution:A:1,3:
:FOM vs. resolution:A:1,4:
$$
1/resol^2 Nrefl R-free FOM $$
$$
0.02 2004 0.25 0.89
0.04 2084 0.23 0.88
0.06 * 0.27 nan
0.08 1949 0.28 0.75
0.1 1783 0.38 *
$$
"""
# Different newline arrangement, otherwise identical
loggraph_input = """\
$TABLE: Resolution shell statistics: $GRAPHS :R-free vs. resolution:A:1,3: :FOM vs. resolution:A:1,4:
$$
1/resol^2 Nrefl """ + """
R-free FOM $$ $$
0.02 2004 """ + """
0.25 0.89
0.04 2084 0.23
0.88
0.06 nan 0.27 nan
0.08 1949 0.28 0.75 0.1 1783 0.38 * $$
"""
t = data_plots.table_data(None)
t.import_loggraph(loggraph_input)
# print t.format_loggraph()
# print "---"
# print loggraph1
assert (len(t.data) == 4)
assert (t.data[0] == [0.02, 0.04, 0.06, 0.08, 0.10])
assert (t.data[3][4] is None)
assert (t.format_loggraph() == loggraph1), t.format_loggraph()
assert (t.export_rows()[-1] == ['0.1', '1783', '0.38', '*'])
json_t = t.export_json_table()
json_d = json.loads(json_t)
assert (json_d['rows'] == [["1/resol^2", "Nrefl", "R-free", "FOM"],
["0.02", "2004", "0.25", "0.89"],
["0.04", "2084", "0.23", "0.88"],
["0.06", "*", "0.27", "nan"],
["0.08", "1949", "0.28", "0.75"],
["0.1", "1783", "0.38", "*"]]), json_d['rows']
assert (json_d['title'] == "Resolution shell statistics"), json_d['title']
f = open("_tst_data_plots.log", "w")
f.write("\nRandom non-loggraph text\n\n")
f.write(loggraph1)
f.write("\n\n")
f.write(loggraph1)
f.close()
tables = data_plots.import_ccp4i_logfile("_tst_data_plots.log")
assert len(tables) == 2
assert tables[0].format_loggraph() == loggraph1
assert tables[0].format_loggraph() == tables[1].format_loggraph()
t2 = data_plots.table_data(
title="Resolution shell statistics",
column_labels=["1/resol^2", "Nrefl", "R-free", "FOM"],
graph_names=["R-free vs. resolution", "FOM vs. resolution"],
graph_columns=[[0, 2], [0, 3]],
data=[[0.02, 0.04, 0.06, 0.08, 0.10], [2004, 2084, None, 1949, 1783],
[0.25, 0.23, 0.27, 0.28, 0.38],
[0.89, 0.88, float('NaN'), 0.75, None]])
#print loggraph1
#print "---"
#print t2.format_loggraph()
assert t2.format_loggraph() == loggraph1
g1 = t2.get_graph("R-free vs. resolution")
g2 = t2.get_graph("FOM vs. resolution")
assert len(g1.data) == 2 and len(g2.data) == 2
p1 = g1.get_plots(fill_in_missing_y=None)
p2 = g2.get_plots(fill_in_missing_y=None)
assert len(p1) == 1 and len(p2) == 1
(plot_x, plot_y) = p1.pop()
assert len(plot_x) == 5
(plot_x, plot_y) = p2.pop()
assert len(plot_x) == 4
formatted_table = """\
-------------------------------------------------
| Resolution shell statistics |
|-----------------------------------------------|
| 1/resol^2 | Nrefl | R-free | FOM |
|-----------------------------------------------|
| 0.02 | 2004 | 0.25 | 0.89 |
| 0.04 | 2084 | 0.23 | 0.88 |
| 0.06 | * | 0.27 | nan |
| 0.08 | 1949 | 0.28 | 0.75 |
| 0.1 | 1783 | 0.38 | * |
-------------------------------------------------
"""
#print t2.format()
assert t2.format(indent=2) == formatted_table
simple_table = """\
Resolution shell statistics
1/resol^2 Nrefl R-free FOM
0.02 2004 0.25 0.89
0.04 2084 0.23 0.88
0.06 * 0.27 nan
0.08 1949 0.28 0.75
0.1 1783 0.38 *
"""
# as above, but without indentation
simpler_table = """\
Resolution shell statistics
1/resol^2 Nrefl R-free FOM
0.02 2004 0.25 0.89
0.04 2084 0.23 0.88
0.06 * 0.27 nan
0.08 1949 0.28 0.75
0.1 1783 0.38 *
"""
formatted = t2.format_simple(indent=2)
assert (formatted == simple_table), formatted
assert str(t2) == simpler_table
json_str = t2.export_json()
json_dict = json.loads(json_str)
expected_dict = {
"graph_types": ["A", "A"],
"graph_columns": [[0, 2], [0, 3]],
"title":
"Resolution shell statistics",
"column_labels": ["1/resol^2", "Nrefl", "R-free", "FOM"],
"data": [[0.02, 0.04, 0.06, 0.08, 0.1], [2004, 2084, None, 1949, 1783],
[0.25, 0.23, 0.27, 0.28, 0.38],
[0.89, 0.88, float('nan'), 0.75, None]],
"graph_names": ["R-free vs. resolution", "FOM vs. resolution"],
"x_is_inverse_d_min":
False
}
for key in expected_dict:
if key != 'data':
assert (key in json_dict), key
assert (json_dict[key] == expected_dict[key]), \
(key, json_dict[key], expected_dict[key])
assert (
'"data": [[0.02, 0.04, 0.06, 0.08, 0.1], [2004, 2084, null, 1949, 1783], [0.25, 0.23, 0.27, 0.28, 0.38], [0.89, 0.88, NaN, 0.75, null]]'
in json_str), json_str
def parse_scala (lines) :
from iotbx import data_plots
info = scaling_info("SCALA")
tables = data_plots.import_ccp4i_logfile(log_lines=lines)
d_max = None
for i, line in enumerate(lines) :
if ("Summary data for " in line) :
if (lines[i+1].startswith("</p>")) or ("<br" in line) :
continue
j = i
n_refl = None
n_refl_all = None
while (j < len(lines)) :
line = lines[j].strip()
if line.startswith("Low resolution limit") :
d_max = float(line.split()[3])
elif line.startswith("Rmerge") and (not "bin" in line) :
info.add_overall_stat("r_merge", float(line.split()[1]))
elif line.startswith("Total number of observations") :
n_refl_all = float(line.split()[4])
elif line.startswith("Total number unique") :
n_refl = float(line.split()[3])
info.set_n_refl(n_refl, n_refl_all)
elif (line.startswith("Mean((I)/sd(I))") or
line.startswith("Mean(I)/sd(I)")) :
info.add_overall_stat("i/sigma", float(line.split()[1]))
elif line.startswith("Completeness") :
info.add_overall_stat("completeness", float(line.split()[1]))
elif line.startswith("Multiplicity") :
info.add_overall_stat("multiplicity", float(line.split()[1]))
elif ("Outlier rejection" in line) or ("$$" in line) :
break
j += 1
assert (d_max is not None)
for table in tables :
if table.title.startswith("Analysis against resolution") :
d_min_by_bin = table.get_column_by_label("Dmin(A)")
bin_d_max = d_max
bins = []
for bin_d_min in d_min_by_bin :
bins.append((bin_d_max, bin_d_min))
bin_d_max = bin_d_min
info.set_bins(bins)
rmerge = table.get_column_by_label("Rmrg")
for (rmerge_bin, bin) in zip(rmerge, bins) :
info.add_bin_stat(bin, "r_merge", rmerge_bin)
try :
s2n = table.get_column_by_label("Mn(I/sd)")
except Exception :
s2n = table.get_column_by_label("Mn(I)/sd")
for (s2n_bin, bin) in zip(s2n, bins) :
info.add_bin_stat(bin, "i/sigma", s2n_bin)
elif table.title.startswith("Completeness, multiplicity, Rmeas") :
completeness = table.get_column_by_label("%poss")
for (comp_bin, bin) in zip(completeness, bins) :
info.add_bin_stat(bin, "completeness", comp_bin)
multiplicity = table.get_column_by_label("Mlplct")
for (mult_bin, bin) in zip(multiplicity, bins) :
info.add_bin_stat(bin, "multiplicity", mult_bin)
break
return info
def extract_loggraph_tables(logfile):
from iotbx import data_plots
return data_plots.import_ccp4i_logfile(file_name=logfile)
def exercise_inline () :
loggraph1 = """\
$TABLE: Resolution shell statistics
$GRAPHS
:R-free vs. resolution:A:1,3:
:FOM vs. resolution:A:1,4:
$$
1/resol^2 Nrefl R-free FOM $$
$$
0.02 2004 0.25 0.89
0.04 2084 0.23 0.88
0.06 2037 0.27 0.83
0.08 1949 0.28 0.75
0.1 1783 0.38 *
$$
"""
t = data_plots.table_data(None)
t.import_loggraph(loggraph1)
# print t.format_loggraph()
# print "---"
# print loggraph1
assert (len(t.data) == 4)
assert (t.data[0] == [0.02, 0.04, 0.06, 0.08, 0.10])
assert (t.data[3][4] is None)
assert (t.format_loggraph() == loggraph1), t.format_loggraph()
assert (t.export_rows()[-1] == ['0.1', '1783', '0.38', '*'])
assert (t.export_json_table() == """{"rows": [["1/resol^2", "Nrefl", "R-free", "FOM"], ["0.02", "2004", "0.25", "0.89"], ["0.04", "2084", "0.23", "0.88"], ["0.06", "2037", "0.27", "0.83"], ["0.08", "1949", "0.28", "0.75"], ["0.1", "1783", "0.38", "*"]], "title": "Resolution shell statistics"}""")
f = open("_tst_data_plots.log", "w")
f.write("\nRandom non-loggraph text\n\n")
f.write(loggraph1)
f.write("\n\n")
f.write(loggraph1)
f.close()
tables = data_plots.import_ccp4i_logfile("_tst_data_plots.log")
assert len(tables) == 2
assert tables[0].format_loggraph() == loggraph1
assert tables[0].format_loggraph() == tables[1].format_loggraph()
t2 = data_plots.table_data(
title = "Resolution shell statistics",
column_labels = ["1/resol^2", "Nrefl", "R-free", "FOM"],
graph_names = ["R-free vs. resolution", "FOM vs. resolution"],
graph_columns = [[0,2], [0,3]],
data = [[0.02, 0.04, 0.06, 0.08, 0.10],
[2004, 2084, 2037, 1949, 1783],
[0.25, 0.23, 0.27, 0.28, 0.38],
[0.89, 0.88, 0.83, 0.75, None]]
)
#print loggraph1
#print "---"
#print t2.format_loggraph()
assert t2.format_loggraph() == loggraph1
g1 = t2.get_graph("R-free vs. resolution")
g2 = t2.get_graph("FOM vs. resolution")
assert len(g1.data) == 2 and len(g2.data) == 2
p1 = g1.get_plots(fill_in_missing_y=None)
p2 = g2.get_plots(fill_in_missing_y=None)
assert len(p1) == 1 and len(p2) == 1
(plot_x, plot_y) = p1.pop()
assert len(plot_x) == 5
(plot_x, plot_y) = p2.pop()
assert len(plot_x) == 4
formatted_table = """\
-------------------------------------------------
| Resolution shell statistics |
|-----------------------------------------------|
| 1/resol^2 | Nrefl | R-free | FOM |
|-----------------------------------------------|
| 0.02 | 2004 | 0.25 | 0.89 |
| 0.04 | 2084 | 0.23 | 0.88 |
| 0.06 | 2037 | 0.27 | 0.83 |
| 0.08 | 1949 | 0.28 | 0.75 |
| 0.1 | 1783 | 0.38 | * |
-------------------------------------------------
"""
#print t2.format()
assert t2.format(indent=2) == formatted_table
simple_table = """\
Resolution shell statistics
1/resol^2 Nrefl R-free FOM
0.02 2004 0.25 0.89
0.04 2084 0.23 0.88
0.06 2037 0.27 0.83
0.08 1949 0.28 0.75
0.1 1783 0.38 *
"""
# as above, but without indentation
simpler_table = """\
Resolution shell statistics
1/resol^2 Nrefl R-free FOM
0.02 2004 0.25 0.89
0.04 2084 0.23 0.88
0.06 2037 0.27 0.83
0.08 1949 0.28 0.75
0.1 1783 0.38 *
"""
formatted = t2.format_simple(indent=2)
assert (formatted == simple_table), formatted
assert str(t2) == simpler_table
json_str = t2.export_json()
assert (json_str == '{"graph_types": ["A", "A"], "graph_columns": [[0, 2], [0, 3]], "title": "Resolution shell statistics", "column_labels": ["1/resol^2", "Nrefl", "R-free", "FOM"], "data": [[0.02, 0.04, 0.06, 0.08, 0.1], [2004, 2084, 2037, 1949, 1783], [0.25, 0.23, 0.27, 0.28, 0.38], [0.89, 0.88, 0.83, 0.75, null]], "graph_names": ["R-free vs. resolution", "FOM vs. resolution"], "x_is_inverse_d_min": false}')
def extract_loggraph_tables(logfile): from iotbx import data_plots return data_plots.import_ccp4i_logfile(file_name=logfile)