def report(dials_data):
data_dir = dials_data("pychef")
mtz = data_dir.join("insulin_dials_scaled_unmerged.mtz").strpath
params = phil_scope.extract()
params.batch = []
params.dose.batch = []
return Report.from_unmerged_mtz(mtz, params)
def report(dials_data, tmpdir_factory):
data_dir = dials_data("pychef")
mtz = data_dir.join("insulin_dials_scaled_unmerged.mtz").strpath
temp_path = tmpdir_factory.mktemp("test_report")
with temp_path.as_cwd():
from xia2.Modules.Analysis import (
phil_scope,
) # import creates /xia2-debug.txt dropping
from xia2.Modules.Report import Report # import creates /DEFAULT/ dropping
params = phil_scope.extract()
params.batch = []
params.dose.batch = []
yield Report.from_unmerged_mtz(mtz, params, report_dir=temp_path.strpath)
def generate_xia2_html(xinfo, filename="xia2.html", params=None, args=[]):
assert params is None or len(args) == 0
if params is None:
from xia2.Modules.Analysis import phil_scope
interp = phil_scope.command_line_argument_interpreter()
params, unhandled = interp.process_and_fetch(
args, custom_processor="collect_remaining")
params = params.extract()
xia2_txt = os.path.join(os.path.abspath(os.path.curdir), "xia2.txt")
assert os.path.isfile(xia2_txt), xia2_txt
with open(xia2_txt, "r") as f:
xia2_output = html.escape(f.read())
styles = {}
columns = []
columns.append([
"",
"Wavelength (Å)",
"Resolution range (Å)",
"Completeness (%)",
"Multiplicity",
"CC-half",
"I/sigma",
"Rmerge(I)",
# anomalous statistics
"Anomalous completeness (%)",
"Anomalous multiplicity",
])
individual_dataset_reports = {}
for cname, xcryst in xinfo.get_crystals().items():
reflection_files = xcryst.get_scaled_merged_reflections()
for wname, unmerged_mtz in reflection_files["mtz_unmerged"].items():
xwav = xcryst.get_xwavelength(wname)
from xia2.Modules.MultiCrystalAnalysis import batch_phil_scope
scope = phil.parse(batch_phil_scope)
scaler = xcryst._scaler
try:
for si in scaler._sweep_information.values():
batch_params = scope.extract().batch[0]
batch_params.id = si["sname"]
batch_params.range = si["batches"]
params.batch.append(batch_params)
except AttributeError:
for si in scaler._sweep_handler._sweep_information.values():
batch_params = scope.extract().batch[0]
batch_params.id = si.get_sweep_name()
batch_params.range = si.get_batch_range()
params.batch.append(batch_params)
report_path = xinfo.path.joinpath(cname, "report")
report_path.mkdir(parents=True, exist_ok=True)
report = Report.from_unmerged_mtz(unmerged_mtz,
params,
report_dir=str(report_path))
xtriage_success, xtriage_warnings, xtriage_danger = None, None, None
if params.xtriage_analysis:
try:
(
xtriage_success,
xtriage_warnings,
xtriage_danger,
) = report.xtriage_report()
except Exception as e:
params.xtriage_analysis = False
logger.debug("Exception running xtriage:")
logger.debug(e, exc_info=True)
(
overall_stats_table,
merging_stats_table,
stats_plots,
) = report.resolution_plots_and_stats()
d = {}
d["merging_statistics_table"] = merging_stats_table
d["overall_statistics_table"] = overall_stats_table
individual_dataset_reports[wname] = d
json_data = {}
if params.xtriage_analysis:
json_data["xtriage"] = (xtriage_success + xtriage_warnings +
xtriage_danger)
json_data.update(stats_plots)
json_data.update(report.batch_dependent_plots())
json_data.update(report.intensity_stats_plots(run_xtriage=False))
json_data.update(report.pychef_plots())
json_data.update(report.pychef_plots(n_bins=1))
from scitbx.array_family import flex
max_points = 500
for g in (
"scale_rmerge_vs_batch",
"completeness_vs_dose",
"rcp_vs_dose",
"scp_vs_dose",
"rd_vs_batch_difference",
):
for i, data in enumerate(json_data[g]["data"]):
x = data["x"]
n = len(x)
if n > max_points:
step = n // max_points
sel = (flex.int_range(n) % step) == 0
data["x"] = list(flex.int(data["x"]).select(sel))
data["y"] = list(flex.double(data["y"]).select(sel))
resolution_graphs = OrderedDict((k + "_" + wname, json_data[k])
for k in (
"cc_one_half",
"i_over_sig_i",
"second_moments",
"wilson_intensity_plot",
"completeness",
"multiplicity_vs_resolution",
) if k in json_data)
if params.include_radiation_damage:
batch_graphs = OrderedDict((k + "_" + wname, json_data[k])
for k in (
"scale_rmerge_vs_batch",
"i_over_sig_i_vs_batch",
"completeness_vs_dose",
"rcp_vs_dose",
"scp_vs_dose",
"rd_vs_batch_difference",
))
else:
batch_graphs = OrderedDict((k + "_" + wname, json_data[k])
for k in ("scale_rmerge_vs_batch",
"i_over_sig_i_vs_batch"))
misc_graphs = OrderedDict((k, json_data[k]) for k in (
"cumulative_intensity_distribution",
"l_test",
"multiplicities",
) if k in json_data)
for k, v in report.multiplicity_plots().items():
misc_graphs[k + "_" + wname] = {"img": v}
d["resolution_graphs"] = resolution_graphs
d["batch_graphs"] = batch_graphs
d["misc_graphs"] = misc_graphs
d["xtriage"] = {
"success": xtriage_success,
"warnings": xtriage_warnings,
"danger": xtriage_danger,
}
merging_stats = report.merging_stats
merging_stats_anom = report.merging_stats_anom
overall = merging_stats.overall
overall_anom = merging_stats_anom.overall
outer_shell = merging_stats.bins[-1]
outer_shell_anom = merging_stats_anom.bins[-1]
column = [
wname,
str(xwav.get_wavelength()),
"%.2f - %.2f (%.2f - %.2f)" %
(overall.d_max, overall.d_min, outer_shell.d_max,
outer_shell.d_min),
"%.2f (%.2f)" %
(overall.completeness * 100, outer_shell.completeness * 100),
f"{overall.mean_redundancy:.2f} ({outer_shell.mean_redundancy:.2f})",
f"{overall.cc_one_half:.4f} ({outer_shell.cc_one_half:.4f})",
"%.2f (%.2f)" %
(overall.i_over_sigma_mean, outer_shell.i_over_sigma_mean),
f"{overall.r_merge:.4f} ({outer_shell.r_merge:.4f})",
# anomalous statistics
"%.2f (%.2f)" % (
overall_anom.anom_completeness * 100,
outer_shell_anom.anom_completeness * 100,
),
"%.2f (%.2f)" % (overall_anom.mean_redundancy,
outer_shell_anom.mean_redundancy),
]
columns.append(column)
table = [[c[i] for c in columns] for i in range(len(columns[0]))]
from cctbx import sgtbx
space_groups = xcryst.get_likely_spacegroups()
space_groups = [
sgtbx.space_group_info(symbol=str(symbol)) for symbol in space_groups
]
space_group = space_groups[0].symbol_and_number()
alternative_space_groups = [
sg.symbol_and_number() for sg in space_groups[1:]
]
unit_cell = str(report.intensities.unit_cell())
# reflection files
for cname, xcryst in xinfo.get_crystals().items():
# hack to replace path to reflection files with DataFiles directory
data_dir = os.path.join(os.path.abspath(os.path.curdir), "DataFiles")
g = glob.glob(os.path.join(data_dir, "*"))
reflection_files = xcryst.get_scaled_merged_reflections()
for k, rfile in reflection_files.items():
if isinstance(rfile, str):
for datafile in g:
if os.path.basename(datafile) == os.path.basename(rfile):
reflection_files[k] = datafile
break
else:
for kk in rfile:
for datafile in g:
if os.path.basename(datafile) == os.path.basename(
rfile[kk]):
reflection_files[k][kk] = datafile
break
headers = ["Dataset", "File name"]
merged_mtz = reflection_files["mtz"]
mtz_files = [
headers,
[
"All datasets",
'<a href="%s">%s</a>' %
(os.path.relpath(merged_mtz), os.path.basename(merged_mtz)),
],
]
for wname, unmerged_mtz in reflection_files["mtz_unmerged"].items():
mtz_files.append([
wname,
'<a href="%s">%s</a>' % (os.path.relpath(unmerged_mtz),
os.path.basename(unmerged_mtz)),
])
sca_files = [headers]
if "sca" in reflection_files:
for wname, merged_sca in reflection_files["sca"].items():
sca_files.append([
wname,
'<a href="%s">%s</a>' % (os.path.relpath(merged_sca),
os.path.basename(merged_sca)),
])
unmerged_sca_files = [headers]
if "sca_unmerged" in reflection_files:
for wname, unmerged_sca in reflection_files["sca_unmerged"].items(
):
unmerged_sca_files.append([
wname,
'<a href="%s">%s</a>' % (
os.path.relpath(unmerged_sca),
os.path.basename(unmerged_sca),
),
])
# other files
other_files = []
other_files.append(["File name", "Description"])
for other_file, description in sorted([
("xia2.cif", "Crystallographic information file"),
("xia2.mmcif", "Macromolecular crystallographic information file"),
("shelxt.hkl", "merged structure factors for SHELXT"),
("shelxt.ins", "SHELXT instruction file"),
] + [(fn, "XPREP input file")
for fn in os.listdir(os.path.join(data_dir)) if fn.endswith(".p4p")]):
if os.path.exists(os.path.join(data_dir, other_file)):
other_files.append([
'<a href="DataFiles/{filename}">{filename}</a>'.format(
filename=other_file),
description,
])
# log files
log_files_table = []
log_dir = os.path.join(os.path.abspath(os.path.curdir), "LogFiles")
g = glob.glob(os.path.join(log_dir, "*.log"))
for logfile in g:
html_file = make_logfile_html(logfile)
html_file = os.path.splitext(logfile)[0] + ".html"
if os.path.exists(html_file):
log_files_table.append([
os.path.basename(logfile),
'<a href="%s">original</a>' % os.path.relpath(logfile),
'<a href="%s">html</a>' % os.path.relpath(html_file),
])
else:
log_files_table.append([
os.path.basename(logfile),
'<a href="%s">original</a>' % os.path.relpath(logfile),
" ",
])
references = {
cdict["acta"]: cdict.get("url")
for cdict in Citations.get_citations_dicts()
}
from jinja2 import Environment, ChoiceLoader, PackageLoader
loader = ChoiceLoader([
PackageLoader("xia2", "templates"),
PackageLoader("dials", "templates")
])
env = Environment(loader=loader)
template = env.get_template("xia2.html")
html_source = template.render(
page_title="xia2 processing report",
xia2_output=xia2_output,
space_group=space_group,
alternative_space_groups=alternative_space_groups,
unit_cell=unit_cell,
overall_stats_table=table,
cc_half_significance_level=params.cc_half_significance_level,
mtz_files=mtz_files,
sca_files=sca_files,
unmerged_sca_files=unmerged_sca_files,
other_files=other_files,
log_files_table=log_files_table,
individual_dataset_reports=individual_dataset_reports,
references=references,
styles=styles,
)
with open("%s-report.json" % os.path.splitext(filename)[0], "w") as fh:
json.dump(json_data, fh, indent=2)
with open(filename, "wb") as f:
f.write(html_source.encode("utf-8", "xmlcharrefreplace"))
def run(args):
usage = "xia2.report [options] scaled_unmerged.mtz"
parser = OptionParser(
usage=usage, phil=phil_scope, check_format=False, epilog=help_message
)
params, options, args = parser.parse_args(
show_diff_phil=True, return_unhandled=True
)
if len(args) == 0:
parser.print_help()
return
unmerged_mtz = args[0]
report = Report.from_unmerged_mtz(unmerged_mtz, params, report_dir=".")
# xtriage
xtriage_success, xtriage_warnings, xtriage_danger = None, None, None
if params.xtriage_analysis:
try:
xtriage_success, xtriage_warnings, xtriage_danger = report.xtriage_report()
except Exception as e:
params.xtriage_analysis = False
print("Exception runnning xtriage:")
print(e)
json_data = {}
if params.xtriage_analysis:
json_data["xtriage"] = xtriage_success + xtriage_warnings + xtriage_danger
(
overall_stats_table,
merging_stats_table,
stats_plots,
) = report.resolution_plots_and_stats()
json_data.update(stats_plots)
json_data.update(report.batch_dependent_plots())
json_data.update(report.intensity_stats_plots(run_xtriage=False))
json_data.update(report.pychef_plots())
resolution_graphs = OrderedDict(
(k, json_data[k])
for k in (
"cc_one_half",
"i_over_sig_i",
"second_moments",
"wilson_intensity_plot",
"completeness",
"multiplicity_vs_resolution",
)
if k in json_data
)
if params.include_radiation_damage:
batch_graphs = OrderedDict(
(k, json_data[k])
for k in (
"scale_rmerge_vs_batch",
"i_over_sig_i_vs_batch",
"completeness_vs_dose",
"rcp_vs_dose",
"scp_vs_dose",
"rd_vs_batch_difference",
)
)
else:
batch_graphs = OrderedDict(
(k, json_data[k])
for k in ("scale_rmerge_vs_batch", "i_over_sig_i_vs_batch")
)
misc_graphs = OrderedDict(
(k, json_data[k])
for k in ("cumulative_intensity_distribution", "l_test", "multiplicities")
if k in json_data
)
for k, v in report.multiplicity_plots().items():
misc_graphs[k] = {"img": v}
styles = {}
for axis in ("h", "k", "l"):
styles["multiplicity_%s" % axis] = "square-plot"
loader = ChoiceLoader(
[PackageLoader("xia2", "templates"), PackageLoader("dials", "templates")]
)
env = Environment(loader=loader)
if params.log_include:
with open(params.log_include, "rb") as fh:
log_text = fh.read().decode("utf-8")
else:
log_text = ""
template = env.get_template("report.html")
html = template.render(
page_title=params.title,
filename=os.path.abspath(unmerged_mtz),
space_group=report.intensities.space_group_info().symbol_and_number(),
unit_cell=str(report.intensities.unit_cell()),
mtz_history=[h.strip() for h in report.mtz_object.history()],
xtriage_success=xtriage_success,
xtriage_warnings=xtriage_warnings,
xtriage_danger=xtriage_danger,
overall_stats_table=overall_stats_table,
merging_stats_table=merging_stats_table,
cc_half_significance_level=params.cc_half_significance_level,
resolution_graphs=resolution_graphs,
batch_graphs=batch_graphs,
misc_graphs=misc_graphs,
styles=styles,
xia2_version=Version,
log_text=log_text,
)
with open("%s-report.json" % params.prefix, "w") as fh:
json.dump(json_data, fh, indent=params.json.indent)
with open("%s-report.html" % params.prefix, "wb") as fh:
fh.write(html.encode("utf-8", "xmlcharrefreplace"))
def report(self):
from xia2.Modules.Report import Report
return Report.from_data_manager(self._data_manager)