def citations(xproject): lines = [] lines.append('\n') lines.append('References') lines.append('=' * len(lines[-1])) # Sort citations citations = {} for cdict in Citations.get_citations_dicts(): citations[cdict['acta']] = cdict.get('url') for citation in sorted(citations.iterkeys()): if citations[citation]: lines.append("`%s\n<%s>`_\n" % (citation, citations[citation])) else: lines.append("%s\n" % citation) return lines
def citations(xproject): lines = [] lines.append('\n') lines.append('References') lines.append('=' * len(lines[-1])) # Sort citations citations = {} for cdict in Citations.get_citations_dicts(): citations[cdict['acta']] = cdict.get('url') for citation in sorted(citations.iterkeys()): if citations[citation]: lines.append("`%s\n<%s>`_\n" % (citation, citations[citation])) else: lines.append("%s\n" % citation) return lines
def generate_xia2_html(xinfo, filename="xia2.html", params=None, args=[]): assert params is None or len(args) == 0 if params is None: from xia2.Modules.Analysis import phil_scope interp = phil_scope.command_line_argument_interpreter() params, unhandled = interp.process_and_fetch( args, custom_processor="collect_remaining") params = params.extract() xia2_txt = os.path.join(os.path.abspath(os.path.curdir), "xia2.txt") assert os.path.isfile(xia2_txt), xia2_txt with open(xia2_txt, "r") as f: xia2_output = html.escape(f.read()) styles = {} columns = [] columns.append([ "", "Wavelength (Å)", "Resolution range (Å)", "Completeness (%)", "Multiplicity", "CC-half", "I/sigma", "Rmerge(I)", # anomalous statistics "Anomalous completeness (%)", "Anomalous multiplicity", ]) individual_dataset_reports = {} for cname, xcryst in xinfo.get_crystals().items(): reflection_files = xcryst.get_scaled_merged_reflections() for wname, unmerged_mtz in reflection_files["mtz_unmerged"].items(): xwav = xcryst.get_xwavelength(wname) from xia2.Modules.MultiCrystalAnalysis import batch_phil_scope scope = phil.parse(batch_phil_scope) scaler = xcryst._scaler try: for si in scaler._sweep_information.values(): batch_params = scope.extract().batch[0] batch_params.id = si["sname"] batch_params.range = si["batches"] params.batch.append(batch_params) except AttributeError: for si in scaler._sweep_handler._sweep_information.values(): batch_params = scope.extract().batch[0] batch_params.id = si.get_sweep_name() batch_params.range = si.get_batch_range() params.batch.append(batch_params) report_path = xinfo.path.joinpath(cname, "report") report_path.mkdir(parents=True, exist_ok=True) report = Report.from_unmerged_mtz(unmerged_mtz, params, report_dir=str(report_path)) xtriage_success, xtriage_warnings, xtriage_danger = None, None, None if params.xtriage_analysis: try: ( xtriage_success, xtriage_warnings, xtriage_danger, ) = report.xtriage_report() except Exception as e: params.xtriage_analysis = False logger.debug("Exception running xtriage:") logger.debug(e, exc_info=True) ( overall_stats_table, merging_stats_table, stats_plots, ) = report.resolution_plots_and_stats() d = {} d["merging_statistics_table"] = merging_stats_table d["overall_statistics_table"] = overall_stats_table individual_dataset_reports[wname] = d json_data = {} if params.xtriage_analysis: json_data["xtriage"] = (xtriage_success + xtriage_warnings + xtriage_danger) json_data.update(stats_plots) json_data.update(report.batch_dependent_plots()) json_data.update(report.intensity_stats_plots(run_xtriage=False)) json_data.update(report.pychef_plots()) json_data.update(report.pychef_plots(n_bins=1)) from scitbx.array_family import flex max_points = 500 for g in ( "scale_rmerge_vs_batch", "completeness_vs_dose", "rcp_vs_dose", "scp_vs_dose", "rd_vs_batch_difference", ): for i, data in enumerate(json_data[g]["data"]): x = data["x"] n = len(x) if n > max_points: step = n // max_points sel = (flex.int_range(n) % step) == 0 data["x"] = list(flex.int(data["x"]).select(sel)) data["y"] = list(flex.double(data["y"]).select(sel)) resolution_graphs = OrderedDict((k + "_" + wname, json_data[k]) for k in ( "cc_one_half", "i_over_sig_i", "second_moments", "wilson_intensity_plot", "completeness", "multiplicity_vs_resolution", ) if k in json_data) if params.include_radiation_damage: batch_graphs = OrderedDict((k + "_" + wname, json_data[k]) for k in ( "scale_rmerge_vs_batch", "i_over_sig_i_vs_batch", "completeness_vs_dose", "rcp_vs_dose", "scp_vs_dose", "rd_vs_batch_difference", )) else: batch_graphs = OrderedDict((k + "_" + wname, json_data[k]) for k in ("scale_rmerge_vs_batch", "i_over_sig_i_vs_batch")) misc_graphs = OrderedDict((k, json_data[k]) for k in ( "cumulative_intensity_distribution", "l_test", "multiplicities", ) if k in json_data) for k, v in report.multiplicity_plots().items(): misc_graphs[k + "_" + wname] = {"img": v} d["resolution_graphs"] = resolution_graphs d["batch_graphs"] = batch_graphs d["misc_graphs"] = misc_graphs d["xtriage"] = { "success": xtriage_success, "warnings": xtriage_warnings, "danger": xtriage_danger, } merging_stats = report.merging_stats merging_stats_anom = report.merging_stats_anom overall = merging_stats.overall overall_anom = merging_stats_anom.overall outer_shell = merging_stats.bins[-1] outer_shell_anom = merging_stats_anom.bins[-1] column = [ wname, str(xwav.get_wavelength()), "%.2f - %.2f (%.2f - %.2f)" % (overall.d_max, overall.d_min, outer_shell.d_max, outer_shell.d_min), "%.2f (%.2f)" % (overall.completeness * 100, outer_shell.completeness * 100), f"{overall.mean_redundancy:.2f} ({outer_shell.mean_redundancy:.2f})", f"{overall.cc_one_half:.4f} ({outer_shell.cc_one_half:.4f})", "%.2f (%.2f)" % (overall.i_over_sigma_mean, outer_shell.i_over_sigma_mean), f"{overall.r_merge:.4f} ({outer_shell.r_merge:.4f})", # anomalous statistics "%.2f (%.2f)" % ( overall_anom.anom_completeness * 100, outer_shell_anom.anom_completeness * 100, ), "%.2f (%.2f)" % (overall_anom.mean_redundancy, outer_shell_anom.mean_redundancy), ] columns.append(column) table = [[c[i] for c in columns] for i in range(len(columns[0]))] from cctbx import sgtbx space_groups = xcryst.get_likely_spacegroups() space_groups = [ sgtbx.space_group_info(symbol=str(symbol)) for symbol in space_groups ] space_group = space_groups[0].symbol_and_number() alternative_space_groups = [ sg.symbol_and_number() for sg in space_groups[1:] ] unit_cell = str(report.intensities.unit_cell()) # reflection files for cname, xcryst in xinfo.get_crystals().items(): # hack to replace path to reflection files with DataFiles directory data_dir = os.path.join(os.path.abspath(os.path.curdir), "DataFiles") g = glob.glob(os.path.join(data_dir, "*")) reflection_files = xcryst.get_scaled_merged_reflections() for k, rfile in reflection_files.items(): if isinstance(rfile, str): for datafile in g: if os.path.basename(datafile) == os.path.basename(rfile): reflection_files[k] = datafile break else: for kk in rfile: for datafile in g: if os.path.basename(datafile) == os.path.basename( rfile[kk]): reflection_files[k][kk] = datafile break headers = ["Dataset", "File name"] merged_mtz = reflection_files["mtz"] mtz_files = [ headers, [ "All datasets", '<a href="%s">%s</a>' % (os.path.relpath(merged_mtz), os.path.basename(merged_mtz)), ], ] for wname, unmerged_mtz in reflection_files["mtz_unmerged"].items(): mtz_files.append([ wname, '<a href="%s">%s</a>' % (os.path.relpath(unmerged_mtz), os.path.basename(unmerged_mtz)), ]) sca_files = [headers] if "sca" in reflection_files: for wname, merged_sca in reflection_files["sca"].items(): sca_files.append([ wname, '<a href="%s">%s</a>' % (os.path.relpath(merged_sca), os.path.basename(merged_sca)), ]) unmerged_sca_files = [headers] if "sca_unmerged" in reflection_files: for wname, unmerged_sca in reflection_files["sca_unmerged"].items( ): unmerged_sca_files.append([ wname, '<a href="%s">%s</a>' % ( os.path.relpath(unmerged_sca), os.path.basename(unmerged_sca), ), ]) # other files other_files = [] other_files.append(["File name", "Description"]) for other_file, description in sorted([ ("xia2.cif", "Crystallographic information file"), ("xia2.mmcif", "Macromolecular crystallographic information file"), ("shelxt.hkl", "merged structure factors for SHELXT"), ("shelxt.ins", "SHELXT instruction file"), ] + [(fn, "XPREP input file") for fn in os.listdir(os.path.join(data_dir)) if fn.endswith(".p4p")]): if os.path.exists(os.path.join(data_dir, other_file)): other_files.append([ '<a href="DataFiles/{filename}">{filename}</a>'.format( filename=other_file), description, ]) # log files log_files_table = [] log_dir = os.path.join(os.path.abspath(os.path.curdir), "LogFiles") g = glob.glob(os.path.join(log_dir, "*.log")) for logfile in g: html_file = make_logfile_html(logfile) html_file = os.path.splitext(logfile)[0] + ".html" if os.path.exists(html_file): log_files_table.append([ os.path.basename(logfile), '<a href="%s">original</a>' % os.path.relpath(logfile), '<a href="%s">html</a>' % os.path.relpath(html_file), ]) else: log_files_table.append([ os.path.basename(logfile), '<a href="%s">original</a>' % os.path.relpath(logfile), " ", ]) references = { cdict["acta"]: cdict.get("url") for cdict in Citations.get_citations_dicts() } from jinja2 import Environment, ChoiceLoader, PackageLoader loader = ChoiceLoader([ PackageLoader("xia2", "templates"), PackageLoader("dials", "templates") ]) env = Environment(loader=loader) template = env.get_template("xia2.html") html_source = template.render( page_title="xia2 processing report", xia2_output=xia2_output, space_group=space_group, alternative_space_groups=alternative_space_groups, unit_cell=unit_cell, overall_stats_table=table, cc_half_significance_level=params.cc_half_significance_level, mtz_files=mtz_files, sca_files=sca_files, unmerged_sca_files=unmerged_sca_files, other_files=other_files, log_files_table=log_files_table, individual_dataset_reports=individual_dataset_reports, references=references, styles=styles, ) with open("%s-report.json" % os.path.splitext(filename)[0], "w") as fh: json.dump(json_data, fh, indent=2) with open(filename, "wb") as f: f.write(html_source.encode("utf-8", "xmlcharrefreplace"))
def generate_xia2_html(xinfo, filename='xia2.html', params=None, args=[]): assert params is None or len(args) == 0 if params is None: from xia2.Modules.Analysis import phil_scope interp = phil_scope.command_line_argument_interpreter() params, unhandled = interp.process_and_fetch( args, custom_processor='collect_remaining') params = params.extract() from xia2.command_line.report import xia2_report crystal = xinfo.get_crystals().values()[0] xia2_txt = os.path.join(os.path.abspath(os.path.curdir), 'xia2.txt') assert os.path.isfile(xia2_txt), xia2_txt with open(xia2_txt, 'rb') as f: xia2_output = f.read().encode('ascii', 'xmlcharrefreplace') xia2_output = cgi.escape(xia2_output) styles = {} reports = [] columns = [] columns.append([ '', u'Wavelength (Å)', u'Resolution range (Å)', 'Completeness (%)', 'Multiplicity', 'CC-half', 'I/sigma', 'Rmerge(I)', # anomalous statistics 'Anomalous completeness (%)', 'Anomalous multiplicity' ]) individual_dataset_reports = {} for cname, xcryst in xinfo.get_crystals().iteritems(): reflection_files = xcryst.get_scaled_merged_reflections() for wname, unmerged_mtz in reflection_files['mtz_unmerged'].iteritems( ): xwav = xcryst.get_xwavelength(wname) report = xia2_report(unmerged_mtz, params) reports.append(report) merging_stats = report.merging_stats merging_stats_anom = report.merging_stats_anom overall = merging_stats.overall overall_anom = merging_stats_anom.overall outer_shell = merging_stats.bins[-1] outer_shell_anom = merging_stats_anom.bins[-1] column = [ wname, str(xwav.get_wavelength()), '%.2f - %.2f (%.2f - %.2f)' % (overall.d_max, overall.d_min, outer_shell.d_max, outer_shell.d_min), '%.2f (%.2f)' % (overall.completeness * 100, outer_shell.completeness * 100), '%.2f (%.2f)' % (overall.mean_redundancy, outer_shell.mean_redundancy), '%.4f (%.4f)' % (overall.cc_one_half, outer_shell.cc_one_half), '%.2f (%.2f)' % (overall.i_over_sigma_mean, outer_shell.i_over_sigma_mean), '%.4f (%.4f)' % (overall.r_merge, outer_shell.r_merge), # anomalous statistics '%.2f (%.2f)' % (overall_anom.anom_completeness * 100, outer_shell_anom.anom_completeness * 100), '%.2f (%.2f)' % (overall_anom.mean_redundancy, outer_shell_anom.mean_redundancy), ] columns.append(column) xtriage_success, xtriage_warnings, xtriage_danger = None, None, None if params.xtriage_analysis: try: xtriage_success, xtriage_warnings, xtriage_danger = report.xtriage_report( ) except Exception as e: from xia2.Handlers.Phil import PhilIndex if PhilIndex.params.xia2.settings.small_molecule == True: print("Xtriage output not available: %s" % str(e)) else: raise d = {} d['merging_statistics_table'] = report.merging_statistics_table() d['overall_statistics_table'] = report.overall_statistics_table() individual_dataset_reports[wname] = d json_data = {} json_data.update(report.multiplicity_vs_resolution_plot()) json_data.update(report.multiplicity_histogram()) json_data.update(report.completeness_plot()) json_data.update(report.scale_rmerge_vs_batch_plot()) json_data.update(report.cc_one_half_plot()) json_data.update(report.i_over_sig_i_plot()) json_data.update(report.i_over_sig_i_vs_batch_plot()) json_data.update(report.second_moments_plot()) json_data.update(report.cumulative_intensity_distribution_plot()) json_data.update(report.l_test_plot()) json_data.update(report.wilson_plot()) json_data.update(report.pychef_plots(n_bins=1)) from scitbx.array_family import flex max_points = 500 for g in ('scale_rmerge_vs_batch', 'completeness_vs_dose', 'rcp_vs_dose', 'scp_vs_dose', 'rd_vs_batch_difference'): for i, data in enumerate(json_data[g]['data']): x = data['x'] n = len(x) if n > max_points: step = n // max_points sel = (flex.int_range(n) % step) == 0 data['x'] = list(flex.int(data['x']).select(sel)) data['y'] = list(flex.double(data['y']).select(sel)) resolution_graphs = collections.OrderedDict( (k + '_' + wname, json_data[k]) for k in ('cc_one_half', 'i_over_sig_i', 'second_moments', 'wilson_intensity_plot', 'completeness', 'multiplicity_vs_resolution') if k in json_data) if params.include_radiation_damage: batch_graphs = collections.OrderedDict( (k + '_' + wname, json_data[k]) for k in ('scale_rmerge_vs_batch', 'i_over_sig_i_vs_batch', 'completeness_vs_dose', 'rcp_vs_dose', 'scp_vs_dose', 'rd_vs_batch_difference')) else: batch_graphs = collections.OrderedDict( (k + '_' + wname, json_data[k]) for k in ('scale_rmerge_vs_batch', 'i_over_sig_i_vs_batch')) misc_graphs = collections.OrderedDict( (k + '_' + wname, json_data[k]) for k in ('cumulative_intensity_distribution', 'l_test', 'multiplicities') if k in json_data) for k, v in report.multiplicity_plots().iteritems(): misc_graphs[k + '_' + wname] = {'img': v} d['resolution_graphs'] = resolution_graphs d['batch_graphs'] = batch_graphs d['misc_graphs'] = misc_graphs d['xtriage'] = { 'success': xtriage_success, 'warnings': xtriage_warnings, 'danger': xtriage_danger } table = [[c[i] for c in columns] for i in range(len(columns[0]))] cell = xcryst.get_cell() from cctbx import sgtbx space_groups = xcryst.get_likely_spacegroups() space_groups = [ sgtbx.space_group_info(symbol=str(symbol)) for symbol in space_groups ] space_group = space_groups[0].symbol_and_number() alternative_space_groups = [ sg.symbol_and_number() for sg in space_groups[1:] ] unit_cell = str(report.intensities.unit_cell()) #twinning_score = xcryst._get_scaler()._scalr_twinning_score #twinning_conclusion = xcryst._get_scaler()._scalr_twinning_conclusion #if twinning_score is not None: #table.append(['','','']) #table.append(['Twinning score', '%.2f' %twinning_score, '']) #if twinning_conclusion is not None: #table.append(['', twinning_conclusion, '']) for row in table: for i in range(len(row)): row[i] = row[i].encode('ascii', 'xmlcharrefreplace') #from libtbx import table_utils #print table_utils.format(rows=table, has_header=True) # reflection files for cname, xcryst in xinfo.get_crystals().iteritems(): # hack to replace path to reflection files with DataFiles directory data_dir = os.path.join(os.path.abspath(os.path.curdir), 'DataFiles') g = glob.glob(os.path.join(data_dir, '*')) reflection_files = xcryst.get_scaled_merged_reflections() for k, rfile in reflection_files.iteritems(): if isinstance(rfile, basestring): for datafile in g: if os.path.basename(datafile) == os.path.basename(rfile): reflection_files[k] = datafile break else: for kk in rfile.keys(): for datafile in g: if os.path.basename(datafile) == os.path.basename( rfile[kk]): reflection_files[k][kk] = datafile break headers = ['Dataset', 'File name'] merged_mtz = reflection_files['mtz'] mtz_files = [ headers, [ 'All datasets', '<a href="%s">%s</a>' % (os.path.relpath(merged_mtz), os.path.basename(merged_mtz)) ] ] for wname, unmerged_mtz in reflection_files['mtz_unmerged'].iteritems( ): mtz_files.append([ wname, '<a href="%s">%s</a>' % (os.path.relpath(unmerged_mtz), os.path.basename(unmerged_mtz)) ]) sca_files = [headers] for wname, merged_sca in reflection_files['sca'].iteritems(): sca_files.append([ wname, '<a href="%s">%s</a>' % (os.path.relpath(merged_sca), os.path.basename(merged_sca)) ]) unmerged_sca_files = [headers] for wname, unmerged_sca in reflection_files['sca_unmerged'].iteritems( ): unmerged_sca_files.append([ wname, '<a href="%s">%s</a>' % (os.path.relpath(unmerged_sca), os.path.basename(unmerged_sca)) ]) # other files other_files = [] other_files.append(['File name', 'Description']) for other_file, description in sorted([ ('xia2.cif', 'Crystallographic information file'), ('xia2.mmcif', 'Macromolecular crystallographic information file'), ('shelxt.hkl', 'merged structure factors for SHELXT'), ('shelxt.ins', 'SHELXT instruction file'), ] + [ (fn, 'XPREP input file') for fn in os.listdir(os.path.join(data_dir)) \ if fn.endswith('.p4p') ]): if os.path.exists(os.path.join(data_dir, other_file)): other_files.append([ '<a href="DataFiles/{filename}">{filename}</a>'.format( filename=other_file), description ]) # log files log_files_table = [] log_dir = os.path.join(os.path.abspath(os.path.curdir), 'LogFiles') g = glob.glob(os.path.join(log_dir, '*.log')) for logfile in g: html_file = make_logfile_html(logfile) html_file = os.path.splitext(logfile)[0] + '.html' if os.path.exists(html_file): log_files_table.append([ os.path.basename(logfile), '<a href="%s">original</a>' % os.path.relpath(logfile), '<a href="%s">html</a>' % os.path.relpath(html_file), ]) else: log_files_table.append([ os.path.basename(logfile), '<a href="%s">original</a>' % os.path.relpath(logfile), ' ', ]) # references references = {} for cdict in Citations.get_citations_dicts(): references[cdict['acta']] = cdict.get('url') from jinja2 import Environment, ChoiceLoader, PackageLoader loader = ChoiceLoader([ PackageLoader('xia2', 'templates'), PackageLoader('dials', 'templates') ]) env = Environment(loader=loader) template = env.get_template('xia2.html') html = template.render( page_title='xia2 processing report', xia2_output=xia2_output, space_group=space_group, alternative_space_groups=alternative_space_groups, unit_cell=unit_cell, xtriage_success=xtriage_success, xtriage_warnings=xtriage_warnings, xtriage_danger=xtriage_danger, overall_stats_table=table, cc_half_significance_level=params.cc_half_significance_level, mtz_files=mtz_files, sca_files=sca_files, unmerged_sca_files=unmerged_sca_files, other_files=other_files, log_files_table=log_files_table, individual_dataset_reports=individual_dataset_reports, references=references, styles=styles) with open(filename, 'wb') as f: f.write(html.encode('ascii', 'xmlcharrefreplace'))