def exercise_twin_detection (verbose=False) :
  # simple model with translational pseudosymmetry
  # XXX one big disadvantage: no centric reflections!
  pdb_str = """\
CRYST1   12.000    8.000   12.000  90.02  89.96  90.05 P 1           1
ATOM     39  N   ASN A   6       5.514   2.664   4.856  1.00 11.99           N
ATOM     40  CA  ASN A   6       6.831   2.310   4.318  1.00 12.30           C
ATOM     41  C   ASN A   6       7.854   2.761   5.324  1.00 13.40           C
ATOM     42  O   ASN A   6       8.219   3.943   5.374  1.00 13.92           O
ATOM     43  CB  ASN A   6       7.065   3.016   2.993  1.00 12.13           C
ATOM     44  CG  ASN A   6       5.961   2.735   2.003  1.00 12.77           C
ATOM     45  OD1 ASN A   6       5.798   1.604   1.551  1.00 14.27           O
ATOM     46  ND2 ASN A   6       5.195   3.747   1.679  1.00 10.07           N
ATOM     47  N   TYR A   7       8.292   1.817   6.147  1.00 14.70           N
ATOM     48  CA  TYR A   7       9.159   2.144   7.299  1.00 15.18           C
ATOM     49  C   TYR A   7      10.603   2.331   6.885  1.00 15.91           C
ATOM     50  O   TYR A   7      11.041   1.811   5.855  1.00 15.76           O
ATOM     51  CB  TYR A   7       9.061   1.065   8.369  1.00 15.35           C
ATOM     52  CG  TYR A   7       7.665   0.929   8.902  1.00 14.45           C
ATOM     53  CD1 TYR A   7       6.771   0.021   8.327  1.00 15.68           C
ATOM     54  CD2 TYR A   7       7.210   1.756   9.920  1.00 14.80           C
ATOM     55  CE1 TYR A   7       5.480  -0.094   8.796  1.00 13.46           C
ATOM     56  CE2 TYR A   7       5.904   1.649  10.416  1.00 14.33           C
ATOM     57  CZ  TYR A   7       5.047   0.729   9.831  1.00 15.09           C
ATOM     58  OH  TYR A   7       3.766   0.589  10.291  1.00 14.39           O
ATOM     59  OXT TYR A   7      11.358   2.999   7.612  1.00 17.49           O
TER
ATOM      1  N   ASN B   6       1.414   5.113   6.019  1.00 12.99           N
ATOM      2  CA  ASN B   6       2.720   4.776   5.445  1.00 13.30           C
ATOM      3  C   ASN B   6       3.763   5.209   6.438  1.00 14.40           C
ATOM      4  O   ASN B   6       4.125   6.391   6.507  1.00 14.92           O
ATOM      5  CB  ASN B   6       2.922   5.513   4.131  1.00 13.13           C
ATOM      6  CG  ASN B   6       1.798   5.250   3.160  1.00 13.77           C
ATOM      7  OD1 ASN B   6       1.629   4.129   2.686  1.00 15.27           O
ATOM      8  ND2 ASN B   6       1.022   6.266   2.875  1.00 11.07           N
ATOM      9  N   TYR B   7       4.222   4.248   7.230  1.00 15.70           N
ATOM     10  CA  TYR B   7       5.113   4.552   8.370  1.00 16.18           C
ATOM     11  C   TYR B   7       6.547   4.754   7.929  1.00 16.91           C
ATOM     12  O   TYR B   7       6.964   4.259   6.878  1.00 16.76           O
ATOM     13  CB  TYR B   7       5.042   3.449   9.417  1.00 16.35           C
ATOM     14  CG  TYR B   7       3.659   3.296   9.977  1.00 15.45           C
ATOM     15  CD1 TYR B   7       2.756   2.398   9.402  1.00 16.68           C
ATOM     16  CD2 TYR B   7       3.224   4.098  11.023  1.00 15.80           C
ATOM     17  CE1 TYR B   7       1.476   2.267   9.896  1.00 14.46           C
ATOM     18  CE2 TYR B   7       1.929   3.975  11.545  1.00 15.33           C
ATOM     19  CZ  TYR B   7       1.063   3.065  10.959  1.00 16.09           C
ATOM     20  OH  TYR B   7      -0.207   2.910  11.443  1.00 15.39           O
ATOM     21  OXT TYR B   7       7.316   5.408   8.654  1.00 18.49           O
END
"""
  pdb_in = iotbx.pdb.hierarchy.input(pdb_string=pdb_str)
  xrs = pdb_in.input.xray_structure_simple()
  fc = abs(xrs.structure_factors(d_min=2.5).f_calc())
  fc = fc.set_observation_type_xray_amplitude()
  sigf = flex.double(fc.size(), 0.1) + (fc.data() * 0.03)
  fc = fc.customized_copy(sigmas=sigf)
  # and now add twinning
  fc_twin = fc.twin_data(twin_law='-l,-k,-h', alpha=0.4)
  fc_twin.as_mtz_dataset(column_root_label="F").mtz_object().write(
    "tmp_xtriage_twinned.mtz")
  # twin_laws
  laws = twin_analyses.twin_laws(miller_array=fc_twin)
  assert (len(laws.operators) == 7)
  delta_santoro = [ tl.delta_santoro for tl in laws.operators ]
  assert approx_equal(delta_santoro,
    [0.104655, 0.104655, 0.066599, 0.104655, 0.076113, 0.066599, 0.028542])
  delta_le_page = [ tl.delta_le_page for tl in laws.operators ]
  assert approx_equal(delta_le_page,
    [0.053839, 0.053839, 0.053839, 0.064020, 0.044706, 0.049480, 0.021221])
  delta_lebedev = [ tl.delta_lebedev for tl in laws.operators ]
  assert approx_equal(delta_lebedev,
    [0.000787, 0.000787, 0.000767, 0.000912, 0.000637, 0.000705, 0.000302])
  # TNCS
  tps = twin_analyses.detect_pseudo_translations(
    miller_array=fc_twin, out=null_out())
  assert ([tps.mod_h, tps.mod_k, tps.mod_l] == [3,2,3])
  assert approx_equal(tps.high_peak, 47.152352)
  assert approx_equal(tps.high_p_value, 0.000103)
  # L test
  fc_norm_acentric = twin_analyses.wilson_normalised_intensities(
    miller_array=fc_twin, normalise=True, out=null_out()).acentric
  ltest = twin_analyses.l_test(miller_array=fc_norm_acentric,
    parity_h=3, parity_l=3)
  assert approx_equal(ltest.mean_l, 0.498974)
  assert approx_equal(ltest.mean_l2, 0.371674)
  # we need to go through the wrapper class for a lot of these...
  out = StringIO()
  tw = twin_analyses.twin_analyses(miller_array=fc_twin, out=out)
  tw2 = twin_analyses.twin_analyses(miller_array=fc_twin, out=out,
    d_hkl_for_l_test=[3,2,3])
  # Wilson moments
  wm = tw.wilson_moments
  assert ([wm.centric_i_ratio, wm.centric_f_ratio, wm.centric_e_sq_minus_one]
          == [None, None, None])
  assert approx_equal(wm.acentric_i_ratio, 2.878383)
  assert approx_equal(wm.acentric_f_ratio, 0.717738)
  assert approx_equal(wm.acentric_abs_e_sq_minus_one, 0.808899)
  assert (not wm.centric_present)
  # Overall analyses
  out = StringIO()
  tw.show(out=out)
  assert ("significantly different than is expected" in out.getvalue())
  summary = tw.twin_summary
  assert approx_equal(summary.l_mean, 0.4989738)
  assert approx_equal(summary.patterson_height, 47.152352)
  for k, twin_r_factor in enumerate(summary.r_obs) :
    if (k == 6) : assert twin_r_factor < 0.11
    else : assert twin_r_factor > 0.3
  out2 = StringIO()
  tw.l_test.show(out=out2)
  assert ("""\
  Mean |L|   :0.499  (untwinned: 0.500; perfect twin: 0.375)
  Mean  L^2  :0.372  (untwinned: 0.333; perfect twin: 0.200)
""" in out2.getvalue()), out2.getvalue()
  out3 = StringIO()
  tw2.l_test.show(out=out3)
  assert not show_diff(out2.getvalue(), out3.getvalue())
  if (verbose) :
    print out.getvalue()
  # twin_results_interpretation object via cctbx.miller.array API extension
  # XXX I get slightly different numbers here versus running through the
  # twin_analyses call above - this seems to be caused by the resolution
  # limits passed here.  Need to confirm these with PHZ.
  result = fc_twin.analyze_intensity_statistics()
  assert approx_equal(result.maha_l, 27.580674)
  assert approx_equal(result.r_obs,
    [0.514901, 0.514901, 0.397741, 0.580353, 0.579294, 0.420914, 0.114999])
  assert approx_equal(result.h_alpha,
    [0.019980, 0.019980, 0.211788, 0.073926, 0.079920, 0.150849, 0.389610])
  assert result.has_twinning()
def exercise_twin_detection(verbose=False):
    # simple model with translational pseudosymmetry
    # XXX one big disadvantage: no centric reflections!
    pdb_str = """\
CRYST1   12.000    8.000   12.000  90.02  89.96  90.05 P 1           1
ATOM     39  N   ASN A   6       5.514   2.664   4.856  1.00 11.99           N
ATOM     40  CA  ASN A   6       6.831   2.310   4.318  1.00 12.30           C
ATOM     41  C   ASN A   6       7.854   2.761   5.324  1.00 13.40           C
ATOM     42  O   ASN A   6       8.219   3.943   5.374  1.00 13.92           O
ATOM     43  CB  ASN A   6       7.065   3.016   2.993  1.00 12.13           C
ATOM     44  CG  ASN A   6       5.961   2.735   2.003  1.00 12.77           C
ATOM     45  OD1 ASN A   6       5.798   1.604   1.551  1.00 14.27           O
ATOM     46  ND2 ASN A   6       5.195   3.747   1.679  1.00 10.07           N
ATOM     47  N   TYR A   7       8.292   1.817   6.147  1.00 14.70           N
ATOM     48  CA  TYR A   7       9.159   2.144   7.299  1.00 15.18           C
ATOM     49  C   TYR A   7      10.603   2.331   6.885  1.00 15.91           C
ATOM     50  O   TYR A   7      11.041   1.811   5.855  1.00 15.76           O
ATOM     51  CB  TYR A   7       9.061   1.065   8.369  1.00 15.35           C
ATOM     52  CG  TYR A   7       7.665   0.929   8.902  1.00 14.45           C
ATOM     53  CD1 TYR A   7       6.771   0.021   8.327  1.00 15.68           C
ATOM     54  CD2 TYR A   7       7.210   1.756   9.920  1.00 14.80           C
ATOM     55  CE1 TYR A   7       5.480  -0.094   8.796  1.00 13.46           C
ATOM     56  CE2 TYR A   7       5.904   1.649  10.416  1.00 14.33           C
ATOM     57  CZ  TYR A   7       5.047   0.729   9.831  1.00 15.09           C
ATOM     58  OH  TYR A   7       3.766   0.589  10.291  1.00 14.39           O
ATOM     59  OXT TYR A   7      11.358   2.999   7.612  1.00 17.49           O
TER
ATOM      1  N   ASN B   6       1.414   5.113   6.019  1.00 12.99           N
ATOM      2  CA  ASN B   6       2.720   4.776   5.445  1.00 13.30           C
ATOM      3  C   ASN B   6       3.763   5.209   6.438  1.00 14.40           C
ATOM      4  O   ASN B   6       4.125   6.391   6.507  1.00 14.92           O
ATOM      5  CB  ASN B   6       2.922   5.513   4.131  1.00 13.13           C
ATOM      6  CG  ASN B   6       1.798   5.250   3.160  1.00 13.77           C
ATOM      7  OD1 ASN B   6       1.629   4.129   2.686  1.00 15.27           O
ATOM      8  ND2 ASN B   6       1.022   6.266   2.875  1.00 11.07           N
ATOM      9  N   TYR B   7       4.222   4.248   7.230  1.00 15.70           N
ATOM     10  CA  TYR B   7       5.113   4.552   8.370  1.00 16.18           C
ATOM     11  C   TYR B   7       6.547   4.754   7.929  1.00 16.91           C
ATOM     12  O   TYR B   7       6.964   4.259   6.878  1.00 16.76           O
ATOM     13  CB  TYR B   7       5.042   3.449   9.417  1.00 16.35           C
ATOM     14  CG  TYR B   7       3.659   3.296   9.977  1.00 15.45           C
ATOM     15  CD1 TYR B   7       2.756   2.398   9.402  1.00 16.68           C
ATOM     16  CD2 TYR B   7       3.224   4.098  11.023  1.00 15.80           C
ATOM     17  CE1 TYR B   7       1.476   2.267   9.896  1.00 14.46           C
ATOM     18  CE2 TYR B   7       1.929   3.975  11.545  1.00 15.33           C
ATOM     19  CZ  TYR B   7       1.063   3.065  10.959  1.00 16.09           C
ATOM     20  OH  TYR B   7      -0.207   2.910  11.443  1.00 15.39           O
ATOM     21  OXT TYR B   7       7.316   5.408   8.654  1.00 18.49           O
END
"""
    pdb_in = iotbx.pdb.hierarchy.input(pdb_string=pdb_str)
    xrs = pdb_in.input.xray_structure_simple()
    fc = abs(xrs.structure_factors(d_min=2.5).f_calc())
    fc = fc.set_observation_type_xray_amplitude()
    sigf = flex.double(fc.size(), 0.1) + (fc.data() * 0.03)
    fc = fc.customized_copy(sigmas=sigf)
    # and now add twinning
    fc_twin = fc.twin_data(twin_law='-l,-k,-h', alpha=0.4)
    fc_twin.as_mtz_dataset(
        column_root_label="F").mtz_object().write("tmp_xtriage_twinned.mtz")
    # twin_laws
    laws = twin_analyses.twin_laws(miller_array=fc_twin)
    assert (len(laws.operators) == 7)
    delta_santoro = [tl.delta_santoro for tl in laws.operators]
    assert approx_equal(
        delta_santoro,
        [0.104655, 0.104655, 0.066599, 0.104655, 0.076113, 0.066599, 0.028542])
    delta_le_page = [tl.delta_le_page for tl in laws.operators]
    assert approx_equal(
        delta_le_page,
        [0.053839, 0.053839, 0.053839, 0.064020, 0.044706, 0.049480, 0.021221])
    delta_lebedev = [tl.delta_lebedev for tl in laws.operators]
    assert approx_equal(
        delta_lebedev,
        [0.000787, 0.000787, 0.000767, 0.000912, 0.000637, 0.000705, 0.000302])
    # TNCS
    tps = twin_analyses.detect_pseudo_translations(miller_array=fc_twin,
                                                   out=null_out())
    assert ([tps.mod_h, tps.mod_k, tps.mod_l] == [3, 2, 3])
    assert approx_equal(tps.high_peak, 47.152352)
    assert approx_equal(tps.high_p_value, 0.000103)
    # L test
    fc_norm_acentric = twin_analyses.wilson_normalised_intensities(
        miller_array=fc_twin, normalise=True, out=null_out()).acentric
    ltest = twin_analyses.l_test(miller_array=fc_norm_acentric,
                                 parity_h=3,
                                 parity_l=3)
    assert approx_equal(ltest.mean_l, 0.498974)
    assert approx_equal(ltest.mean_l2, 0.371674)
    # we need to go through the wrapper class for a lot of these...
    out = StringIO()
    tw = twin_analyses.twin_analyses(miller_array=fc_twin, out=out)
    tw2 = twin_analyses.twin_analyses(miller_array=fc_twin,
                                      out=out,
                                      d_hkl_for_l_test=[3, 2, 3])
    # Wilson moments
    wm = tw.wilson_moments
    assert ([
        wm.centric_i_ratio, wm.centric_f_ratio, wm.centric_e_sq_minus_one
    ] == [None, None, None])
    assert approx_equal(wm.acentric_i_ratio, 2.878383)
    assert approx_equal(wm.acentric_f_ratio, 0.717738)
    assert approx_equal(wm.acentric_abs_e_sq_minus_one, 0.808899)
    assert (not wm.centric_present)
    # Overall analyses
    out = StringIO()
    tw.show(out=out)
    assert ("significantly different than is expected" in out.getvalue())
    summary = tw.twin_summary
    assert approx_equal(summary.l_mean, 0.4989738)
    assert approx_equal(summary.patterson_height, 47.152352)
    for k, twin_r_factor in enumerate(summary.r_obs):
        if (k == 6): assert twin_r_factor < 0.11
        else: assert twin_r_factor > 0.3
    out2 = StringIO()
    tw.l_test.show(out=out2)
    assert ("""\
  Mean |L|   :0.499  (untwinned: 0.500; perfect twin: 0.375)
  Mean  L^2  :0.372  (untwinned: 0.333; perfect twin: 0.200)
""" in out2.getvalue()), out2.getvalue()
    out3 = StringIO()
    tw2.l_test.show(out=out3)
    assert not show_diff(out2.getvalue(), out3.getvalue())
    if (verbose):
        print out.getvalue()
    # twin_results_interpretation object via cctbx.miller.array API extension
    # XXX I get slightly different numbers here versus running through the
    # twin_analyses call above - this seems to be caused by the resolution
    # limits passed here.  Need to confirm these with PHZ.
    result = fc_twin.analyze_intensity_statistics()
    assert approx_equal(result.maha_l, 27.580674)
    assert approx_equal(
        result.r_obs,
        [0.514901, 0.514901, 0.397741, 0.580353, 0.579294, 0.420914, 0.114999])
    assert approx_equal(
        result.h_alpha,
        [0.019980, 0.019980, 0.211788, 0.073926, 0.079920, 0.150849, 0.389610])
    assert result.has_twinning()
Esempio n. 3
0
File: report.py Progetto: hainm/xia2
def run(args):
  from iotbx.reflection_file_reader import any_reflection_file

  from xia2.Modules.Analysis import phil_scope
  interp = phil_scope.command_line_argument_interpreter()
  params, unhandled = interp.process_and_fetch(
    args, custom_processor='collect_remaining')
  params = params.extract()
  n_bins = params.resolution_bins

  args = unhandled

  intensities = None
  batches = None
  scales = None
  dose = None

  reader = any_reflection_file(args[0])
  assert reader.file_type() == 'ccp4_mtz'
  arrays = reader.as_miller_arrays(merge_equivalents=False)
  for ma in arrays:
    if ma.info().labels == ['BATCH']:
      batches = ma
    elif ma.info().labels == ['DOSE']:
      dose = ma
    elif ma.info().labels == ['I', 'SIGI']:
      intensities = ma
    elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']:
      intensities = ma
    elif ma.info().labels == ['SCALEUSED']:
      scales = ma

  assert intensities is not None
  assert batches is not None
  mtz_object = reader.file_content()

  indices = mtz_object.extract_original_index_miller_indices()
  intensities = intensities.customized_copy(
    indices=indices, info=intensities.info())
  batches = batches.customized_copy(indices=indices, info=batches.info())

  from iotbx import merging_statistics
  merging_stats = merging_statistics.dataset_statistics(
    intensities, n_bins=n_bins)

  merging_acentric = intensities.select_acentric().merge_equivalents()
  merging_centric = intensities.select_centric().merge_equivalents()

  multiplicities_acentric = {}
  multiplicities_centric = {}

  for x in sorted(set(merging_acentric.redundancies().data())):
    multiplicities_acentric[x] = merging_acentric.redundancies().data().count(x)
  for x in sorted(set(merging_centric.redundancies().data())):
    multiplicities_centric[x] = merging_centric.redundancies().data().count(x)

  headers = [u'Resolution (Å)', 'N(obs)', 'N(unique)', 'Multiplicity', 'Completeness',
             'Mean(I)', 'Mean(I/sigma)', 'Rmerge', 'Rmeas', 'Rpim', 'CC1/2', 'CCano']
  rows = []
  for bin_stats in merging_stats.bins:
    row = ['%.2f - %.2f' %(bin_stats.d_max, bin_stats.d_min),
           bin_stats.n_obs, bin_stats.n_uniq, '%.2f' %bin_stats.mean_redundancy,
           '%.2f' %(100*bin_stats.completeness), '%.1f' %bin_stats.i_mean,
           '%.1f' %bin_stats.i_over_sigma_mean, '%.3f' %bin_stats.r_merge,
           '%.3f' %bin_stats.r_meas, '%.3f' %bin_stats.r_pim,
           '%.3f' %bin_stats.cc_one_half, '%.3f' %bin_stats.cc_anom]
    rows.append(row)

  from xia2.lib.tabulate import tabulate
  merging_stats_table_html = tabulate(rows, headers, tablefmt='html')
  merging_stats_table_html = merging_stats_table_html.replace(
    '<table>', '<table class="table table-hover table-condensed">')

  unit_cell_params = intensities.unit_cell().parameters()

  headers = ['', 'Overall', 'Low resolution', 'High resolution']

  stats = (merging_stats.overall, merging_stats.bins[0], merging_stats.bins[-1])

  rows = [
    [u'Resolution (Å)'] + [
      '%.2f - %.2f' %(s.d_max, s.d_min) for s in stats],
    ['Observations'] + ['%i' %s.n_obs for s in stats],
    ['Unique reflections'] + ['%i' %s.n_uniq for s in stats],
    ['Multiplicity'] + ['%.1f' %s.mean_redundancy for s in stats],
    ['Completeness'] + ['%.2f%%' %(s.completeness * 100) for s in stats],
    #['Mean intensity'] + ['%.1f' %s.i_mean for s in stats],
    ['Mean I/sigma(I)'] + ['%.1f' %s.i_over_sigma_mean for s in stats],
    ['Rmerge'] + ['%.3f' %s.r_merge for s in stats],
    ['Rmeas'] + ['%.3f' %s.r_meas for s in stats],
    ['Rpim'] + ['%.3f' %s.r_pim for s in stats],
    ['CC1/2'] + ['%.3f' %s.cc_one_half for s in stats],
  ]
  rows = [[u'<strong>%s</strong>' %r[0]] + r[1:] for r in rows]

  overall_stats_table_html = tabulate(rows, headers, tablefmt='html')
  overall_stats_table_html = overall_stats_table_html.replace(
    '<table>', '<table class="table table-hover table-condensed">')

  #headers = ['Crystal symmetry', '']
  #rows = [
    #[u'Unit cell: a (Å)', '%.3f' %unit_cell_params[0]],
    #[u'b (Å)', '%.3f' %unit_cell_params[1]],
    #[u'c (Å)', '%.3f' %unit_cell_params[2]],
    #[u'α (°)', '%.3f' %unit_cell_params[3]],
    #[u'β (°)', '%.3f' %unit_cell_params[4]],
    #[u'γ (°)', '%.3f' %unit_cell_params[5]],
    #['Space group', intensities.space_group_info().symbol_and_number()],
  #]

  #symmetry_table_html = tabulate(rows, headers, tablefmt='html')
  symmetry_table_html = """
  <p>
    <b>Filename:</b> %s
    <br>
    <b>Unit cell:</b> %s
    <br>
    <b>Space group:</b> %s
  </p>
""" %(os.path.abspath(reader.file_name()),
      intensities.space_group_info().symbol_and_number(),
      str(intensities.unit_cell()))

  if params.anomalous:
    intensities = intensities.as_anomalous_array()
    batches = batches.as_anomalous_array()


  from xia2.Modules.PyChef2.PyChef import remove_batch_gaps
  new_batch_data = remove_batch_gaps(batches.data())
  new_batches = batches.customized_copy(data=new_batch_data)
  sc_vs_b = scales_vs_batch(scales, new_batches)
  rmerge_vs_b = rmerge_vs_batch(intensities, new_batches)

  intensities.setup_binner(n_bins=n_bins)

  merged_intensities = intensities.merge_equivalents().array()
  from mmtbx.scaling import twin_analyses
  normalised_intensities = twin_analyses.wilson_normalised_intensities(
    miller_array=merged_intensities)
  nz_test = twin_analyses.n_z_test(
    normalised_acentric=normalised_intensities.acentric,
    normalised_centric=normalised_intensities.centric)

  from mmtbx.scaling import data_statistics
  if not intensities.space_group().is_centric():
    wilson_scaling = data_statistics.wilson_scaling(
      miller_array=merged_intensities, n_residues=200) # XXX default n_residues?

  acentric = intensities.select_acentric()
  centric = intensities.select_centric()
  if acentric.size():
    acentric.setup_binner(n_bins=n_bins)
    second_moments_acentric = acentric.second_moment_of_intensities(use_binning=True)
  if centric.size():
    centric.setup_binner(n_bins=n_bins)
    second_moments_centric = centric.second_moment_of_intensities(use_binning=True)

  d_star_sq_bins = [
    (1/bin_stats.d_min**2) for bin_stats in merging_stats.bins]
  i_over_sig_i_bins = [
    bin_stats.i_over_sigma_mean for bin_stats in merging_stats.bins]
  cc_one_half_bins = [
    bin_stats.cc_one_half for bin_stats in merging_stats.bins]
  cc_anom_bins = [
    bin_stats.cc_anom for bin_stats in merging_stats.bins]

  from xia2.Modules.PyChef2 import PyChef
  if params.chef_min_completeness:
    d_min = PyChef.resolution_limit(
      mtz_file=args[0], min_completeness=params.chef_min_completeness, n_bins=8)
    print 'Estimated d_min for CHEF analysis: %.2f' %d_min
    sel = flex.bool(intensities.size(), True)
    d_spacings = intensities.d_spacings().data()
    sel &= d_spacings >= d_min
    intensities = intensities.select(sel)
    batches = batches.select(sel)
    if dose is not None:
      dose = dose.select(sel)

  if dose is None:
    dose = PyChef.batches_to_dose(batches.data(), params.dose)
  else:
    dose = dose.data()
  pychef_stats = PyChef.Statistics(intensities, dose)

  pychef_dict = pychef_stats.to_dict()

  def d_star_sq_to_d_ticks(d_star_sq, nticks):
    from cctbx import uctbx
    d_spacings = uctbx.d_star_sq_as_d(flex.double(d_star_sq))
    min_d_star_sq = min(d_star_sq)
    dstep = (max(d_star_sq) - min_d_star_sq)/nticks
    tickvals = list(min_d_star_sq + (i*dstep) for i in range(nticks))
    ticktext = ['%.2f' %(uctbx.d_star_sq_as_d(dsq)) for dsq in tickvals]
    return tickvals, ticktext

  tickvals, ticktext = d_star_sq_to_d_ticks(d_star_sq_bins, nticks=5)
  tickvals_wilson, ticktext_wilson = d_star_sq_to_d_ticks(
    wilson_scaling.d_star_sq, nticks=5)
  second_moment_d_star_sq = []
  if acentric.size():
    second_moment_d_star_sq.extend(second_moments_acentric.binner.bin_centers(2))
  if centric.size():
    second_moment_d_star_sq.extend(second_moments_centric.binner.bin_centers(2))
  tickvals_2nd_moment, ticktext_2nd_moment = d_star_sq_to_d_ticks(
    second_moment_d_star_sq, nticks=5)

  json_data = {

    'multiplicities': {
      'data': [
        {
          'x': multiplicities_acentric.keys(),
          'y': multiplicities_acentric.values(),
          'type': 'bar',
          'name': 'Acentric',
          'opacity': 0.75,
        },
        {
          'x': multiplicities_centric.keys(),
          'y': multiplicities_centric.values(),
          'type': 'bar',
          'name': 'Centric',
          'opacity': 0.75,
        },
      ],
      'layout': {
        'title': 'Distribution of multiplicities',
        'xaxis': {'title': 'Multiplicity'},
        'yaxis': {
          'title': 'Frequency',
          #'rangemode': 'tozero'
        },
        'bargap': 0,
        'barmode': 'overlay',
      },
    },

    'scale_rmerge_vs_batch': {
      'data': [
        {
          'x': sc_vs_b.batches,
          'y': sc_vs_b.data,
          'type': 'scatter',
          'name': 'Scale',
          'opacity': 0.75,
        },
        {
          'x': rmerge_vs_b.batches,
          'y': rmerge_vs_b.data,
          'yaxis': 'y2',
          'type': 'scatter',
          'name': 'Rmerge',
          'opacity': 0.75,
        },
      ],
      'layout': {
        'title': 'Scale and Rmerge vs batch',
        'xaxis': {'title': 'N'},
        'yaxis': {
          'title': 'Scale',
          'rangemode': 'tozero'
        },
        'yaxis2': {
          'title': 'Rmerge',
          'overlaying': 'y',
          'side': 'right',
          'rangemode': 'tozero'
        }
      },
    },

    'cc_one_half': {
      'data': [
        {
          'x': d_star_sq_bins, # d_star_sq
          'y': cc_one_half_bins,
          'type': 'scatter',
          'name': 'CC-half',
        },
        ({
          'x': d_star_sq_bins, # d_star_sq
          'y': cc_anom_bins,
          'type': 'scatter',
          'name': 'CC-anom',
        } if not intensities.space_group().is_centric() else {}),
      ],
      'layout':{
        'title': 'CC-half vs resolution',
        'xaxis': {
          'title': u'Resolution (Å)',
          'tickvals': tickvals,
          'ticktext': ticktext,
        },
        'yaxis': {
          'title': 'CC-half',
          'range': [min(cc_one_half_bins + cc_anom_bins + [0]), 1]
          },
        },
    },

    'i_over_sig_i': {
      'data': [{
        'x': d_star_sq_bins, # d_star_sq
        'y': i_over_sig_i_bins,
        'type': 'scatter',
        'name': 'Scales vs batch',
      }],
      'layout': {
        'title': '<I/sig(I)> vs resolution',
        'xaxis': {
          'title': u'Resolution (Å)',
          'tickvals': tickvals,
          'ticktext': ticktext,
        },
        'yaxis': {
          'title': '<I/sig(I)>',
          'rangemode': 'tozero'
        },
      }
    },

    'second_moments': {
      'data': [
        ({
          'x': list(second_moments_acentric.binner.bin_centers(2)), # d_star_sq
          'y': second_moments_acentric.data[1:-1],
          'type': 'scatter',
          'name': '<I^2> acentric',
        } if acentric.size() else {}),
        ({
          'x': list(second_moments_centric.binner.bin_centers(2)), # d_star_sq
          'y': second_moments_centric.data[1:-1],
          'type': 'scatter',
          'name': '<I^2> centric',
          } if centric.size() else {})
      ],
      'layout': {
        'title': 'Second moment of I',
        'xaxis': {
          'title': u'Resolution (Å)',
          'tickvals': tickvals_2nd_moment,
          'ticktext': ticktext_2nd_moment,
        },
        'yaxis': {
          'title': '<I^2>',
          'rangemode': 'tozero'
        },
      }
    },

    'cumulative_intensity_distribution': {
      'data': [
        {
          'x': list(nz_test.z),
          'y': list(nz_test.ac_obs),
          'type': 'scatter',
          'name': 'Acentric observed',
          'mode': 'lines',
          'line': {
            'color': 'rgb(31, 119, 180)',
          },
        },
        {
          'x': list(nz_test.z),
          'y': list(nz_test.c_obs),
          'type': 'scatter',
          'name': 'Centric observed',
          'mode': 'lines',
          'line': {
            'color': 'rgb(255, 127, 14)',
          },
        },
        {
          'x': list(nz_test.z),
          'y': list(nz_test.ac_untwinned),
          'type': 'scatter',
          'name': 'Acentric theory',
          'mode': 'lines',
          'line': {
            'color': 'rgb(31, 119, 180)',
            'dash': 'dot',
          },
          'opacity': 0.8,
        },
        {
          'x': list(nz_test.z),
          'y': list(nz_test.c_untwinned),
          'type': 'scatter',
          'name': 'Centric theory',
          'mode': 'lines',
          'line': {
            'color': 'rgb(255, 127, 14)',
            'dash': 'dot',
          },
          'opacity': 0.8,
        },
      ],
      'layout': {
        'title': 'Cumulative intensity distribution',
        'xaxis': {'title': 'z'},
        'yaxis': {
          'title': 'P(Z <= Z)',
          'rangemode': 'tozero'
        },
      }
    },

    'wilson_intensity_plot': {
      'data': ([
        {
          'x': list(wilson_scaling.d_star_sq),
          'y': list(wilson_scaling.mean_I_obs_data),
          'type': 'scatter',
          'name': 'Observed',
        },
        {
          'x': list(wilson_scaling.d_star_sq),
          'y': list(wilson_scaling.mean_I_obs_theory),
          'type': 'scatter',
          'name': 'Expected',
        },
        {
          'x': list(wilson_scaling.d_star_sq),
          'y': list(wilson_scaling.mean_I_normalisation),
          'type': 'scatter',
          'name': 'Smoothed',
        }] if not intensities.space_group().is_centric() else []),
      'layout': {
        'title': 'Wilson intensity plot',
        'xaxis': {
          'title': u'Resolution (Å)',
          'tickvals': tickvals_wilson,
          'ticktext': ticktext_wilson,
        },
        'yaxis': {
          'type': 'log',
          'title': 'Mean(I)',
          'rangemode': 'tozero',
        },
      },
    },
  }

  json_data.update(pychef_dict)

  from dials.report import html_report
  report = html_report.html_report()

  page_header = html_report.page_header('xia2 report')
  report.add_content(page_header)

  overall_panel = html_report.panel('Overall', 'overall', show=True)
  overall_table = html_report.table_responsive(
    overall_stats_table_html, width=800)
  overall_panel.add_content(overall_table)

  merging_stats_panel = html_report.panel('Resolution shells', 'merging_stats')
  merging_stats_table = html_report.table_responsive(merging_stats_table_html)
  merging_stats_panel.add_content(merging_stats_table)

  merging_stats_panel_group = html_report.panel_group(
    [overall_panel, merging_stats_panel])
  div = html_report.div()
  div.add_content(html_report.raw_html('<h2>Merging statistics</h2>'))
  div.add_content(html_report.raw_html(symmetry_table_html))
  div.add_content(merging_stats_panel_group)
  report.add_content(div)

  resolution_plots_panel = html_report.panel('Analysis by resolution', 'resolution')
  for graph in ('cc_one_half', 'i_over_sig_i', 'second_moments',
                'wilson_intensity_plot'):
    resolution_plots_panel.add_content(html_report.plotly_graph(
      json_data[graph], graph))

  batch_plots_panel = html_report.panel('Analysis by batch', 'batch')
  for graph in ('scale_rmerge_vs_batch', 'completeness_vs_dose',
                'rcp_vs_dose', 'scp_vs_dose', 'rd_vs_batch_difference'):
    batch_plots_panel.add_content(html_report.plotly_graph(
      json_data[graph], graph))

  misc_plots_panel = html_report.panel('Miscellaneous', 'misc')
  for graph in ('multiplicities', 'cumulative_intensity_distribution'):
    misc_plots_panel.add_content(html_report.plotly_graph(
      json_data[graph], graph))

  analysis_plots_panel_group = html_report.panel_group(
    [resolution_plots_panel, batch_plots_panel, misc_plots_panel])
  div = html_report.div()
  div.add_content(html_report.raw_html('<h2>Analysis plots</h2>'))
  div.add_content(analysis_plots_panel_group)
  report.add_content(div)

  html = report.html()

  import json
  json_str = json.dumps(json_data)
  with open('xia2-report.json', 'wb') as f:
    print >> f, json_str

  with open('xia2-report.html', 'wb') as f:
    print >> f, html.encode('ascii', 'xmlcharrefreplace')

  return