def _compute_merging_stats(self): from iotbx import merging_statistics self.merging_stats = merging_statistics.dataset_statistics( self.intensities, n_bins=self.params.resolution_bins, cc_one_half_significance_level=self.params. cc_half_significance_level, eliminate_sys_absent=self.params.eliminate_sys_absent, use_internal_variance=self.params.use_internal_variance, assert_is_not_unique_set_under_symmetry=False) intensities_anom = self.intensities.as_anomalous_array() intensities_anom = intensities_anom.map_to_asu().customized_copy( info=self.intensities.info()) self.merging_stats_anom = merging_statistics.dataset_statistics( intensities_anom, n_bins=self.params.resolution_bins, anomalous=True, cc_one_half_significance_level=self.params. cc_half_significance_level, eliminate_sys_absent=self.params.eliminate_sys_absent, use_internal_variance=self.params.use_internal_variance, assert_is_not_unique_set_under_symmetry=False) self.d_star_sq_bins = [(1 / bin_stats.d_min**2) for bin_stats in self.merging_stats.bins] self.d_star_sq_tickvals, self.d_star_sq_ticktext = d_star_sq_to_d_ticks( self.d_star_sq_bins, nticks=5)
def resolution_plots_and_stats(self): self.merging_stats = merging_statistics.dataset_statistics( self.intensities, n_bins=self.params.resolution_bins, cc_one_half_significance_level=self.params.cc_half_significance_level, eliminate_sys_absent=self.params.eliminate_sys_absent, use_internal_variance=self.params.use_internal_variance, assert_is_not_unique_set_under_symmetry=False, ) intensities_anom = self.intensities.as_anomalous_array() intensities_anom = intensities_anom.map_to_asu().customized_copy( info=self.intensities.info() ) self.merging_stats_anom = merging_statistics.dataset_statistics( intensities_anom, n_bins=self.params.resolution_bins, anomalous=True, cc_one_half_significance_level=self.params.cc_half_significance_level, eliminate_sys_absent=self.params.eliminate_sys_absent, use_internal_variance=self.params.use_internal_variance, assert_is_not_unique_set_under_symmetry=False, ) is_centric = self.intensities.space_group().is_centric() plotter = ResolutionPlotsAndStats( self.merging_stats, self.merging_stats_anom, is_centric ) d = OrderedDict() d.update(plotter.make_all_plots(cc_one_half_method=self.params.cc_half_method)) overall_stats = plotter.overall_statistics_table(self.params.cc_half_method) merging_stats = plotter.merging_statistics_table(self.params.cc_half_method) return overall_stats, merging_stats, d
def run(files): assert len(files) == 2 hkl1 = xds_ascii.XDS_ASCII(files[0], sys.stdout) hkl2 = xds_ascii.XDS_ASCII(files[1], sys.stdout) hkl1_points = numpy.column_stack((hkl1.xd, hkl1.yd, hkl1.zd)) tree1 = spatial.cKDTree(hkl1_points) n_ovl, n_nonovl = 0, 0 novl_indices, novl_i, novl_sigma = flex.miller_index(), flex.double(), flex.double() for i in xrange(len(hkl2.indices)): x, y, z = hkl2.xd[i], hkl2.yd[i], hkl2.zd[i] #if z > 180: # continue dists, idxs = tree1.query((x,y,z), k=3, p=1) overlaps = [] for dist, idx in zip(dists, idxs): idx = int(idx) xo, yo, zo = hkl1.xd[idx], hkl1.yd[idx], hkl1.zd[idx] if abs(z-zo) < 2.5 and (xo-x)**2+(yo-y)**2 < 15**2: # FIXME MAGIC NUMBER! overlaps.append((dist,idx)) if len(overlaps) == 0: novl_indices.append(hkl2.indices[i]) novl_i.append(hkl2.iobs[i]) novl_sigma.append(hkl2.sigma_iobs[i]) n_nonovl += 1 else: print hkl2.indices[i], x, y, z for dist, idx in overlaps: xo, yo, zo = hkl1.xd[idx], hkl1.yd[idx], hkl1.zd[idx] print hkl1.indices[idx], xo, yo, zo print dist, idx print print n_ref = len(hkl2.indices) print "%.2f%% overlap!" % (100.*(n_ref-n_nonovl)/n_ref) novl_array = miller.array(miller_set=miller.set(crystal_symmetry=hkl2.symm, indices=novl_indices), data=novl_i, sigmas=novl_sigma) stats = dataset_statistics(novl_array, anomalous=False, sigma_filtering="xds") stats.show(out=sys.stdout) novl_array = novl_array.customized_copy(anomalous_flag=False).map_to_asu() novl_array = novl_array.eliminate_sys_absent() novl_array = novl_array.select(novl_array.sigmas() >= 0) filtr = filter_intensities_by_sigma(novl_array, "xds") hklout = os.path.splitext(os.path.basename(files[1]))[0] + "_novl.mtz" filtr.array_merged.set_observation_type_xray_intensity().as_mtz_dataset(column_root_label="IMEAN").mtz_object().write(hklout)
def __init__( self, intensities, batches, n_bins=20, d_min=None, cc_one_half_method="sigma_tau", group_size=None, ): self.intensities = intensities self.batches = batches self._cc_one_half_method = cc_one_half_method self._n_bins = n_bins unmerged_intensities = None for ma in intensities: if unmerged_intensities is None: unmerged_intensities = ma else: unmerged_intensities = unmerged_intensities.concatenate( ma, assert_is_similar_symmetry=False).set_observation_type( unmerged_intensities.observation_type()) self.binner = unmerged_intensities.eliminate_sys_absent( ).setup_binner_counting_sorted(n_bins=self._n_bins) self.merging_statistics = dataset_statistics( unmerged_intensities, n_bins=n_bins, cc_one_half_significance_level=0.01, binning_method="counting_sorted", anomalous=True, use_internal_variance=False, eliminate_sys_absent=False, cc_one_half_method=self._cc_one_half_method, assert_is_not_unique_set_under_symmetry=False, ) if self._cc_one_half_method == "sigma_tau": self.cc_overall = self.merging_statistics.cc_one_half_sigma_tau_overall else: self.cc_overall = self.merging_statistics.cc_one_half_overall self._group_size = group_size self._setup_processing_groups() self.delta_cc = self._compute_delta_ccs()
def test_ResolutionPlotsAndStats(iobs): i_obs_anom = iobs.as_anomalous_array() iobs_anom = i_obs_anom.map_to_asu().customized_copy(info=iobs.info()) n_bins = 2 result = dataset_statistics(iobs, assert_is_not_unique_set_under_symmetry=False, n_bins=n_bins) anom_result = dataset_statistics( iobs_anom, assert_is_not_unique_set_under_symmetry=False, anomalous=True, n_bins=n_bins, ) plotter = ResolutionPlotsAndStats(result, anom_result) assert plotter.d_star_sq_ticktext == [ "1.26", "1.19", "1.13", "1.08", "1.04" ] assert plotter.d_star_sq_tickvals == pytest.approx( [0.6319, 0.7055, 0.7792, 0.8528, 0.9264], 1e-4) tables = plotter.statistics_tables() assert len(tables) == 2 # overall and per resolution # test plots individually d = plotter.cc_one_half_plot() assert len(d["cc_one_half"]["data"]) == 4 assert all(len(x["x"]) == n_bins for x in d["cc_one_half"]["data"]) d = plotter.i_over_sig_i_plot() assert len(d["i_over_sig_i"]["data"]) == 1 assert len(d["i_over_sig_i"]["data"][0]["y"]) == n_bins d = plotter.r_pim_plot() assert len(d["r_pim"]["data"]) == 1 assert len(d["r_pim"]["data"][0]["y"]) == n_bins d = plotter.completeness_plot() assert len(d["completeness"]["data"]) == 2 assert len(d["completeness"]["data"][0]["y"]) == n_bins d = plotter.multiplicity_vs_resolution_plot() assert len(d["multiplicity_vs_resolution"]["data"]) == 2 assert len(d["multiplicity_vs_resolution"]["data"][0]["y"]) == n_bins # now try centric options and sigma tau for cc_one_half plotter = ResolutionPlotsAndStats(result, anom_result, is_centric=True) d = plotter.cc_one_half_plot(method="sigma_tau") assert len(d["cc_one_half"]["data"]) == 4 assert all(len(x["x"]) == n_bins for x in d["cc_one_half"]["data"][:2]) assert d["cc_one_half"]["data"][2] == {} # no anomalous plots assert d["cc_one_half"]["data"][3] == {} # no anomalous plots d = plotter.completeness_plot() assert len(d["completeness"]["data"]) == 2 assert len(d["completeness"]["data"][0]["y"]) == n_bins assert d["completeness"]["data"][1] == {} d = plotter.multiplicity_vs_resolution_plot() assert len(d["multiplicity_vs_resolution"]["data"]) == 2 assert len(d["multiplicity_vs_resolution"]["data"][0]["y"]) == n_bins assert d["multiplicity_vs_resolution"]["data"][1] == {} plots = plotter.make_all_plots() assert list(plots.keys()) == [ "cc_one_half", "i_over_sig_i", "completeness", "multiplicity_vs_resolution", "r_pim", ] for plot in plots.values(): assert plot["layout"]["xaxis"][ "ticktext"] == plotter.d_star_sq_ticktext assert plot["layout"]["xaxis"][ "tickvals"] == plotter.d_star_sq_tickvals
def __init__(self, unmerged_intensities, batches_all, n_bins=20, d_min=None, cc_one_half_method='sigma_tau', id_to_batches=None): sel = unmerged_intensities.sigmas() > 0 unmerged_intensities = unmerged_intensities.select(sel).set_info( unmerged_intensities.info()) batches_all = batches_all.select(sel) unmerged_intensities.setup_binner(n_bins=n_bins) self.unmerged_intensities = unmerged_intensities self.merged_intensities = unmerged_intensities.merge_equivalents().array() separate = separate_unmerged(unmerged_intensities, batches_all, id_to_batches=id_to_batches) self.intensities = separate.intensities self.batches = separate.batches self.run_id_to_batch_id = separate.run_id_to_batch_id from iotbx.merging_statistics import dataset_statistics self.merging_statistics = dataset_statistics( unmerged_intensities, n_bins=n_bins, cc_one_half_significance_level=0.01, binning_method='counting_sorted', anomalous=True, use_internal_variance=False, eliminate_sys_absent=False, cc_one_half_method=cc_one_half_method) if cc_one_half_method == 'sigma_tau': cc_overall = self.merging_statistics.cc_one_half_sigma_tau_overall else: cc_overall = self.merging_statistics.cc_one_half_overall self.merging_statistics.show() self.delta_cc = flex.double() for test_k in self.intensities.keys(): #print test_k indices_i = flex.miller_index() data_i = flex.double() sigmas_i = flex.double() for k, unmerged in self.intensities.iteritems(): if k == test_k: continue indices_i.extend(unmerged.indices()) data_i.extend(unmerged.data()) sigmas_i.extend(unmerged.sigmas()) unmerged_i = unmerged_intensities.customized_copy( indices=indices_i, data=data_i, sigmas=sigmas_i).set_info( unmerged_intensities.info()) unmerged_i.setup_binner_counting_sorted(n_bins=n_bins) if cc_one_half_method == 'sigma_tau': cc_bins = unmerged_i.cc_one_half_sigma_tau( use_binning=True, return_n_refl=True) else: cc_bins = unmerged_i.cc_one_half(use_binning=True, return_n_refl=True) cc_i = flex.mean_weighted( flex.double(b[0] for b in cc_bins.data[1:-1]), flex.double(b[1] for b in cc_bins.data[1:-1])) delta_cc_i = cc_i - cc_overall self.delta_cc.append(delta_cc_i)
def merging_and_model_statistics(f_obs, f_model, r_free_flags, unmerged_i_obs, n_bins=20, sigma_filtering=Auto, anomalous=False, use_internal_variance=True): """ Compute merging statistics - CC* in particular - together with measures of model quality using reciprocal-space data (R-factors and CCs). See Karplus & Diederichs 2012 for rationale. """ from iotbx import merging_statistics free_sel = r_free_flags # very important: must use original intensities for i_obs, not squared f_obs, # because French-Wilson treatment is one-way assert (unmerged_i_obs.sigmas() is not None) info = unmerged_i_obs.info() assert (info is not None) unmerged_i_obs = unmerged_i_obs.customized_copy(crystal_symmetry=f_obs) unmerged_i_obs = unmerged_i_obs.select( unmerged_i_obs.sigmas() >= 0).set_info(info) filter = merging_statistics.filter_intensities_by_sigma( array=unmerged_i_obs, sigma_filtering=sigma_filtering) i_obs = filter.array_merged unmerged_i_obs = filter.array # average Bijvoet pairs if not anomalous if (not anomalous): if (i_obs.anomalous_flag()): i_obs = i_obs.average_bijvoet_mates() if (f_obs.anomalous_flag()): f_obs = f_obs.average_bijvoet_mates() if (f_model.anomalous_flag()): f_model = f_model.average_bijvoet_mates() if (free_sel.anomalous_flag()): free_sel = free_sel.average_bijvoet_mates() # create Bijvoet pairs if an array is not anomalous else: if (not i_obs.anomalous_flag()): i_obs = i_obs.generate_bijvoet_mates() if (not f_obs.anomalous_flag()): f_obs = f_obs.generate_bijvoet_mates() if (not f_model.anomalous_flag()): f_model = f_model.generate_bijvoet_mates() if (not free_sel.anomalous_flag()): free_sel = free_sel.generate_bijvoet_mates() if (free_sel.data().count(True) == 0): raise Sorry( "R-free array does not select any reflections. To calculate " + "CC* and related statistics, a valid set of R-free flags must be used." ) work_sel = free_sel.customized_copy(data=~free_sel.data()) i_obs, f_obs = i_obs.common_sets(other=f_obs) i_obs, f_model = i_obs.common_sets(other=f_model) i_obs, work_sel = i_obs.common_sets(other=work_sel) i_obs, free_sel = i_obs.common_sets(other=free_sel) i_calc = abs(f_model).f_as_f_sq() d_max, d_min = i_calc.d_max_min() model_arrays = merging_statistics.model_based_arrays(f_obs=f_obs, i_obs=i_obs, i_calc=i_calc, work_sel=work_sel, free_sel=free_sel) return merging_statistics.dataset_statistics( i_obs=unmerged_i_obs, crystal_symmetry=i_calc, d_min=d_min, d_max=d_max, n_bins=n_bins, model_arrays=model_arrays, anomalous=anomalous, use_internal_variance=use_internal_variance, sigma_filtering=None) # no need, since it was done here
def write(self, experiments, reflections): """ Write the experiments and reflections to file """ # if mmmcif filename is auto, then choose scaled.cif or integrated.cif if self.params.mmcif.hklout in (None, Auto, "auto"): if ("intensity.scale.value" in reflections) and ("intensity.scale.variance" in reflections): filename = "scaled.cif" logger.info( "Data appears to be scaled, setting mmcif.hklout = 'scaled_unmerged.cif'" ) else: filename = "integrated.cif" logger.info( "Data appears to be unscaled, setting mmcif.hklout = 'integrated.cif'" ) # Select reflections selection = reflections.get_flags(reflections.flags.integrated, all=True) reflections = reflections.select(selection) # Filter out bad variances and other issues, but don't filter on ice rings # or alter partialities. ### Assumes you want to apply the lp and dqe corrections to sum and prf ### Do we want to combine partials? reflections = filter_reflection_table( reflections, self.params.intensity, combine_partials=False, partiality_threshold=0.0, d_min=self.params.mtz.d_min, ) # Get the cif block cif_block = iotbx.cif.model.block() # Audit trail dials_version = dials.util.version.dials_version() cif_block["_audit.creation_method"] = dials_version cif_block["_audit.creation_date"] = datetime.date.today().isoformat() cif_block["_computing.data_reduction"] = ( "%s (Winter, G. et al., 2018)" % dials_version) cif_block[ "_publ.section_references"] = "Winter, G. et al. (2018) Acta Cryst. D74, 85-97." # Hard coding X-ray cif_block["_pdbx_diffrn_data_section.id"] = "dials" cif_block["_pdbx_diffrn_data_section.type_scattering"] = "x-ray" cif_block["_pdbx_diffrn_data_section.type_merged"] = "false" cif_block["_pdbx_diffrn_data_section.type_scaled"] = str( "scale" in self.params.intensity).lower() # FIXME Haven't put in any of these bits yet # # Facility/beamline proposal tracking details # # cif_block["_pdbx_diffrn_data_section_experiment.ordinal"] = 1 # cif_block["_pdbx_diffrn_data_section_experiment.data_section_id"] = "dials" # cif_block["_pdbx_diffrn_data_section_experiment.proposal_id"] = "<PROPOSAL ID> # Facility/beamline details for this data collection # # cif_block["_pdbx_diffrn_data_section_site.data_section_id"] = 'dials' # cif_block["_pdbx_diffrn_data_section_site.facility"] = "DIAMOND" # cif_block["_pdbx_diffrn_data_section_site.beamline"] = "VMX-M" # cif_block["_pdbx_diffrn_data_section_site.collection_date"] = scan.epochs()[0] # cif_block["_pdbx_diffrn_data_section_site.detector"] = detector[0].name() # cif_block["_pdbx_diffrn_data_section_site.detector_type"] = detector[0].type() # Write the crystal information cif_loop = iotbx.cif.model.loop(header=( "_pdbx_diffrn_unmerged_cell.ordinal", "_pdbx_diffrn_unmerged_cell.crystal_id", "_pdbx_diffrn_unmerged_cell.wavelength", "_pdbx_diffrn_unmerged_cell.cell_length_a", "_pdbx_diffrn_unmerged_cell.cell_length_b", "_pdbx_diffrn_unmerged_cell.cell_length_c", "_pdbx_diffrn_unmerged_cell.cell_angle_alpha", "_pdbx_diffrn_unmerged_cell.cell_angle_beta", "_pdbx_diffrn_unmerged_cell.cell_angle_gamma", "_pdbx_diffrn_unmerged_cell.Bravais_lattice", )) crystals = experiments.crystals() crystal_to_id = {crystal: i + 1 for i, crystal in enumerate(crystals)} for i, exp in enumerate(experiments): crystal = exp.crystal crystal_id = crystal_to_id[crystal] wavelength = exp.beam.get_wavelength() a, b, c, alpha, beta, gamma = crystal.get_unit_cell().parameters() latt_type = str( bravais_types.bravais_lattice(group=crystal.get_space_group())) cif_loop.add_row((i + 1, crystal_id, wavelength, a, b, c, alpha, beta, gamma, latt_type)) cif_block.add_loop(cif_loop) # Write the scan information cif_loop = iotbx.cif.model.loop(header=( "_pdbx_diffrn_scan.scan_id", "_pdbx_diffrn_scan.crystal_id", "_pdbx_diffrn_scan.image_id_begin", "_pdbx_diffrn_scan.image_id_end", "_pdbx_diffrn_scan.scan_angle_begin", "_pdbx_diffrn_scan.scan_angle_end", )) for i, exp in enumerate(experiments): scan = exp.scan crystal_id = crystal_to_id[exp.crystal] image_range = scan.get_image_range() osc_range = scan.get_oscillation_range(deg=True) cif_loop.add_row(( i + 1, crystal_id, image_range[0], image_range[1], osc_range[0], osc_range[1], )) cif_block.add_loop(cif_loop) # Make a dict of unit_cell parameters unit_cell_parameters = {} if crystal.num_scan_points > 1: for i in range(crystal.num_scan_points): a, b, c, alpha, beta, gamma = crystal.get_unit_cell_at_scan_point( i).parameters() unit_cell_parameters[i] = (a, b, c, alpha, beta, gamma) else: unit_cell_parameters[0] = (a, b, c, alpha, beta, gamma) ### _pdbx_diffrn_image_proc has been removed from the dictionary extension. ### Keeping this section commented out as it may be added back in some ### form in future # # Write the image data # scan = experiments[0].scan # z0 = scan.get_image_range()[0] # # cif_loop = iotbx.cif.model.loop( # header=("_pdbx_diffrn_image_proc.image_id", # "_pdbx_diffrn_image_proc.crystal_id", # "_pdbx_diffrn_image_proc.image_number", # "_pdbx_diffrn_image_proc.phi_value", # "_pdbx_diffrn_image_proc.wavelength", # "_pdbx_diffrn_image_proc.cell_length_a", # "_pdbx_diffrn_image_proc.cell_length_b", # "_pdbx_diffrn_image_proc.cell_length_c", # "_pdbx_diffrn_image_proc.cell_angle_alpha", # "_pdbx_diffrn_image_proc.cell_angle_beta", # "_pdbx_diffrn_image_proc.cell_angle_gamma")) # for i in range(len(scan)): # z = z0 + i # if crystal.num_scan_points > 1: # a, b, c, alpha, beta, gamma = unit_cell_parameters[i] # else: # a, b, c, alpha, beta, gamma = unit_cell_parameters[0] # # phi is the angle at the image centre # phi = scan.get_angle_from_image_index(z + 0.5, deg=True) # cif_loop.add_row((i+1, 1, z, phi, wavelength, # a, b, c, alpha, beta, gamma)) # cif_block.add_loop(cif_loop) # Write reflection data # Required columns header = ( "_pdbx_diffrn_unmerged_refln.reflection_id", "_pdbx_diffrn_unmerged_refln.scan_id", "_pdbx_diffrn_unmerged_refln.image_id_begin", "_pdbx_diffrn_unmerged_refln.image_id_end", "_pdbx_diffrn_unmerged_refln.index_h", "_pdbx_diffrn_unmerged_refln.index_k", "_pdbx_diffrn_unmerged_refln.index_l", ) headernames = { "scales": "_pdbx_diffrn_unmerged_refln.scale_value", "intensity.scale.value": "_pdbx_diffrn_unmerged_refln.intensity_meas", "intensity.scale.sigma": "_pdbx_diffrn_unmerged_refln.intensity_sigma", "intensity.sum.value": "_pdbx_diffrn_unmerged_refln.intensity_sum", "intensity.sum.sigma": "_pdbx_diffrn_unmerged_refln.intensity_sum_sigma", "intensity.prf.value": "_pdbx_diffrn_unmerged_refln.intensity_prf", "intensity.prf.sigma": "_pdbx_diffrn_unmerged_refln.intensity_prf_sigma", "angle": "_pdbx_diffrn_unmerged_refln.scan_angle_reflection", "partiality": "_pdbx_diffrn_unmerged_refln.partiality", } variables_present = [] if "scale" in self.params.intensity: reflections["scales"] = 1.0 / reflections["inverse_scale_factor"] reflections["intensity.scale.sigma"] = ( reflections["intensity.scale.variance"]**0.5) variables_present.extend( ["scales", "intensity.scale.value", "intensity.scale.sigma"]) if "sum" in self.params.intensity: reflections["intensity.sum.sigma"] = ( reflections["intensity.sum.variance"]**0.5) variables_present.extend( ["intensity.sum.value", "intensity.sum.sigma"]) if "profile" in self.params.intensity: reflections["intensity.prf.sigma"] = ( reflections["intensity.prf.variance"]**0.5) variables_present.extend( ["intensity.prf.value", "intensity.prf.sigma"]) # Should always exist reflections["angle"] = reflections["xyzcal.mm"].parts()[2] * RAD2DEG variables_present.extend(["angle"]) if "partiality" in reflections: variables_present.extend(["partiality"]) for name in variables_present: if name in reflections: header += (headernames[name], ) if "scale" in self.params.intensity: # Write dataset_statistics - first make a miller array crystal_symmetry = cctbxcrystal.symmetry( space_group=experiments[0].crystal.get_space_group(), unit_cell=experiments[0].crystal.get_unit_cell(), ) miller_set = miller.set( crystal_symmetry=crystal_symmetry, indices=reflections["miller_index"], anomalous_flag=False, ) i_obs = miller.array(miller_set, data=reflections["intensity.scale.value"]) i_obs.set_observation_type_xray_intensity() i_obs.set_sigmas(reflections["intensity.scale.sigma"]) i_obs.set_info( miller.array_info(source="DIALS", source_type="reflection_tables")) result = dataset_statistics( i_obs=i_obs, crystal_symmetry=crystal_symmetry, use_internal_variance=False, eliminate_sys_absent=False, ) cif_block.update(result.as_cif_block()) cif_loop = iotbx.cif.model.loop(header=header) for i, r in enumerate(reflections.rows()): refl_id = i + 1 scan_id = r["id"] + 1 _, _, _, _, z0, z1 = r["bbox"] h, k, l = r["miller_index"] variable_values = tuple((r[name]) for name in variables_present) cif_loop.add_row((refl_id, scan_id, z0, z1, h, k, l) + variable_values) cif_block.add_loop(cif_loop) # Add the block self._cif["dials"] = cif_block # Print to file with open(filename, "w") as fh: self._cif.show(out=fh) # Log logger.info("Wrote reflections to %s" % filename)
def run(files): assert len(files) == 2 hkl1 = xds_ascii.XDS_ASCII(files[0], sys.stdout) hkl2 = xds_ascii.XDS_ASCII(files[1], sys.stdout) hkl1_points = numpy.column_stack((hkl1.xd, hkl1.yd, hkl1.zd)) tree1 = spatial.cKDTree(hkl1_points) n_ovl, n_nonovl = 0, 0 novl_indices, novl_i, novl_sigma = flex.miller_index(), flex.double( ), flex.double() for i in xrange(len(hkl2.indices)): x, y, z = hkl2.xd[i], hkl2.yd[i], hkl2.zd[i] #if z > 180: # continue dists, idxs = tree1.query((x, y, z), k=3, p=1) overlaps = [] for dist, idx in zip(dists, idxs): idx = int(idx) xo, yo, zo = hkl1.xd[idx], hkl1.yd[idx], hkl1.zd[idx] if abs(z - zo) < 2.5 and (xo - x)**2 + ( yo - y)**2 < 15**2: # FIXME MAGIC NUMBER! overlaps.append((dist, idx)) if len(overlaps) == 0: novl_indices.append(hkl2.indices[i]) novl_i.append(hkl2.iobs[i]) novl_sigma.append(hkl2.sigma_iobs[i]) n_nonovl += 1 else: print hkl2.indices[i], x, y, z for dist, idx in overlaps: xo, yo, zo = hkl1.xd[idx], hkl1.yd[idx], hkl1.zd[idx] print hkl1.indices[idx], xo, yo, zo print dist, idx print print n_ref = len(hkl2.indices) print "%.2f%% overlap!" % (100. * (n_ref - n_nonovl) / n_ref) novl_array = miller.array(miller_set=miller.set(crystal_symmetry=hkl2.symm, indices=novl_indices), data=novl_i, sigmas=novl_sigma) stats = dataset_statistics(novl_array, anomalous=False, sigma_filtering="xds") stats.show(out=sys.stdout) novl_array = novl_array.customized_copy(anomalous_flag=False).map_to_asu() novl_array = novl_array.eliminate_sys_absent() novl_array = novl_array.select(novl_array.sigmas() >= 0) filtr = filter_intensities_by_sigma(novl_array, "xds") hklout = os.path.splitext(os.path.basename(files[1]))[0] + "_novl.mtz" filtr.array_merged.set_observation_type_xray_intensity().as_mtz_dataset( column_root_label="IMEAN").mtz_object().write(hklout)
def make_cif_block(self, experiments, reflections): """Write the data to a cif block""" # Select reflections selection = reflections.get_flags(reflections.flags.integrated, all=True) reflections = reflections.select(selection) # Filter out bad variances and other issues, but don't filter on ice rings # or alter partialities. # Assumes you want to apply the lp and dqe corrections to sum and prf # Do we want to combine partials? reflections = filter_reflection_table( reflections, self.params.intensity, combine_partials=False, partiality_threshold=0.0, d_min=self.params.mtz.d_min, ) # Get the cif block cif_block = iotbx.cif.model.block() # Audit trail dials_version = dials.util.version.dials_version() cif_block["_audit.revision_id"] = 1 cif_block["_audit.creation_method"] = dials_version cif_block["_audit.creation_date"] = datetime.date.today().isoformat() cif_block["_entry.id"] = "DIALS" # add software loop mmcif_software_header = ( "_software.pdbx_ordinal", "_software.citation_id", "_software.name", # as defined at [1] "_software.version", "_software.type", "_software.classification", "_software.description", ) mmcif_citations_header = ( "_citation.id", "_citation.journal_abbrev", "_citation.journal_volume", "_citation.journal_issue", "_citation.page_first", "_citation.page_last", "_citation.year", "_citation.title", ) software_loop = iotbx.cif.model.loop(header=mmcif_software_header) citations_loop = iotbx.cif.model.loop(header=mmcif_citations_header) software_loop.add_row( ( 1, 1, "DIALS", dials_version, "package", "data processing", "Data processing and integration within the DIALS software package", ) ) citations_loop.add_row( ( 1, "Acta Cryst. D", 74, 2, 85, 97, 2018, "DIALS: implementation and evaluation of a new integration package", ) ) if "scale" in self.params.intensity: software_loop.add_row( ( 2, 2, "DIALS", dials_version, "program", "data scaling", "Data scaling and merging within the DIALS software package", ) ) citations_loop.add_row( ( 2, "Acta Cryst. D", 76, 4, 385, 399, 2020, "Scaling diffraction data in the DIALS software package: algorithms and new approaches for multi-crystal scaling", ) ) cif_block.add_loop(software_loop) cif_block.add_loop(citations_loop) # Hard coding X-ray if self.params.mmcif.pdb_version == "v5_next": cif_block["_pdbx_diffrn_data_section.id"] = "dials" cif_block["_pdbx_diffrn_data_section.type_scattering"] = "x-ray" cif_block["_pdbx_diffrn_data_section.type_merged"] = "false" cif_block["_pdbx_diffrn_data_section.type_scaled"] = str( "scale" in self.params.intensity ).lower() # FIXME finish metadata addition - detector and source details needed # http://mmcif.wwpdb.org/dictionaries/mmcif_pdbx_v50.dic/Categories/index.html # Add source information; # _diffrn_source.pdbx_wavelength_list = (list of wavelengths) # _diffrn_source.source = (general class of source e.g. synchrotron) # _diffrn_source.type = (specific beamline or instrument e.g DIAMOND BEAMLINE I04) wls = [] epochs = [] for exp in experiments: wls.append(round(exp.beam.get_wavelength(), 5)) epochs.append(exp.scan.get_epochs()[0]) unique_wls = set(wls) cif_block["_exptl_crystal.id"] = 1 # links to crystal_id cif_block["_diffrn.id"] = 1 # links to diffrn_id cif_block["_diffrn.crystal_id"] = 1 cif_block["_diffrn_source.diffrn_id"] = 1 cif_block["_diffrn_source.pdbx_wavelength_list"] = ", ".join( str(w) for w in unique_wls ) # Add detector information; # _diffrn_detector.detector = (general class e.g. PIXEL, PLATE etc) # _diffrn_detector.pdbx_collection_date = (Date of collection yyyy-mm-dd) # _diffrn_detector.type = (full name of detector e.g. DECTRIS PILATUS3 2M) # One date is required, so if multiple just use the first date. min_epoch = min(epochs) date_str = time.strftime("%Y-%m-%d", time.gmtime(min_epoch)) cif_block["_diffrn_detector.diffrn_id"] = 1 cif_block["_diffrn_detector.pdbx_collection_date"] = date_str # Write reflection data # Required columns header = ( "_pdbx_diffrn_unmerged_refln.reflection_id", "_pdbx_diffrn_unmerged_refln.scan_id", "_pdbx_diffrn_unmerged_refln.image_id_begin", "_pdbx_diffrn_unmerged_refln.image_id_end", "_pdbx_diffrn_unmerged_refln.index_h", "_pdbx_diffrn_unmerged_refln.index_k", "_pdbx_diffrn_unmerged_refln.index_l", ) extra_items = { "scales": ("_pdbx_diffrn_unmerged_refln.scale_value", "%5.3f"), "intensity.scale.value": ( "_pdbx_diffrn_unmerged_refln.intensity_meas", "%8.3f", ), "intensity.scale.sigma": ( "_pdbx_diffrn_unmerged_refln.intensity_sigma", "%8.3f", ), "intensity.sum.value": ( "_pdbx_diffrn_unmerged_refln.intensity_sum", "%8.3f", ), "intensity.sum.sigma": ( "_pdbx_diffrn_unmerged_refln.intensity_sum_sigma", "%8.3f", ), "intensity.prf.value": ( "_pdbx_diffrn_unmerged_refln.intensity_prf", "%8.3f", ), "intensity.prf.sigma": ( "_pdbx_diffrn_unmerged_refln.intensity_prf_sigma", "%8.3f", ), "angle": ("_pdbx_diffrn_unmerged_refln.scan_angle_reflection", "%7.4f"), "partiality": ("_pdbx_diffrn_unmerged_refln.partiality", "%7.4f"), } variables_present = [] if "scale" in self.params.intensity: reflections["scales"] = 1.0 / reflections["inverse_scale_factor"] reflections["intensity.scale.sigma"] = flex.sqrt( reflections["intensity.scale.variance"] ) variables_present.extend( ["scales", "intensity.scale.value", "intensity.scale.sigma"] ) if "sum" in self.params.intensity: reflections["intensity.sum.sigma"] = flex.sqrt( reflections["intensity.sum.variance"] ) variables_present.extend(["intensity.sum.value", "intensity.sum.sigma"]) if "profile" in self.params.intensity: reflections["intensity.prf.sigma"] = flex.sqrt( reflections["intensity.prf.variance"] ) variables_present.extend(["intensity.prf.value", "intensity.prf.sigma"]) # Should always exist reflections["angle"] = reflections["xyzcal.mm"].parts()[2] * RAD2DEG variables_present.extend(["angle"]) if "partiality" in reflections: variables_present.extend(["partiality"]) for name in variables_present: if name in reflections: header += (extra_items[name][0],) self._fmt += " " + extra_items[name][1] if "scale" in self.params.intensity: # Write dataset_statistics - first make a miller array crystal_symmetry = cctbxcrystal.symmetry( space_group=experiments[0].crystal.get_space_group(), unit_cell=experiments[0].crystal.get_unit_cell(), ) miller_set = miller.set( crystal_symmetry=crystal_symmetry, indices=reflections["miller_index"], anomalous_flag=False, ) i_obs = miller.array(miller_set, data=reflections["intensity.scale.value"]) i_obs.set_observation_type_xray_intensity() i_obs.set_sigmas(reflections["intensity.scale.sigma"]) i_obs.set_info( miller.array_info(source="DIALS", source_type="reflection_tables") ) result = dataset_statistics( i_obs=i_obs, crystal_symmetry=crystal_symmetry, use_internal_variance=False, eliminate_sys_absent=False, ) merged_block = iotbx.cif.model.block() merged_block["_reflns.pdbx_ordinal"] = 1 merged_block["_reflns.pdbx_diffrn_id"] = 1 merged_block["_reflns.entry_id"] = "DIALS" merged_data = result.as_cif_block() merged_block.update(merged_data) cif_block.update(merged_block) # Write the crystal information # if v5, thats all so return if self.params.mmcif.pdb_version == "v5": return cif_block # continue if v5_next cif_loop = iotbx.cif.model.loop( header=( "_pdbx_diffrn_unmerged_cell.ordinal", "_pdbx_diffrn_unmerged_cell.crystal_id", "_pdbx_diffrn_unmerged_cell.wavelength", "_pdbx_diffrn_unmerged_cell.cell_length_a", "_pdbx_diffrn_unmerged_cell.cell_length_b", "_pdbx_diffrn_unmerged_cell.cell_length_c", "_pdbx_diffrn_unmerged_cell.cell_angle_alpha", "_pdbx_diffrn_unmerged_cell.cell_angle_beta", "_pdbx_diffrn_unmerged_cell.cell_angle_gamma", "_pdbx_diffrn_unmerged_cell.Bravais_lattice", ) ) crystals = experiments.crystals() crystal_to_id = {crystal: i + 1 for i, crystal in enumerate(crystals)} for i, exp in enumerate(experiments): crystal = exp.crystal crystal_id = crystal_to_id[crystal] wavelength = exp.beam.get_wavelength() a, b, c, alpha, beta, gamma = crystal.get_unit_cell().parameters() latt_type = str( bravais_types.bravais_lattice(group=crystal.get_space_group()) ) cif_loop.add_row( (i + 1, crystal_id, wavelength, a, b, c, alpha, beta, gamma, latt_type) ) cif_block.add_loop(cif_loop) # Write the scan information cif_loop = iotbx.cif.model.loop( header=( "_pdbx_diffrn_scan.scan_id", "_pdbx_diffrn_scan.crystal_id", "_pdbx_diffrn_scan.image_id_begin", "_pdbx_diffrn_scan.image_id_end", "_pdbx_diffrn_scan.scan_angle_begin", "_pdbx_diffrn_scan.scan_angle_end", ) ) expid_to_scan_id = {exp.identifier: i + 1 for i, exp in enumerate(experiments)} for i, exp in enumerate(experiments): scan = exp.scan crystal_id = crystal_to_id[exp.crystal] image_range = scan.get_image_range() osc_range = scan.get_oscillation_range(deg=True) cif_loop.add_row( ( i + 1, crystal_id, image_range[0], image_range[1], osc_range[0], osc_range[1], ) ) cif_block.add_loop(cif_loop) _, _, _, _, z0, z1 = reflections["bbox"].parts() h, k, l = [ hkl.iround() for hkl in reflections["miller_index"].as_vec3_double().parts() ] # make scan id consistent with header as defined above scan_id = flex.int(reflections.size(), 0) for id_ in reflections.experiment_identifiers().keys(): expid = reflections.experiment_identifiers()[id_] sel = reflections["id"] == id_ scan_id.set_selected(sel, expid_to_scan_id[expid]) loop_values = [ flex.size_t_range(1, len(reflections) + 1), scan_id, z0, z1, h, k, l, ] + [reflections[name] for name in variables_present] cif_loop = iotbx.cif.model.loop(data=dict(zip(header, loop_values))) cif_block.add_loop(cif_loop) return cif_block
def merging_and_model_statistics ( f_obs, f_model, r_free_flags, unmerged_i_obs, n_bins=20, sigma_filtering=Auto, anomalous=False, use_internal_variance=True) : """ Compute merging statistics - CC* in particular - together with measures of model quality using reciprocal-space data (R-factors and CCs). See Karplus & Diederichs 2012 for rationale. """ from iotbx import merging_statistics free_sel = r_free_flags # very important: must use original intensities for i_obs, not squared f_obs, # because French-Wilson treatment is one-way assert (unmerged_i_obs.sigmas() is not None) info = unmerged_i_obs.info() assert (info is not None) unmerged_i_obs = unmerged_i_obs.customized_copy(crystal_symmetry=f_obs) unmerged_i_obs = unmerged_i_obs.select( unmerged_i_obs.sigmas() >= 0).set_info(info) filter = merging_statistics.filter_intensities_by_sigma( array=unmerged_i_obs, sigma_filtering=sigma_filtering) i_obs = filter.array_merged unmerged_i_obs = filter.array # average Bijvoet pairs if not anomalous if (not anomalous): if (i_obs.anomalous_flag()): i_obs = i_obs.average_bijvoet_mates() if (f_obs.anomalous_flag()): f_obs = f_obs.average_bijvoet_mates() if (f_model.anomalous_flag()): f_model = f_model.average_bijvoet_mates() if (free_sel.anomalous_flag()): free_sel = free_sel.average_bijvoet_mates() # create Bijvoet pairs if an array is not anomalous else: if (not i_obs.anomalous_flag()): i_obs = i_obs.generate_bijvoet_mates() if (not f_obs.anomalous_flag()): f_obs = f_obs.generate_bijvoet_mates() if (not f_model.anomalous_flag()): f_model = f_model.generate_bijvoet_mates() if (not free_sel.anomalous_flag()): free_sel = free_sel.generate_bijvoet_mates() if (free_sel.data().count(True) == 0) : raise Sorry("R-free array does not select any reflections. To calculate "+ "CC* and related statistics, a valid set of R-free flags must be used.") work_sel = free_sel.customized_copy(data=~free_sel.data()) i_obs, f_model = i_obs.common_sets(other=f_model) i_obs, f_obs = i_obs.common_sets(other=f_obs) i_obs, work_sel = i_obs.common_sets(other=work_sel) i_obs, free_sel = i_obs.common_sets(other=free_sel) i_calc = abs(f_model).f_as_f_sq() d_max, d_min = i_calc.d_max_min() model_arrays = merging_statistics.model_based_arrays( f_obs=f_obs, i_obs=i_obs, i_calc=i_calc, work_sel=work_sel, free_sel=free_sel) return merging_statistics.dataset_statistics( i_obs=unmerged_i_obs, crystal_symmetry=i_calc, d_min=d_min, d_max=d_max, n_bins=n_bins, model_arrays=model_arrays, anomalous=anomalous, use_internal_variance=use_internal_variance, sigma_filtering=None) # no need, since it was done here
def write(self, experiments, reflections): """ Write the experiments and reflections to file """ # if mmcif filename is auto, then choose scaled.cif or integrated.cif if self.params.mmcif.hklout in (None, Auto, "auto"): if ("intensity.scale.value" in reflections) and ("intensity.scale.variance" in reflections): filename = "scaled.cif" logger.info( "Data appears to be scaled, setting mmcif.hklout = 'scaled.cif'" ) else: filename = "integrated.cif" logger.info( "Data appears to be unscaled, setting mmcif.hklout = 'integrated.cif'" ) else: filename = self.params.mmcif.hklout # Select reflections selection = reflections.get_flags(reflections.flags.integrated, all=True) reflections = reflections.select(selection) # Filter out bad variances and other issues, but don't filter on ice rings # or alter partialities. # Assumes you want to apply the lp and dqe corrections to sum and prf # Do we want to combine partials? reflections = filter_reflection_table( reflections, self.params.intensity, combine_partials=False, partiality_threshold=0.0, d_min=self.params.mtz.d_min, ) # Get the cif block cif_block = iotbx.cif.model.block() # Audit trail dials_version = dials.util.version.dials_version() cif_block["_audit.creation_method"] = dials_version cif_block["_audit.creation_date"] = datetime.date.today().isoformat() cif_block["_computing.data_reduction"] = ( "%s (Winter, G. et al., 2018)" % dials_version) cif_block[ "_publ.section_references"] = "Winter, G. et al. (2018) Acta Cryst. D74, 85-97." if "scale" in self.params.intensity: cif_block[ "_publ.section_references"] += "\nBeilsten-Edmands, J. et al. (2020) Acta Cryst. D76, 385-399." # Hard coding X-ray cif_block["_pdbx_diffrn_data_section.id"] = "dials" cif_block["_pdbx_diffrn_data_section.type_scattering"] = "x-ray" cif_block["_pdbx_diffrn_data_section.type_merged"] = "false" cif_block["_pdbx_diffrn_data_section.type_scaled"] = str( "scale" in self.params.intensity).lower() # FIXME finish metadata addition - detector and source details needed # http://mmcif.wwpdb.org/dictionaries/mmcif_pdbx_v50.dic/Categories/index.html # Add source information; # _diffrn_source.pdbx_wavelength_list = (list of wavelengths) # _diffrn_source.source = (general class of source e.g. synchrotron) # _diffrn_source.type = (specific beamline or instrument e.g DIAMOND BEAMLINE I04) wls = [] epochs = [] for exp in experiments: wls.append(round(exp.beam.get_wavelength(), 5)) epochs.append(exp.scan.get_epochs()[0]) unique_wls = set(wls) cif_block["_diffrn_source.pdbx_wavelength_list"] = ", ".join( str(w) for w in unique_wls) # Add detector information; # _diffrn_detector.detector = (general class e.g. PIXEL, PLATE etc) # _diffrn_detector.pdbx_collection_date = (Date of collection yyyy-mm-dd) # _diffrn_detector.type = (full name of detector e.g. DECTRIS PILATUS3 2M) # One date is required, so if multiple just use the first date. min_epoch = min(epochs) date_str = time.strftime("%Y-%m-%d", time.gmtime(min_epoch)) cif_block["_diffrn_detector.pdbx_collection_date"] = date_str # Write the crystal information cif_loop = iotbx.cif.model.loop(header=( "_pdbx_diffrn_unmerged_cell.ordinal", "_pdbx_diffrn_unmerged_cell.crystal_id", "_pdbx_diffrn_unmerged_cell.wavelength", "_pdbx_diffrn_unmerged_cell.cell_length_a", "_pdbx_diffrn_unmerged_cell.cell_length_b", "_pdbx_diffrn_unmerged_cell.cell_length_c", "_pdbx_diffrn_unmerged_cell.cell_angle_alpha", "_pdbx_diffrn_unmerged_cell.cell_angle_beta", "_pdbx_diffrn_unmerged_cell.cell_angle_gamma", "_pdbx_diffrn_unmerged_cell.Bravais_lattice", )) crystals = experiments.crystals() crystal_to_id = {crystal: i + 1 for i, crystal in enumerate(crystals)} for i, exp in enumerate(experiments): crystal = exp.crystal crystal_id = crystal_to_id[crystal] wavelength = exp.beam.get_wavelength() a, b, c, alpha, beta, gamma = crystal.get_unit_cell().parameters() latt_type = str( bravais_types.bravais_lattice(group=crystal.get_space_group())) cif_loop.add_row((i + 1, crystal_id, wavelength, a, b, c, alpha, beta, gamma, latt_type)) cif_block.add_loop(cif_loop) # Write the scan information cif_loop = iotbx.cif.model.loop(header=( "_pdbx_diffrn_scan.scan_id", "_pdbx_diffrn_scan.crystal_id", "_pdbx_diffrn_scan.image_id_begin", "_pdbx_diffrn_scan.image_id_end", "_pdbx_diffrn_scan.scan_angle_begin", "_pdbx_diffrn_scan.scan_angle_end", )) for i, exp in enumerate(experiments): scan = exp.scan crystal_id = crystal_to_id[exp.crystal] image_range = scan.get_image_range() osc_range = scan.get_oscillation_range(deg=True) cif_loop.add_row(( i + 1, crystal_id, image_range[0], image_range[1], osc_range[0], osc_range[1], )) cif_block.add_loop(cif_loop) # Make a dict of unit_cell parameters unit_cell_parameters = {} if crystal.num_scan_points > 1: for i in range(crystal.num_scan_points): a, b, c, alpha, beta, gamma = crystal.get_unit_cell_at_scan_point( i).parameters() unit_cell_parameters[i] = (a, b, c, alpha, beta, gamma) else: unit_cell_parameters[0] = (a, b, c, alpha, beta, gamma) ### _pdbx_diffrn_image_proc has been removed from the dictionary extension. ### Keeping this section commented out as it may be added back in some ### form in future # # Write the image data # scan = experiments[0].scan # z0 = scan.get_image_range()[0] # # cif_loop = iotbx.cif.model.loop( # header=("_pdbx_diffrn_image_proc.image_id", # "_pdbx_diffrn_image_proc.crystal_id", # "_pdbx_diffrn_image_proc.image_number", # "_pdbx_diffrn_image_proc.phi_value", # "_pdbx_diffrn_image_proc.wavelength", # "_pdbx_diffrn_image_proc.cell_length_a", # "_pdbx_diffrn_image_proc.cell_length_b", # "_pdbx_diffrn_image_proc.cell_length_c", # "_pdbx_diffrn_image_proc.cell_angle_alpha", # "_pdbx_diffrn_image_proc.cell_angle_beta", # "_pdbx_diffrn_image_proc.cell_angle_gamma")) # for i in range(len(scan)): # z = z0 + i # if crystal.num_scan_points > 1: # a, b, c, alpha, beta, gamma = unit_cell_parameters[i] # else: # a, b, c, alpha, beta, gamma = unit_cell_parameters[0] # # phi is the angle at the image centre # phi = scan.get_angle_from_image_index(z + 0.5, deg=True) # cif_loop.add_row((i+1, 1, z, phi, wavelength, # a, b, c, alpha, beta, gamma)) # cif_block.add_loop(cif_loop) # Write reflection data # Required columns header = ( "_pdbx_diffrn_unmerged_refln.reflection_id", "_pdbx_diffrn_unmerged_refln.scan_id", "_pdbx_diffrn_unmerged_refln.image_id_begin", "_pdbx_diffrn_unmerged_refln.image_id_end", "_pdbx_diffrn_unmerged_refln.index_h", "_pdbx_diffrn_unmerged_refln.index_k", "_pdbx_diffrn_unmerged_refln.index_l", ) headernames = { "scales": "_pdbx_diffrn_unmerged_refln.scale_value", "intensity.scale.value": "_pdbx_diffrn_unmerged_refln.intensity_meas", "intensity.scale.sigma": "_pdbx_diffrn_unmerged_refln.intensity_sigma", "intensity.sum.value": "_pdbx_diffrn_unmerged_refln.intensity_sum", "intensity.sum.sigma": "_pdbx_diffrn_unmerged_refln.intensity_sum_sigma", "intensity.prf.value": "_pdbx_diffrn_unmerged_refln.intensity_prf", "intensity.prf.sigma": "_pdbx_diffrn_unmerged_refln.intensity_prf_sigma", "angle": "_pdbx_diffrn_unmerged_refln.scan_angle_reflection", "partiality": "_pdbx_diffrn_unmerged_refln.partiality", } variables_present = [] if "scale" in self.params.intensity: reflections["scales"] = 1.0 / reflections["inverse_scale_factor"] reflections["intensity.scale.sigma"] = flex.sqrt( reflections["intensity.scale.variance"]) variables_present.extend( ["scales", "intensity.scale.value", "intensity.scale.sigma"]) if "sum" in self.params.intensity: reflections["intensity.sum.sigma"] = flex.sqrt( reflections["intensity.sum.variance"]) variables_present.extend( ["intensity.sum.value", "intensity.sum.sigma"]) if "profile" in self.params.intensity: reflections["intensity.prf.sigma"] = flex.sqrt( reflections["intensity.prf.variance"]) variables_present.extend( ["intensity.prf.value", "intensity.prf.sigma"]) # Should always exist reflections["angle"] = reflections["xyzcal.mm"].parts()[2] * RAD2DEG variables_present.extend(["angle"]) if "partiality" in reflections: variables_present.extend(["partiality"]) for name in variables_present: if name in reflections: header += (headernames[name], ) if "scale" in self.params.intensity: # Write dataset_statistics - first make a miller array crystal_symmetry = cctbxcrystal.symmetry( space_group=experiments[0].crystal.get_space_group(), unit_cell=experiments[0].crystal.get_unit_cell(), ) miller_set = miller.set( crystal_symmetry=crystal_symmetry, indices=reflections["miller_index"], anomalous_flag=False, ) i_obs = miller.array(miller_set, data=reflections["intensity.scale.value"]) i_obs.set_observation_type_xray_intensity() i_obs.set_sigmas(reflections["intensity.scale.sigma"]) i_obs.set_info( miller.array_info(source="DIALS", source_type="reflection_tables")) result = dataset_statistics( i_obs=i_obs, crystal_symmetry=crystal_symmetry, use_internal_variance=False, eliminate_sys_absent=False, ) cif_block.update(result.as_cif_block()) cif_loop = iotbx.cif.model.loop(header=header) for i, r in enumerate(reflections.rows()): refl_id = i + 1 scan_id = r["id"] + 1 _, _, _, _, z0, z1 = r["bbox"] h, k, l = r["miller_index"] variable_values = tuple((r[name]) for name in variables_present) cif_loop.add_row((refl_id, scan_id, z0, z1, h, k, l) + variable_values) cif_block.add_loop(cif_loop) # Add the block self._cif["dials"] = cif_block # Print to file with open(filename, "w") as fh: self._cif.show(out=fh) # Log logger.info("Wrote reflections to %s" % filename)
def run(args): from iotbx.reflection_file_reader import any_reflection_file from xia2.Modules.Analysis import phil_scope interp = phil_scope.command_line_argument_interpreter() params, unhandled = interp.process_and_fetch( args, custom_processor='collect_remaining') params = params.extract() n_bins = params.resolution_bins args = unhandled intensities = None batches = None scales = None dose = None reader = any_reflection_file(args[0]) assert reader.file_type() == 'ccp4_mtz' arrays = reader.as_miller_arrays(merge_equivalents=False) for ma in arrays: if ma.info().labels == ['BATCH']: batches = ma elif ma.info().labels == ['DOSE']: dose = ma elif ma.info().labels == ['I', 'SIGI']: intensities = ma elif ma.info().labels == ['I(+)', 'SIGI(+)', 'I(-)', 'SIGI(-)']: intensities = ma elif ma.info().labels == ['SCALEUSED']: scales = ma assert intensities is not None assert batches is not None mtz_object = reader.file_content() indices = mtz_object.extract_original_index_miller_indices() intensities = intensities.customized_copy( indices=indices, info=intensities.info()) batches = batches.customized_copy(indices=indices, info=batches.info()) from iotbx import merging_statistics merging_stats = merging_statistics.dataset_statistics( intensities, n_bins=n_bins) merging_acentric = intensities.select_acentric().merge_equivalents() merging_centric = intensities.select_centric().merge_equivalents() multiplicities_acentric = {} multiplicities_centric = {} for x in sorted(set(merging_acentric.redundancies().data())): multiplicities_acentric[x] = merging_acentric.redundancies().data().count(x) for x in sorted(set(merging_centric.redundancies().data())): multiplicities_centric[x] = merging_centric.redundancies().data().count(x) headers = [u'Resolution (Å)', 'N(obs)', 'N(unique)', 'Multiplicity', 'Completeness', 'Mean(I)', 'Mean(I/sigma)', 'Rmerge', 'Rmeas', 'Rpim', 'CC1/2', 'CCano'] rows = [] for bin_stats in merging_stats.bins: row = ['%.2f - %.2f' %(bin_stats.d_max, bin_stats.d_min), bin_stats.n_obs, bin_stats.n_uniq, '%.2f' %bin_stats.mean_redundancy, '%.2f' %(100*bin_stats.completeness), '%.1f' %bin_stats.i_mean, '%.1f' %bin_stats.i_over_sigma_mean, '%.3f' %bin_stats.r_merge, '%.3f' %bin_stats.r_meas, '%.3f' %bin_stats.r_pim, '%.3f' %bin_stats.cc_one_half, '%.3f' %bin_stats.cc_anom] rows.append(row) from xia2.lib.tabulate import tabulate merging_stats_table_html = tabulate(rows, headers, tablefmt='html') merging_stats_table_html = merging_stats_table_html.replace( '<table>', '<table class="table table-hover table-condensed">') unit_cell_params = intensities.unit_cell().parameters() headers = ['', 'Overall', 'Low resolution', 'High resolution'] stats = (merging_stats.overall, merging_stats.bins[0], merging_stats.bins[-1]) rows = [ [u'Resolution (Å)'] + [ '%.2f - %.2f' %(s.d_max, s.d_min) for s in stats], ['Observations'] + ['%i' %s.n_obs for s in stats], ['Unique reflections'] + ['%i' %s.n_uniq for s in stats], ['Multiplicity'] + ['%.1f' %s.mean_redundancy for s in stats], ['Completeness'] + ['%.2f%%' %(s.completeness * 100) for s in stats], #['Mean intensity'] + ['%.1f' %s.i_mean for s in stats], ['Mean I/sigma(I)'] + ['%.1f' %s.i_over_sigma_mean for s in stats], ['Rmerge'] + ['%.3f' %s.r_merge for s in stats], ['Rmeas'] + ['%.3f' %s.r_meas for s in stats], ['Rpim'] + ['%.3f' %s.r_pim for s in stats], ['CC1/2'] + ['%.3f' %s.cc_one_half for s in stats], ] rows = [[u'<strong>%s</strong>' %r[0]] + r[1:] for r in rows] overall_stats_table_html = tabulate(rows, headers, tablefmt='html') overall_stats_table_html = overall_stats_table_html.replace( '<table>', '<table class="table table-hover table-condensed">') #headers = ['Crystal symmetry', ''] #rows = [ #[u'Unit cell: a (Å)', '%.3f' %unit_cell_params[0]], #[u'b (Å)', '%.3f' %unit_cell_params[1]], #[u'c (Å)', '%.3f' %unit_cell_params[2]], #[u'α (°)', '%.3f' %unit_cell_params[3]], #[u'β (°)', '%.3f' %unit_cell_params[4]], #[u'γ (°)', '%.3f' %unit_cell_params[5]], #['Space group', intensities.space_group_info().symbol_and_number()], #] #symmetry_table_html = tabulate(rows, headers, tablefmt='html') symmetry_table_html = """ <p> <b>Filename:</b> %s <br> <b>Unit cell:</b> %s <br> <b>Space group:</b> %s </p> """ %(os.path.abspath(reader.file_name()), intensities.space_group_info().symbol_and_number(), str(intensities.unit_cell())) if params.anomalous: intensities = intensities.as_anomalous_array() batches = batches.as_anomalous_array() from xia2.Modules.PyChef2.PyChef import remove_batch_gaps new_batch_data = remove_batch_gaps(batches.data()) new_batches = batches.customized_copy(data=new_batch_data) sc_vs_b = scales_vs_batch(scales, new_batches) rmerge_vs_b = rmerge_vs_batch(intensities, new_batches) intensities.setup_binner(n_bins=n_bins) merged_intensities = intensities.merge_equivalents().array() from mmtbx.scaling import twin_analyses normalised_intensities = twin_analyses.wilson_normalised_intensities( miller_array=merged_intensities) nz_test = twin_analyses.n_z_test( normalised_acentric=normalised_intensities.acentric, normalised_centric=normalised_intensities.centric) from mmtbx.scaling import data_statistics if not intensities.space_group().is_centric(): wilson_scaling = data_statistics.wilson_scaling( miller_array=merged_intensities, n_residues=200) # XXX default n_residues? acentric = intensities.select_acentric() centric = intensities.select_centric() if acentric.size(): acentric.setup_binner(n_bins=n_bins) second_moments_acentric = acentric.second_moment_of_intensities(use_binning=True) if centric.size(): centric.setup_binner(n_bins=n_bins) second_moments_centric = centric.second_moment_of_intensities(use_binning=True) d_star_sq_bins = [ (1/bin_stats.d_min**2) for bin_stats in merging_stats.bins] i_over_sig_i_bins = [ bin_stats.i_over_sigma_mean for bin_stats in merging_stats.bins] cc_one_half_bins = [ bin_stats.cc_one_half for bin_stats in merging_stats.bins] cc_anom_bins = [ bin_stats.cc_anom for bin_stats in merging_stats.bins] from xia2.Modules.PyChef2 import PyChef if params.chef_min_completeness: d_min = PyChef.resolution_limit( mtz_file=args[0], min_completeness=params.chef_min_completeness, n_bins=8) print 'Estimated d_min for CHEF analysis: %.2f' %d_min sel = flex.bool(intensities.size(), True) d_spacings = intensities.d_spacings().data() sel &= d_spacings >= d_min intensities = intensities.select(sel) batches = batches.select(sel) if dose is not None: dose = dose.select(sel) if dose is None: dose = PyChef.batches_to_dose(batches.data(), params.dose) else: dose = dose.data() pychef_stats = PyChef.Statistics(intensities, dose) pychef_dict = pychef_stats.to_dict() def d_star_sq_to_d_ticks(d_star_sq, nticks): from cctbx import uctbx d_spacings = uctbx.d_star_sq_as_d(flex.double(d_star_sq)) min_d_star_sq = min(d_star_sq) dstep = (max(d_star_sq) - min_d_star_sq)/nticks tickvals = list(min_d_star_sq + (i*dstep) for i in range(nticks)) ticktext = ['%.2f' %(uctbx.d_star_sq_as_d(dsq)) for dsq in tickvals] return tickvals, ticktext tickvals, ticktext = d_star_sq_to_d_ticks(d_star_sq_bins, nticks=5) tickvals_wilson, ticktext_wilson = d_star_sq_to_d_ticks( wilson_scaling.d_star_sq, nticks=5) second_moment_d_star_sq = [] if acentric.size(): second_moment_d_star_sq.extend(second_moments_acentric.binner.bin_centers(2)) if centric.size(): second_moment_d_star_sq.extend(second_moments_centric.binner.bin_centers(2)) tickvals_2nd_moment, ticktext_2nd_moment = d_star_sq_to_d_ticks( second_moment_d_star_sq, nticks=5) json_data = { 'multiplicities': { 'data': [ { 'x': multiplicities_acentric.keys(), 'y': multiplicities_acentric.values(), 'type': 'bar', 'name': 'Acentric', 'opacity': 0.75, }, { 'x': multiplicities_centric.keys(), 'y': multiplicities_centric.values(), 'type': 'bar', 'name': 'Centric', 'opacity': 0.75, }, ], 'layout': { 'title': 'Distribution of multiplicities', 'xaxis': {'title': 'Multiplicity'}, 'yaxis': { 'title': 'Frequency', #'rangemode': 'tozero' }, 'bargap': 0, 'barmode': 'overlay', }, }, 'scale_rmerge_vs_batch': { 'data': [ { 'x': sc_vs_b.batches, 'y': sc_vs_b.data, 'type': 'scatter', 'name': 'Scale', 'opacity': 0.75, }, { 'x': rmerge_vs_b.batches, 'y': rmerge_vs_b.data, 'yaxis': 'y2', 'type': 'scatter', 'name': 'Rmerge', 'opacity': 0.75, }, ], 'layout': { 'title': 'Scale and Rmerge vs batch', 'xaxis': {'title': 'N'}, 'yaxis': { 'title': 'Scale', 'rangemode': 'tozero' }, 'yaxis2': { 'title': 'Rmerge', 'overlaying': 'y', 'side': 'right', 'rangemode': 'tozero' } }, }, 'cc_one_half': { 'data': [ { 'x': d_star_sq_bins, # d_star_sq 'y': cc_one_half_bins, 'type': 'scatter', 'name': 'CC-half', }, ({ 'x': d_star_sq_bins, # d_star_sq 'y': cc_anom_bins, 'type': 'scatter', 'name': 'CC-anom', } if not intensities.space_group().is_centric() else {}), ], 'layout':{ 'title': 'CC-half vs resolution', 'xaxis': { 'title': u'Resolution (Å)', 'tickvals': tickvals, 'ticktext': ticktext, }, 'yaxis': { 'title': 'CC-half', 'range': [min(cc_one_half_bins + cc_anom_bins + [0]), 1] }, }, }, 'i_over_sig_i': { 'data': [{ 'x': d_star_sq_bins, # d_star_sq 'y': i_over_sig_i_bins, 'type': 'scatter', 'name': 'Scales vs batch', }], 'layout': { 'title': '<I/sig(I)> vs resolution', 'xaxis': { 'title': u'Resolution (Å)', 'tickvals': tickvals, 'ticktext': ticktext, }, 'yaxis': { 'title': '<I/sig(I)>', 'rangemode': 'tozero' }, } }, 'second_moments': { 'data': [ ({ 'x': list(second_moments_acentric.binner.bin_centers(2)), # d_star_sq 'y': second_moments_acentric.data[1:-1], 'type': 'scatter', 'name': '<I^2> acentric', } if acentric.size() else {}), ({ 'x': list(second_moments_centric.binner.bin_centers(2)), # d_star_sq 'y': second_moments_centric.data[1:-1], 'type': 'scatter', 'name': '<I^2> centric', } if centric.size() else {}) ], 'layout': { 'title': 'Second moment of I', 'xaxis': { 'title': u'Resolution (Å)', 'tickvals': tickvals_2nd_moment, 'ticktext': ticktext_2nd_moment, }, 'yaxis': { 'title': '<I^2>', 'rangemode': 'tozero' }, } }, 'cumulative_intensity_distribution': { 'data': [ { 'x': list(nz_test.z), 'y': list(nz_test.ac_obs), 'type': 'scatter', 'name': 'Acentric observed', 'mode': 'lines', 'line': { 'color': 'rgb(31, 119, 180)', }, }, { 'x': list(nz_test.z), 'y': list(nz_test.c_obs), 'type': 'scatter', 'name': 'Centric observed', 'mode': 'lines', 'line': { 'color': 'rgb(255, 127, 14)', }, }, { 'x': list(nz_test.z), 'y': list(nz_test.ac_untwinned), 'type': 'scatter', 'name': 'Acentric theory', 'mode': 'lines', 'line': { 'color': 'rgb(31, 119, 180)', 'dash': 'dot', }, 'opacity': 0.8, }, { 'x': list(nz_test.z), 'y': list(nz_test.c_untwinned), 'type': 'scatter', 'name': 'Centric theory', 'mode': 'lines', 'line': { 'color': 'rgb(255, 127, 14)', 'dash': 'dot', }, 'opacity': 0.8, }, ], 'layout': { 'title': 'Cumulative intensity distribution', 'xaxis': {'title': 'z'}, 'yaxis': { 'title': 'P(Z <= Z)', 'rangemode': 'tozero' }, } }, 'wilson_intensity_plot': { 'data': ([ { 'x': list(wilson_scaling.d_star_sq), 'y': list(wilson_scaling.mean_I_obs_data), 'type': 'scatter', 'name': 'Observed', }, { 'x': list(wilson_scaling.d_star_sq), 'y': list(wilson_scaling.mean_I_obs_theory), 'type': 'scatter', 'name': 'Expected', }, { 'x': list(wilson_scaling.d_star_sq), 'y': list(wilson_scaling.mean_I_normalisation), 'type': 'scatter', 'name': 'Smoothed', }] if not intensities.space_group().is_centric() else []), 'layout': { 'title': 'Wilson intensity plot', 'xaxis': { 'title': u'Resolution (Å)', 'tickvals': tickvals_wilson, 'ticktext': ticktext_wilson, }, 'yaxis': { 'type': 'log', 'title': 'Mean(I)', 'rangemode': 'tozero', }, }, }, } json_data.update(pychef_dict) from dials.report import html_report report = html_report.html_report() page_header = html_report.page_header('xia2 report') report.add_content(page_header) overall_panel = html_report.panel('Overall', 'overall', show=True) overall_table = html_report.table_responsive( overall_stats_table_html, width=800) overall_panel.add_content(overall_table) merging_stats_panel = html_report.panel('Resolution shells', 'merging_stats') merging_stats_table = html_report.table_responsive(merging_stats_table_html) merging_stats_panel.add_content(merging_stats_table) merging_stats_panel_group = html_report.panel_group( [overall_panel, merging_stats_panel]) div = html_report.div() div.add_content(html_report.raw_html('<h2>Merging statistics</h2>')) div.add_content(html_report.raw_html(symmetry_table_html)) div.add_content(merging_stats_panel_group) report.add_content(div) resolution_plots_panel = html_report.panel('Analysis by resolution', 'resolution') for graph in ('cc_one_half', 'i_over_sig_i', 'second_moments', 'wilson_intensity_plot'): resolution_plots_panel.add_content(html_report.plotly_graph( json_data[graph], graph)) batch_plots_panel = html_report.panel('Analysis by batch', 'batch') for graph in ('scale_rmerge_vs_batch', 'completeness_vs_dose', 'rcp_vs_dose', 'scp_vs_dose', 'rd_vs_batch_difference'): batch_plots_panel.add_content(html_report.plotly_graph( json_data[graph], graph)) misc_plots_panel = html_report.panel('Miscellaneous', 'misc') for graph in ('multiplicities', 'cumulative_intensity_distribution'): misc_plots_panel.add_content(html_report.plotly_graph( json_data[graph], graph)) analysis_plots_panel_group = html_report.panel_group( [resolution_plots_panel, batch_plots_panel, misc_plots_panel]) div = html_report.div() div.add_content(html_report.raw_html('<h2>Analysis plots</h2>')) div.add_content(analysis_plots_panel_group) report.add_content(div) html = report.html() import json json_str = json.dumps(json_data) with open('xia2-report.json', 'wb') as f: print >> f, json_str with open('xia2-report.html', 'wb') as f: print >> f, html.encode('ascii', 'xmlcharrefreplace') return
def test_ResolutionPlotsAndStats(iobs): i_obs_anom = iobs.as_anomalous_array() iobs_anom = i_obs_anom.map_to_asu().customized_copy(info=iobs.info()) n_bins = 2 result = dataset_statistics( iobs, assert_is_not_unique_set_under_symmetry=False, n_bins=n_bins ) anom_result = dataset_statistics( iobs_anom, assert_is_not_unique_set_under_symmetry=False, anomalous=True, n_bins=n_bins, ) plotter = ResolutionPlotsAndStats(result, anom_result) assert plotter.d_star_sq_ticktext == ["1.74", "1.53", "1.38", "1.27", "1.18"] assert plotter.d_star_sq_tickvals == pytest.approx( [ 0.32984033277048164, 0.42706274943714834, 0.524285166103815, 0.6215075827704818, 0.7187299994371485, ], 1e-4, ) tables = plotter.statistics_tables() assert len(tables) == 2 # overall and per resolution # test plots individually d = plotter.cc_one_half_plot() assert len(d["cc_one_half"]["data"]) == 6 assert all(len(x["x"]) == n_bins for x in d["cc_one_half"]["data"][:4]) d["cc_one_half"]["data"][0]["x"] == [ 0.5 * (uctbx.d_as_d_star_sq(b.d_max) + uctbx.d_as_d_star_sq(b.d_min)) for b in result.bins ] d = plotter.i_over_sig_i_plot() assert len(d["i_over_sig_i"]["data"]) == 1 assert len(d["i_over_sig_i"]["data"][0]["y"]) == n_bins d = plotter.r_pim_plot() assert len(d["r_pim"]["data"]) == 1 assert len(d["r_pim"]["data"][0]["y"]) == n_bins d = plotter.completeness_plot() assert len(d["completeness"]["data"]) == 2 assert len(d["completeness"]["data"][0]["y"]) == n_bins d = plotter.multiplicity_vs_resolution_plot() assert len(d["multiplicity_vs_resolution"]["data"]) == 2 assert len(d["multiplicity_vs_resolution"]["data"][0]["y"]) == n_bins # now try centric options and sigma tau for cc_one_half plotter = ResolutionPlotsAndStats(result, anom_result, is_centric=True) d = plotter.cc_one_half_plot(method="sigma_tau") assert len(d["cc_one_half"]["data"]) == 6 assert all(len(x["x"]) == n_bins for x in d["cc_one_half"]["data"][:2]) assert d["cc_one_half"]["data"][2] == {} # no anomalous plots assert d["cc_one_half"]["data"][3] == {} # no anomalous plots assert d["cc_one_half"]["data"][4] == {} # no cc_fit assert d["cc_one_half"]["data"][5] == {} # no d_min d = plotter.completeness_plot() assert len(d["completeness"]["data"]) == 2 assert len(d["completeness"]["data"][0]["y"]) == n_bins assert d["completeness"]["data"][1] == {} d = plotter.multiplicity_vs_resolution_plot() assert len(d["multiplicity_vs_resolution"]["data"]) == 2 assert len(d["multiplicity_vs_resolution"]["data"][0]["y"]) == n_bins assert d["multiplicity_vs_resolution"]["data"][1] == {} plots = plotter.make_all_plots() assert list(plots.keys()) == [ "cc_one_half", "i_over_sig_i", "completeness", "multiplicity_vs_resolution", "r_pim", ] for plot in plots.values(): assert plot["layout"]["xaxis"]["ticktext"] == plotter.d_star_sq_ticktext assert plot["layout"]["xaxis"]["tickvals"] == plotter.d_star_sq_tickvals
def return_scaler(self): """return scaler method""" from dials.algorithms.scaling.scaler import MultiScalerBase print_step_table(self) if self._scaler.id_ == "single": if self._parameters.apm_list[0].var_cov_matrix: self._scaler.update_var_cov(self._parameters.apm_list[0]) self._scaler.experiment.scaling_model.set_scaling_model_as_scaled() elif self._scaler.id_ == "multi" or self._scaler.id_ == "target": if self._parameters.apm_list[0].var_cov_matrix: # test if has been set for i, scaler in enumerate(self._scaler.active_scalers): scaler.update_var_cov(self._parameters.apm_list[i]) scaler.experiment.scaling_model.set_scaling_model_as_scaled() if not isinstance(self._scaler, MultiScalerBase): self._scaler.experiment.scaling_model.normalise_components() logger.debug("\n" + str(self._scaler.experiment.scaling_model)) if self._scaler.Ih_table.free_Ih_table: i_obs = self._scaler.Ih_table.as_miller_array( self._scaler.experiment.crystal.get_unit_cell(), return_free_set_data=True, ) res = merging_statistics.dataset_statistics( i_obs=i_obs, n_bins=20, anomalous=False, sigma_filtering=None, use_internal_variance=False, eliminate_sys_absent=False, cc_one_half_method="sigma_tau", ) free_rpim = res.overall.r_pim free_cc12 = res.overall.cc_one_half ccs = flex.double([b.cc_one_half for b in res.bins]) n_refl = flex.double([b.n_obs for b in res.bins]) n_tot = sum(n_refl) free_wcc12 = sum(ccs * n_refl / n_tot) i_obs = self._scaler.Ih_table.as_miller_array( self._scaler.experiment.crystal.get_unit_cell() ) res = merging_statistics.dataset_statistics( i_obs=i_obs, n_bins=20, anomalous=False, sigma_filtering=None, use_internal_variance=False, eliminate_sys_absent=False, cc_one_half_method="sigma_tau", ) work_rpim = res.overall.r_pim work_cc12 = res.overall.cc_one_half ccs = flex.double([b.cc_one_half for b in res.bins]) n_refl = flex.double([b.n_obs for b in res.bins]) n_tot = sum(n_refl) work_wcc12 = sum(ccs * n_refl / n_tot) rpim_gap = free_rpim - work_rpim cc12_gap = work_cc12 - free_cc12 wcc12_gap = work_wcc12 - free_wcc12 self._scaler.final_rmsds = [ work_rpim, free_rpim, rpim_gap, work_cc12, free_cc12, cc12_gap, work_wcc12, free_wcc12, wcc12_gap, ] header = ["", "Work", "Free", "Gap"] rows = [ [ "Rpim", str(round(work_rpim, 5)), str(round(free_rpim, 5)), str(round(rpim_gap, 5)), ], [ "CC1/2", str(round(work_cc12, 5)), str(round(free_cc12, 5)), str(round(cc12_gap, 5)), ], [ "CC1/2 (weighted-avg)", str(round(work_wcc12, 5)), str(round(free_wcc12, 5)), str(round(wcc12_gap, 5)), ], ] logger.info( """\nWork/Free set quality indicators: (CC1/2 calculated using the sigma-tau method, weighted-avg is the average CC1/2 over resolution bins, weighted by n_obs per bin.) Gaps are defined as Rfree-Rwork and CCWork-CCfree.""" ) st = simple_table(rows, header) logger.info(st.format())