def _prepare_merged_reflection_table(experiments, reflections, d_min=None): """Filter the data and prepare a reflection table with merged data.""" if ("inverse_scale_factor" in reflections[0] and "intensity.scale.value" in reflections[0]): logger.info("Attempting to perform absence checks on scaled data") reflections = filter_reflection_table(reflections[0], intensity_choice=["scale"], d_min=d_min) reflections["intensity"] = reflections["intensity.scale.value"] reflections["variance"] = reflections["intensity.scale.variance"] else: logger.info( "Attempting to perform absence checks on unscaled profile-integrated data" ) reflections = filter_reflection_table(reflections[0], intensity_choice=["profile"], d_min=d_min) reflections["intensity"] = reflections["intensity.prf.value"] reflections["variance"] = reflections["intensity.prf.variance"] # now merge space_group = experiments[0].crystal.get_space_group() reflections["asu_miller_index"] = map_indices_to_asu( reflections["miller_index"], space_group) reflections["inverse_scale_factor"] = flex.double(reflections.size(), 1.0) merged = (_reflection_table_to_iobs( reflections, experiments[0].crystal.get_unit_cell(), space_group).merge_equivalents(use_internal_variance=False).array()) merged_reflections = flex.reflection_table() merged_reflections["intensity"] = merged.data() merged_reflections["variance"] = merged.sigmas()**2 merged_reflections["miller_index"] = merged.indices() return merged_reflections
def from_reflections_and_experiments(cls, reflection_tables, experiments, params): """Construct the resolutionizer from native dials datatypes.""" # add some assertions about data # do batch assignment (same functions as in dials.export) offsets = calculate_batch_offsets(experiments) reflection_tables = assign_batches_to_reflections( reflection_tables, offsets) batches = flex.int() intensities = flex.double() indices = flex.miller_index() variances = flex.double() for table in reflection_tables: if "intensity.scale.value" in table: table = filter_reflection_table(table, ["scale"], partiality_threshold=0.4) intensities.extend(table["intensity.scale.value"]) variances.extend(table["intensity.scale.variance"]) else: table = filter_reflection_table(table, ["profile"], partiality_threshold=0.4) intensities.extend(table["intensity.prf.value"]) variances.extend(table["intensity.prf.variance"]) indices.extend(table["miller_index"]) batches.extend(table["batch"]) crystal_symmetry = miller.crystal.symmetry( unit_cell=determine_best_unit_cell(experiments), space_group=experiments[0].crystal.get_space_group(), assert_is_compatible_unit_cell=False, ) miller_set = miller.set(crystal_symmetry, indices, anomalous_flag=False) i_obs = miller.array(miller_set, data=intensities, sigmas=flex.sqrt(variances)) i_obs.set_observation_type_xray_intensity() i_obs.set_info(miller.array_info(source="DIALS", source_type="refl")) ms = i_obs.customized_copy() batch_array = miller.array(ms, data=batches) if params.reference is not None: reference, _ = miller_array_from_mtz(params.reference, anomalous=params.anomalous, labels=params.labels) else: reference = None return cls(i_obs, params, batches=batch_array, reference=reference)
def test_filter_reflection_table(): """Test the interface function""" reflections = generate_integrated_test_reflections() reflections = filter_reflection_table(reflections, ["sum"]) assert "intensity.sum.value" in reflections assert "intensity.prf.value" not in reflections assert "intensity.scale.value" not in reflections reflections = generate_integrated_test_reflections() reflections = filter_reflection_table(reflections, ["profile"]) assert "intensity.prf.value" in reflections assert "intensity.sum.value" not in reflections assert "intensity.scale.value" not in reflections reflections = generate_integrated_test_reflections() reflections = filter_reflection_table(reflections, ["scale"]) assert "intensity.scale.value" in reflections assert "intensity.prf.value" not in reflections assert "intensity.sum.value" not in reflections reflections = generate_integrated_test_reflections() reflections = filter_reflection_table(reflections, ["sum", "profile"]) assert "intensity.sum.value" in reflections assert "intensity.prf.value" in reflections assert "intensity.scale.value" not in reflections reflections = generate_integrated_test_reflections() reflections = filter_reflection_table(reflections, ["sum", "profile", "scale"]) assert "intensity.sum.value" in reflections assert "intensity.prf.value" in reflections assert "intensity.scale.value" in reflections with pytest.raises(ValueError): reflections = filter_reflection_table(reflections, ["bad"]) # try filter with profile option and no profiles reflections = generate_integrated_test_reflections() reflections.unset_flags( flex.bool(reflections.size(), True), reflections.flags.integrated_prf ) reflections = filter_reflection_table(reflections, ["sum", "profile"]) # should try profile but fail and so retry with just sum assert "intensity.sum.value" in reflections assert "intensity.prf.value" not in reflections reflections = generate_integrated_test_reflections() reflections.unset_flags( flex.bool(reflections.size(), True), reflections.flags.integrated_prf ) reflections = filter_reflection_table(reflections, ["sum", "profile", "scale"]) # should try profile but fail and so retry with just sum assert "intensity.sum.value" in reflections assert "intensity.prf.value" not in reflections assert "intensity.scale.value" in reflections reflections = generate_integrated_test_reflections() reflections.unset_flags( flex.bool(reflections.size(), True), reflections.flags.integrated_prf ) with pytest.raises(ValueError): _ = filter_reflection_table(reflections, ["profile"])
def read_experiments(experiments, reflection_table): """ Get information from experiments and reflections """ # Get space group and unit cell space_group = experiments[0].crystal.get_space_group() sgno = space_group.type().number() unit_cell_list = [] for e in experiments: assert sgno == e.crystal.get_space_group().type().number() unit_cell_list.append(e.crystal.get_unit_cell()) if len(unit_cell_list) > 1: # calc mean unit cell mean_parameters = [0, 0, 0, 0, 0, 0] for uc in unit_cell_list: for i in range(6): mean_parameters[i] += uc.parameters()[i] for i in range(6): mean_parameters[i] /= len(unit_cell_list) mean_unit_cell = uctbx.unit_cell(mean_parameters) else: mean_unit_cell = unit_cell_list[0] # Require a dials scaled experiments file. filtered_table = filter_reflection_table(reflection_table, ["scale"]) filtered_table["intensity"] = filtered_table["intensity.scale.value"] filtered_table["variance"] = filtered_table["intensity.scale.variance"] filtered_table["dataset"] = filtered_table["id"] filtered_table["image"] = ( flex.floor(filtered_table["xyzobs.px.value"].parts()[2]).iround() + 1) return filtered_table, mean_unit_cell, space_group
def from_dials_data_files(cls, params, experiments, reflection_table): """Initialise the class from an experiment list and reflection table. Args: params: A damage-analysis phil params object experiments: An ExperimentList reflection_table: A reflection table. """ reflection_table = filter_reflection_table( reflection_table, intensity_choice=["scale"], partiality_threshold=0.4 ) # get scaled intensities intensities = miller.array( miller.set( crystal.symmetry( unit_cell=median_unit_cell(experiments), space_group=experiments[0].crystal.get_space_group(), ), indices=reflection_table["miller_index"], anomalous_flag=params.anomalous, ), data=reflection_table["intensity.scale.value"], sigmas=flex.sqrt(reflection_table["intensity.scale.variance"]), ) intensities.set_observation_type_xray_intensity() doses = flex.double() start_doses, doses_per_image = interpret_images_to_doses_options( experiments, params.dose.experiments.dose_per_image, params.dose.experiments.starting_doses, params.dose.experiments.shared_crystal, ) logger.info( "Interpreting data using:\n starting_doses=%s\n dose_per_image=%s", ", ".join("%s" % i for i in start_doses) if len(set(start_doses)) > 1 else " all %s" % str(start_doses[0]), ", ".join("%s" % i for i in doses_per_image) if len(set(doses_per_image)) > 1 else " all %s" % str(doses_per_image[0]), ) for expt, starting_dose, dose_per_img in zip( experiments, start_doses, doses_per_image ): refls = reflection_table.select(expt) imgno = flex.ceil(refls["xyzobs.px.value"].parts()[2]) dose = (imgno * dose_per_img) + starting_dose doses.extend(dose) doses = doses.iround() return cls(intensities, doses, params)
def export_mtz(integrated_data, experiment_list, params): """Export data from integrated_data corresponding to experiment_list to an MTZ file hklout.""" # if mtz filename is auto, then choose scaled.mtz or integrated.mtz if params.mtz.hklout in (None, Auto, "auto"): if ("intensity.scale.value" in integrated_data) and ("intensity.scale.variance" in integrated_data): params.mtz.hklout = "scaled.mtz" logger.info( "Data appears to be scaled, setting mtz.hklout = 'scaled.mtz'") else: params.mtz.hklout = "integrated.mtz" logger.info( "Data appears to be unscaled, setting mtz.hklout = 'integrated.mtz'" ) # First get the experiment identifier information out of the data expids_in_table = integrated_data.experiment_identifiers() if not list(expids_in_table.keys()): reflection_tables = parse_multiple_datasets([integrated_data]) experiment_list, refl_list = assign_unique_identifiers( experiment_list, reflection_tables) integrated_data = flex.reflection_table() for reflections in refl_list: integrated_data.extend(reflections) expids_in_table = integrated_data.experiment_identifiers() integrated_data.assert_experiment_identifiers_are_consistent( experiment_list) expids_in_list = list(experiment_list.identifiers()) # Convert experiment_list to a real python list or else identity assumptions # fail like: # assert experiment_list[0] is experiment_list[0] # And assumptions about added attributes break experiment_list = list(experiment_list) # Validate multi-experiment assumptions if len(experiment_list) > 1: # All experiments should match crystals, or else we need multiple crystals/datasets if not all(x.crystal == experiment_list[0].crystal for x in experiment_list[1:]): logger.warning( "Experiment crystals differ. Using first experiment crystal for file-level data." ) wavelengths = match_wavelengths(experiment_list) if len(wavelengths.keys()) > 1: logger.info( "Multiple wavelengths found: \n%s", "\n".join(" Wavlength: %.5f, experiment numbers: %s " % (k, ",".join(map(str, v))) for k, v in wavelengths.items()), ) else: wavelengths = OrderedDict( {experiment_list[0].beam.get_wavelength(): [0]}) # also only work correctly with one panel (for the moment) if any(len(experiment.detector) != 1 for experiment in experiment_list): logger.warning("Ignoring multiple panels in output MTZ") best_unit_cell = params.mtz.best_unit_cell if best_unit_cell is None: best_unit_cell = determine_best_unit_cell(experiment_list) integrated_data["d"] = best_unit_cell.d(integrated_data["miller_index"]) # Clean up the data with the passed in options integrated_data = filter_reflection_table( integrated_data, intensity_choice=params.intensity, partiality_threshold=params.mtz.partiality_threshold, combine_partials=params.mtz.combine_partials, min_isigi=params.mtz.min_isigi, filter_ice_rings=params.mtz.filter_ice_rings, d_min=params.mtz.d_min, ) # get batch offsets and image ranges - even for scanless experiments batch_offsets = [ expt.scan.get_batch_offset() for expt in experiment_list if expt.scan is not None ] unique_offsets = set(batch_offsets) if len(set(unique_offsets)) <= 1: logger.debug("Calculating new batches") batch_offsets = calculate_batch_offsets(experiment_list) batch_starts = [ e.scan.get_image_range()[0] if e.scan else 0 for e in experiment_list ] effective_offsets = [ o + s for o, s in zip(batch_offsets, batch_starts) ] unique_offsets = set(effective_offsets) else: logger.debug("Keeping existing batches") image_ranges = get_image_ranges(experiment_list) if len(unique_offsets) != len(batch_offsets): raise ValueError("Duplicate batch offsets detected: %s" % ", ".join( str(item) for item, count in Counter(batch_offsets).items() if count > 1)) # Create the mtz file mtz_writer = UnmergedMTZWriter( experiment_list[0].crystal.get_space_group()) # FIXME TODO for more than one experiment into an MTZ file: # # - add an epoch (or recover an epoch) from the scan and add this as an extra # column to the MTZ file for scaling, so we know that the two lattices were # integrated at the same time # ✓ decide a sensible BATCH increment to apply to the BATCH value between # experiments and add this for id_ in expids_in_table.keys(): # Grab our subset of the data loc = expids_in_list.index( expids_in_table[id_]) # get strid and use to find loc in list experiment = experiment_list[loc] if len(list(wavelengths.keys())) > 1: for i, (wl, exps) in enumerate(wavelengths.items()): if loc in exps: wavelength = wl dataset_id = i + 1 break else: wavelength = list(wavelengths.keys())[0] dataset_id = 1 reflections = integrated_data.select(integrated_data["id"] == id_) batch_offset = batch_offsets[loc] image_range = image_ranges[loc] reflections = assign_batches_to_reflections([reflections], [batch_offset])[0] experiment.data = dict(reflections) s0n = matrix.col(experiment.beam.get_s0()).normalize().elems logger.debug("Beam vector: %.4f %.4f %.4f" % s0n) mtz_writer.add_batch_list( image_range, experiment, wavelength, dataset_id, batch_offset=batch_offset, force_static_model=params.mtz.force_static_model, ) # Create the batch offset array. This gives us an experiment (id)-dependent # batch offset to calculate the correct batch from image number. experiment.data["batch_offset"] = flex.int(len(experiment.data["id"]), batch_offset) # Calculate whether we have a ROT value for this experiment, and set the column _, _, z = experiment.data["xyzcal.px"].parts() if experiment.scan: experiment.data[ "ROT"] = experiment.scan.get_angle_from_array_index(z) else: experiment.data["ROT"] = z mtz_writer.add_crystal( crystal_name=params.mtz.crystal_name, project_name=params.mtz.project_name, unit_cell=best_unit_cell, ) # Note: add unit cell here as may have changed basis since creating mtz. # For multi-wave unmerged mtz, we add an empty dataset for each wavelength, # but only write the data into the final dataset (for unmerged the batches # link the unmerged data to the individual wavelengths). for wavelength in wavelengths: mtz_writer.add_empty_dataset(wavelength) # Combine all of the experiment data columns before writing combined_data = { k: v.deep_copy() for k, v in experiment_list[0].data.items() } for experiment in experiment_list[1:]: for k, v in experiment.data.items(): combined_data[k].extend(v) # ALL columns must be the same length assert len({len(v) for v in combined_data.values() }) == 1, "Column length mismatch" assert len(combined_data["id"]) == len( integrated_data["id"]), "Lost rows in split/combine" # Write all the data and columns to the mtz file mtz_writer.write_columns(combined_data) logger.info("Saving {} integrated reflections to {}".format( len(combined_data["id"]), params.mtz.hklout)) mtz_file = mtz_writer.mtz_file mtz_file.write(params.mtz.hklout) return mtz_file
def merge( experiments, reflections, d_min=None, d_max=None, combine_partials=True, partiality_threshold=0.4, best_unit_cell=None, anomalous=True, use_internal_variance=False, assess_space_group=False, n_bins=20, ): """ Merge reflection table data and generate a summary of the merging statistics. This procedure filters the input data, merges the data (normal and optionally anomalous), assesses the space group symmetry and generates a summary of the merging statistics. """ logger.info("\nMerging scaled reflection data\n") # first filter bad reflections using dials.util.filter methods reflections = filter_reflection_table( reflections, intensity_choice=["scale"], d_min=d_min, d_max=d_max, combine_partials=combine_partials, partiality_threshold=partiality_threshold, ) # ^ scale factor has been applied, so now set to 1.0 - okay as not # going to output scale factor in merged mtz. reflections["inverse_scale_factor"] = flex.double(reflections.size(), 1.0) scaled_array = scaled_data_as_miller_array([reflections], experiments, best_unit_cell) # Note, merge_equivalents does not raise an error if data is unique. merged = scaled_array.merge_equivalents( use_internal_variance=use_internal_variance).array() merged_anom = None if anomalous: anomalous_scaled = scaled_array.as_anomalous_array() merged_anom = anomalous_scaled.merge_equivalents( use_internal_variance=use_internal_variance).array() # Before merge, do assessment of the space_group if assess_space_group: merged_reflections = flex.reflection_table() merged_reflections["intensity"] = merged.data() merged_reflections["variance"] = flex.pow2(merged.sigmas()) merged_reflections["miller_index"] = merged.indices() logger.info("Running systematic absences check") run_systematic_absences_checks(experiments, merged_reflections) try: stats, anom_stats = merging_stats_from_scaled_array( scaled_array, n_bins, use_internal_variance, ) except DialsMergingStatisticsError as e: logger.error(e, exc_info=True) stats_summary = None else: stats_summary = make_merging_statistics_summary(stats) stats_summary += table_1_summary(stats, anom_stats) return merged, merged_anom, stats_summary
def run_sys_abs_checks(experiments, reflections, d_min=None, significance_level=0.95): """Check for systematic absences in the data for the laue group. Select the good data, merge, test screw axes and score possible space groups. The crystals are updated with the most likely space group. """ if ( "inverse_scale_factor" in reflections[0] and "intensity.scale.value" in reflections[0] ): logger.info("Attempting to perform absence checks on scaled data") reflections = filter_reflection_table( reflections[0], intensity_choice=["scale"], d_min=d_min ) reflections["intensity"] = reflections["intensity.scale.value"] reflections["variance"] = reflections["intensity.scale.variance"] else: logger.info( "Attempting to perform absence checks on unscaled profile-integrated data" ) reflections = filter_reflection_table( reflections[0], intensity_choice=["profile"], d_min=d_min ) reflections["intensity"] = reflections["intensity.prf.value"] reflections["variance"] = reflections["intensity.prf.variance"] # now merge space_group = experiments[0].crystal.get_space_group() reflections["asu_miller_index"] = map_indices_to_asu( reflections["miller_index"], space_group ) reflections["inverse_scale_factor"] = flex.double(reflections.size(), 1.0) merged = ( _reflection_table_to_iobs( reflections, experiments[0].crystal.get_unit_cell(), space_group ) .merge_equivalents(use_internal_variance=False) .array() ) merged_reflections = flex.reflection_table() merged_reflections["intensity"] = merged.data() merged_reflections["variance"] = merged.sigmas() ** 2 merged_reflections["miller_index"] = merged.indices() # Get the laue class from the space group. laue_group = str(space_group.build_derived_patterson_group().info()) logger.info("Laue group: %s", laue_group) if laue_group not in laue_groups: logger.info("No absences to check for this laue group") return # Score the screw axes. screw_axes, screw_axis_scores = score_screw_axes( laue_groups[laue_group], merged_reflections, significance_level ) logger.info( simple_table( [ [ a.name, "%.3f" % score, str(a.n_refl_used[0]), str(a.n_refl_used[1]), "%.3f" % a.mean_I, "%.3f" % a.mean_I_abs, "%.3f" % a.mean_I_sigma, "%.3f" % a.mean_I_sigma_abs, ] for a, score in zip(screw_axes, screw_axis_scores) ], column_headers=[ "Screw axis", "Score", "No. present", "No. absent", "<I> present", "<I> absent", "<I/sig> present", "<I/sig> absent", ], ).format() ) # Score the space groups from the screw axis scores. space_groups, scores = score_space_groups( screw_axis_scores, laue_groups[laue_group] ) logger.info( simple_table( [[sg, "%.4f" % score] for sg, score in zip(space_groups, scores)], column_headers=["Space group", "score"], ).format() ) # Find the best space group and update the experiments. best_sg = space_groups[scores.index(max(scores))] logger.info("Recommended space group: %s", best_sg) if "enantiomorphic pairs" in laue_groups[laue_group]: if best_sg in laue_groups[laue_group]["enantiomorphic pairs"]: logger.info( "Space group with equivalent score (enantiomorphic pair): %s", laue_groups[laue_group]["enantiomorphic pairs"][best_sg], ) new_sg = sgtbx.space_group_info(symbol=best_sg).group() for experiment in experiments: experiment.crystal.set_space_group(new_sg)
def write(self, experiments, reflections): """ Write the experiments and reflections to file """ # if mmmcif filename is auto, then choose scaled.cif or integrated.cif if self.params.mmcif.hklout in (None, Auto, "auto"): if ("intensity.scale.value" in reflections) and ("intensity.scale.variance" in reflections): filename = "scaled.cif" logger.info( "Data appears to be scaled, setting mmcif.hklout = 'scaled_unmerged.cif'" ) else: filename = "integrated.cif" logger.info( "Data appears to be unscaled, setting mmcif.hklout = 'integrated.cif'" ) # Select reflections selection = reflections.get_flags(reflections.flags.integrated, all=True) reflections = reflections.select(selection) # Filter out bad variances and other issues, but don't filter on ice rings # or alter partialities. ### Assumes you want to apply the lp and dqe corrections to sum and prf ### Do we want to combine partials? reflections = filter_reflection_table( reflections, self.params.intensity, combine_partials=False, partiality_threshold=0.0, d_min=self.params.mtz.d_min, ) # Get the cif block cif_block = iotbx.cif.model.block() # Audit trail dials_version = dials.util.version.dials_version() cif_block["_audit.creation_method"] = dials_version cif_block["_audit.creation_date"] = datetime.date.today().isoformat() cif_block["_computing.data_reduction"] = ( "%s (Winter, G. et al., 2018)" % dials_version) cif_block[ "_publ.section_references"] = "Winter, G. et al. (2018) Acta Cryst. D74, 85-97." # Hard coding X-ray cif_block["_pdbx_diffrn_data_section.id"] = "dials" cif_block["_pdbx_diffrn_data_section.type_scattering"] = "x-ray" cif_block["_pdbx_diffrn_data_section.type_merged"] = "false" cif_block["_pdbx_diffrn_data_section.type_scaled"] = str( "scale" in self.params.intensity).lower() # FIXME Haven't put in any of these bits yet # # Facility/beamline proposal tracking details # # cif_block["_pdbx_diffrn_data_section_experiment.ordinal"] = 1 # cif_block["_pdbx_diffrn_data_section_experiment.data_section_id"] = "dials" # cif_block["_pdbx_diffrn_data_section_experiment.proposal_id"] = "<PROPOSAL ID> # Facility/beamline details for this data collection # # cif_block["_pdbx_diffrn_data_section_site.data_section_id"] = 'dials' # cif_block["_pdbx_diffrn_data_section_site.facility"] = "DIAMOND" # cif_block["_pdbx_diffrn_data_section_site.beamline"] = "VMX-M" # cif_block["_pdbx_diffrn_data_section_site.collection_date"] = scan.epochs()[0] # cif_block["_pdbx_diffrn_data_section_site.detector"] = detector[0].name() # cif_block["_pdbx_diffrn_data_section_site.detector_type"] = detector[0].type() # Write the crystal information cif_loop = iotbx.cif.model.loop(header=( "_pdbx_diffrn_unmerged_cell.ordinal", "_pdbx_diffrn_unmerged_cell.crystal_id", "_pdbx_diffrn_unmerged_cell.wavelength", "_pdbx_diffrn_unmerged_cell.cell_length_a", "_pdbx_diffrn_unmerged_cell.cell_length_b", "_pdbx_diffrn_unmerged_cell.cell_length_c", "_pdbx_diffrn_unmerged_cell.cell_angle_alpha", "_pdbx_diffrn_unmerged_cell.cell_angle_beta", "_pdbx_diffrn_unmerged_cell.cell_angle_gamma", "_pdbx_diffrn_unmerged_cell.Bravais_lattice", )) crystals = experiments.crystals() crystal_to_id = {crystal: i + 1 for i, crystal in enumerate(crystals)} for i, exp in enumerate(experiments): crystal = exp.crystal crystal_id = crystal_to_id[crystal] wavelength = exp.beam.get_wavelength() a, b, c, alpha, beta, gamma = crystal.get_unit_cell().parameters() latt_type = str( bravais_types.bravais_lattice(group=crystal.get_space_group())) cif_loop.add_row((i + 1, crystal_id, wavelength, a, b, c, alpha, beta, gamma, latt_type)) cif_block.add_loop(cif_loop) # Write the scan information cif_loop = iotbx.cif.model.loop(header=( "_pdbx_diffrn_scan.scan_id", "_pdbx_diffrn_scan.crystal_id", "_pdbx_diffrn_scan.image_id_begin", "_pdbx_diffrn_scan.image_id_end", "_pdbx_diffrn_scan.scan_angle_begin", "_pdbx_diffrn_scan.scan_angle_end", )) for i, exp in enumerate(experiments): scan = exp.scan crystal_id = crystal_to_id[exp.crystal] image_range = scan.get_image_range() osc_range = scan.get_oscillation_range(deg=True) cif_loop.add_row(( i + 1, crystal_id, image_range[0], image_range[1], osc_range[0], osc_range[1], )) cif_block.add_loop(cif_loop) # Make a dict of unit_cell parameters unit_cell_parameters = {} if crystal.num_scan_points > 1: for i in range(crystal.num_scan_points): a, b, c, alpha, beta, gamma = crystal.get_unit_cell_at_scan_point( i).parameters() unit_cell_parameters[i] = (a, b, c, alpha, beta, gamma) else: unit_cell_parameters[0] = (a, b, c, alpha, beta, gamma) ### _pdbx_diffrn_image_proc has been removed from the dictionary extension. ### Keeping this section commented out as it may be added back in some ### form in future # # Write the image data # scan = experiments[0].scan # z0 = scan.get_image_range()[0] # # cif_loop = iotbx.cif.model.loop( # header=("_pdbx_diffrn_image_proc.image_id", # "_pdbx_diffrn_image_proc.crystal_id", # "_pdbx_diffrn_image_proc.image_number", # "_pdbx_diffrn_image_proc.phi_value", # "_pdbx_diffrn_image_proc.wavelength", # "_pdbx_diffrn_image_proc.cell_length_a", # "_pdbx_diffrn_image_proc.cell_length_b", # "_pdbx_diffrn_image_proc.cell_length_c", # "_pdbx_diffrn_image_proc.cell_angle_alpha", # "_pdbx_diffrn_image_proc.cell_angle_beta", # "_pdbx_diffrn_image_proc.cell_angle_gamma")) # for i in range(len(scan)): # z = z0 + i # if crystal.num_scan_points > 1: # a, b, c, alpha, beta, gamma = unit_cell_parameters[i] # else: # a, b, c, alpha, beta, gamma = unit_cell_parameters[0] # # phi is the angle at the image centre # phi = scan.get_angle_from_image_index(z + 0.5, deg=True) # cif_loop.add_row((i+1, 1, z, phi, wavelength, # a, b, c, alpha, beta, gamma)) # cif_block.add_loop(cif_loop) # Write reflection data # Required columns header = ( "_pdbx_diffrn_unmerged_refln.reflection_id", "_pdbx_diffrn_unmerged_refln.scan_id", "_pdbx_diffrn_unmerged_refln.image_id_begin", "_pdbx_diffrn_unmerged_refln.image_id_end", "_pdbx_diffrn_unmerged_refln.index_h", "_pdbx_diffrn_unmerged_refln.index_k", "_pdbx_diffrn_unmerged_refln.index_l", ) headernames = { "scales": "_pdbx_diffrn_unmerged_refln.scale_value", "intensity.scale.value": "_pdbx_diffrn_unmerged_refln.intensity_meas", "intensity.scale.sigma": "_pdbx_diffrn_unmerged_refln.intensity_sigma", "intensity.sum.value": "_pdbx_diffrn_unmerged_refln.intensity_sum", "intensity.sum.sigma": "_pdbx_diffrn_unmerged_refln.intensity_sum_sigma", "intensity.prf.value": "_pdbx_diffrn_unmerged_refln.intensity_prf", "intensity.prf.sigma": "_pdbx_diffrn_unmerged_refln.intensity_prf_sigma", "angle": "_pdbx_diffrn_unmerged_refln.scan_angle_reflection", "partiality": "_pdbx_diffrn_unmerged_refln.partiality", } variables_present = [] if "scale" in self.params.intensity: reflections["scales"] = 1.0 / reflections["inverse_scale_factor"] reflections["intensity.scale.sigma"] = ( reflections["intensity.scale.variance"]**0.5) variables_present.extend( ["scales", "intensity.scale.value", "intensity.scale.sigma"]) if "sum" in self.params.intensity: reflections["intensity.sum.sigma"] = ( reflections["intensity.sum.variance"]**0.5) variables_present.extend( ["intensity.sum.value", "intensity.sum.sigma"]) if "profile" in self.params.intensity: reflections["intensity.prf.sigma"] = ( reflections["intensity.prf.variance"]**0.5) variables_present.extend( ["intensity.prf.value", "intensity.prf.sigma"]) # Should always exist reflections["angle"] = reflections["xyzcal.mm"].parts()[2] * RAD2DEG variables_present.extend(["angle"]) if "partiality" in reflections: variables_present.extend(["partiality"]) for name in variables_present: if name in reflections: header += (headernames[name], ) if "scale" in self.params.intensity: # Write dataset_statistics - first make a miller array crystal_symmetry = cctbxcrystal.symmetry( space_group=experiments[0].crystal.get_space_group(), unit_cell=experiments[0].crystal.get_unit_cell(), ) miller_set = miller.set( crystal_symmetry=crystal_symmetry, indices=reflections["miller_index"], anomalous_flag=False, ) i_obs = miller.array(miller_set, data=reflections["intensity.scale.value"]) i_obs.set_observation_type_xray_intensity() i_obs.set_sigmas(reflections["intensity.scale.sigma"]) i_obs.set_info( miller.array_info(source="DIALS", source_type="reflection_tables")) result = dataset_statistics( i_obs=i_obs, crystal_symmetry=crystal_symmetry, use_internal_variance=False, eliminate_sys_absent=False, ) cif_block.update(result.as_cif_block()) cif_loop = iotbx.cif.model.loop(header=header) for i, r in enumerate(reflections.rows()): refl_id = i + 1 scan_id = r["id"] + 1 _, _, _, _, z0, z1 = r["bbox"] h, k, l = r["miller_index"] variable_values = tuple((r[name]) for name in variables_present) cif_loop.add_row((refl_id, scan_id, z0, z1, h, k, l) + variable_values) cif_block.add_loop(cif_loop) # Add the block self._cif["dials"] = cif_block # Print to file with open(filename, "w") as fh: self._cif.show(out=fh) # Log logger.info("Wrote reflections to %s" % filename)
def run(self): """Execute the script.""" start_time = time() # Parse the command line params, _ = self.parser.parse_args(show_diff_phil=False) # set up global reflections list reflections = flex.reflection_table() # loop through the input, building up the global lists reflections_list = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) reflections_list = parse_multiple_datasets(reflections_list) for refs in reflections_list: reflections.extend(refs) # Try to load the models and data nexp = len(experiments) if nexp == 0: print("No Experiments found in the input") self.parser.print_help() return if not reflections: print("No reflection data found in the input") self.parser.print_help() return self.check_input(reflections) # Configure the logging log.config(logfile=params.output.log) logger.info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) # Convert to P 1? if params.refinement.triclinic: reflections, experiments = self.convert_to_P1(reflections, experiments) # Combine crystals? if params.refinement.combine_crystal_models and len(experiments) > 1: logger.info("Combining {0} crystal models".format(len(experiments))) experiments = self.combine_crystals(experiments) # Filter integrated centroids? if params.refinement.filter_integrated_centroids: reflections = self.filter_integrated_centroids(reflections) # Filter data if scaled to remove outliers if "inverse_scale_factor" in reflections: try: reflections = filter_reflection_table(reflections, ["scale"]) except ValueError as e: logger.warn(e) logger.info( "Filtering on scaled data failed, proceeding with integrated data." ) # Get the refiner logger.info("Configuring refiner") refiner = self.create_refiner(params, reflections, experiments) # Refine the geometry if nexp == 1: logger.info("Performing refinement of a single Experiment...") else: logger.info("Performing refinement of {} Experiments...".format(nexp)) refiner.run() # get the refined experiments experiments = refiner.get_experiments() crystals = experiments.crystals() if len(crystals) == 1: # output the refined model for information logger.info("") logger.info("Final refined crystal model:") logger.info(crystals[0]) logger.info(self.cell_param_table(crystals[0])) # Save the refined experiments to file output_experiments_filename = params.output.experiments logger.info( "Saving refined experiments to {}".format(output_experiments_filename) ) experiments.as_file(output_experiments_filename) # Create correlation plots if params.output.correlation_plot.filename is not None: create_correlation_plots(refiner, params.output) if params.output.cif is not None: self.generate_cif(crystals[0], refiner, filename=params.output.cif) if params.output.p4p is not None: self.generate_p4p( crystals[0], experiments[0].beam, filename=params.output.p4p ) if params.output.mmcif is not None: self.generate_mmcif(crystals[0], refiner, filename=params.output.mmcif) # Log the total time taken logger.info("\nTotal time taken: {:.2f}s".format(time() - start_time))
def export_xds_ascii(integrated_data, experiment_list, params, var_model=(1, 0)): """Export data from integrated_data corresponding to experiment_list to an XDS_ASCII.HKL formatted text file.""" from dials.array_family import flex # for the moment assume (and assert) that we will convert data from exactly # one lattice... assert len(experiment_list) == 1 # select reflections that are assigned to an experiment (i.e. non-negative id) integrated_data = integrated_data.select(integrated_data["id"] >= 0) assert max(integrated_data["id"]) == 0 # export for xds_ascii should only be for non-scaled reflections assert any([ i in integrated_data for i in ["intensity.sum.value", "intensity.prf.value"] ]) integrated_data = filter_reflection_table( integrated_data, intensity_choice=params.intensity, partiality_threshold=params.mtz.partiality_threshold, combine_partials=params.mtz.combine_partials, min_isigi=params.mtz.min_isigi, filter_ice_rings=params.mtz.filter_ice_rings, d_min=params.mtz.d_min, ) # calculate the scl = lp/dqe correction for outputting but don't apply it as # it has already been applied in filter_reflection_table integrated_data, scl = FilteringReductionMethods.calculate_lp_qe_correction_and_filter( integrated_data) experiment = experiment_list[0] # sort data before output nref = len(integrated_data["miller_index"]) indices = flex.size_t_range(nref) import copy unique = copy.deepcopy(integrated_data["miller_index"]) from cctbx.miller import map_to_asu map_to_asu(experiment.crystal.get_space_group().type(), False, unique) perm = sorted(indices, key=lambda k: unique[k]) integrated_data = integrated_data.select(flex.size_t(perm)) from scitbx import matrix from rstbx.cftbx.coordinate_frame_helpers import align_reference_frame assert not experiment.goniometer is None unit_cell = experiment.crystal.get_unit_cell() from scitbx.array_family import flex assert not experiment.scan is None image_range = experiment.scan.get_image_range() phi_start, phi_range = experiment.scan.get_image_oscillation( image_range[0]) # gather the required information for the reflection file nref = len(integrated_data["miller_index"]) zdet = flex.double(integrated_data["xyzcal.px"].parts()[2]) miller_index = integrated_data["miller_index"] # profile correlation if "profile.correlation" in integrated_data: prof_corr = 100.0 * integrated_data["profile.correlation"] else: prof_corr = flex.double(nref, 100.0) # partiality if "partiality" in integrated_data: partiality = 100 * integrated_data["partiality"] else: prof_corr = flex.double(nref, 100.0) if "intensity.sum.value" in integrated_data: I = integrated_data["intensity.sum.value"] V = integrated_data["intensity.sum.variance"] assert V.all_gt(0) V = var_model[0] * (V + var_model[1] * I * I) sigI = flex.sqrt(V) else: I = integrated_data["intensity.prf.value"] V = integrated_data["intensity.prf.variance"] assert V.all_gt(0) V = var_model[0] * (V + var_model[1] * I * I) sigI = flex.sqrt(V) fout = open(params.xds_ascii.hklout, "w") # first write the header - in the "standard" coordinate frame... panel = experiment.detector[0] fast = panel.get_fast_axis() slow = panel.get_slow_axis() Rd = align_reference_frame(fast, (1, 0, 0), slow, (0, 1, 0)) print("Coordinate change:") print("%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n" % Rd.elems) fast = Rd * fast slow = Rd * slow qx, qy = panel.get_pixel_size() nx, ny = panel.get_image_size() distance = matrix.col(Rd * panel.get_origin()).dot( matrix.col(Rd * panel.get_normal())) org = Rd * (matrix.col(panel.get_origin()) - distance * matrix.col(panel.get_normal())) orgx = -org.dot(fast) / qx orgy = -org.dot(slow) / qy UB = Rd * matrix.sqr(experiment.crystal.get_A()) real_space_ABC = UB.inverse().elems axis = Rd * experiment.goniometer.get_rotation_axis() beam = Rd * experiment.beam.get_s0() cell_fmt = "%9.3f %9.3f %9.3f %7.3f %7.3f %7.3f" axis_fmt = "%9.3f %9.3f %9.3f" fout.write("\n".join([ "!FORMAT=XDS_ASCII MERGE=FALSE FRIEDEL'S_LAW=TRUE", "!Generated by dials.export", "!DATA_RANGE= %d %d" % image_range, "!ROTATION_AXIS= %9.6f %9.6f %9.6f" % axis.elems, "!OSCILLATION_RANGE= %f" % phi_range, "!STARTING_ANGLE= %f" % phi_start, "!STARTING_FRAME= %d" % image_range[0], "!SPACE_GROUP_NUMBER= %d" % experiment.crystal.get_space_group().type().number(), "!UNIT_CELL_CONSTANTS= %s" % (cell_fmt % unit_cell.parameters()), "!UNIT_CELL_A-AXIS= %s" % (axis_fmt % real_space_ABC[0:3]), "!UNIT_CELL_B-AXIS= %s" % (axis_fmt % real_space_ABC[3:6]), "!UNIT_CELL_C-AXIS= %s" % (axis_fmt % real_space_ABC[6:9]), "!X-RAY_WAVELENGTH= %f" % experiment.beam.get_wavelength(), "!INCIDENT_BEAM_DIRECTION= %f %f %f" % beam.elems, "!NX= %d NY= %d QX= %f QY= %f" % (nx, ny, qx, qy), "!ORGX= %9.2f ORGY= %9.2f" % (orgx, orgy), "!DETECTOR_DISTANCE= %8.3f" % distance, "!DIRECTION_OF_DETECTOR_X-AXIS= %9.5f %9.5f %9.5f" % fast.elems, "!DIRECTION_OF_DETECTOR_Y-AXIS= %9.5f %9.5f %9.5f" % slow.elems, "!VARIANCE_MODEL= %7.3e %7.3e" % var_model, "!NUMBER_OF_ITEMS_IN_EACH_DATA_RECORD=12", "!ITEM_H=1", "!ITEM_K=2", "!ITEM_L=3", "!ITEM_IOBS=4", "!ITEM_SIGMA(IOBS)=5", "!ITEM_XD=6", "!ITEM_YD=7", "!ITEM_ZD=8", "!ITEM_RLP=9", "!ITEM_PEAK=10", "!ITEM_CORR=11", "!ITEM_PSI=12", "!END_OF_HEADER", "", ])) # then write the data records s0 = Rd * matrix.col(experiment.beam.get_s0()) for j in range(nref): x, y, z = integrated_data["xyzcal.px"][j] phi = phi_start + z * phi_range h, k, l = miller_index[j] X = (UB * (h, k, l)).rotate(axis, phi, deg=True) s = s0 + X g = s.cross(s0).normalize() f = (s - s0).normalize() # find component of beam perpendicular to f, e e = -(s + s0).normalize() if h == k and k == l: u = (h, -h, 0) else: u = (k - l, l - h, h - k) q = ((matrix.col(u).transpose() * UB.inverse()).normalize().transpose().rotate(axis, phi, deg=True)) psi = q.angle(g, deg=True) if q.dot(e) < 0: psi *= -1 fout.write("%d %d %d %f %f %f %f %f %f %.1f %.1f %f\n" % ( h, k, l, I[j], sigI[j], x, y, z, scl[j], partiality[j], prof_corr[j], psi, )) fout.write("!END_OF_DATA\n") fout.close() logger.info("Output %d reflections to %s" % (nref, params.xds_ascii.hklout))
def _export_experiment(filename, integrated_data, experiment, params, var_model=(1, 0)): # type: (str, flex.reflection_table, dxtbx.model.Experiment, libtbx.phil.scope_extract, Tuple) """Export a single experiment to an XDS_ASCII.HKL format file. Args: filename: The file to write to integrated_data: The reflection table, pre-selected to one experiment experiment: The experiment list entry to export params: The PHIL configuration object var_model: """ # export for xds_ascii should only be for non-scaled reflections assert any(i in integrated_data for i in ["intensity.sum.value", "intensity.prf.value"]) # Handle requesting profile intensities (default via auto) but no column if "profile" in params.intensity and "intensity.prf.value" not in integrated_data: raise Sorry( "Requested profile intensity data but only summed present. Use intensity=sum." ) integrated_data = filter_reflection_table( integrated_data, intensity_choice=params.intensity, partiality_threshold=params.mtz.partiality_threshold, combine_partials=params.mtz.combine_partials, min_isigi=params.mtz.min_isigi, filter_ice_rings=params.mtz.filter_ice_rings, d_min=params.mtz.d_min, ) # calculate the scl = lp/dqe correction for outputting but don't apply it as # it has already been applied in filter_reflection_table ( integrated_data, scl, ) = FilteringReductionMethods.calculate_lp_qe_correction_and_filter( integrated_data) # sort data before output nref = len(integrated_data["miller_index"]) indices = flex.size_t_range(nref) unique = copy.deepcopy(integrated_data["miller_index"]) map_to_asu(experiment.crystal.get_space_group().type(), False, unique) perm = sorted(indices, key=lambda k: unique[k]) integrated_data = integrated_data.select(flex.size_t(perm)) if experiment.goniometer is None: print( "Warning: No goniometer. Experimentally exporting with (1 0 0) axis" ) unit_cell = experiment.crystal.get_unit_cell() if experiment.scan is None: print( "Warning: No Scan. Experimentally exporting no-oscillation values") image_range = (1, 1) phi_start, phi_range = 0.0, 0.0 else: image_range = experiment.scan.get_image_range() phi_start, phi_range = experiment.scan.get_image_oscillation( image_range[0]) # gather the required information for the reflection file nref = len(integrated_data["miller_index"]) miller_index = integrated_data["miller_index"] # profile correlation if "profile.correlation" in integrated_data: prof_corr = 100.0 * integrated_data["profile.correlation"] else: prof_corr = flex.double(nref, 100.0) # partiality if "partiality" in integrated_data: partiality = 100 * integrated_data["partiality"] else: prof_corr = flex.double(nref, 100.0) if "intensity.sum.value" in integrated_data: I = integrated_data["intensity.sum.value"] V = integrated_data["intensity.sum.variance"] assert V.all_gt(0) V = var_model[0] * (V + var_model[1] * I * I) sigI = flex.sqrt(V) else: I = integrated_data["intensity.prf.value"] V = integrated_data["intensity.prf.variance"] assert V.all_gt(0) V = var_model[0] * (V + var_model[1] * I * I) sigI = flex.sqrt(V) fout = open(filename, "w") # first write the header - in the "standard" coordinate frame... panel = experiment.detector[0] fast = panel.get_fast_axis() slow = panel.get_slow_axis() Rd = align_reference_frame(fast, (1, 0, 0), slow, (0, 1, 0)) print("Coordinate change:") print("%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n" % Rd.elems) fast = Rd * fast slow = Rd * slow qx, qy = panel.get_pixel_size() nx, ny = panel.get_image_size() distance = matrix.col(Rd * panel.get_origin()).dot( matrix.col(Rd * panel.get_normal())) org = Rd * (matrix.col(panel.get_origin()) - distance * matrix.col(panel.get_normal())) orgx = -org.dot(fast) / qx orgy = -org.dot(slow) / qy UB = Rd * matrix.sqr(experiment.crystal.get_A()) real_space_ABC = UB.inverse().elems if experiment.goniometer is not None: axis = Rd * experiment.goniometer.get_rotation_axis() else: axis = Rd * (1, 0, 0) beam = Rd * experiment.beam.get_s0() cell_fmt = "%9.3f %9.3f %9.3f %7.3f %7.3f %7.3f" axis_fmt = "%9.3f %9.3f %9.3f" fout.write("\n".join([ "!FORMAT=XDS_ASCII MERGE=FALSE FRIEDEL'S_LAW=TRUE", "!Generated by dials.export", "!DATA_RANGE= %d %d" % image_range, "!ROTATION_AXIS= %9.6f %9.6f %9.6f" % axis.elems, "!OSCILLATION_RANGE= %f" % phi_range, "!STARTING_ANGLE= %f" % phi_start, "!STARTING_FRAME= %d" % image_range[0], "!SPACE_GROUP_NUMBER= %d" % experiment.crystal.get_space_group().type().number(), "!UNIT_CELL_CONSTANTS= %s" % (cell_fmt % unit_cell.parameters()), "!UNIT_CELL_A-AXIS= %s" % (axis_fmt % real_space_ABC[0:3]), "!UNIT_CELL_B-AXIS= %s" % (axis_fmt % real_space_ABC[3:6]), "!UNIT_CELL_C-AXIS= %s" % (axis_fmt % real_space_ABC[6:9]), "!X-RAY_WAVELENGTH= %f" % experiment.beam.get_wavelength(), "!INCIDENT_BEAM_DIRECTION= %f %f %f" % beam.elems, "!NX= %d NY= %d QX= %f QY= %f" % (nx, ny, qx, qy), "!ORGX= %9.2f ORGY= %9.2f" % (orgx, orgy), "!DETECTOR_DISTANCE= %8.3f" % distance, "!DIRECTION_OF_DETECTOR_X-AXIS= %9.5f %9.5f %9.5f" % fast.elems, "!DIRECTION_OF_DETECTOR_Y-AXIS= %9.5f %9.5f %9.5f" % slow.elems, "!VARIANCE_MODEL= %7.3e %7.3e" % var_model, "!NUMBER_OF_ITEMS_IN_EACH_DATA_RECORD=12", "!ITEM_H=1", "!ITEM_K=2", "!ITEM_L=3", "!ITEM_IOBS=4", "!ITEM_SIGMA(IOBS)=5", "!ITEM_XD=6", "!ITEM_YD=7", "!ITEM_ZD=8", "!ITEM_RLP=9", "!ITEM_PEAK=10", "!ITEM_CORR=11", "!ITEM_PSI=12", "!END_OF_HEADER", "", ])) # then write the data records s0 = Rd * matrix.col(experiment.beam.get_s0()) for j in range(nref): x, y, z = integrated_data["xyzcal.px"][j] phi = phi_start + z * phi_range h, k, l = miller_index[j] X = (UB * (h, k, l)).rotate(axis, phi, deg=True) s = s0 + X g = s.cross(s0).normalize() # find component of beam perpendicular to f, e e = -(s + s0).normalize() if h == k and k == l: u = (h, -h, 0) else: u = (k - l, l - h, h - k) q = ((matrix.col(u).transpose() * UB.inverse()).normalize().transpose().rotate(axis, phi, deg=True)) psi = q.angle(g, deg=True) if q.dot(e) < 0: psi *= -1 fout.write("%d %d %d %f %f %f %f %f %f %.1f %.1f %f\n" % ( h, k, l, I[j], sigI[j], x, y, z, scl[j], partiality[j], prof_corr[j], psi, )) fout.write("!END_OF_DATA\n") fout.close() logger.info("Output %d reflections to %s" % (nref, filename))
def merge_and_truncate(params, experiments, reflections): """Filter data, assess space group, run french wilson and Wilson stats.""" logger.info("\nMerging scaled reflection data\n") # first filter bad reflections using dials.util.filter methods reflections = filter_reflection_table( reflections[0], intensity_choice=["scale"], d_min=params.d_min, combine_partials=params.combine_partials, partiality_threshold=params.partiality_threshold, ) # ^ scale factor has been applied, so now set to 1.0 - okay as not # going to output scale factor in merged mtz. reflections["inverse_scale_factor"] = flex.double(reflections.size(), 1.0) scaled_array = scaled_data_as_miller_array([reflections], experiments) if params.anomalous: anomalous_scaled = scaled_array.as_anomalous_array() merged = scaled_array.merge_equivalents( use_internal_variance=params.merging.use_internal_variance).array() merged_anom = None if params.anomalous: merged_anom = anomalous_scaled.merge_equivalents( use_internal_variance=params.merging.use_internal_variance).array( ) # Before merge, do some assessment of the space_group if params.assess_space_group: merged_reflections = flex.reflection_table() merged_reflections["intensity"] = merged.data() merged_reflections["variance"] = merged.sigmas()**2 merged_reflections["miller_index"] = merged.indices() logger.info("Running systematic absences check") run_sys_abs_checks(experiments, merged_reflections) # Run the stats on truncating on anomalous or non anomalous? if params.anomalous: intensities = merged_anom else: intensities = merged assert intensities.is_xray_intensity_array() amplitudes = None anom_amplitudes = None if params.truncate: logger.info("\nScaling input intensities via French-Wilson Method") out = StringIO() if params.anomalous: anom_amplitudes = intensities.french_wilson(params=params, log=out) n_removed = intensities.size() - anom_amplitudes.size() assert anom_amplitudes.is_xray_amplitude_array() amplitudes = anom_amplitudes.as_non_anomalous_array() amplitudes = amplitudes.merge_equivalents().array() else: amplitudes = intensities.french_wilson(params=params, log=out) n_removed = intensities.size() - amplitudes.size() logger.info("Total number of rejected intensities %s", n_removed) logger.debug(out.getvalue()) if params.reporting.wilson_stats: if not intensities.space_group().is_centric(): wilson_scaling = data_statistics.wilson_scaling( miller_array=intensities, n_residues=params.n_residues) # XXX default n_residues? # Divert output through logger - do with StringIO rather than # info_handle else get way too much whitespace in output. out = StringIO() wilson_scaling.show(out=out) logger.info(out.getvalue()) # Apply wilson B to give absolute scale? # Show merging stats again. if params.reporting.merging_stats: stats, anom_stats = merging_stats_from_scaled_array( scaled_array, params.merging.n_bins, params.merging.use_internal_variance) if params.merging.anomalous: logger.info(make_merging_statistics_summary(anom_stats)) else: logger.info(make_merging_statistics_summary(stats)) return merged, merged_anom, amplitudes, anom_amplitudes
def write(self, experiments, reflections): """ Write the experiments and reflections to file """ # if mmcif filename is auto, then choose scaled.cif or integrated.cif if self.params.mmcif.hklout in (None, Auto, "auto"): if ("intensity.scale.value" in reflections) and ("intensity.scale.variance" in reflections): filename = "scaled.cif" logger.info( "Data appears to be scaled, setting mmcif.hklout = 'scaled.cif'" ) else: filename = "integrated.cif" logger.info( "Data appears to be unscaled, setting mmcif.hklout = 'integrated.cif'" ) else: filename = self.params.mmcif.hklout # Select reflections selection = reflections.get_flags(reflections.flags.integrated, all=True) reflections = reflections.select(selection) # Filter out bad variances and other issues, but don't filter on ice rings # or alter partialities. # Assumes you want to apply the lp and dqe corrections to sum and prf # Do we want to combine partials? reflections = filter_reflection_table( reflections, self.params.intensity, combine_partials=False, partiality_threshold=0.0, d_min=self.params.mtz.d_min, ) # Get the cif block cif_block = iotbx.cif.model.block() # Audit trail dials_version = dials.util.version.dials_version() cif_block["_audit.creation_method"] = dials_version cif_block["_audit.creation_date"] = datetime.date.today().isoformat() cif_block["_computing.data_reduction"] = ( "%s (Winter, G. et al., 2018)" % dials_version) cif_block[ "_publ.section_references"] = "Winter, G. et al. (2018) Acta Cryst. D74, 85-97." if "scale" in self.params.intensity: cif_block[ "_publ.section_references"] += "\nBeilsten-Edmands, J. et al. (2020) Acta Cryst. D76, 385-399." # Hard coding X-ray cif_block["_pdbx_diffrn_data_section.id"] = "dials" cif_block["_pdbx_diffrn_data_section.type_scattering"] = "x-ray" cif_block["_pdbx_diffrn_data_section.type_merged"] = "false" cif_block["_pdbx_diffrn_data_section.type_scaled"] = str( "scale" in self.params.intensity).lower() # FIXME finish metadata addition - detector and source details needed # http://mmcif.wwpdb.org/dictionaries/mmcif_pdbx_v50.dic/Categories/index.html # Add source information; # _diffrn_source.pdbx_wavelength_list = (list of wavelengths) # _diffrn_source.source = (general class of source e.g. synchrotron) # _diffrn_source.type = (specific beamline or instrument e.g DIAMOND BEAMLINE I04) wls = [] epochs = [] for exp in experiments: wls.append(round(exp.beam.get_wavelength(), 5)) epochs.append(exp.scan.get_epochs()[0]) unique_wls = set(wls) cif_block["_diffrn_source.pdbx_wavelength_list"] = ", ".join( str(w) for w in unique_wls) # Add detector information; # _diffrn_detector.detector = (general class e.g. PIXEL, PLATE etc) # _diffrn_detector.pdbx_collection_date = (Date of collection yyyy-mm-dd) # _diffrn_detector.type = (full name of detector e.g. DECTRIS PILATUS3 2M) # One date is required, so if multiple just use the first date. min_epoch = min(epochs) date_str = time.strftime("%Y-%m-%d", time.gmtime(min_epoch)) cif_block["_diffrn_detector.pdbx_collection_date"] = date_str # Write the crystal information cif_loop = iotbx.cif.model.loop(header=( "_pdbx_diffrn_unmerged_cell.ordinal", "_pdbx_diffrn_unmerged_cell.crystal_id", "_pdbx_diffrn_unmerged_cell.wavelength", "_pdbx_diffrn_unmerged_cell.cell_length_a", "_pdbx_diffrn_unmerged_cell.cell_length_b", "_pdbx_diffrn_unmerged_cell.cell_length_c", "_pdbx_diffrn_unmerged_cell.cell_angle_alpha", "_pdbx_diffrn_unmerged_cell.cell_angle_beta", "_pdbx_diffrn_unmerged_cell.cell_angle_gamma", "_pdbx_diffrn_unmerged_cell.Bravais_lattice", )) crystals = experiments.crystals() crystal_to_id = {crystal: i + 1 for i, crystal in enumerate(crystals)} for i, exp in enumerate(experiments): crystal = exp.crystal crystal_id = crystal_to_id[crystal] wavelength = exp.beam.get_wavelength() a, b, c, alpha, beta, gamma = crystal.get_unit_cell().parameters() latt_type = str( bravais_types.bravais_lattice(group=crystal.get_space_group())) cif_loop.add_row((i + 1, crystal_id, wavelength, a, b, c, alpha, beta, gamma, latt_type)) cif_block.add_loop(cif_loop) # Write the scan information cif_loop = iotbx.cif.model.loop(header=( "_pdbx_diffrn_scan.scan_id", "_pdbx_diffrn_scan.crystal_id", "_pdbx_diffrn_scan.image_id_begin", "_pdbx_diffrn_scan.image_id_end", "_pdbx_diffrn_scan.scan_angle_begin", "_pdbx_diffrn_scan.scan_angle_end", )) for i, exp in enumerate(experiments): scan = exp.scan crystal_id = crystal_to_id[exp.crystal] image_range = scan.get_image_range() osc_range = scan.get_oscillation_range(deg=True) cif_loop.add_row(( i + 1, crystal_id, image_range[0], image_range[1], osc_range[0], osc_range[1], )) cif_block.add_loop(cif_loop) # Make a dict of unit_cell parameters unit_cell_parameters = {} if crystal.num_scan_points > 1: for i in range(crystal.num_scan_points): a, b, c, alpha, beta, gamma = crystal.get_unit_cell_at_scan_point( i).parameters() unit_cell_parameters[i] = (a, b, c, alpha, beta, gamma) else: unit_cell_parameters[0] = (a, b, c, alpha, beta, gamma) ### _pdbx_diffrn_image_proc has been removed from the dictionary extension. ### Keeping this section commented out as it may be added back in some ### form in future # # Write the image data # scan = experiments[0].scan # z0 = scan.get_image_range()[0] # # cif_loop = iotbx.cif.model.loop( # header=("_pdbx_diffrn_image_proc.image_id", # "_pdbx_diffrn_image_proc.crystal_id", # "_pdbx_diffrn_image_proc.image_number", # "_pdbx_diffrn_image_proc.phi_value", # "_pdbx_diffrn_image_proc.wavelength", # "_pdbx_diffrn_image_proc.cell_length_a", # "_pdbx_diffrn_image_proc.cell_length_b", # "_pdbx_diffrn_image_proc.cell_length_c", # "_pdbx_diffrn_image_proc.cell_angle_alpha", # "_pdbx_diffrn_image_proc.cell_angle_beta", # "_pdbx_diffrn_image_proc.cell_angle_gamma")) # for i in range(len(scan)): # z = z0 + i # if crystal.num_scan_points > 1: # a, b, c, alpha, beta, gamma = unit_cell_parameters[i] # else: # a, b, c, alpha, beta, gamma = unit_cell_parameters[0] # # phi is the angle at the image centre # phi = scan.get_angle_from_image_index(z + 0.5, deg=True) # cif_loop.add_row((i+1, 1, z, phi, wavelength, # a, b, c, alpha, beta, gamma)) # cif_block.add_loop(cif_loop) # Write reflection data # Required columns header = ( "_pdbx_diffrn_unmerged_refln.reflection_id", "_pdbx_diffrn_unmerged_refln.scan_id", "_pdbx_diffrn_unmerged_refln.image_id_begin", "_pdbx_diffrn_unmerged_refln.image_id_end", "_pdbx_diffrn_unmerged_refln.index_h", "_pdbx_diffrn_unmerged_refln.index_k", "_pdbx_diffrn_unmerged_refln.index_l", ) headernames = { "scales": "_pdbx_diffrn_unmerged_refln.scale_value", "intensity.scale.value": "_pdbx_diffrn_unmerged_refln.intensity_meas", "intensity.scale.sigma": "_pdbx_diffrn_unmerged_refln.intensity_sigma", "intensity.sum.value": "_pdbx_diffrn_unmerged_refln.intensity_sum", "intensity.sum.sigma": "_pdbx_diffrn_unmerged_refln.intensity_sum_sigma", "intensity.prf.value": "_pdbx_diffrn_unmerged_refln.intensity_prf", "intensity.prf.sigma": "_pdbx_diffrn_unmerged_refln.intensity_prf_sigma", "angle": "_pdbx_diffrn_unmerged_refln.scan_angle_reflection", "partiality": "_pdbx_diffrn_unmerged_refln.partiality", } variables_present = [] if "scale" in self.params.intensity: reflections["scales"] = 1.0 / reflections["inverse_scale_factor"] reflections["intensity.scale.sigma"] = flex.sqrt( reflections["intensity.scale.variance"]) variables_present.extend( ["scales", "intensity.scale.value", "intensity.scale.sigma"]) if "sum" in self.params.intensity: reflections["intensity.sum.sigma"] = flex.sqrt( reflections["intensity.sum.variance"]) variables_present.extend( ["intensity.sum.value", "intensity.sum.sigma"]) if "profile" in self.params.intensity: reflections["intensity.prf.sigma"] = flex.sqrt( reflections["intensity.prf.variance"]) variables_present.extend( ["intensity.prf.value", "intensity.prf.sigma"]) # Should always exist reflections["angle"] = reflections["xyzcal.mm"].parts()[2] * RAD2DEG variables_present.extend(["angle"]) if "partiality" in reflections: variables_present.extend(["partiality"]) for name in variables_present: if name in reflections: header += (headernames[name], ) if "scale" in self.params.intensity: # Write dataset_statistics - first make a miller array crystal_symmetry = cctbxcrystal.symmetry( space_group=experiments[0].crystal.get_space_group(), unit_cell=experiments[0].crystal.get_unit_cell(), ) miller_set = miller.set( crystal_symmetry=crystal_symmetry, indices=reflections["miller_index"], anomalous_flag=False, ) i_obs = miller.array(miller_set, data=reflections["intensity.scale.value"]) i_obs.set_observation_type_xray_intensity() i_obs.set_sigmas(reflections["intensity.scale.sigma"]) i_obs.set_info( miller.array_info(source="DIALS", source_type="reflection_tables")) result = dataset_statistics( i_obs=i_obs, crystal_symmetry=crystal_symmetry, use_internal_variance=False, eliminate_sys_absent=False, ) cif_block.update(result.as_cif_block()) cif_loop = iotbx.cif.model.loop(header=header) for i, r in enumerate(reflections.rows()): refl_id = i + 1 scan_id = r["id"] + 1 _, _, _, _, z0, z1 = r["bbox"] h, k, l = r["miller_index"] variable_values = tuple((r[name]) for name in variables_present) cif_loop.add_row((refl_id, scan_id, z0, z1, h, k, l) + variable_values) cif_block.add_loop(cif_loop) # Add the block self._cif["dials"] = cif_block # Print to file with open(filename, "w") as fh: self._cif.show(out=fh) # Log logger.info("Wrote reflections to %s" % filename)
def make_cif_block(self, experiments, reflections): """Write the data to a cif block""" # Select reflections selection = reflections.get_flags(reflections.flags.integrated, all=True) reflections = reflections.select(selection) # Filter out bad variances and other issues, but don't filter on ice rings # or alter partialities. # Assumes you want to apply the lp and dqe corrections to sum and prf # Do we want to combine partials? reflections = filter_reflection_table( reflections, self.params.intensity, combine_partials=False, partiality_threshold=0.0, d_min=self.params.mtz.d_min, ) # Get the cif block cif_block = iotbx.cif.model.block() # Audit trail dials_version = dials.util.version.dials_version() cif_block["_audit.revision_id"] = 1 cif_block["_audit.creation_method"] = dials_version cif_block["_audit.creation_date"] = datetime.date.today().isoformat() cif_block["_entry.id"] = "DIALS" # add software loop mmcif_software_header = ( "_software.pdbx_ordinal", "_software.citation_id", "_software.name", # as defined at [1] "_software.version", "_software.type", "_software.classification", "_software.description", ) mmcif_citations_header = ( "_citation.id", "_citation.journal_abbrev", "_citation.journal_volume", "_citation.journal_issue", "_citation.page_first", "_citation.page_last", "_citation.year", "_citation.title", ) software_loop = iotbx.cif.model.loop(header=mmcif_software_header) citations_loop = iotbx.cif.model.loop(header=mmcif_citations_header) software_loop.add_row( ( 1, 1, "DIALS", dials_version, "package", "data processing", "Data processing and integration within the DIALS software package", ) ) citations_loop.add_row( ( 1, "Acta Cryst. D", 74, 2, 85, 97, 2018, "DIALS: implementation and evaluation of a new integration package", ) ) if "scale" in self.params.intensity: software_loop.add_row( ( 2, 2, "DIALS", dials_version, "program", "data scaling", "Data scaling and merging within the DIALS software package", ) ) citations_loop.add_row( ( 2, "Acta Cryst. D", 76, 4, 385, 399, 2020, "Scaling diffraction data in the DIALS software package: algorithms and new approaches for multi-crystal scaling", ) ) cif_block.add_loop(software_loop) cif_block.add_loop(citations_loop) # Hard coding X-ray if self.params.mmcif.pdb_version == "v5_next": cif_block["_pdbx_diffrn_data_section.id"] = "dials" cif_block["_pdbx_diffrn_data_section.type_scattering"] = "x-ray" cif_block["_pdbx_diffrn_data_section.type_merged"] = "false" cif_block["_pdbx_diffrn_data_section.type_scaled"] = str( "scale" in self.params.intensity ).lower() # FIXME finish metadata addition - detector and source details needed # http://mmcif.wwpdb.org/dictionaries/mmcif_pdbx_v50.dic/Categories/index.html # Add source information; # _diffrn_source.pdbx_wavelength_list = (list of wavelengths) # _diffrn_source.source = (general class of source e.g. synchrotron) # _diffrn_source.type = (specific beamline or instrument e.g DIAMOND BEAMLINE I04) wls = [] epochs = [] for exp in experiments: wls.append(round(exp.beam.get_wavelength(), 5)) epochs.append(exp.scan.get_epochs()[0]) unique_wls = set(wls) cif_block["_exptl_crystal.id"] = 1 # links to crystal_id cif_block["_diffrn.id"] = 1 # links to diffrn_id cif_block["_diffrn.crystal_id"] = 1 cif_block["_diffrn_source.diffrn_id"] = 1 cif_block["_diffrn_source.pdbx_wavelength_list"] = ", ".join( str(w) for w in unique_wls ) # Add detector information; # _diffrn_detector.detector = (general class e.g. PIXEL, PLATE etc) # _diffrn_detector.pdbx_collection_date = (Date of collection yyyy-mm-dd) # _diffrn_detector.type = (full name of detector e.g. DECTRIS PILATUS3 2M) # One date is required, so if multiple just use the first date. min_epoch = min(epochs) date_str = time.strftime("%Y-%m-%d", time.gmtime(min_epoch)) cif_block["_diffrn_detector.diffrn_id"] = 1 cif_block["_diffrn_detector.pdbx_collection_date"] = date_str # Write reflection data # Required columns header = ( "_pdbx_diffrn_unmerged_refln.reflection_id", "_pdbx_diffrn_unmerged_refln.scan_id", "_pdbx_diffrn_unmerged_refln.image_id_begin", "_pdbx_diffrn_unmerged_refln.image_id_end", "_pdbx_diffrn_unmerged_refln.index_h", "_pdbx_diffrn_unmerged_refln.index_k", "_pdbx_diffrn_unmerged_refln.index_l", ) extra_items = { "scales": ("_pdbx_diffrn_unmerged_refln.scale_value", "%5.3f"), "intensity.scale.value": ( "_pdbx_diffrn_unmerged_refln.intensity_meas", "%8.3f", ), "intensity.scale.sigma": ( "_pdbx_diffrn_unmerged_refln.intensity_sigma", "%8.3f", ), "intensity.sum.value": ( "_pdbx_diffrn_unmerged_refln.intensity_sum", "%8.3f", ), "intensity.sum.sigma": ( "_pdbx_diffrn_unmerged_refln.intensity_sum_sigma", "%8.3f", ), "intensity.prf.value": ( "_pdbx_diffrn_unmerged_refln.intensity_prf", "%8.3f", ), "intensity.prf.sigma": ( "_pdbx_diffrn_unmerged_refln.intensity_prf_sigma", "%8.3f", ), "angle": ("_pdbx_diffrn_unmerged_refln.scan_angle_reflection", "%7.4f"), "partiality": ("_pdbx_diffrn_unmerged_refln.partiality", "%7.4f"), } variables_present = [] if "scale" in self.params.intensity: reflections["scales"] = 1.0 / reflections["inverse_scale_factor"] reflections["intensity.scale.sigma"] = flex.sqrt( reflections["intensity.scale.variance"] ) variables_present.extend( ["scales", "intensity.scale.value", "intensity.scale.sigma"] ) if "sum" in self.params.intensity: reflections["intensity.sum.sigma"] = flex.sqrt( reflections["intensity.sum.variance"] ) variables_present.extend(["intensity.sum.value", "intensity.sum.sigma"]) if "profile" in self.params.intensity: reflections["intensity.prf.sigma"] = flex.sqrt( reflections["intensity.prf.variance"] ) variables_present.extend(["intensity.prf.value", "intensity.prf.sigma"]) # Should always exist reflections["angle"] = reflections["xyzcal.mm"].parts()[2] * RAD2DEG variables_present.extend(["angle"]) if "partiality" in reflections: variables_present.extend(["partiality"]) for name in variables_present: if name in reflections: header += (extra_items[name][0],) self._fmt += " " + extra_items[name][1] if "scale" in self.params.intensity: # Write dataset_statistics - first make a miller array crystal_symmetry = cctbxcrystal.symmetry( space_group=experiments[0].crystal.get_space_group(), unit_cell=experiments[0].crystal.get_unit_cell(), ) miller_set = miller.set( crystal_symmetry=crystal_symmetry, indices=reflections["miller_index"], anomalous_flag=False, ) i_obs = miller.array(miller_set, data=reflections["intensity.scale.value"]) i_obs.set_observation_type_xray_intensity() i_obs.set_sigmas(reflections["intensity.scale.sigma"]) i_obs.set_info( miller.array_info(source="DIALS", source_type="reflection_tables") ) result = dataset_statistics( i_obs=i_obs, crystal_symmetry=crystal_symmetry, use_internal_variance=False, eliminate_sys_absent=False, ) merged_block = iotbx.cif.model.block() merged_block["_reflns.pdbx_ordinal"] = 1 merged_block["_reflns.pdbx_diffrn_id"] = 1 merged_block["_reflns.entry_id"] = "DIALS" merged_data = result.as_cif_block() merged_block.update(merged_data) cif_block.update(merged_block) # Write the crystal information # if v5, thats all so return if self.params.mmcif.pdb_version == "v5": return cif_block # continue if v5_next cif_loop = iotbx.cif.model.loop( header=( "_pdbx_diffrn_unmerged_cell.ordinal", "_pdbx_diffrn_unmerged_cell.crystal_id", "_pdbx_diffrn_unmerged_cell.wavelength", "_pdbx_diffrn_unmerged_cell.cell_length_a", "_pdbx_diffrn_unmerged_cell.cell_length_b", "_pdbx_diffrn_unmerged_cell.cell_length_c", "_pdbx_diffrn_unmerged_cell.cell_angle_alpha", "_pdbx_diffrn_unmerged_cell.cell_angle_beta", "_pdbx_diffrn_unmerged_cell.cell_angle_gamma", "_pdbx_diffrn_unmerged_cell.Bravais_lattice", ) ) crystals = experiments.crystals() crystal_to_id = {crystal: i + 1 for i, crystal in enumerate(crystals)} for i, exp in enumerate(experiments): crystal = exp.crystal crystal_id = crystal_to_id[crystal] wavelength = exp.beam.get_wavelength() a, b, c, alpha, beta, gamma = crystal.get_unit_cell().parameters() latt_type = str( bravais_types.bravais_lattice(group=crystal.get_space_group()) ) cif_loop.add_row( (i + 1, crystal_id, wavelength, a, b, c, alpha, beta, gamma, latt_type) ) cif_block.add_loop(cif_loop) # Write the scan information cif_loop = iotbx.cif.model.loop( header=( "_pdbx_diffrn_scan.scan_id", "_pdbx_diffrn_scan.crystal_id", "_pdbx_diffrn_scan.image_id_begin", "_pdbx_diffrn_scan.image_id_end", "_pdbx_diffrn_scan.scan_angle_begin", "_pdbx_diffrn_scan.scan_angle_end", ) ) expid_to_scan_id = {exp.identifier: i + 1 for i, exp in enumerate(experiments)} for i, exp in enumerate(experiments): scan = exp.scan crystal_id = crystal_to_id[exp.crystal] image_range = scan.get_image_range() osc_range = scan.get_oscillation_range(deg=True) cif_loop.add_row( ( i + 1, crystal_id, image_range[0], image_range[1], osc_range[0], osc_range[1], ) ) cif_block.add_loop(cif_loop) _, _, _, _, z0, z1 = reflections["bbox"].parts() h, k, l = [ hkl.iround() for hkl in reflections["miller_index"].as_vec3_double().parts() ] # make scan id consistent with header as defined above scan_id = flex.int(reflections.size(), 0) for id_ in reflections.experiment_identifiers().keys(): expid = reflections.experiment_identifiers()[id_] sel = reflections["id"] == id_ scan_id.set_selected(sel, expid_to_scan_id[expid]) loop_values = [ flex.size_t_range(1, len(reflections) + 1), scan_id, z0, z1, h, k, l, ] + [reflections[name] for name in variables_present] cif_loop = iotbx.cif.model.loop(data=dict(zip(header, loop_values))) cif_block.add_loop(cif_loop) return cif_block
def export_sadabs(integrated_data, experiment_list, params): """Export data from integrated_data corresponding to experiment_list to a file for input to SADABS. FIXME probably need to make a .p4p file as well...""" from dials.array_family import flex # for the moment assume (and assert) that we will convert data from exactly # one lattice... assert len(experiment_list) == 1 # select reflections that are assigned to an experiment (i.e. non-negative id) integrated_data = integrated_data.select(integrated_data["id"] >= 0) assert max(integrated_data["id"]) == 0 # export for sadabs should only be for non-scaled reflections assert any(i in integrated_data for i in ["intensity.sum.value", "intensity.prf.value"]) integrated_data = filter_reflection_table( integrated_data, intensity_choice=params.intensity, partiality_threshold=params.mtz.partiality_threshold, combine_partials=params.mtz.combine_partials, min_isigi=params.mtz.min_isigi, filter_ice_rings=params.mtz.filter_ice_rings, d_min=params.mtz.d_min, ) experiment = experiment_list[0] assert experiment.scan is not None # sort data before output nref = len(integrated_data["miller_index"]) indices = flex.size_t_range(nref) perm = sorted(indices, key=lambda k: integrated_data["miller_index"][k]) integrated_data = integrated_data.select(flex.size_t(perm)) assert experiment.goniometer is not None # Warn of unhelpful SADABS behaviour for certain multi-sequence data sets hkl_file_root, _ = os.path.splitext(params.sadabs.hklout) if not params.sadabs.run or re.search("_0+$", hkl_file_root): logger.warning( "It seems SADABS rejects multi-sequence data when the first " "filename ends " "'_0', '_00', etc., with a cryptic error message:\n" "\t'Inconsistent 2theta values in same scan'.\n" "You may need to begin the numbering of your SADABS HKL files from 1, " "rather than 0, and ensure the SADABS run/batch number is greater than 0." ) axis = matrix.col(experiment.goniometer.get_rotation_axis_datum()) beam = matrix.col(experiment.beam.get_sample_to_source_direction()) s0 = matrix.col(experiment.beam.get_s0()) F = matrix.sqr(experiment.goniometer.get_fixed_rotation()) S = matrix.sqr(experiment.goniometer.get_setting_rotation()) unit_cell = experiment.crystal.get_unit_cell() if params.debug: m_format = "%6.3f%6.3f%6.3f\n%6.3f%6.3f%6.3f\n%6.3f%6.3f%6.3f" c_format = "%.2f %.2f %.2f %.2f %.2f %.2f" logger.info( "Unit cell parameters from experiment: %s", c_format % unit_cell.parameters(), ) logger.info( "Symmetry: %s", experiment.crystal.get_space_group().type().lookup_symbol()) logger.info("Goniometer fixed matrix:\n%s", m_format % F.elems) logger.info("Goniometer setting matrix:\n%s", m_format % S.elems) logger.info("Goniometer scan axis:\n%6.3f%6.3f%6.3f", axis.elems) # detector scaling info assert len(experiment.detector) == 1 panel = experiment.detector[0] dims = panel.get_image_size() pixel = panel.get_pixel_size() fast_axis = matrix.col(panel.get_fast_axis()) slow_axis = matrix.col(panel.get_slow_axis()) normal = fast_axis.cross(slow_axis) detector2t = s0.angle(normal, deg=True) if params.debug: logger.info("Detector fast, slow axes:") logger.info("%6.3f%6.3f%6.3f", fast_axis.elems) logger.info("%6.3f%6.3f%6.3f", slow_axis.elems) logger.info("Detector two theta (degrees): %.2f", detector2t) scl_x = 512.0 / (dims[0] * pixel[0]) scl_y = 512.0 / (dims[1] * pixel[1]) image_range = experiment.scan.get_image_range() from cctbx.array_family import flex as cflex # implicit import # noqa: F401 from cctbx.miller import map_to_asu_isym # implicit import # noqa: F401 # gather the required information for the reflection file nref = len(integrated_data["miller_index"]) miller_index = integrated_data["miller_index"] if "intensity.sum.value" in integrated_data: I = integrated_data["intensity.sum.value"] V = integrated_data["intensity.sum.variance"] assert V.all_gt(0) sigI = flex.sqrt(V) else: I = integrated_data["intensity.prf.value"] V = integrated_data["intensity.prf.variance"] assert V.all_gt(0) sigI = flex.sqrt(V) # figure out scaling to make sure data fit into format 2F8.2 i.e. Imax < 1e5 Imax = flex.max(I) if params.debug: logger.info("Maximum intensity in file: %8.2f", Imax) if Imax > 99999.0: scale = 99999.0 / Imax I = I * scale sigI = sigI * scale phi_start, phi_range = experiment.scan.get_image_oscillation( image_range[0]) if params.sadabs.predict: logger.info("Using scan static predicted spot locations") from dials.algorithms.spot_prediction import ScanStaticReflectionPredictor predictor = ScanStaticReflectionPredictor(experiment) UB = experiment.crystal.get_A() predictor.for_reflection_table(integrated_data, UB) if not experiment.crystal.num_scan_points: logger.info("No scan varying model: use static") static = True else: static = False with open(params.sadabs.hklout, "w") as fout: for j in range(nref): h, k, l = miller_index[j] if params.sadabs.predict: x_mm, y_mm, z_rad = integrated_data["xyzcal.mm"][j] else: x_mm, y_mm, z_rad = integrated_data["xyzobs.mm.value"][j] z0 = integrated_data["xyzcal.px"][j][2] istol = int(round(10000 * unit_cell.stol((h, k, l)))) if params.sadabs.predict or static: # work from a scan static model & assume perfect goniometer # FIXME maybe should work back in the option to predict spot positions UB = matrix.sqr(experiment.crystal.get_A()) phi = phi_start + z0 * phi_range R = axis.axis_and_angle_as_r3_rotation_matrix(phi, deg=True) RUB = S * R * F * UB else: # properly compute RUB for every reflection UB = matrix.sqr( experiment.crystal.get_A_at_scan_point(int(round(z0)))) phi = phi_start + z0 * phi_range R = axis.axis_and_angle_as_r3_rotation_matrix(phi, deg=True) RUB = S * R * F * UB x = RUB * (h, k, l) s = (s0 + x).normalize() # can also compute s based on centre of mass of spot # s = (origin + x_mm * fast_axis + y_mm * slow_axis).normalize() astar = (RUB * (1, 0, 0)).normalize() bstar = (RUB * (0, 1, 0)).normalize() cstar = (RUB * (0, 0, 1)).normalize() ix = beam.dot(astar) iy = beam.dot(bstar) iz = beam.dot(cstar) dx = s.dot(astar) dy = s.dot(bstar) dz = s.dot(cstar) x = x_mm * scl_x y = y_mm * scl_y z = (z_rad * 180 / math.pi - phi_start) / phi_range fout.write("%4d%4d%4d%8.2f%8.2f%4d%8.5f%8.5f%8.5f%8.5f%8.5f%8.5f" % (h, k, l, I[j], sigI[j], params.sadabs.run, ix, dx, iy, dy, iz, dz)) fout.write("%7.2f%7.2f%8.2f%7.2f%5d\n" % (x, y, z, detector2t, istol)) fout.close() logger.info("Output %d reflections to %s", nref, params.sadabs.hklout)
def export_shelx(scaled_data, experiment_list, params): """Export scaled data corresponding to experiment_list to a SHELX HKL formatted text file.""" # Handle requesting profile intensities (default via auto) but no column if "profile" in params.intensity and "intensity.prf.value" not in scaled_data: raise Sorry( "Requested profile intensity data but only summed present. Use intensity=sum." ) # use supplied best unit cell or that determined from experiment list to define d in reflection table. best_unit_cell = params.mtz.best_unit_cell if best_unit_cell is None: best_unit_cell = determine_best_unit_cell(experiment_list) else: logger.info("Using supplied unit cell across experiments : %s", best_unit_cell) scaled_data["d"] = best_unit_cell.d(scaled_data["miller_index"]) # Clean up reflection table with mtz defaults (as in export_xds_ascii) scaled_data = filter_reflection_table( scaled_data, intensity_choice=params.intensity, partiality_threshold=params.mtz.partiality_threshold, combine_partials=params.mtz.combine_partials, min_isigi=params.mtz.min_isigi, filter_ice_rings=params.mtz.filter_ice_rings, d_min=params.mtz.d_min, ) # Check that all experiments have the same space group if len({x.crystal.get_space_group().make_tidy() for x in experiment_list}) != 1: raise ValueError("Experiments do not have a unique space group") # Create miller set with space group from 1st crystal in experiment list and best unit cell miller_set = miller.set( crystal_symmetry=crystal.symmetry( unit_cell=best_unit_cell, space_group=experiment_list[0].crystal.get_space_group(), ), indices=scaled_data["miller_index"], anomalous_flag=False, ) intensity_choice = ( params.intensity[0] if params.intensity[0] != "profile" else "prf" ) intensities, variances = ( scaled_data["intensity." + intensity_choice + ".value"], scaled_data["intensity." + intensity_choice + ".variance"], ) assert variances.all_gt(0) i_obs = miller.array(miller_set, data=intensities) i_obs.set_observation_type_xray_intensity() i_obs.set_sigmas(flex.sqrt(variances)) # If requested scale up data to maximise use of scale_range if params.shelx.scale: max_val = max( i_obs.data().min_max_mean().max, i_obs.sigmas().min_max_mean().max ) min_val = min( i_obs.data().min_max_mean().min, i_obs.sigmas().min_max_mean().min ) min_scale = params.shelx.scale_range[0] / min_val max_scale = params.shelx.scale_range[1] / max_val scale = min(min_scale, max_scale) i_obs = i_obs.apply_scaling(factor=scale) # write the SHELX HKL file hkl_file = params.shelx.hklout with open(hkl_file, "w") as hkl_file_object: i_obs.export_as_shelx_hklf( file_object=hkl_file_object, scale_range=params.shelx.scale_range, normalise_if_format_overflow=True, ) logger.info(f"Written {i_obs.size()} relflections to {hkl_file}") # and a stub of an .ins file with information from the .expt file _write_ins( experiment_list, best_unit_cell=params.mtz.best_unit_cell, ins_file=params.shelx.ins, ) logger.info(f"Written {params.shelx.ins}")