def get_map_and_model(params=None, map_data=None, crystal_symmetry=None, pdb_inp=None, ncs_obj=None, half_map_data_list=None, map_coords_inside_cell=True, out=sys.stdout): acc = None # accessor used to shift map back to original location if desired origin_frac = (0, 0, 0) if map_data and crystal_symmetry: original_crystal_symmetry = crystal_symmetry original_unit_cell_grid = None elif params.input_files.map_file: print >> out, "\nReading map from %s\n" % (params.input_files.map_file) from cctbx.maptbx.segment_and_split_map import get_map_object map_data,space_group,unit_cell,crystal_symmetry,origin_frac,acc,\ original_crystal_symmetry,original_unit_cell_grid=\ get_map_object(file_name=params.input_files.map_file,out=out) map_data = map_data.as_double() if origin_frac != (0, 0, 0) and acc is None: print >>out,"\nWARNING: Unable to place output map at position of "+\ "input map though input map has non-zero origin at %s\n" %( str(origin_frac)) elif params.input_files.map_coeffs_file: map_coeffs = get_map_coeffs_from_file( map_coeffs_file=params.input_files.map_coeffs_file, map_coeffs_labels=params.input_files.map_coeffs_labels) if not map_coeffs: raise Sorry("Could not get map coeffs from %s with labels %s" % (params.input_files.map_coeffs_file, params.input_files.map_coeffs_labels)) print >> out, "Map coefficients read from %s with labels %s" % ( params.input_files.map_coeffs_file, str(params.input_files.map_coeffs_labels)) crystal_symmetry = map_coeffs.crystal_symmetry() from cctbx.maptbx.segment_and_split_map import get_map_from_map_coeffs map_data = get_map_from_map_coeffs(map_coeffs=map_coeffs, crystal_symmetry=crystal_symmetry) acc = map_data.accessor() original_crystal_symmetry = crystal_symmetry original_unit_cell_grid = None if not params.crystal_info.resolution: params.crystal_info.resolution = map_coeffs.d_min() print >> out, "Resolution from map_coeffs is %7.2f A" % ( params.crystal_info.resolution) else: raise Sorry("Need ccp4 map or map_coeffs") if params.input_files.half_map_file: if len(params.input_files.half_map_file) != 2: raise Sorry("Please supply zero or two half_map files") half_map_data_list = [] from cctbx.maptbx.segment_and_split_map import get_map_object for file_name in params.input_files.half_map_file: print >> out, "\nReading half-map from %s\n" % (file_name) half_map_data,half_map_space_group,half_map_unit_cell,\ half_map_crystal_symmetry,half_map_origin_frac,half_map_acc,\ half_map_original_crystal_symmetry,half_map_original_unit_cell_grid=\ get_map_object(file_name=file_name,out=out) half_map_data = half_map_data.as_double() assert half_map_crystal_symmetry.is_similar_symmetry( crystal_symmetry) half_map_data_list.append(half_map_data) if params.crystal_info.resolution is None: raise Sorry("Need resolution if map is supplied") if params.crystal_info.resolution >= 10: print >>out,"\n** WARNING: auto_sharpen is designed for maps at a "+\ "resolution of about 4.5 A\nor better. Sharpening may be"+\ "poor at %7.0f A" %(params.crystal_info.resolution) if params.input_files.pdb_file and not pdb_inp: # get model model_file = params.input_files.pdb_file if not os.path.isfile(model_file): raise Sorry("Missing the model file: %s" % (model_file)) pdb_inp = iotbx.pdb.input(file_name=model_file) if origin_frac != (0, 0, 0): print >> out, "Shifting model by %s" % (str(origin_frac)) from cctbx.maptbx.segment_and_split_map import \ apply_shift_to_pdb_hierarchy origin_shift = crystal_symmetry.unit_cell().orthogonalize( (-origin_frac[0], -origin_frac[1], -origin_frac[2])) pdb_inp = apply_shift_to_pdb_hierarchy( origin_shift=origin_shift, crystal_symmetry=crystal_symmetry, pdb_hierarchy=pdb_inp.construct_hierarchy(), out=out).as_pdb_input() if map_coords_inside_cell: # put inside (0,1) pdb_inp = map_inside_cell(pdb_inp, crystal_symmetry=crystal_symmetry) if params.input_files.ncs_file and not ncs_obj: # NCS from cctbx.maptbx.segment_and_split_map import get_ncs ncs_obj, dummy_tracking_data = get_ncs(params, out=out) if origin_frac != (0, 0, 0): origin_shift = crystal_symmetry.unit_cell().orthogonalize( (-origin_frac[0], -origin_frac[1], -origin_frac[2])) print >> out, "Shifting NCS by (%7.2f,%7.2f,%7.2f) " % ( (origin_shift)) from scitbx.math import matrix ncs_obj = ncs_obj.coordinate_offset( coordinate_offset=matrix.col(origin_shift)) return pdb_inp,map_data,half_map_data_list,ncs_obj,crystal_symmetry,acc,\ original_crystal_symmetry,original_unit_cell_grid
def run(args, command_name="phenix.cif_as_mtz", out=sys.stdout, return_as_miller_arrays=False): if (len(args) == 0): args = ["--help"] try: command_line = (iotbx_option_parser( usage="%s [reflection_cif_file] [options]" % command_name, description='Example: %s r1o9ksf.ent --symmetry=pdb1o9k.ent' % command_name ).enable_symmetry_comprehensive().option( None, "--output_file_name", action="store", default=False, type="string", help="Output mtz file name." ).option( None, "--wavelength_id", action="store", default=None, type="int", help="Extract data set with given wavelength_id." ).option( None, "--crystal_id", action="store", default=None, type="int", help="Extract data set with given crystal_id." ).option( None, "--output_r_free_label", action="store", default="R-free-flags", type="string", help= "MTZ column label to use for R-free flags (default: R-free-flags)" ).option( None, "--merge", action="store_true", help="Merge non-unique data where present." ).option( None, "--incompatible_flags_to_work_set", action="store_true", help= "When merging place reflections with incompatible flags into the " "working set." ).option( None, "--remove_systematic_absences", action="store_true", help="Remove systematic absent reflections." ).option( None, "--map_to_asu", action="store_true", help="Map to asymmetric unit." ).option( "--show_details_if_error", action="store_true", help="Show data details for some errors." ).option( "--show_log", action="store_true", help="Show some output." ).option( "--ignore_bad_sigmas", action="store_true", help= "Set sigmas to None instead of raising an error when bad sigmas " "are present." ).option( "--extend_flags", action="store_true", help="Extend R-free flags to cover all reflections if necessary.") ).process(args=args) except Exception as e: if (str(e) != "0"): print(str(e)) sys.exit(0) crystal_symmetry = command_line.symmetry if (len(command_line.args) > 1): print("%d arguments are given from the command line:"% \ len(command_line.args), command_line.args, file=out) raise Sorry("Please specify one reflection cif file.") file_name = command_line.args[0] if (not os.path.isfile(file_name)): raise Sorry("File is not found: %s" % file_name) output_r_free_label = command_line.options.output_r_free_label if ((not output_r_free_label[0] in string.ascii_uppercase) or (re.search("[^a-zA-Z0-9_\-]", output_r_free_label))): raise Sorry(( "%s is not a suitable column label. MTZ format requires " + "an uppercase letter as the first character, and only alphanumeric " + "characters or hyphens in the rest of the string.") % output_r_free_label) result = process_files( file_name=file_name, crystal_symmetry=crystal_symmetry, output_file_name=command_line.options.output_file_name, wavelength_id=command_line.options.wavelength_id, crystal_id=command_line.options.crystal_id, show_details_if_error=command_line.options.show_details_if_error, output_r_free_label=command_line.options.output_r_free_label, merge_non_unique_under_symmetry=command_line.options.merge, map_to_asu=command_line.options.map_to_asu, remove_systematic_absences=command_line.options. remove_systematic_absences, incompatible_flags_to_work_set=command_line.options. incompatible_flags_to_work_set, return_as_miller_arrays=return_as_miller_arrays, ignore_bad_sigmas=command_line.options.ignore_bad_sigmas, extend_flags=command_line.options.extend_flags, log=out) if return_as_miller_arrays: return result
def __init__(self, fo2, fc, scale_factor=None, outlier_cutoff_factor=None, probability_plot_slope=None): self.probability_plot_slope = probability_plot_slope assert fo2.is_xray_intensity_array() assert fc.is_complex_array() assert not fo2.space_group().is_centric() if scale_factor is None: scale_factor = fo2.scale_factor(fc) fc2 = fc.as_intensity_array() self.delta_fc2 = fc2.anomalous_differences() self.delta_fo2 = fo2.anomalous_differences() self.n_bijvoet_pairs = self.delta_fo2.size() if outlier_cutoff_factor is not None: cutoff_sel = flex.abs(self.delta_fo2.data()) > ( outlier_cutoff_factor * scale_factor) * flex.max( flex.abs(self.delta_fc2.data())) self.delta_fo2 = self.delta_fo2.select(~cutoff_sel) self.delta_fc2 = self.delta_fc2.select(~cutoff_sel) self.delta_fc2 = self.delta_fc2.customized_copy( data=self.delta_fc2.data() * scale_factor) if not self.delta_fo2.size(): raise Sorry("Absolute structure could not be determined") min_gamma = -10 max_gamma = 10 # quick and dirty to find better min, max gammas max_log_p_obs = -1e100 while True: # search for the maximum width = max_gamma - min_gamma if width < 0.0001: break middle = (min_gamma + max_gamma)/2 a = middle - width/4 b = middle + width/4 value_a = self.log_p_obs_given_gamma(a) value_b = self.log_p_obs_given_gamma(b) if value_a > value_b: max_gamma = middle elif value_a == value_b: min_gamma = a max_gamma = b else: min_gamma = middle max_log_p_obs = max([max_log_p_obs, value_a, value_b]) while True: # search for where the curve becomes close to zero on the left min_gamma = middle - width/2 if (width > 100 or self.log_p_obs_given_gamma(min_gamma) - max_log_p_obs < -10): break width *= 2 width = max_gamma - min_gamma while True: # search for where the curve becomes close to zero on the right max_gamma = middle + width/2 if (width > 100 or self.log_p_obs_given_gamma(max_gamma) - max_log_p_obs < -10): break width *= 2 n_steps = 500 d_gamma = (max_gamma - min_gamma)/n_steps # now do it properly log_p_obs_given_gammas = flex.double() for gamma in xfrange(min_gamma, max_gamma, d_gamma): log_p_obs_given_gammas.append(self.log_p_obs_given_gamma(gamma)) max_log_p_obs = flex.max(log_p_obs_given_gammas) G_numerator = 0 G_denominator = 0 p_u_gammas = flex.double() # Numerical integration using trapezoidal rule for i, gamma in enumerate(xfrange(min_gamma, max_gamma, d_gamma)): p_u_gamma = math.exp(log_p_obs_given_gammas[i] - max_log_p_obs) p_u_gammas.append(p_u_gamma) if i == 0: continue G_numerator += 0.5 * d_gamma * ( (gamma-d_gamma) * p_u_gammas[-2] + gamma * p_u_gammas[-1]) G_denominator += 0.5 * (p_u_gammas[-2] + p_u_gammas[-1]) * d_gamma self.G = G_numerator/G_denominator sigma_squared_G_numerator = 0 # Numerical integration using trapezoidal rule next = None for i, gamma in enumerate(xfrange(min_gamma, max_gamma, d_gamma)): previous = next next = math.pow((gamma - self.G), 2) * p_u_gammas[i] * d_gamma if i == 0: continue sigma_squared_G_numerator += 0.5 * (previous + next) self.hooft_y = (1-self.G)/2 self.sigma_G = math.sqrt(sigma_squared_G_numerator/G_denominator) self.sigma_y = self.sigma_G/2 # Now calculate P2, P3 values log_p_obs_given_gamma_is_minus_1 = self.log_p_obs_given_gamma(-1) log_p_obs_given_gamma_is_0 = self.log_p_obs_given_gamma(0) log_p_obs_given_gamma_is_1 = self.log_p_obs_given_gamma(1) max_log_p_obs = max([log_p_obs_given_gamma_is_minus_1, log_p_obs_given_gamma_is_0, log_p_obs_given_gamma_is_1]) # all values normalised by max_log_p_obs for numerical stability log_p_obs_given_gamma_is_minus_1 -= max_log_p_obs log_p_obs_given_gamma_is_0 -= max_log_p_obs log_p_obs_given_gamma_is_1 -= max_log_p_obs p2_denominator = math.exp(log_p_obs_given_gamma_is_1) \ + math.exp(log_p_obs_given_gamma_is_minus_1) p3_denominator = math.exp(log_p_obs_given_gamma_is_1) \ + math.exp(log_p_obs_given_gamma_is_minus_1) \ + math.exp(log_p_obs_given_gamma_is_0) # if p2_denominator == 0: self.p2_true = self.p2_false = None else: self.p2_true = ( math.exp(log_p_obs_given_gamma_is_1)) / p2_denominator self.p2_false = ( math.exp(log_p_obs_given_gamma_is_minus_1)) / p2_denominator self.p3_true = ( math.exp(log_p_obs_given_gamma_is_1)) / p3_denominator self.p3_false = ( math.exp(log_p_obs_given_gamma_is_minus_1)) / p3_denominator self.p3_racemic_twin = ( math.exp(log_p_obs_given_gamma_is_0)) / p3_denominator
def _target_function05b(args, kwds, connection): raise Sorry("_target_function05b")
def find_data_arrays(mtz_file, log=None, merge_anomalous=False, preferred_labels=None, crystal_symmetry=None): """ Guess an appropriate data array to use for refinement, plus optional Hendrickson-Lattman coefficients and R-free flags if present. """ from iotbx import reflection_file_utils from iotbx.file_reader import any_file if (log is None) : log = sys.stdout phases = data = flags = flag_value = None hkl_in = any_file(mtz_file, force_type="hkl") hkl_server = hkl_in.file_server # always use anomalous data if available! also, prefer amplitudes over # intensities if possible, as they may already be on an absolute scale data_arrays = hkl_server.get_xray_data( file_name = None, labels = None, ignore_all_zeros = False, parameter_scope = "", return_all_valid_arrays = True, minimum_score = 4, prefer_amplitudes = True, prefer_anomalous = True) if (len(data_arrays) == 0): raise Sorry("No data arrays found in %s." % mtz_file) data = data_arrays[0] if (preferred_labels is not None): for array in data_arrays : array_labels = array.info().label_string() if (array_label== preferred_labels): data = array break else : raise Sorry("Can't find label string '%s'!" % preferred_labels) else : print("Defaulting to using %s" % data.info().label_string(), file=log) hl_arrays = hkl_server.get_experimental_phases( file_name = None, labels = None, ignore_all_zeros = True, parameter_scope = "", return_all_valid_arrays = True, minimum_score = 1) if (len(hl_arrays) > 0): phases = hl_arrays[0] flags_and_values = hkl_server.get_r_free_flags( file_name=None, label=None, test_flag_value=None, disable_suitability_test=False, parameter_scope="", return_all_valid_arrays=True, minimum_score=1) if (len(flags_and_values) > 0): flags, flag_value = flags_and_values[0] if (crystal_symmetry is not None): data = data.customized_copy(crystal_symmetry=crystal_symmetry) if (flags is not None): flags = flags.customized_copy(crystal_symmetry=crystal_symmetry) if (phases is not None): phases = phases.customized_copy(crystal_symmetry=crystal_symmetry) return reflection_file_utils.process_raw_data( obs=data, r_free_flags=flags, test_flag_value=flag_value, phases=phases, log=log, merge_anomalous=merge_anomalous)
def parse_phil_args(master_phil, args, blank_arg_prepend=None, home_scope=None): if blank_arg_prepend is None: pass elif isinstance(blank_arg_prepend, dict): for item in blank_arg_prepend.values(): assert '=' in item elif isinstance(blank_arg_prepend, str): assert '=' in blank_arg_prepend else: raise Exception('blank_arg_prepend must be str or dict') # Copy the args so that we can remove items from the list without affecting args etc args = copy.copy(args) # Construct interpreter cmd_interpr = master_phil.command_line_argument_interpreter( home_scope=home_scope) # Process any args that are eff files eff_files = [ f for f in args if ((f.endswith('.eff') or f.endswith('.def')) and ( not f.count('=')) and os.path.isfile(f)) ] # Remove them from the original lists [args.remove(f) for f in eff_files] # Parse the 'eff' files - these should contain phils #eff_sources = [libtbx.phil.parse(open(f, 'r').read()) for f in eff_files] eff_sources = [cmd_interpr.process(open(f, 'r').read()) for f in eff_files] # Process input arguments arg_sources = [] for arg in args: try: # Prepend if blank if '=' not in arg: if isinstance(blank_arg_prepend, dict): found_key = False for key in blank_arg_prepend.keys(): if key is None: continue if arg.endswith(key): arg = blank_arg_prepend[key] + arg found_key = True break if (found_key == False) and (None in blank_arg_prepend.keys()): arg = blank_arg_prepend[None] + arg elif isinstance(blank_arg_prepend, str): arg = blank_arg_prepend + arg # Attempt to process arg cmd_line_args = cmd_interpr.process(arg=arg) except KeyboardInterrupt: raise except Exception: raise Sorry("Unknown file or keyword: %s" % arg) else: arg_sources.append(cmd_line_args) # Extract Scope object (putting eff sources first so that they're overridden if double-defined) working_phil = master_phil.fetch(sources=eff_sources + arg_sources) return working_phil
def run(params): log = Log(log_file=params.output.log_file, verbose=True) # Process MTZs if params.input.mtz: log.heading('Processing {} MTZ Files'.format(len(params.input.mtz))) if params.input.file_label=='filename': labels = [os.path.basename(os.path.splitext(f)[0]) for f in params.input.mtz] elif params.input.file_label=='foldername': labels = [os.path.basename(os.path.dirname(f)) for f in params.input.mtz] else: raise Exception('MTZ labelling function not supported: {}'.format(params.input.file_label)) log.bar() log('Grouping {} mtz files by space group'.format(len(params.input.mtz))) crystal_groups = CrystalGroup.by_space_group(crystals=[CrystalSummary.from_mtz(mtz_file=f, id=lab) for f,lab in zip(params.input.mtz, labels)]) log('> Clustered into {} space group(s)'.format(len(crystal_groups))) log.bar() for cg in crystal_groups: log.subheading('Space group {} - {} datasets'.format(','.join(cg.space_groups), len(cg.crystals))) error = False for c in cg.crystals: for label in params.check_for.column_label: if label is None: continue if label not in c.column_labels: log('Checking: column "{}" not in diffraction data of {}. columns present are {}'.format(label, c.mtz_file, c.column_labels)) for label in params.summary.column_label: if label is None: continue if label not in c.column_labels: log('Required: column "{}" not in diffraction data of {}. columns present are {}'.format(label, c.mtz_file, c.column_labels)) error = True if error is True: raise Sorry('There are datasets that do not contain the right columns.') log(crystal_statistics('Wavelength', cg.crystals, value_func=lambda c: c.mtz_object().crystals()[1].datasets()[0].wavelength(), header=True)) log(crystal_statistics('Resolution (high)', cg.crystals, value_func=lambda c: c.high_res, header=False)) log(crystal_statistics('Resolution (low)', cg.crystals, value_func=lambda c: c.low_res, header=False)) log(crystal_statistics('Unit cell - vol', cg.crystals, value_func=lambda c: c.unit_cell.volume(), header=False)) log(crystal_statistics('Unit cell - a', cg.crystals, value_func=lambda c: c.unit_cell.parameters()[0], header=False)) log(crystal_statistics('Unit cell - b', cg.crystals, value_func=lambda c: c.unit_cell.parameters()[1], header=False)) log(crystal_statistics('Unit cell - c', cg.crystals, value_func=lambda c: c.unit_cell.parameters()[2], header=False)) log(crystal_statistics('Unit cell - alpha', cg.crystals, value_func=lambda c: c.unit_cell.parameters()[3], header=False)) log(crystal_statistics('Unit cell - beta', cg.crystals, value_func=lambda c: c.unit_cell.parameters()[4], header=False)) log(crystal_statistics('Unit cell - gamma', cg.crystals, value_func=lambda c: c.unit_cell.parameters()[5], header=False, footer=True)) for label in params.summary.column_label: if label is None: continue log(crystal_statistics('Column: {}'.format(label), cg.crystals, value_func=lambda c: c.mtz_object().get_column(label).n_valid_values(), header=False, footer=True)) log.bar(True, False) log('Smallest + Largest Values') log.bar() log(crystal_min_max('Resolution', cg.crystals, value_func=lambda c: c.high_res)) # Process PDBs if params.input.pdb: log.heading('Processing {} PDB Files'.format(len(params.input.pdb))) if params.input.file_label=='filename': labels = [os.path.basename(os.path.splitext(f)[0]) for f in params.input.pdb] elif params.input.file_label=='foldername': labels = [os.path.basename(os.path.dirname(f)) for f in params.input.pdb] else: raise Exception('PDB labelling function not supported: {}'.format(params.input.file_label)) log.bar() log('Grouping {} pdb files by space group'.format(len(params.input.pdb))) crystal_groups = CrystalGroup.by_space_group(crystals=[CrystalSummary.from_pdb(pdb_file=f, id=lab) for f,lab in zip(params.input.pdb, labels)]) log('> Clustered into {} space group(s)'.format(len(crystal_groups))) for cg in crystal_groups: log.subheading('Space group: {} - {} datasets'.format(','.join(cg.space_groups), len(cg.crystals))) log(crystal_statistics('R-work', cg.crystals, value_func=lambda c: c.pdb_input().get_r_rfree_sigma().r_work, header=True)) log(crystal_statistics('R-free', cg.crystals, value_func=lambda c: c.pdb_input().get_r_rfree_sigma().r_free, header=False, footer=True)) log.bar(True, False) log('Smallest + Largest Values') log.bar() log(crystal_min_max('R-free', cg.crystals, value_func=lambda c: c.pdb_input().get_r_rfree_sigma().r_free)) log.heading('finished')
def as_miller_arrays( self, crystal_symmetry=None, force_symmetry=False, merge_equivalents=True, base_array_info=None, assume_shelx_observation_type_is=None, enforce_positive_sigmas=False, anomalous=None, ): """ Convert the contents of the reflection file into a list of :py:class:`cctbx.miller.array` objects, each of which may contain multiple columns of data from the underlying file. By default this will automatically merge redundant observations to obtain a unique set under symmetry. :param crystal_symmetry: :py:class:`cctbx.crystal.symmetry` object (defaults to using internally specified symmetry, if any) :param force_symmetry: TODO :param merge_equivalents: merge redundant obervations (default=True) :param base_array_info: :py:class:`cctbx.miller.array_info` object containing basic information to be propagated to the arrays :param assume_shelx_observation_type_is: if specified, instead of raising an exception if the SHELX file type is not known from the file name plus data type tag, the function will force the specified data type. """ assert (assume_shelx_observation_type_is in [None, "amplitudes", "intensities"]) if (self._file_type is None): return [] info_source = self._file_name if (info_source.startswith("./") or info_source.startswith(".\\")): info_source = info_source[2:] if (base_array_info is None): base_array_info = miller.array_info(source=info_source, source_type=self._file_type) if (self._file_type == "cctbx.miller.array"): result = [] for miller_array in self._file_content: info = miller_array.info() if (info is None or not isinstance(info, miller.array_info)): info = base_array_info info.source = info_source info.crystal_symmetry_from_file = crystal.symmetry( unit_cell=miller_array.unit_cell(), space_group_info=miller_array.space_group_info(), raise_sorry_if_incompatible_unit_cell=True) result.append( miller_array.customized_copy( crystal_symmetry=miller_array.join_symmetry( other_symmetry=crystal_symmetry, force=force_symmetry, raise_sorry_if_incompatible_unit_cell=True)). set_info(info).set_observation_type( miller_array.observation_type())) return result if ((crystal_symmetry is None or crystal_symmetry.unit_cell() is None) and self._observation_type == 'hklf+ins/res'): name, ext = os.path.splitext(self._file_name) if ext != '.hkl': # it may be compressed: name.hkl.gz name, ext = os.path.splitext(name) for shelx_file_name in ('%s.ins' % name, '%s.res' % name): try: shelx_file = open(shelx_file_name) break except IOError: continue else: raise Sorry("Can't open files %s.ins or %s.res" "required by the option hklf+ins/res" % ((name, ) * 2)) crystal_symmetry = crystal_symmetry_from_ins.extract_from( file=shelx_file) shelx_file.seek(0) remaining = shelx_file.read() shelx_file.close() m = re.search(r"^HKLF\s*(\d)", remaining, re.X | re.M | re.S) if m is None: raise Sorry( "%s does not contain the mandatory HKLF instruction" % shelx_file.name) if m.group(1) == "4": self._observation_type = "intensities" elif m.group(1) == "3": self._observation_type = "amplitudes" else: raise Sorry("HKLF %s not supported" % m.group(1)) result = self._file_content.as_miller_arrays( crystal_symmetry=crystal_symmetry, force_symmetry=force_symmetry, merge_equivalents=merge_equivalents, base_array_info=base_array_info, anomalous=anomalous, ) if (self.file_type() == "shelx_hklf"): if ((self._observation_type == "intensities") or (assume_shelx_observation_type_is == "intensities")): result[0].set_info(result[0].info().customized_copy( labels=["Iobs", "SigIobs"])) result[0].set_observation_type_xray_intensity() elif ((self._observation_type == "amplitudes") or (assume_shelx_observation_type_is == "amplitudes")): result[0].set_info(result[0].info().customized_copy( labels=["Fobs", "SigFobs"])) result[0].set_observation_type_xray_amplitude() else: raise Sorry( "Unresolved amplitude/intensity ambiguity: %s\n" " SHELX reflection files may contain amplitudes or intensities.\n" " Please append =amplitudes\n" " or =hklf3\n" " or =intensities\n" " or =hklf4\n" " to the file name argument or parameter to resolve the" " ambiguity.\n" " If a corresponding .ins file is available, look for the" " HKLF codeword.\n" " Alternatively, run the phenix.reflection_statistics" " command twice,\n" " once with =amplitudes and once with =intensities. Inspect" " the <I^2>/(<I>)^2\n" " statistics. For acentric structures the values should" " fluctuate around\n" " 2.0, for centric structures around 3.0. If the statistics" " are not conclusive\n" " it will be best to recover the original reflection data, such" " as SCALEPACK,\n" " SCALA MTZ, XDS, or d*TREK files." % self._file_name) # discard reflections where sigma <= 0 # XXX note that this will happen after data merging, so for unmerged data # it is better to specify merge_equivalents=False! if (enforce_positive_sigmas): result_ = [] for array in result: result_.append(array.enforce_positive_sigmas()) result = result_ return result
def check_if_atoms_superposed(self, r1, r2, i1, i2): if abs((r1 - r2).length()) < 0.001: sorry_str = '''Atoms %s and %s are superposed or very close. Fix your model before proceeding.''' % (self.site_labels[i1], self.site_labels[i2]) raise Sorry(sorry_str)
def run(args, command_name="phenix.fest"): print_banner(command_name=command_name) scenarios = [ "SAD",#0 "SIR",#1 "RIP",#2 "SIRAS",#3 "RIPAS",#4 "MIR",#5 "MIRAS",#6 "2WMAD",#7 "3WMAD",#8 "4WMAD",#9 "2WMAD+NAT",#10 "3WMAD+NAT",#11 "4WMAD+NAT",#12 "2WSAD",#13 "3WSAD",#14 "4WSAD" ]#15 if len(args)==0: print() print("usage: %s <EXPERIMENT TYPE> <FLAGS and/or PARAMETER FILE>" \ % command_name) print() raise Sorry("No instructions received") if args[0] in scenarios: print("experiment: " , args[0]) else: print("Unknown experiment type") print(" Choose from the following list:") for method in scenarios: print(method) raise Sorry("Unmknown experimenttype") ## this is a rather ugly enumeration of all cases if args[0]==scenarios[0]: sad_scale.run( args[1:] ) elif args[0]==scenarios[1]: sir_scale.run( args[1:] ) elif args[0]==scenarios[2]: rip_scale.run( args[1:] ) elif args[0]==scenarios[3]: siras_scale.run( args[1:] ) elif args[0]==scenarios[4]: siras_scale.run( args[1:] ) elif args[0]==scenarios[7]: twmad_scale.run( args[1:] ) else: print() print("Sorry, no time. This method has not yet been implemented.") print("Currently, only SAD,SIR,RIP and 2WMAD are supported.") print()
def run(args, out=sys.stdout, quiet=False): cmdline = iotbx.phil.process_command_line_with_files( args=args, master_phil=get_master_phil(), pdb_file_def="model", usage_string=usage_string) params = cmdline.work.extract() if (params.model is None and params.model_list is None): raise Usage(usage_string) if params.model: models = [params.model] elif params.model_list: if os.path.isfile(params.model_list): with open(params.model_list, 'r') as f: models = f.read().split('\n') models = [m for m in models if m != ""] else: models = params.model_list.split(',') params.verbose = False params.model = models[0] results = [] for model in models: if not os.path.isfile(model) and params.model_list: print "Cannot find '%s', skipping." % model continue pdb_in = cmdline.get_file(model, force_type="pdb") hierarchy = pdb_in.file_object.hierarchy hierarchy.atoms().reset_i_seq() result = mmtbx.validation.ramalyze.ramalyze( pdb_hierarchy=hierarchy, show_errors=None, outliers_only=params.outliers_only, out=out, quiet=quiet) results.append(result) if params.model_list: print '\nmodel : %s' % model result.show_summary() # combine result = results[0] for i in range(1, len(results)): result += results[i] if params.verbose: result.show_old_output(out=out, verbose=True) if params.plot: plot_file_base = params.output_prefix if plot_file_base is None: plot_file_base = os.path.splitext(os.path.basename( params.model))[0] result.write_plots(plot_file_base=plot_file_base, out=out, show_labels=params.show_labels, point_style=params.point_style) if params.wxplot: try: import wxtbx.app except ImportError, e: raise Sorry("wxPython not available.") else: app = wxtbx.app.CCTBXApp(0) result.display_wx_plots() app.MainLoop()
def split_for_scan_range(self, experiments, reference, scan_range): ''' Update experiments when scan range is set. ''' from dxtbx.model.experiment_list import ExperimentList from dxtbx.model.experiment_list import Experiment from dials.array_family import flex # Only do anything is the scan range is set if scan_range is not None and len(scan_range) > 0: # Ensure that all experiments have the same imageset and scan iset = [e.imageset for e in experiments] scan = [e.scan for e in experiments] assert (all(x == iset[0] for x in iset)) assert (all(x == scan[0] for x in scan)) # Get the imageset and scan iset = experiments[0].imageset scan = experiments[0].scan # Get the array range if scan is not None: frames_start, frames_end = scan.get_array_range() assert (scan.get_num_images() == len(iset)) else: frames_start, frames_end = (0, len(iset)) # Create the new lists new_experiments = ExperimentList() new_reference_all = reference.split_by_experiment_id() new_reference = flex.reflection_table() for i in range(len(new_reference_all) - len(experiments)): new_reference_all.append(flex.reflection_table()) assert (len(new_reference_all) == len(experiments)) # Loop through all the scan ranges and create a new experiment list with # the requested scan ranges. for scan_start, scan_end in scan_range: # Validate the requested scan range if scan_end == scan_start: raise Sorry( "Scan range end must be higher than start; pass {},{} for single image" .format(scan_start, scan_start + 1)) if scan_end < scan_start: raise Sorry("Scan range must be in ascending order") elif scan_start < frames_start or scan_end > frames_end: raise Sorry( "Scan range must be within image range {}..{}".format( frames_start, frames_end)) assert (scan_end > scan_start) assert (scan_start >= frames_start) assert (scan_end <= frames_end) index_start = scan_start - frames_start index_end = index_start + (scan_end - scan_start) assert (index_start < index_end) assert (index_start >= 0) assert (index_end <= len(iset)) new_iset = iset[index_start:index_end] if scan is None: new_scan = None else: new_scan = scan[index_start:index_end] for i, e1 in enumerate(experiments): e2 = Experiment() e2.beam = e1.beam e2.detector = e1.detector e2.goniometer = e1.goniometer e2.crystal = e1.crystal e2.profile = e1.profile e2.imageset = new_iset e2.scan = new_scan new_reference_all[i]['id'] = flex.int( len(new_reference_all[i]), len(new_experiments)) new_reference.extend(new_reference_all[i]) new_experiments.append(e2) experiments = new_experiments reference = new_reference # Print some information logger.info( 'Modified experiment list to integrate over requested scan range' ) for scan_start, scan_end in scan_range: logger.info(' scan_range = %d -> %d' % (scan_start, scan_end)) logger.info('') # Return the experiments return experiments, reference
def run(self): ''' Perform the integration. ''' from dials.util.command_line import heading from dials.util.options import flatten_reflections, flatten_experiments from dials.util import log from time import time from libtbx.utils import Sorry # Check the number of arguments is correct start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) reference = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) if len(reference) == 0 and len(experiments) == 0: self.parser.print_help() return if len(reference) == 0: reference = None elif len(reference) != 1: raise Sorry('more than 1 reflection file was given') else: reference = reference[0] if len(experiments) == 0: raise Sorry('no experiment list was specified') # Save phil parameters if params.output.phil is not None: with open(params.output.phil, "w") as outfile: outfile.write(self.parser.diff_phil.as_str()) # Configure logging log.config(params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) for abs_params in params.absorption_correction: if abs_params.apply: if not (params.integration.debug.output and not params.integration.debug.separate_files): raise Sorry('Shoeboxes must be saved to integration intermediates to apply an absorption correction. '\ +'Set integration.debug.output=True, integration.debug.separate_files=False and '\ +'integration.debug.delete_shoeboxes=True to temporarily store shoeboxes.') # Print if we're using a mask for i, exp in enumerate(experiments): mask = exp.imageset.external_lookup.mask if mask.filename is not None: if mask.data: logger.info('Using external mask: %s' % mask.filename) for tile in mask.data: logger.info(' Mask has %d pixels masked' % tile.data().count(False)) # Print the experimental models for i, exp in enumerate(experiments): logger.debug("Models for experiment %d" % i) logger.debug("") logger.debug(str(exp.beam)) logger.debug(str(exp.detector)) if exp.goniometer: logger.debug(str(exp.goniometer)) if exp.scan: logger.debug(str(exp.scan)) logger.debug(str(exp.crystal)) logger.info("=" * 80) logger.info("") logger.info(heading("Initialising")) logger.info("") # Load the data reference, rubbish = self.process_reference(reference) # Check pixels don't belong to neighbours if reference is not None: if exp.goniometer is not None and exp.scan is not None: self.filter_reference_pixels(reference, experiments) logger.info("") # Initialise the integrator from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.array_family import flex # Modify experiment list if scan range is set. experiments, reference = self.split_for_scan_range( experiments, reference, params.scan_range) # Modify experiment list if exclude images is set experiments = self.exclude_images(experiments, params.exclude_images) # Predict the reflections logger.info("") logger.info("=" * 80) logger.info("") logger.info(heading("Predicting reflections")) logger.info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=params.prediction.d_min, dmax=params.prediction.d_max, margin=params.prediction.margin, force_static=params.prediction.force_static, padding=params.prediction.padding) # Match reference with predicted if reference: matched, reference, unmatched = predicted.match_with_reference( reference) assert (len(matched) == len(predicted)) assert (matched.count(True) <= len(reference)) if matched.count(True) == 0: raise Sorry(''' Invalid input for reference reflections. Zero reference spots were matched to predictions ''') elif len(unmatched) != 0: logger.info('') logger.info('*' * 80) logger.info( 'Warning: %d reference spots were not matched to predictions' % (len(unmatched))) logger.info('*' * 80) logger.info('') rubbish.extend(unmatched) if len(experiments) > 1: # filter out any experiments without matched reference reflections # f_: filtered from dxtbx.model.experiment_list import ExperimentList f_reference = flex.reflection_table() f_predicted = flex.reflection_table() f_rubbish = flex.reflection_table() f_experiments = ExperimentList() good_expt_count = 0 def refl_extend(src, dest, eid): tmp = src.select(src['id'] == eid) tmp['id'] = flex.int(len(tmp), good_expt_count) dest.extend(tmp) for expt_id, experiment in enumerate(experiments): if len(reference.select(reference['id'] == expt_id)) != 0: refl_extend(reference, f_reference, expt_id) refl_extend(predicted, f_predicted, expt_id) refl_extend(rubbish, f_rubbish, expt_id) f_experiments.append(experiment) good_expt_count += 1 else: logger.info( "Removing experiment %d: no reference reflections matched to predictions" % expt_id) reference = f_reference predicted = f_predicted experiments = f_experiments rubbish = f_rubbish # Select a random sample of the predicted reflections if not params.sampling.integrate_all_reflections: predicted = self.sample_predictions(experiments, predicted, params) # Compute the profile model if (params.create_profile_model and reference is not None and "shoebox" in reference): experiments = ProfileModelFactory.create(params, experiments, reference) else: experiments = ProfileModelFactory.create(params, experiments) for expr in experiments: if expr.profile is None: raise Sorry('No profile information in experiment list') del reference # Compute the bounding box predicted.compute_bbox(experiments) # Create the integrator logger.info("") integrator = IntegratorFactory.create(params, experiments, predicted) # Integrate the reflections reflections = integrator.integrate() # Append rubbish data onto the end if rubbish is not None and params.output.include_bad_reference: mask = flex.bool(len(rubbish), True) rubbish.unset_flags(mask, rubbish.flags.integrated_sum) rubbish.unset_flags(mask, rubbish.flags.integrated_prf) rubbish.set_flags(mask, rubbish.flags.bad_reference) reflections.extend(rubbish) # Correct integrated intensities for absorption correction, if necessary for abs_params in params.absorption_correction: if abs_params.apply and abs_params.algorithm == "fuller_kapton": from dials.algorithms.integration.kapton_correction import multi_kapton_correction experiments, reflections = multi_kapton_correction( experiments, reflections, abs_params.fuller_kapton, logger=logger)() if params.significance_filter.enable: from dials.algorithms.integration.stills_significance_filter import SignificanceFilter sig_filter = SignificanceFilter(params) refls = sig_filter(experiments, reflections) logger.info( "Removed %d reflections out of %d when applying significance filter" % (len(reflections) - len(refls), len(reflections))) reflections = refls # Delete the shoeboxes used for intermediate calculations, if requested if params.integration.debug.delete_shoeboxes and 'shoebox' in reflections: del reflections['shoebox'] # Save the reflections self.save_reflections(reflections, params.output.reflections) self.save_experiments(experiments, params.output.experiments) # Write a report if requested if params.output.report is not None: integrator.report().as_file(params.output.report) # Print the total time taken logger.info("\nTotal time taken: %f" % (time() - start_time))
# This code is distributed under the BSD license, a copy of which is # included in the root directory of this package. # import libtbx from libtbx.utils import Sorry from scitbx.array_family import flex import logging logger = logging.getLogger(__name__) from dials.algorithms.refinement.engine import DisableMPmixin try: from scitbx.examples.bevington import non_linear_ls_eigen_wrapper except ImportError, e: raise Sorry( """Eigen package is not available. Please untar the Eigen source package (http://eigen.tuxfamily.org) and place a link to it (eigen--> Eigen source dir) in the modules directory of your developer install; then recompile. """) from dials.algorithms.refinement.engine import AdaptLstbx as AdaptLstbxBase class AdaptLstbxSparse(DisableMPmixin, AdaptLstbxBase, non_linear_ls_eigen_wrapper): """Adapt the base class for Eigen""" def __init__(self, target, prediction_parameterisation, constraints_manager=None, log=None, verbosity=0,
def integration_concept(self,image_number=0,cb_op_to_primitive=None,verbose=False,**kwargs): self.image_number = image_number NEAR = 10 pxlsz = self.pixel_size self.get_predictions_accounting_for_centering(cb_op_to_primitive,**kwargs) FWMOSAICITY = self.inputai.getMosaicity() DOMAIN_SZ_ANG = kwargs.get("domain_size_ang", self.__dict__.get("actual",0) ) refineflag = {True:0,False:1}[kwargs.get("domain_size_ang",0)==0] self.inputpd["symmetry"].show_summary(prefix="EXCURSION%1d REPORT FWMOS= %6.4f DOMAIN= %6.1f "%(refineflag,FWMOSAICITY,DOMAIN_SZ_ANG)) from annlib_ext import AnnAdaptor self.cell = self.inputai.getOrientation().unit_cell() query = flex.double() for pred in self.predicted: # predicted spot coord in pixels query.append(pred[0]/pxlsz) query.append(pred[1]/pxlsz) self.reserve_hkllist_for_signal_search = self.hkllist reference = flex.double() spots = self.get_observations_with_outlier_removal() assert len(spots)>NEAR# Can't do spot/pred matching with too few spots for spot in spots: reference.append(spot.ctr_mass_x()) reference.append(spot.ctr_mass_y()) IS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR) IS_adapt.query(query) print("Calculate correction vectors for %d observations & %d predictions"%(len(spots),len(self.predicted))) indexed_pairs_provisional = [] correction_vectors_provisional = [] c_v_p_flex = flex.vec3_double() idx_cutoff = float(min(self.mask_focus[image_number])) if verbose: print("idx_cutoff distance in pixels",idx_cutoff) if not self.horizons_phil.integration.enable_one_to_one_safeguard: # legacy code, no safeguard against many-to-one predicted-to-observation mapping for i in range(len(self.predicted)): # loop over predicteds #for n in range(NEAR): # loop over near spotfinder spots for n in range(1): # only consider the nearest spotfinder spot Match = dict(spot=IS_adapt.nn[i*NEAR+n],pred=i) if n==0 and math.sqrt(IS_adapt.distances[i*NEAR+n]) < idx_cutoff: indexed_pairs_provisional.append(Match) vector = matrix.col( [spots[Match["spot"]].ctr_mass_x() - self.predicted[Match["pred"]][0]/pxlsz, spots[Match["spot"]].ctr_mass_y() - self.predicted[Match["pred"]][1]/pxlsz]) correction_vectors_provisional.append(vector) c_v_p_flex.append((vector[0],vector[1],0.)) else: one_to_one = {} for i in range(len(self.predicted)): # loop over predicteds annresultidx = i*NEAR obsidx = IS_adapt.nn[annresultidx] this_distancesq = IS_adapt.distances[annresultidx] if obsidx not in one_to_one or \ this_distancesq < one_to_one[obsidx]["distancesq"]: if math.sqrt(this_distancesq) < idx_cutoff: one_to_one[obsidx] = dict(spot=obsidx,pred=i,distancesq=this_distancesq) for key,value in one_to_one.items(): indexed_pairs_provisional.append(value) vector = matrix.col( [spots[value["spot"]].ctr_mass_x() - self.predicted[value["pred"]][0]/pxlsz, spots[value["spot"]].ctr_mass_y() - self.predicted[value["pred"]][1]/pxlsz]) correction_vectors_provisional.append(vector) c_v_p_flex.append((vector[0],vector[1],0.)) print("... %d provisional matches"%len(correction_vectors_provisional), end=' ') print("r.m.s.d. in pixels: %5.2f"%(math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))))) if self.horizons_phil.integration.enable_residual_scatter: from matplotlib import pyplot as plt fig = plt.figure() for cv in correction_vectors_provisional: plt.plot([cv[1]],[-cv[0]],"b.") plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))))) plt.axes().set_aspect("equal") self.show_figure(plt,fig,"res") plt.close() if self.horizons_phil.integration.enable_residual_map: from matplotlib import pyplot as plt fig = plt.figure() for match,cv in zip(indexed_pairs_provisional,correction_vectors_provisional): plt.plot([spots[match["spot"]].ctr_mass_y()],[-spots[match["spot"]].ctr_mass_x()],"r.") plt.plot([self.predicted[match["pred"]][1]/pxlsz],[-self.predicted[match["pred"]][0]/pxlsz],"g.") plt.plot([spots[match["spot"]].ctr_mass_y(), spots[match["spot"]].ctr_mass_y() + 10.*cv[1]], [-spots[match["spot"]].ctr_mass_x(), -spots[match["spot"]].ctr_mass_x() - 10.*cv[0]],'b-') plt.xlim([0,float(self.inputpd["size2"])]) plt.ylim([-float(self.inputpd["size1"]),0]) plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))))) plt.axes().set_aspect("equal") self.show_figure(plt,fig,"map") plt.close() # insert code here to remove correction length outliers... # they are causing terrible # problems for finding legitimate correction vectors (print out the list) # also remove outliers for the purpose of reporting RMS outlier_rejection = True cache_refinement_spots = getattr(slip_callbacks.slip_callback,"requires_refinement_spots",False) if outlier_rejection: correction_lengths = flex.double([v.length() for v in correction_vectors_provisional]) clorder = flex.sort_permutation(correction_lengths) sorted_cl = correction_lengths.select(clorder) ACCEPTABLE_LIMIT = 2 limit = int(0.33 * len(sorted_cl)) # best 1/3 of data are assumed to be correctly modeled. if (limit <= ACCEPTABLE_LIMIT): raise Sorry("Not enough indexed spots to reject outliers; have %d need >%d" % (limit, ACCEPTABLE_LIMIT)) y_data = flex.double(len(sorted_cl)) for i in range(len(y_data)): y_data[i] = float(i)/float(len(y_data)) # ideas are explained in Sauter & Poon (2010) J Appl Cryst 43, 611-616. from rstbx.outlier_spots.fit_distribution import fit_cdf,rayleigh fitted_rayleigh = fit_cdf(x_data = sorted_cl[0:limit], y_data = y_data[0:limit], distribution=rayleigh) inv_cdf = [fitted_rayleigh.distribution.inv_cdf(cdf) for cdf in y_data] #print "SORTED LIST OF ",len(sorted_cl), "with sigma",fitted_rayleigh.distribution.sigma indexed_pairs = [] correction_vectors = [] self.correction_vectors = [] for icand in range(len(sorted_cl)): # somewhat arbitrary sigma = 1.0 cutoff for outliers if (sorted_cl[icand]-inv_cdf[icand])/fitted_rayleigh.distribution.sigma > 1.0: break indexed_pairs.append(indexed_pairs_provisional[clorder[icand]]) correction_vectors.append(correction_vectors_provisional[clorder[icand]]) if cache_refinement_spots: self.spotfinder.images[self.frame_numbers[self.image_number]]["refinement_spots"].append( spots[indexed_pairs[-1]["spot"]]) if kwargs.get("verbose_cv")==True: print("CV OBSCENTER %7.2f %7.2f REFINEDCENTER %7.2f %7.2f"%( float(self.inputpd["size1"])/2.,float(self.inputpd["size2"])/2., self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz), end=' ') print("OBSSPOT %7.2f %7.2f PREDSPOT %7.2f %7.2f"%( spots[indexed_pairs[-1]["spot"]].ctr_mass_x(), spots[indexed_pairs[-1]["spot"]].ctr_mass_y(), self.predicted[indexed_pairs[-1]["pred"]][0]/pxlsz, self.predicted[indexed_pairs[-1]["pred"]][1]/pxlsz), end=' ') the_hkl = self.hkllist[indexed_pairs[-1]["pred"]] print("HKL %4d %4d %4d"%the_hkl,"%2d"%self.setting_id, end=' ') radial, azimuthal = spots[indexed_pairs[-1]["spot"]].get_radial_and_azimuthal_size( self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz) print("RADIALpx %5.3f AZIMUTpx %5.3f"%(radial,azimuthal)) # Store a list of correction vectors in self. radial, azimuthal = spots[indexed_pairs[-1]['spot']].get_radial_and_azimuthal_size( self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz) self.correction_vectors.append( dict(obscenter=(float(self.inputpd['size1']) / 2, float(self.inputpd['size2']) / 2), refinedcenter=(self.inputai.xbeam() / pxlsz, self.inputai.ybeam() / pxlsz), obsspot=(spots[indexed_pairs[-1]['spot']].ctr_mass_x(), spots[indexed_pairs[-1]['spot']].ctr_mass_y()), predspot=(self.predicted[indexed_pairs[-1]['pred']][0] / pxlsz, self.predicted[indexed_pairs[-1]['pred']][1] / pxlsz), hkl=(self.hkllist[indexed_pairs[-1]['pred']][0], self.hkllist[indexed_pairs[-1]['pred']][1], self.hkllist[indexed_pairs[-1]['pred']][2]), setting_id=self.setting_id, radial=radial, azimuthal=azimuthal)) print("After outlier rejection %d indexed spotfinder spots remain."%len(indexed_pairs)) if False: rayleigh_cdf = [ fitted_rayleigh.distribution.cdf(x=sorted_cl[c]) for c in range(len(sorted_cl))] from matplotlib import pyplot as plt plt.plot(sorted_cl,y_data,"r+") #plt.plot(sorted_cl,rayleigh_cdf,"g.") plt.plot(inv_cdf,y_data,"b.") plt.show() else: indexed_pairs = indexed_pairs_provisional correction_vectors = correction_vectors_provisional ########### finished with outlier rejection self.inputpd["symmetry"].show_summary(prefix="SETTING ") is_triclinic = (self.setting_id==1) if is_triclinic: self.triclinic_pairs = [ dict(pred=self.hkllist[a["pred"]],spot=a["spot"]) for a in indexed_pairs ] if self.horizons_phil.integration.model == "user_supplied": if kwargs.get("user-reentrant",None)==None: from cxi_user import post_outlier_rejection self.indexed_pairs = indexed_pairs self.spots = spots post_outlier_rejection(self,image_number,cb_op_to_primitive,self.horizons_phil,kwargs) return ########### finished with user-supplied code if self.horizons_phil.integration.spot_shape_verbose: from rstbx.new_horizons.spot_shape import spot_shape_verbose spot_shape_verbose(rawdata = self.imagefiles.images[self.image_number].linearintdata, beam_center_pix = matrix.col((self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz)), indexed_pairs = indexed_pairs, spotfinder_observations = spots, distance_mm = self.inputai.distance(), mm_per_pixel = pxlsz, hkllist = self.hkllist, unit_cell = self.cell, wavelength_ang = self.inputai.wavelength ) #Other checks to be implemented (future): # spot is within active area of detector on a circular detector such as the Mar IP # integration masks do not overlap; or deconvolute correction_lengths=flex.double([v.length() for v in correction_vectors]) if verbose: print("average correction %5.2f over %d vectors"%(flex.mean(correction_lengths), len(correction_lengths)), end=' ') print("or %5.2f mm."%(pxlsz*flex.mean(correction_lengths))) self.r_residual = pxlsz*flex.mean(correction_lengths) #assert len(indexed_pairs)>NEAR # must have enough indexed spots if (len(indexed_pairs) <= NEAR): raise Sorry("Not enough indexed spots, only found %d, need %d" % (len(indexed_pairs), NEAR)) reference = flex.double() for item in indexed_pairs: reference.append(spots[item["spot"]].ctr_mass_x()) reference.append(spots[item["spot"]].ctr_mass_y()) PS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR) PS_adapt.query(query) self.BSmasks = [] #self.null_correction_mapping( predicted=self.predicted, # correction_vectors = correction_vectors, # IS_adapt = IS_adapt, # spots = spots) self.positional_correction_mapping( predicted=self.predicted, correction_vectors = correction_vectors, PS_adapt = PS_adapt, IS_adapt = IS_adapt, spots = spots) # which spots are close enough to interfere with background? MAXOVER=6 OS_adapt = AnnAdaptor(data=query,dim=2,k=MAXOVER) #six near nbrs OS_adapt.query(query) if self.mask_focus[image_number] is None: raise Sorry("No observed/predicted spot agreement; no Spotfinder masks; skip integration") nbr_cutoff = 2.0* max(self.mask_focus[image_number]) FRAME = int(nbr_cutoff/2) #print "The overlap cutoff is %d pixels"%nbr_cutoff nbr_cutoff_sq = nbr_cutoff * nbr_cutoff #print "Optimized C++ section...", self.set_frame(FRAME) self.set_background_factor(kwargs["background_factor"]) self.set_nbr_cutoff_sq(nbr_cutoff_sq) self.set_guard_width_sq(self.horizons_phil.integration.guard_width_sq) self.set_detector_gain(self.horizons_phil.integration.detector_gain) flex_sorted = flex.int() for item in self.sorted: flex_sorted.append(item[0]);flex_sorted.append(item[1]); if self.horizons_phil.integration.mask_pixel_value is not None: self.set_mask_pixel_val(self.horizons_phil.integration.mask_pixel_value) image_obj = self.imagefiles.imageindex(self.frame_numbers[self.image_number]) image_obj.read() rawdata = image_obj.linearintdata # assume image #1 if self.inputai.active_areas != None: self.detector_xy_draft = self.safe_background( rawdata=rawdata, predicted=self.predicted, OS_adapt=OS_adapt, sorted=flex_sorted, tiles=self.inputai.active_areas.IT, tile_id=self.inputai.active_areas.tile_id); else: self.detector_xy_draft = self.safe_background( rawdata=rawdata, predicted=self.predicted, OS_adapt=OS_adapt, sorted=flex_sorted); for i in range(len(self.predicted)): # loop over predicteds B_S_mask = {} keys = self.get_bsmask(i) for k in range(0,len(keys),2): B_S_mask[(keys[k],keys[k+1])]=True self.BSmasks.append(B_S_mask) #print "Done" return
.multiple = True apply_detector_distance = False .type = bool .help = If True, copy detector distance """) if (__name__ == "__main__"): user_phil = [] for arg in sys.argv[1:]: if (os.path.isfile(arg)): user_phil.append(libtbx.phil.parse("""dest_cbf=\"%s\"""" % arg)) else: try: user_phil.append(libtbx.phil.parse(arg)) except RuntimeError, e: raise Sorry("Unrecognized argument '%s' (error: %s)" % (arg, str(e))) params = master_phil.fetch(sources=user_phil).extract() if (params.source_cbf is None) or not os.path.isfile(params.source_cbf): master_phil.show() raise Usage("source_cbf must be a file") if (params.source_cbf is None): master_phil.show() raise Usage( "dest_cbf must be a file (either dest_cbf=XXX, or the file path(s) alone)." ) print "Source file:", params.source_cbf print "Destination file(s):", for path in params.dest_cbf: print path,
def get_predictions_accounting_for_centering(self,cb_op_to_primitive=None,**kwargs): # interface requires this function to set current_orientation # in the actual setting used for Miller index calculation if (self.horizons_phil.known_setting is None or self.horizons_phil.known_setting == self.setting_id ) and \ self.horizons_phil.integration.model in ["use_case_3_simulated_annealing", "use_case_3_simulated_annealing_7", "use_case_3_simulated_annealing_9"]: if cb_op_to_primitive==None: raise Sorry("Can't use model_3 simulated annealing for non-primitive cells, contact authors.") if self.horizons_phil.integration.model=="use_case_3_simulated_annealing": self.best_params = dict(zip(("half_mosaicity_deg","wave_HE_ang","wave_LE_ang", "reserve_orientation","rotation100_rad","rotation010_rad","rotation001_rad"), self.use_case_3_simulated_annealing(self.horizons_phil.integration.use_subpixel_translations)) ) elif self.horizons_phil.integration.model=="use_case_3_simulated_annealing_7": self.best_params = dict(zip(("half_mosaicity_deg","wave_HE_ang","wave_LE_ang", "reserve_orientation","rotation100_rad","rotation010_rad","rotation001_rad", "domain_size_ang"), self.use_case_3_simulated_annealing_7(self.horizons_phil.integration.use_subpixel_translations)) ) elif self.horizons_phil.integration.model=="use_case_3_simulated_annealing_9": self.best_params = dict(zip(("half_mosaicity_deg","wave_HE_ang","wave_LE_ang", "reserve_orientation","rotation100_rad","rotation010_rad","rotation001_rad", "domain_size_ang","ab_factor","c_factor"), self.use_case_3_simulated_annealing_9(self.horizons_phil.integration.use_subpixel_translations)) ) self.current_orientation = self.best_params["reserve_orientation"] self.current_cb_op_to_primitive = cb_op_to_primitive BPpredicted = self.bp3_wrapper.ucbp3.selected_predictions_labelit_format() BPhkllist = self.bp3_wrapper.ucbp3.selected_hkls() self.predicted,self.hkllist = BPpredicted, BPhkllist self.partialities = dict(indices=BPhkllist.deep_copy(), data=self.bp3_wrapper.ucbp3.selected_partialities()) #self.hi = self.bp3_wrapper.ucbp3.selected_hi_predictions() #self.lo = self.bp3_wrapper.ucbp3.selected_lo_predictions() #print list(self.partialities) #for x in range(len(self.hi)): # print "%4d %4d %4d %7.1f %7.1f %7.1f %7.1f PARTIAL %7.2f"%( # self.hkllist[x][0], self.hkllist[x][1],self.hkllist[x][2], # self.hi[x][0], self.hi[x][1], # self.lo[x][0], self.lo[x][1], self.partialities[x]) if self.inputai.active_areas != None: self.predicted,self.hkllist = self.inputai.active_areas( self.predicted,self.hkllist,self.pixel_size) return if self.horizons_phil.integration.model == "user_supplied": lower_limit_domain_size = math.pow( self.inputai.getOrientation().unit_cell().volume(), 1./3.)*self.horizons_phil.integration.mosaic.domain_size_lower_limit # default 10-unit cell block size minimum reasonable domain actual_used_domain_size = kwargs.get("domain_size_ang",lower_limit_domain_size) if cb_op_to_primitive==None: from cxi_user import pre_get_predictions self.bp3_wrapper = pre_get_predictions(self.inputai, self.horizons_phil, raw_image = self.imagefiles.images[self.image_number], imageindex = self.frame_numbers[self.image_number], spotfinder = self.spotfinder, limiting_resolution = self.limiting_resolution, domain_size_ang = actual_used_domain_size) self.current_orientation = self.inputai.getOrientation() self.current_cb_op_to_primitive = cb_op_to_primitive BPpredicted = self.bp3_wrapper.ucbp3.selected_predictions_labelit_format() BPhkllist = self.bp3_wrapper.ucbp3.selected_hkls() self.predicted,self.hkllist = BPpredicted, BPhkllist if self.inputai.active_areas != None: self.predicted,self.hkllist = self.inputai.active_areas( self.predicted,self.hkllist,self.pixel_size) return else: self.block_counter+=1 rot_mat = matrix.sqr(cb_op_to_primitive.c().r().as_double()).transpose() centered_orientation = self.inputai.getOrientation() self.current_orientation = centered_orientation self.current_cb_op_to_primitive = cb_op_to_primitive primitive_orientation = centered_orientation.change_basis(rot_mat) self.inputai.setOrientation(primitive_orientation) from cxi_user import pre_get_predictions if self.block_counter < 2: KLUDGE = self.horizons_phil.integration.mosaic.kludge1 # bugfix 1 of 2 for protocol 6, equation 2 self.inputai.setMosaicity(KLUDGE*self.inputai.getMosaicity()) self.bp3_wrapper = pre_get_predictions(self.inputai, self.horizons_phil, raw_image = self.imagefiles.images[self.image_number], imageindex = self.frame_numbers[self.image_number], spotfinder = self.spotfinder, limiting_resolution = self.limiting_resolution, domain_size_ang = actual_used_domain_size) BPpredicted = self.bp3_wrapper.ucbp3.selected_predictions_labelit_format() BPhkllist = self.bp3_wrapper.ucbp3.selected_hkls() self.actual = actual_used_domain_size self.predicted = BPpredicted primitive_hkllist = BPhkllist #not sure if matrix needs to be transposed first for outputting HKL's???: self.hkllist = cb_op_to_primitive.inverse().apply(primitive_hkllist) self.inputai.setOrientation(centered_orientation) if self.inputai.active_areas != None: self.predicted,self.hkllist = self.inputai.active_areas( self.predicted,self.hkllist,self.pixel_size) if self.block_counter < 2: down = self.inputai.getMosaicity()/KLUDGE print("Readjusting mosaicity back down to ",down) self.inputai.setMosaicity(down) return if cb_op_to_primitive==None: predicted = self.inputai.predict_all( self.image_centers[self.image_number],self.limiting_resolution) self.predicted = predicted.vec3() #only good for integrating one frame... self.hkllist = predicted.hkl() self.current_orientation = self.inputai.getOrientation() from cctbx import sgtbx self.cb_op_to_primitive = sgtbx.change_of_basis_op() else: rot_mat = matrix.sqr(cb_op_to_primitive.c().r().as_double()).transpose() centered_orientation = self.inputai.getOrientation() self.current_orientation = centered_orientation self.current_cb_op_to_primitive = cb_op_to_primitive primitive_orientation = centered_orientation.change_basis(rot_mat) self.inputai.setOrientation(primitive_orientation) predicted = self.inputai.predict_all( self.image_centers[self.image_number],self.limiting_resolution) self.predicted = predicted.vec3() #only good for integrating one frame... primitive_hkllist = predicted.hkl() #not sure if matrix needs to be transposed first for outputting HKL's???: self.hkllist = cb_op_to_primitive.inverse().apply(primitive_hkllist) self.inputai.setOrientation(centered_orientation) if self.inputai.active_areas != None: self.predicted,self.hkllist = self.inputai.active_areas( self.predicted,self.hkllist,self.pixel_size) if False: #development only; compare the two methods: from matplotlib import pyplot as plt plt.plot([i[0] for i in BPpredicted],[i[1] for i in BPpredicted],"r.") plt.plot([i[0] for i in predicted],[i[1] for i in predicted],"b.") plt.show()
def write_map( self, file_name=None, # Name of file to be written verbose=None, ): ''' Simple version of write file_name is output file name map_data is map_data object with 3D values for map. If not supplied, use self.map_data() Normally call with file_name (file to be written) Output labels are generated from existing self.labels, self.program_name, and self.limitations ''' if not file_name: raise Sorry("Need file_name for write_map") if not self.map_data(): raise Sorry("Need map_data for write_map") map_data = self.map_data() from iotbx.mrcfile import create_output_labels labels = create_output_labels(program_name=self.program_name, input_file_name=self.input_file_name, input_labels=self.labels, limitations=self.limitations) crystal_symmetry = self.unit_cell_crystal_symmetry() unit_cell_grid = self.unit_cell_grid origin_shift_grid_units = self.origin_shift_grid_units if map_data.origin() == (0, 0, 0): # Usual self._print( "Writing map with origin at %s and size of %s to %s" % (str(origin_shift_grid_units), str(map_data.all()), file_name)) write_ccp4_map( file_name=file_name, crystal_symmetry=crystal_symmetry, # unit cell and space group map_data=map_data, unit_cell_grid= unit_cell_grid, # optional gridding of full unit cell origin_shift_grid_units= origin_shift_grid_units, # optional origin shift labels=labels, verbose=verbose) else: # map_data has not been shifted to (0,0,0). Shift it and then write self._print("Writing map after shifting origin") if self.origin_shift_grid_units and origin_shift_grid_units != ( 0, 0, 0): self._print( "WARNING: map_data has origin at %s " % (str(map_data.origin())), " and this map_manager will apply additional origin shift of %s " % (str(self.origin_shift_grid_units))) new_mm = self.deep_copy() new_mm.set_log(self.log) new_mm.shift_origin() new_mm.write_map(file_name=file_name)
def __init__( self, model, pdb_hierarchy=None, # keep for mmtbx.validation_summary (multiple models) fmodel=None, fmodel_neutron=None, sequences=None, flags=None, header_info=None, raw_data=None, unmerged_data=None, keep_hydrogens=True, nuclear=False, save_probe_unformatted_file=None, show_hydrogen_outliers=False, min_cc_two_fofc=0.8, n_bins_data=10, count_anomalous_pairs_separately=False, use_internal_variance=True, outliers_only=True, use_pdb_header_resolution_cutoffs=False, file_name=None, ligand_selection=None, rotamer_library="8000", map_params=None): assert rotamer_library == "8000", "data_version given to RotamerEval not recognized." for name in self.__slots__: setattr(self, name, None) # use objects from model self.model = model if (self.model is not None): pdb_hierarchy = self.model.get_hierarchy() xray_structure = self.model.get_xray_structure() geometry_restraints_manager = self.model.get_restraints_manager( ).geometry crystal_symmetry = self.model.crystal_symmetry() all_chain_proxies = self.model.all_chain_proxies else: assert (pdb_hierarchy is not None) xray_structure = None geometry_restraints_manager = None crystal_symmetry = None all_chain_proxies = None # very important - the i_seq attributes may be extracted later pdb_hierarchy.atoms().reset_i_seq() self.pdb_hierarchy = pdb_hierarchy if (xray_structure is None): if (fmodel is not None): xray_structure = fmodel.xray_structure elif (crystal_symmetry is not None): xray_structure = pdb_hierarchy.extract_xray_structure( crystal_symmetry=crystal_symmetry) self.crystal_symmetry = crystal_symmetry if (crystal_symmetry is None) and (fmodel is not None): self.crystal_symmetry = fmodel.f_obs().crystal_symmetry() # use maps (fmodel is not used) # run earlier since pdb_hierarchy gets modified use_maps = False if (map_params is not None): use_maps = ((map_params.input.maps.map_file_name) or ((map_params.input.maps.map_coefficients_file_name) and (map_params.input.maps.map_coefficients_label))) if (use_maps): if (flags.real_space): self.real_space = experimental.real_space( fmodel=None, model=self.model, cc_min=min_cc_two_fofc, molprobity_map_params=map_params.input.maps) if (flags.waters): self.waters = waters.waters( pdb_hierarchy=pdb_hierarchy, xray_structure=xray_structure, fmodel=None, collect_all=True, molprobity_map_params=map_params.input.maps) self.header_info = header_info if (flags is None): flags = molprobity_flags() import mmtbx.model.statistics self.model_statistics_geometry = mmtbx.model.statistics.geometry( pdb_hierarchy=pdb_hierarchy, geometry_restraints_manager=geometry_restraints_manager, use_hydrogens=keep_hydrogens, use_nuclear=nuclear) self.model_statistics_geometry_result = \ self.model_statistics_geometry.result() self.ramalyze = self.model_statistics_geometry_result.ramachandran.ramalyze self.omegalyze = self.model_statistics_geometry_result.omega.omegalyze self.rotalyze = self.model_statistics_geometry_result.rotamer.rotalyze self.cbetadev = self.model_statistics_geometry_result.c_beta.cbetadev self.clashes = self.model_statistics_geometry_result.clash.clashes if pdb_hierarchy.contains_protein(): self.find_missing_atoms(out=null_out()) if (flags.nqh): self.nqh_flips = clashscore.nqh_flips( pdb_hierarchy=pdb_hierarchy) if (pdb_hierarchy.contains_rna() and flags.rna and libtbx.env.has_module(name="suitename")): if (geometry_restraints_manager is not None): self.rna = rna_validate.rna_validation( pdb_hierarchy=pdb_hierarchy, geometry_restraints_manager=geometry_restraints_manager, outliers_only=outliers_only, params=None) if (flags.model_stats) and (xray_structure is not None): self.model_stats = model_properties.model_statistics( pdb_hierarchy=pdb_hierarchy, xray_structure=xray_structure, all_chain_proxies=all_chain_proxies, ignore_hd=(not nuclear), ligand_selection=ligand_selection) if (geometry_restraints_manager is not None) and (flags.restraints): assert (xray_structure is not None) self.restraints = restraints.combined( pdb_hierarchy=pdb_hierarchy, xray_structure=xray_structure, geometry_restraints_manager=geometry_restraints_manager, ignore_hd=(not nuclear), cdl=getattr(all_chain_proxies, "use_cdl", None)) if (sequences is not None) and (flags.seq): self.sequence = sequence.validation( pdb_hierarchy=pdb_hierarchy, sequences=sequences, log=null_out(), include_secondary_structure=True, extract_coordinates=True) if (fmodel is not None): if (use_pdb_header_resolution_cutoffs) and (header_info is not None): fmodel = fmodel.resolution_filter(d_min=header_info.d_min, d_max=header_info.d_max) if (flags.rfactors): self.data_stats = experimental.data_statistics( fmodel, raw_data=raw_data, n_bins=n_bins_data, count_anomalous_pairs_separately= count_anomalous_pairs_separately) if (not use_maps): # if maps are used, keep previous results if (flags.real_space): self.real_space = experimental.real_space( model=model, fmodel=fmodel, cc_min=min_cc_two_fofc) if (flags.waters): self.waters = waters.waters(pdb_hierarchy=pdb_hierarchy, xray_structure=xray_structure, fmodel=fmodel, collect_all=True) if (unmerged_data is not None): self.merging = experimental.merging_and_model_statistics( f_obs=fmodel.f_obs(), f_model=fmodel.f_model(), r_free_flags=fmodel.r_free_flags(), unmerged_i_obs=unmerged_data, anomalous=count_anomalous_pairs_separately, use_internal_variance=use_internal_variance, n_bins=n_bins_data) if (flags.xtriage): import mmtbx.scaling.xtriage f_model = abs( fmodel.f_model()).set_observation_type_xray_amplitude() if (raw_data is not None): f_model, obs = f_model.common_sets(other=raw_data) else: obs = fmodel.f_obs() self.xtriage = mmtbx.scaling.xtriage.xtriage_analyses( miller_obs=obs, miller_calc=f_model, unmerged_obs=unmerged_data, # XXX some redundancy here... text_out=null_out()) if (fmodel_neutron is not None) and (flags.rfactors): self.neutron_stats = experimental.data_statistics( fmodel_neutron, n_bins=n_bins_data, count_anomalous_pairs_separately=False) if (pdb_hierarchy.models_size() == 1): self._multi_criterion = multi_criterion_view(pdb_hierarchy) # wilson B self.wilson_b = None if (fmodel is not None): self.wilson_b = fmodel.wilson_b() elif (fmodel_neutron is not None): self.wilson_b = fmodel_neutron.wilson_b() # validate hydrogens self.hydrogens = None if self.model is not None and self.model.has_hd(): # import here to avoid circular import issues from mmtbx.hydrogens.validate_H import validate_H, validate_H_results hydrogens = validate_H(model, nuclear) hydrogens.validate_inputs() hydrogens.run() self.hydrogens = validate_H_results(hydrogens.get_results()) # write probe file if needed (CLI and GUI) if (save_probe_unformatted_file is not None): pcm = self.clashes.probe_clashscore_manager try: with open(save_probe_unformatted_file, 'w') as f: f.write(pcm.probe_unformatted) self.clashes.probe_file = save_probe_unformatted_file except IOError as err: raise Sorry('%s could not be written correctly.\n%s' % (save_probe_unformatted_file, err))
def run(args): if len(args) == 0 or '-h' in args or '--help' in args or '-c' in args: print("Usage: %s [-p] files" % libtbx.env.dispatcher_name) phil_scope.show(attributes_level=2) return files = [arg for arg in args if os.path.isfile(arg)] arguments = [arg for arg in args if not os.path.isfile(arg)] user_phil = [] for arg in arguments: if arg == '-p': user_phil.append(parse("show_plots=True")) else: try: user_phil.append(parse(arg)) except Exception as e: raise Sorry("Unrecognized argument: %s" % arg) params = phil_scope.fetch(sources=user_phil).extract() for file in files: message = """Based on the file %s, this program will compute incremental quadrant translations to circularize powder rings on the inner four sensors. The algorithm treats each quadrant independently and scores based on self-correlation upon 45-degree rotation. """ % file print(message) image = dxtbx.load(file) detector = image.get_detector() beam = image.get_beam() from xfel.metrology.quadrant import one_panel ccs = flex.double() for i_quad, quad in enumerate(detector.hierarchy()): # find panel closest to the beam center panels = [] def recursive_get_panels(group): if group.is_group(): for child in group: recursive_get_panels(child) else: panels.append(group) recursive_get_panels(quad) smallest_dist = float("inf") for panel in panels: p_w, p_h = panel.get_image_size() c = center([ col(panel.get_pixel_lab_coord((0, 0))), col(panel.get_pixel_lab_coord((p_w - 1, 0))), col(panel.get_pixel_lab_coord((p_w - 1, p_h - 1))), col(panel.get_pixel_lab_coord((0, p_h - 1))) ]) beam_center = col(panel.get_beam_centre_lab(beam.get_s0())) dist = (c - beam_center).length() if dist < smallest_dist: smallest_dist = dist key_panel = panel print("Doing cross-correlation on panel", key_panel.get_name()) Q = one_panel(image, key_panel, i_quad, quad, params.show_plots, params.multi_angle, params.plot_range, params.pdf_file is None) delta = panel.pixel_to_millimeter((Q.coordmax[0], Q.coordmax[1])) quad.set_frame( quad.get_fast_axis(), quad.get_slow_axis(), col(quad.get_origin()) - col((delta[0], delta[1], 0))) ccs.append(Q.ccmax) print("Average CC: %7.4f" % flex.mean(ccs)) if params.pdf_file is not None: print("Saving plots to", params.pdf_file) from matplotlib import pyplot as plt from matplotlib.backends.backend_pdf import PdfPages pp = PdfPages(params.pdf_file) for i in plt.get_fignums(): pp.savefig(plt.figure(i), dpi=300) pp.close() if params.save_cbf: import pycbf image.sync_detector_to_cbf() dest_path = os.path.splitext(file)[0] + "_cc.cbf" print("Saving result to", dest_path) image._cbf_handle.write_widefile(dest_path,pycbf.CBF,\ pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K,0)
def __init__(self, i_obs, crystal_symmetry=None, d_min=None, d_max=None, anomalous=False, n_bins=10, binning_method='volume', debug=False, file_name=None, model_arrays=None, sigma_filtering=Auto, use_internal_variance=True, eliminate_sys_absent=True, d_min_tolerance=1.e-6, extend_d_max_min=False, cc_one_half_significance_level=None, cc_one_half_method='half_dataset', assert_is_not_unique_set_under_symmetry=True, log=None): self.file_name = file_name if (log is None) : log = null_out() assert (i_obs.sigmas() is not None) info = i_obs.info() sigma_filtering = get_filtering_convention(i_obs, sigma_filtering) if (crystal_symmetry is None): assert (i_obs.space_group() is not None) crystal_symmetry = i_obs.crystal_symmetry() self.crystal_symmetry = crystal_symmetry i_obs = i_obs.customized_copy( crystal_symmetry=crystal_symmetry).set_info(info) if (assert_is_not_unique_set_under_symmetry and i_obs.is_unique_set_under_symmetry()): raise Sorry(("The data in %s are already merged. Only unmerged (but "+ "scaled) data may be used in this program.")% i_obs.info().label_string()) d_min_cutoff = d_min d_max_cutoff = d_max if (d_min is not None): d_min_cutoff *= (1-d_min_tolerance) if (d_max is not None): assert (d_max > d_min) if (d_max is not None): d_max_cutoff *= 1+d_min_tolerance i_obs = i_obs.resolution_filter( d_min=d_min_cutoff, d_max=d_max_cutoff).set_info(info) if (i_obs.size() == 0): raise Sorry("No reflections left after applying resolution cutoffs.") i_obs.show_summary(f=log) self.anom_extra = "" if (not anomalous): i_obs = i_obs.customized_copy(anomalous_flag=False).set_info(info) self.anom_extra = " (non-anomalous)" overall_d_max_min = None # eliminate_sys_absent() before setting up binner to ensure consistency # between reported overall d_min/max and d_min/max for resolution bins" if eliminate_sys_absent: i_obs = i_obs.eliminate_sys_absent() if extend_d_max_min : i_obs.setup_binner( n_bins=n_bins, d_max=d_max_cutoff, d_min=d_min_cutoff) overall_d_max_min = d_max_cutoff, d_min_cutoff else : if binning_method == 'volume': i_obs.setup_binner(n_bins=n_bins) elif binning_method == 'counting_sorted': i_obs.setup_binner_counting_sorted(n_bins=n_bins) self.overall = merging_stats(i_obs, d_max_min=overall_d_max_min, model_arrays=model_arrays, anomalous=anomalous, debug=debug, sigma_filtering=sigma_filtering, use_internal_variance=use_internal_variance, cc_one_half_significance_level=cc_one_half_significance_level, cc_one_half_method=cc_one_half_method) self.bins = [] title = "Intensity merging statistics" column_labels = ["1/d**2","N(obs)","N(unique)","Redundancy","Completeness", "Mean(I)", "Mean(I/sigma)", "R-merge", "R-meas", "R-pim", "CC1/2", "CC(anom)"] graph_names = ["Reflection counts", "Redundancy", "Completeness", "Mean(I)", "Mean(I/sigma)", "R-factors", "CC1/2", "CC(anom)"] graph_columns = [[0,1,2],[0,3],[0,4],[0,5],[0,6],[0,7,8,9],[0,10],[0,13]] if cc_one_half_significance_level is not None: column_labels.extend(["CC1/2 significance", "CC1/2 critical value"]) graph_names.extend(["CC1/2 significance", "CC1/2 critical value"]) graph_columns[-2] = [0,10,12] #--- CC* mode if (model_arrays is not None): title = "Model quality and intensity merging statistics" column_labels.extend(["CC*", "CC(work)", "CC(free)", "R-work", "R-free"]) graph_names.extend(["CC*", "Model R-factors"]) graph_columns.extend([[0,11,14,15],[0,16,17]]) #--- self.table = data_plots.table_data( title=title, column_labels=column_labels, graph_names=graph_names, graph_columns=graph_columns, x_is_inverse_d_min=True, force_exact_x_labels=True) last_bin = None for bin in i_obs.binner().range_used(): sele_unmerged = i_obs.binner().selection(bin) bin_stats = merging_stats(i_obs.select(sele_unmerged), d_max_min=i_obs.binner().bin_d_range(bin), model_arrays=model_arrays, anomalous=anomalous, debug=debug, sigma_filtering=sigma_filtering, use_internal_variance=use_internal_variance, cc_one_half_significance_level=cc_one_half_significance_level, cc_one_half_method=cc_one_half_method) self.bins.append(bin_stats) self.table.add_row(bin_stats.table_data()) from scitbx.array_family import flex self.cc_one_half_overall = flex.mean_weighted( flex.double(b.cc_one_half for b in self.bins), flex.double(b.cc_one_half_n_refl for b in self.bins)) self.cc_one_half_sigma_tau_overall = flex.mean_weighted( flex.double(b.cc_one_half_sigma_tau for b in self.bins), flex.double(b.cc_one_half_sigma_tau_n_refl for b in self.bins))
def run(args, crystal_symmetry=None, ncs_object=None, pdb_hierarchy=None, map_data=None, half_map_data_list=None, half_map_labels_list=None, lower_bounds=None, upper_bounds=None, write_output_files=True, log=None): h = "phenix.map_box: extract box with model and map around selected atoms" if (log is None): log = sys.stdout print_statistics.make_header(h, out=log) default_message = """\ %s. Usage: phenix.map_box model.pdb map_coefficients.mtz selection="chain A and resseq 1:10" or phenix.map_box map.ccp4 density_select=True Parameters:""" % h if (len(args) == 0 and not pdb_hierarchy): print(default_message) master_phil.show(prefix=" ") return # Process inputs ignoring symmetry conflicts just to get the value of # ignore_symmetry_conflicts... inputs = mmtbx.utils.process_command_line_args( args=args, cmd_cs=crystal_symmetry, master_params=master_phil, suppress_symmetry_related_errors=True) params = inputs.params.extract() # Now process inputs for real and write a nice error message if necessary. try: inputs = mmtbx.utils.process_command_line_args( args=args, cmd_cs=crystal_symmetry, master_params=master_phil, suppress_symmetry_related_errors=params.ignore_symmetry_conflicts) except Exception as e: if str(e).find("symmetry mismatch ") > 1: raise Sorry(str(e) + "\nTry 'ignore_symmetry_conflicts=True'") else: raise e params = inputs.params.extract() master_phil.format(python_object=params).show(out=log) # Overwrite params with parameters in call if available if lower_bounds: params.lower_bounds = lower_bounds if upper_bounds: params.upper_bounds = upper_bounds # PDB file if params.pdb_file and not inputs.pdb_file_names and not pdb_hierarchy: inputs.pdb_file_names = [params.pdb_file] if (len(inputs.pdb_file_names) != 1 and not params.density_select and not params.mask_select and not pdb_hierarchy and not params.keep_map_size and not params.upper_bounds and not params.extract_unique and not params.bounds_match_this_file): raise Sorry( "PDB file is needed unless extract_unique, " + "density_select, mask_select, keep_map_size \nor bounds are set .") if (len(inputs.pdb_file_names)!=1 and not pdb_hierarchy and \ (params.mask_atoms )): raise Sorry("PDB file is needed for mask_atoms") if params.soft_mask and (not params.resolution) and \ (len(inputs.pdb_file_names)!=1 and not pdb_hierarchy): raise Sorry("Need resolution for soft_mask without PDB file") if ((params.density_select or params.mask_select) and params.keep_map_size): raise Sorry( "Cannot set both density_select/mask_select and keep_map_size") if ((params.density_select or params.mask_select) and params.upper_bounds): raise Sorry("Cannot set both density_select/mask_select and bounds") if (params.keep_map_size and params.upper_bounds): raise Sorry("Cannot set both keep_map_size and bounds") if (params.upper_bounds and not params.lower_bounds): raise Sorry("Please set lower_bounds if you set upper_bounds") if (params.extract_unique): if (not params.resolution): raise Sorry("Please set resolution for extract_unique") if (not params.symmetry) and (not params.symmetry_file) and \ (not ncs_object): raise Sorry( "Please supply a symmetry file or symmetry for extract_unique (you " + "\ncan try symmetry=ALL if you do not know your symmetry or " + "symmetry=C1 if \nthere is none)") from mmtbx.ncs.ncs import ncs ncs_object = ncs() ncs_object.set_unit_ncs() if params.keep_input_unit_cell_and_grid and ( (params.output_unit_cell_grid is not None) or (params.output_unit_cell is not None)): raise Sorry("If you set keep_input_unit_cell_and_grid then you cannot "+\ "set \noutput_unit_cell_grid or output_unit_cell") if (write_output_files) and ("mtz" in params.output_format) and ( (params.keep_origin) and (not params.keep_map_size)): print("\nNOTE: Skipping write of mtz file as keep_origin=True and \n"+\ "keep_map_size is False\n") params.output_format = remove_element(params.output_format, element='mtz') if (write_output_files) and ("mtz" in params.output_format) and ( (params.extract_unique)): print("\nNOTE: Skipping write of mtz file as extract_unique=True\n") params.output_format = remove_element(params.output_format, element='mtz') if params.output_origin_match_this_file or params.bounds_match_this_file: if params.output_origin_match_this_file: fn = params.output_origin_match_this_file if params.bounds_match_this_file: raise Sorry("Cannot match origin and bounds at same time") else: fn = params.bounds_match_this_file if not params.ccp4_map_file: raise Sorry( "Need to specify your input file with ccp4_map_file=xxx if you use " + "output_origin_match_this_file=xxxx or bounds_match_this_file=xxxx" ) af = any_file(fn) if (af.file_type == 'ccp4_map'): origin = af.file_content.data.origin() if params.output_origin_match_this_file: params.output_origin_grid_units = origin print("Origin of (%s,%s,%s) taken from %s" % (origin[0], origin[1], origin[2], fn)) else: all = af.file_content.data.all() params.lower_bounds = origin print("Lower bounds of (%s,%s,%s) taken from %s" % (params.lower_bounds[0], params.lower_bounds[1], params.lower_bounds[2], fn)) params.upper_bounds = list( col(origin) + col(all) - col((1, 1, 1))) print("upper bounds of (%s,%s,%s) taken from %s" % (params.upper_bounds[0], params.upper_bounds[1], params.upper_bounds[2], fn)) params.bounds_are_absolute = True else: raise Sorry("Unable to interpret %s as map file" % (fn)) if params.output_origin_grid_units is not None and params.keep_origin: params.keep_origin = False print("Setting keep_origin=False as output_origin_grid_units is set") print_statistics.make_sub_header("pdb model", out=log) if len(inputs.pdb_file_names) > 0: pdb_inp = iotbx.pdb.input(file_name=inputs.pdb_file_names[0]) pdb_hierarchy = pdb_inp.construct_hierarchy() if pdb_hierarchy: pdb_atoms = pdb_hierarchy.atoms() pdb_atoms.reset_i_seq() else: pdb_hierarchy = None # Map or map coefficients map_coeff = None input_unit_cell_grid = None input_unit_cell = None input_map_labels = None if (not map_data): # read first mtz file if ((len(inputs.reflection_file_names) > 0) or (params.map_coefficients_file is not None)): # file in phil takes precedent if (params.map_coefficients_file is not None): if (len(inputs.reflection_file_names) == 0): inputs.reflection_file_names.append( params.map_coefficients_file) else: inputs.reflection_file_names[ 0] = params.map_coefficients_file map_coeff = reflection_file_utils.extract_miller_array_from_file( file_name=inputs.reflection_file_names[0], label=params.label, type="complex", log=log) if not crystal_symmetry: crystal_symmetry = map_coeff.crystal_symmetry() fft_map = map_coeff.fft_map( resolution_factor=params.resolution_factor) fft_map.apply_sigma_scaling() map_data = fft_map.real_map_unpadded() map_or_map_coeffs_prefix = os.path.basename( inputs.reflection_file_names[0][:-4]) # or read CCP4 map elif ((inputs.ccp4_map is not None) or (params.ccp4_map_file is not None)): if (params.ccp4_map_file is not None): af = any_file(params.ccp4_map_file) if (af.file_type == 'ccp4_map'): inputs.ccp4_map = af.file_content inputs.ccp4_map_file_name = params.ccp4_map_file print_statistics.make_sub_header("CCP4 map", out=log) ccp4_map = inputs.ccp4_map ccp4_map.show_summary(prefix=" ", out=log) if not crystal_symmetry: crystal_symmetry = ccp4_map.crystal_symmetry() map_data = ccp4_map.data #map_data() input_unit_cell_grid = ccp4_map.unit_cell_grid input_unit_cell = ccp4_map.unit_cell_parameters input_map_labels = ccp4_map.get_labels() if inputs.ccp4_map_file_name.endswith(".ccp4"): map_or_map_coeffs_prefix = os.path.basename( inputs.ccp4_map_file_name[:-5]) else: map_or_map_coeffs_prefix = os.path.basename( inputs.ccp4_map_file_name[:-4]) else: # have map_data map_or_map_coeffs_prefix = None if params.half_map_list and (not half_map_data_list): if not params.extract_unique: raise Sorry("Can only use half_map_with extract_unique") print("Reading half-maps", params.half_map_list) half_map_data_list = [] half_map_labels_list = [] for fn in params.half_map_list: print("Reading half map from %s" % (fn), file=log) af = any_file(fn) print_statistics.make_sub_header("CCP4 map", out=log) h_ccp4_map = af.file_content h_ccp4_map.show_summary(prefix=" ", out=log) h_map_data = h_ccp4_map.data half_map_data_list.append(h_map_data) half_map_labels_list.append(h_ccp4_map.get_labels()) if params.map_scale_factor: print("Applying scale factor of %s to map data on read-in" % (params.map_scale_factor)) map_data = map_data * params.map_scale_factor if params.output_origin_grid_units is not None: origin_to_match = tuple(params.output_origin_grid_units) else: origin_to_match = None if origin_to_match: sc = [] for x, o, a in zip(crystal_symmetry.unit_cell().parameters()[:3], origin_to_match, map_data.all()): sc.append(-x * o / a) shift_cart_for_origin_to_match = tuple(sc) else: origin_to_match = None shift_cart_for_origin_to_match = None if crystal_symmetry and not inputs.crystal_symmetry: inputs.crystal_symmetry = crystal_symmetry # final check that map_data exists if (map_data is None): raise Sorry("Map or map coefficients file is needed.") if len(inputs.pdb_file_names) > 0: output_prefix = os.path.basename(inputs.pdb_file_names[0])[:-4] else: output_prefix = map_or_map_coeffs_prefix if not pdb_hierarchy: # get an empty hierarchy from cctbx.array_family import flex pdb_hierarchy = iotbx.pdb.input( source_info='', lines=flex.split_lines('')).construct_hierarchy() xray_structure = pdb_hierarchy.extract_xray_structure( crystal_symmetry=inputs.crystal_symmetry) xray_structure.show_summary(f=log) # if not params.selection: params.selection = "all" selection = pdb_hierarchy.atom_selection_cache().selection( string=params.selection) if selection.size(): print_statistics.make_sub_header("atom selection", out=log) print("Selection string: selection='%s'" % params.selection, file=log) print(" selects %d atoms from total %d atoms." % (selection.count(True), selection.size()), file=log) sites_cart_all = xray_structure.sites_cart() sites_cart = sites_cart_all.select(selection) selection = xray_structure.selection_within(radius=params.selection_radius, selection=selection) if not ncs_object: from mmtbx.ncs.ncs import ncs ncs_object = ncs() if params.symmetry_file: ncs_object.read_ncs(params.symmetry_file, log=log) print("Total of %s operators read" % (ncs_object.max_operators()), file=log) if not ncs_object or ncs_object.max_operators() < 1: print("No symmetry available", file=log) if ncs_object: n_ops = max(1, ncs_object.max_operators()) else: n_ops = 1 # Get sequence if extract_unique is set sequence = None if params.extract_unique or params.mask_select: if params.sequence_file: if n_ops > 1: # get unique part of sequence remove_duplicates = True else: remove_duplicates = False from iotbx.bioinformatics import get_sequences sequence = (" ".join( get_sequences(file_name=params.sequence_file, remove_duplicates=remove_duplicates))) if params.chain_type in ['None', None]: params.chain_type = None if sequence and not params.molecular_mass: # get molecular mass from sequence from iotbx.bioinformatics import text_from_chains_matching_chain_type if params.chain_type in [None, 'PROTEIN']: n_protein = len( text_from_chains_matching_chain_type(text=sequence, chain_type='PROTEIN')) else: n_protein = 0 if params.chain_type in [None, 'RNA']: n_rna = len( text_from_chains_matching_chain_type(text=sequence, chain_type='RNA')) else: n_rna = 0 if params.chain_type in [None, 'DNA']: n_dna = len( text_from_chains_matching_chain_type(text=sequence, chain_type='DNA')) else: n_dna = 0 params.molecular_mass = n_ops * (n_protein * 110 + (n_rna + n_dna) * 330) print("\nEstimate of molecular mass is %.0f " % (params.molecular_mass), file=log) if params.density_select or params.mask_select: print_statistics.make_sub_header( "Extracting box around selected density and writing output files", out=log) else: print_statistics.make_sub_header( "Extracting box around selected atoms and writing output files", out=log) # if params.value_outside_atoms == 'mean': print("\nValue outside atoms mask will be set to mean inside mask", file=log) if params.get_half_height_width and params.density_select: print("\nHalf width at half height will be used to id boundaries", file=log) if params.soft_mask and sites_cart_all.size() > 0: print("\nSoft mask will be applied to model-based mask", file=log) elif params.soft_mask: print("\nSoft mask will be applied to outside of map box", file=log) if params.keep_map_size: print("\nEntire map will be kept (not cutting out region)", file=log) if params.restrict_map_size: print("\nOutput map will be within input map", file=log) if params.lower_bounds and params.upper_bounds: print("Bounds for cut out map are (%s,%s,%s) to (%s,%s,%s)" % (tuple(list(params.lower_bounds) + list(params.upper_bounds))), file=log) box = mmtbx.utils.extract_box_around_model_and_map( xray_structure=xray_structure, map_data=map_data.as_double(), box_cushion=params.box_cushion, selection=selection, mask_select=params.mask_select, density_select=params.density_select, threshold=params.density_select_threshold, get_half_height_width=params.get_half_height_width, mask_atoms=params.mask_atoms, soft_mask=params.soft_mask, soft_mask_radius=params.soft_mask_radius, mask_atoms_atom_radius=params.mask_atoms_atom_radius, value_outside_atoms=params.value_outside_atoms, keep_map_size=params.keep_map_size, restrict_map_size=params.restrict_map_size, lower_bounds=params.lower_bounds, upper_bounds=params.upper_bounds, bounds_are_absolute=params.bounds_are_absolute, zero_outside_original_map=params.zero_outside_original_map, extract_unique=params.extract_unique, target_ncs_au_file=params.target_ncs_au_file, regions_to_keep=params.regions_to_keep, box_buffer=params.box_buffer, soft_mask_extract_unique=params.soft_mask_extract_unique, mask_expand_ratio=params.mask_expand_ratio, keep_low_density=params.keep_low_density, chain_type=params.chain_type, sequence=sequence, solvent_content=params.solvent_content, molecular_mass=params.molecular_mass, resolution=params.resolution, ncs_object=ncs_object, symmetry=params.symmetry, half_map_data_list=half_map_data_list, ) ph_box = pdb_hierarchy.select(selection) ph_box.adopt_xray_structure(box.xray_structure_box) box.hierarchy = ph_box if params.mask_select: print("\nSolvent content used in mask_select: %.3f " % (box.get_solvent_content()), file=log) if (inputs and inputs.crystal_symmetry and inputs.ccp4_map and inputs.crystal_symmetry.unit_cell().parameters() and inputs.ccp4_map.unit_cell_parameters) and ( inputs.crystal_symmetry.unit_cell().parameters() != inputs.ccp4_map.unit_cell_parameters): print("\nNOTE: Input CCP4 map is only part of unit cell:", file=log) print("Full unit cell ('unit cell parameters'): "+\ "(%.1f, %.1f, %.1f, %.1f, %.1f, %.1f) A" %tuple( inputs.ccp4_map.unit_cell_parameters), file=log) print("Size of CCP4 map 'map unit cell': "+\ "(%.1f, %.1f, %.1f, %.1f, %.1f, %.1f) A" %tuple( inputs.crystal_symmetry.unit_cell().parameters()), file=log) print("Full unit cell as grid units: (%s, %s, %s)" % (inputs.ccp4_map.unit_cell_grid), file=log) print("Map unit cell as grid units: (%s, %s, %s)" % (map_data.all()), file=log) box.unit_cell_parameters_from_ccp4_map = inputs.ccp4_map.unit_cell_parameters box.unit_cell_parameters_deduced_from_map_grid=\ inputs.crystal_symmetry.unit_cell().parameters() else: box.unit_cell_parameters_from_ccp4_map = None box.unit_cell_parameters_deduced_from_map_grid = None if box.pdb_outside_box_msg: print(box.pdb_outside_box_msg, file=log) # NOTE: box object is always shifted to place origin at (0,0,0) # NOTE ON ORIGIN SHIFTS: The shifts are applied locally here. The box # object is not affected and always has the origin at (0,0,0) # output_box is copy of box with shift_cart corresponding to the output # files. Normally this is the same as the original shift_cart. However # if user has specified a new output origin it will differ. # For output files ONLY: # keep_origin==False leave origin at (0,0,0) # keep_origin==True: we shift everything back to where it was, # output_origin_grid_units=10,10,10: output origin is at (10,10,10) # ncs_object is original # box.ncs_object is shifted by shift_cart # output_box.ncs_object is shifted back by -new shift_cart # Additional note on output unit_cell and grid_units. # The ccp4-style output map can specify the unit cell and grid units # corresponding to that cell. This can be separate from the origin and # number of grid points in the map as written. If specified, write these # out to the output ccp4 map and also use this unit cell for writing # any output PDB files from copy import deepcopy output_box = deepcopy(box) # won't use box below here except to return it print("\nBox cell dimensions: (%.2f, %.2f, %.2f) A" % (box.box_crystal_symmetry.unit_cell().parameters()[:3]), file=log) if box.shift_cart: print("Working origin moved from grid position of"+\ ": (%d, %d, %d) to (0,0,0) " %( tuple(box.origin_shift_grid_units(reverse=True))), file=log) print("Working origin moved from coordinates of:"+\ " (%.2f, %.2f, %.2f) A to (0,0,0)\n" %( tuple(-col(box.shift_cart))), file=log) if (params.keep_origin): print("\nRestoring original position for output files", file=log) print("Origin will be at grid position of"+\ ": (%d, %d, %d) " %( tuple(box.origin_shift_grid_units(reverse=True))), file=log) print("\nOutput files will be in same location as original", end=' ', file=log) if not params.keep_map_size: print("just cut out.", file=log) else: print("keeping entire map", file=log) print("Note that output maps are only valid in the cut out region.\n", file=log) else: if origin_to_match: output_box.shift_cart = shift_cart_for_origin_to_match if params.output_origin_grid_units: print("Output map origin to be shifted to match target", file=log) print("Placing origin at grid point (%s, %s, %s)" %( origin_to_match)+"\n"+ \ "Final coordinate shift for output files: (%.2f,%.2f,%.2f) A\n" %( tuple(col(output_box.shift_cart)-col(box.shift_cart))), file=log) elif box.shift_cart: output_box.shift_cart = (0, 0, 0) # not shifting back print("Final origin will be at (0,0,0)", file=log) print( "Final coordinate shift for output files: (%.2f,%.2f,%.2f) A\n" % (tuple(col(output_box.shift_cart) - col(box.shift_cart))), file=log) else: print("\nOutput files are in same location as original and origin "+\ "is at (0,0,0)\n", file=log) print("\nBox grid: (%s, %s, %s) " % (output_box.map_box.all())) ph_output_box_output_location = ph_box.deep_copy() if output_box.shift_cart: # shift coordinates and NCS back by shift_cart # NOTE output_box.shift_cart could be different than box.shift_cart if # there is a target position for the origin and it is not the same as the # original origin. sites_cart = output_box.shift_sites_cart_back( output_box.xray_structure_box.sites_cart()) xrs_offset = ph_output_box_output_location.extract_xray_structure( crystal_symmetry=output_box.xray_structure_box.crystal_symmetry( )).replace_sites_cart(new_sites=sites_cart) ph_output_box_output_location.adopt_xray_structure(xrs_offset) if output_box.ncs_object: output_box.ncs_object = output_box.ncs_object.coordinate_offset( tuple(-col(output_box.shift_cart))) shift_back = True else: shift_back = False if params.keep_input_unit_cell_and_grid and \ (input_unit_cell_grid is not None) and \ (input_unit_cell is not None): params.output_unit_cell = input_unit_cell params.output_unit_cell_grid = input_unit_cell_grid print("Setting output unit cell parameters and unit cell grid to"+\ " match\ninput map file", file=log) if params.output_unit_cell: # Set output unit cell parameters from cctbx import crystal output_crystal_symmetry = crystal.symmetry( unit_cell=params.output_unit_cell, space_group="P1") output_unit_cell = output_crystal_symmetry.unit_cell() print("Output unit cell set to: %.2f, %.2f, %.2f, %.2f, %.2f, %.2f)" % tuple(output_crystal_symmetry.unit_cell().parameters()), file=log) else: output_crystal_symmetry = None # ============= Check/set output unit cell grid and cell parameters ======= if params.output_unit_cell_grid or output_crystal_symmetry: if params.output_unit_cell_grid: output_unit_cell_grid = params.output_unit_cell_grid else: output_unit_cell_grid = output_box.map_box.all() print("Output unit cell grid set to: (%s, %s, %s)" % tuple(output_unit_cell_grid), file=log) expected_output_abc = [] box_spacing = [] output_spacing = [] box_abc=output_box.xray_structure_box.\ crystal_symmetry().unit_cell().parameters()[:3] if output_crystal_symmetry: output_abc = output_crystal_symmetry.unit_cell().parameters()[:3] else: output_abc = [None, None, None] for a_box, a_output, n_box, n_output in zip(box_abc, output_abc, output_box.map_box.all(), output_unit_cell_grid): expected_output_abc.append(a_box * n_output / n_box) box_spacing.append(a_box / n_box) if output_crystal_symmetry: output_spacing.append(a_output / n_output) else: output_spacing.append(a_box / n_box) if output_crystal_symmetry: # make sure it is compatible... r0 = expected_output_abc[0] / output_abc[0] r1 = expected_output_abc[1] / output_abc[1] r2 = expected_output_abc[2] / output_abc[2] from libtbx.test_utils import approx_equal if not approx_equal(r0, r1, eps=0.001) or not approx_equal( r0, r2, eps=0.001): print("WARNING: output_unit_cell and cell_grid will "+\ "change ratio of grid spacing.\nOld spacings: "+\ "(%.2f, %.2f, %.2f) A " %(tuple(box_spacing))+\ "\nNew spacings: (%.2f, %.2f, %.2f) A \n" %(tuple(output_spacing)), file=log) else: output_abc = expected_output_abc from cctbx import crystal output_crystal_symmetry = crystal.symmetry(unit_cell=list(output_abc) + [90, 90, 90], space_group="P1") print( "Output unit cell will be: (%.2f, %.2f, %.2f, %.2f, %.2f, %.2f)\n" % (tuple(output_crystal_symmetry.unit_cell().parameters())), file=log) else: output_unit_cell_grid = output_box.map_box.all() output_crystal_symmetry = output_box.xray_structure_box.crystal_symmetry( ) # ========== Done check/set output unit cell grid and cell parameters ===== if write_output_files: # Write PDB file if ph_box.overall_counts().n_residues > 0: if (params.output_file_name_prefix is None): file_name = "%s_box.pdb" % output_prefix else: file_name = "%s.pdb" % params.output_file_name_prefix ph_output_box_output_location.write_pdb_file( file_name=file_name, crystal_symmetry=output_crystal_symmetry) print("Writing boxed PDB with box unit cell to %s" % (file_name), file=log) # Write NCS file if NCS if output_box.ncs_object and output_box.ncs_object.max_operators() > 0: if (params.output_file_name_prefix is None): output_symmetry_file = "%s_box.ncs_spec" % output_prefix else: output_symmetry_file = "%s.ncs_spec" % params.output_file_name_prefix output_box.ncs_object.format_all_for_group_specification( file_name=output_symmetry_file) print("\nWriting symmetry to %s" % (output_symmetry_file), file=log) # Write ccp4 map. if ("ccp4" in params.output_format): if (params.output_file_name_prefix is None): file_name = "%s_box.ccp4" % output_prefix else: file_name = "%s.ccp4" % params.output_file_name_prefix from iotbx.mrcfile import create_output_labels if params.extract_unique: program_name = 'map_box using extract_unique' limitations = ["extract_unique"] else: program_name = 'map_box' limitations = [] labels = create_output_labels( program_name=program_name, input_file_name=inputs.ccp4_map_file_name, input_labels=input_map_labels, limitations=limitations, output_labels=params.output_map_labels) output_box.write_ccp4_map( file_name=file_name, output_crystal_symmetry=output_crystal_symmetry, output_mean=params.output_ccp4_map_mean, output_sd=params.output_ccp4_map_sd, output_unit_cell_grid=output_unit_cell_grid, shift_back=shift_back, output_map_labels=labels, output_external_origin=params.output_external_origin) print("Writing boxed map "+\ "to CCP4 formatted file: %s"%file_name, file=log) if not params.half_map_list: params.half_map_list = [] if not output_box.map_box_half_map_list: output_box.map_box_half_map_list = [] if not half_map_labels_list: half_map_labels_list = len( output_box.map_box_half_map_list) * [None] for hm, labels, fn in zip( output_box.map_box_half_map_list, half_map_labels_list, params.half_map_list): # half maps matching labels = create_output_labels( program_name=program_name, input_file_name=fn, input_labels=labels, limitations=limitations, output_labels=params.output_map_labels) hm_fn = "%s_box.ccp4" % (".".join( os.path.basename(fn).split(".")[:-1])) output_box.write_ccp4_map( file_name=hm_fn, map_data=hm, output_crystal_symmetry=output_crystal_symmetry, output_mean=params.output_ccp4_map_mean, output_sd=params.output_ccp4_map_sd, output_unit_cell_grid=output_unit_cell_grid, shift_back=shift_back, output_map_labels=labels, output_external_origin=params.output_external_origin) print("Writing boxed half map to: %s " % (hm_fn), file=log) # Write xplor map. Shift back if keep_origin=True if ("xplor" in params.output_format): if (params.output_file_name_prefix is None): file_name = "%s_box.xplor" % output_prefix else: file_name = "%s.xplor" % params.output_file_name_prefix output_box.write_xplor_map( file_name=file_name, output_crystal_symmetry=output_crystal_symmetry, output_unit_cell_grid=output_unit_cell_grid, shift_back=shift_back, ) print("Writing boxed map "+\ "to X-plor formatted file: %s"%file_name, file=log) # Write mtz map coeffs. Shift back if keep_origin=True if ("mtz" in params.output_format): if (params.output_file_name_prefix is None): file_name = "%s_box.mtz" % output_prefix else: file_name = "%s.mtz" % params.output_file_name_prefix print("Writing map coefficients "+\ "to MTZ file: %s"%file_name, file=log) if (map_coeff is not None): d_min = map_coeff.d_min() elif params.resolution is not None: d_min = params.resolution else: d_min = maptbx.d_min_from_map( map_data=output_box.map_box, unit_cell=output_box.xray_structure_box.unit_cell()) output_box.map_coefficients( d_min=d_min, scale_max=params.scale_max, resolution_factor=params.resolution_factor, file_name=file_name, shift_back=shift_back) print(file=log) return box
def __init__(self, pdb_file, output_file=None, log=None, quiet=False, set_se_occ=True, remove_atoms_with_zero_occupancy=False): from iotbx.file_reader import any_file import iotbx.pdb if (log is None): log = null_out() pdb_in = any_file(pdb_file, force_type="pdb") pdb_in.assert_file_type("pdb") hierarchy = pdb_in.file_object.hierarchy if (len(hierarchy.models()) > 1): raise Sorry("Multi-MODEL PDB files are not supported.") n_unknown = 0 all_atoms = hierarchy.atoms() cache = hierarchy.atom_selection_cache() # resname UNK is now okay (with some restrictions) known_sel = cache.selection("not (element X or resname UNX or resname UNL)") semet_sel = cache.selection("element SE and resname MSE") zero_occ_sel = all_atoms.extract_occ() == 0 self.n_unknown = known_sel.count(False) self.n_semet = semet_sel.count(True) self.n_zero_occ = zero_occ_sel.count(True) keep_sel = known_sel modified = False if ((self.n_unknown > 0) or ((self.n_semet > 0) and (set_se_occ)) or (self.n_zero_occ > 0) and (remove_atoms_with_zero_occupancy)): modified = True if (output_file is None): output_file = pdb_file if (self.n_unknown > 0) and (not quiet): print("Warning: %d unknown atoms or ligands removed:" % \ self.n_unknown, file=log) for i_seq in (~known_sel).iselection(): print(" %s" % all_atoms[i_seq].id_str(), file=log) if (self.n_zero_occ > 0): msg = "Warning: %d atoms with zero occupancy present in structure:" if (remove_atoms_with_zero_occupancy): msg = "Warning: %d atoms with zero occupancy removed:" keep_sel &= ~zero_occ_sel if (not quiet): print(msg % self.n_zero_occ, file=log) for i_seq in zero_occ_sel.iselection(): print(" %s" % all_atoms[i_seq].id_str(), file=log) hierarchy_filtered = hierarchy.select(keep_sel) if (self.n_semet > 0) and (set_se_occ): for atom in hierarchy_filtered.atoms(): if (atom.element == "SE") and (atom.fetch_labels().resname == "MSE"): if (atom.occ == 1.0): if (not quiet): print("Set occupancy of %s to 0.99" % atom.id_str(), file=log) atom.occ = 0.99 # just enough to trigger occupancy refinement if (modified): f = open(output_file, "w") # if the input file is actually from the PDB, we need to preserve the # header information for downstream code. print("\n".join(pdb_in.file_object.input.title_section()), file=f) print("\n".join(pdb_in.file_object.input.remark_section()), file=f) print(iotbx.pdb.format_cryst1_record( crystal_symmetry=pdb_in.file_object.crystal_symmetry()), file=f) print(hierarchy_filtered.as_pdb_string(), file=f) f.close()
def __setitem__(self, key, value): assert isinstance(value, block) if not re.match(tag_re, '_' + key): raise Sorry("%s is not a valid data block name" % key) self.blocks[key] = value self.keys_lower[key.lower()] = key
def extract(file_name, crystal_symmetry, wavelength_id, crystal_id, show_details_if_error, output_r_free_label, merge_non_unique_under_symmetry, map_to_asu, remove_systematic_absences, all_miller_arrays=None, incompatible_flags_to_work_set=False, ignore_bad_sigmas=False, extend_flags=False, return_as_miller_arrays=False, log=sys.stdout): import iotbx.cif from cctbx import miller if all_miller_arrays is None: base_array_info = miller.array_info( crystal_symmetry_from_file=crystal_symmetry) all_miller_arrays = iotbx.cif.reader( file_path=file_name).build_miller_arrays( base_array_info=base_array_info) if (len(all_miller_arrays) == 0): raise Sorry( "No data arrays were found in this CIF file. Please make " + "sure that the file contains reflection data, rather than the refined " + "model.") column_labels = set() if (extend_flags): map_to_asu = True # TODO: is all_mille_arrays a dict ? If not change back for (data_name, miller_arrays) in six.iteritems(all_miller_arrays): for ma in miller_arrays.values(): other_symmetry = crystal_symmetry try: crystal_symmetry = other_symmetry.join_symmetry( other_symmetry=ma.crystal_symmetry(), force=True) except AssertionError as e: str_e = str(e) from six.moves import cStringIO as StringIO s = StringIO() if "Space group is incompatible with unit cell parameters." in str_e: other_symmetry.show_summary(f=s) ma.crystal_symmetry().show_summary(f=s) str_e += "\n%s" % (s.getvalue()) raise Sorry(str_e) else: raise if (crystal_symmetry.unit_cell() is None or crystal_symmetry.space_group_info() is None): raise Sorry( "Crystal symmetry is not defined. Please use the --symmetry option." ) mtz_object = iotbx.mtz.object() \ .set_title(title="phenix.cif_as_mtz") \ .set_space_group_info(space_group_info=crystal_symmetry.space_group_info()) unit_cell = crystal_symmetry.unit_cell() mtz_crystals = {} mtz_object.set_hkl_base(unit_cell=unit_cell) from iotbx.reflection_file_utils import cif_status_flags_as_int_r_free_flags # generate list of all reflections (for checking R-free flags) from iotbx.reflection_file_utils import make_joined_set all_arrays = [] for (data_name, miller_arrays) in six.iteritems(all_miller_arrays): for ma in miller_arrays.values(): all_arrays.append(ma) complete_set = make_joined_set(all_arrays) if return_as_miller_arrays: miller_array_list = [] for i, (data_name, miller_arrays) in enumerate(six.iteritems(all_miller_arrays)): for ma in miller_arrays.values(): ma = ma.customized_copy( crystal_symmetry=crystal_symmetry).set_info(ma.info()) labels = ma.info().labels label = get_label(miller_array=ma, output_r_free_label=output_r_free_label) if label is None: print("Can't determine output label for %s - skipping." % \ ma.info().label_string(), file=log) continue elif label.startswith(output_r_free_label): ma, _ = cif_status_flags_as_int_r_free_flags( ma, test_flag_value="f") if isinstance(ma.data(), flex.double): data_int = ma.data().iround() assert data_int.as_double().all_eq(ma.data()) ma = ma.customized_copy(data=data_int).set_info(ma.info()) elif ( (ma.is_xray_amplitude_array() or ma.is_xray_intensity_array()) and isinstance(ma.data(), flex.int)): ma = ma.customized_copy(data=ma.data().as_double()).set_info( ma.info()) crys_id = 0 for l in labels: if 'crystal_id' in l: crys_id = int(l.split('=')[-1]) break if crys_id > 0 and crystal_id is None: label += "%i" % crys_id if crystal_id is not None and crys_id > 0 and crys_id != crystal_id: continue if crys_id not in mtz_crystals: mtz_crystals[crys_id] = (mtz_object.add_crystal( name="crystal_%i" % crys_id, project_name="project", unit_cell=unit_cell), {}) crystal, datasets = mtz_crystals[crys_id] w_id = 0 for l in labels: if 'wavelength_id' in l: w_id = int(l.split('=')[-1]) break if wavelength_id is not None and w_id > 0 and w_id != wavelength_id: continue if w_id > 1 and wavelength_id is None: if (label in column_labels): label += "%i" % w_id #print "label is", label if w_id not in datasets: wavelength = ma.info().wavelength if (wavelength is None): wavelength = 0 datasets[w_id] = crystal.add_dataset(name="dataset", wavelength=wavelength) dataset = datasets[w_id] # if all sigmas for an array are set to zero either raise an error, or set sigmas to None if ma.sigmas() is not None and (ma.sigmas() == 0).count(False) == 0: if ignore_bad_sigmas: print("Warning: bad sigmas, setting sigmas to None.", file=log) ma.set_sigmas(None) else: raise Sorry("""Bad sigmas: all sigmas are equal to zero. Add --ignore_bad_sigmas to command arguments to leave out sigmas from mtz file.""" ) if not ma.is_unique_set_under_symmetry(): if merge_non_unique_under_symmetry: print("Warning: merging non-unique data", file=log) if (label.startswith(output_r_free_label) and incompatible_flags_to_work_set): merging = ma.merge_equivalents( incompatible_flags_replacement=0) if merging.n_incompatible_flags > 0: print("Warning: %i reflections were placed in the working set " \ "because of incompatible flags between equivalents." %( merging.n_incompatible_flags), file=log) else: try: merging = ma.merge_equivalents() except Sorry as e: if ("merge_equivalents_exact: incompatible" in str(e)): raise Sorry( str(e) + " for %s" % ma.info().labels[-1] + "\n" + "Add --incompatible_flags_to_work_set to command line " "arguments to place incompatible flags to working set." ) raise ma = merging.array().customized_copy( crystal_symmetry=ma).set_info(ma.info()) elif return_as_miller_arrays: # allow non-unique set pass else: n_all = ma.indices().size() sel_unique = ma.unique_under_symmetry_selection() sel_dup = ~flex.bool(n_all, sel_unique) n_duplicate = sel_dup.count(True) n_uus = sel_unique.size() msg = ( "Miller indices not unique under symmetry: " + file_name + \ "(%d redundant indices out of %d)" % (n_all-n_uus, n_all) + "Add --merge to command arguments to force merging data.") if (show_details_if_error): print(msg) ma.show_comprehensive_summary(prefix=" ") ma.map_to_asu().sort().show_array(prefix=" ") raise Sorry(msg) if (map_to_asu): ma = ma.map_to_asu().set_info(ma.info()) if (remove_systematic_absences): ma = ma.remove_systematic_absences() if (label.startswith(output_r_free_label) and complete_set is not None): n_missing = len(complete_set.lone_set(other=ma).indices()) if (n_missing > 0): if (extend_flags): from cctbx import r_free_utils # determine flag values fvals = list(set(ma.data())) print("fvals", fvals) fval = None if (len(fvals) == 1): fval = fvals[0] elif (len(fvals) == 2): f1 = (ma.data() == fvals[0]).count(True) / ma.data().size() f2 = (ma.data() == fvals[1]).count(True) / ma.data().size() if (f1 < f2): fval = fvals[0] else: fval = fvals[1] elif (len(fvals) == 0): fval = None else: fval = 0 if (not fval in fvals): raise Sorry( "Cannot determine free-R flag value.") # if (fval is not None): ma = r_free_utils.extend_flags( r_free_flags=ma, test_flag_value=fval, array_label=label, complete_set=complete_set, preserve_input_values=True, allow_uniform_flags=True, log=sys.stdout) else: ma = None else: libtbx.warn(( "%d reflections do not have R-free flags in the " + "array '%s' - this may " + "cause problems if you try to use the MTZ file for refinement " + "or map calculation. We recommend that you extend the flags " + "to cover all reflections (--extend_flags on the command line)." ) % (n_missing, label)) # Get rid of fake (0,0,0) reflection in some CIFs if (ma is not None): ma = ma.select_indices(indices=flex.miller_index( ((0, 0, 0), )), negate=True).set_info(ma.info()) if return_as_miller_arrays: miller_array_list.append(ma) continue # don't make a dataset dec = None if ("FWT" in label): dec = iotbx.mtz.ccp4_label_decorator() column_types = None if ("PHI" in label or "PHWT" in label) and (ma.is_real_array()): column_types = "P" elif (label.startswith("DANO") and ma.is_real_array()): if (ma.sigmas() is not None): column_types = "DQ" else: column_types = "D" label_base = label i = 1 while label in column_labels: label = label_base + "-%i" % (i) i += 1 if (ma is not None): column_labels.add(label) dataset.add_miller_array(ma, column_root_label=label, label_decorator=dec, column_types=column_types) if return_as_miller_arrays: return miller_array_list else: return mtz_object
def run(args): import libtbx.load_env usage = "%s [options] datablock.json strong.pickle" % libtbx.env.dispatcher_name parser = OptionParser(usage=usage, read_reflections=True, read_datablocks=True, read_experiments=True, phil=phil_scope, check_format=False, epilog=help_message) from libtbx.utils import Sorry params, options = parser.parse_args(show_diff_phil=False) reflections = flatten_reflections(params.input.reflections) datablocks = flatten_datablocks(params.input.datablock) experiments = flatten_experiments(params.input.experiments) if not any([reflections, experiments, datablocks]): parser.print_help() return if len(reflections) != 1: raise Sorry('exactly 1 reflection table must be specified') if len(datablocks) != 1: if experiments: if len(experiments.imagesets()) != 1: raise Sorry('exactly 1 datablock must be specified') imageset = experiments.imagesets()[0] else: raise Sorry('exactly 1 datablock must be specified') else: imageset = datablocks[0].extract_imagesets()[0] reflections = reflections[0] if params.id is not None: reflections = reflections.select(reflections['id'] == params.id) stats = per_image_analysis.stats_imageset( imageset, reflections, resolution_analysis=params.resolution_analysis, plot=params.individual_plots) per_image_analysis.print_table(stats) from libtbx import table_utils overall_stats = per_image_analysis.stats_single_image( imageset, reflections, resolution_analysis=params.resolution_analysis) rows = [ ("Overall statistics", ""), ("#spots", "%i" % overall_stats.n_spots_total), ("#spots_no_ice", "%i" % overall_stats.n_spots_no_ice), ("d_min", "%.2f" % overall_stats.estimated_d_min), ("d_min (distl method 1)", "%.2f (%.2f)" % (overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1)), ("d_min (distl method 2)", "%.2f (%.2f)" % (overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1)), ] print(table_utils.format(rows, has_header=True, prefix="| ", postfix=" |")) if params.json: import json if params.split_json: for k in stats.__dict__: start, end = params.json.split('.') with open('%s_%s.%s' % (start, k, end), 'wb') as fp: json.dump(stats.__dict__[k], fp) if params.joint_json: with open(params.json, 'wb') as fp: json.dump(stats.__dict__, fp) if params.plot: per_image_analysis.plot_stats(stats, filename=params.plot)
class MainWindow(wx.Frame): ''' Frame housing the entire app; all windows open from this one ''' def __init__(self, parent, id, title): wx.Frame.__init__(self, parent, id, title, size=(800, 500)) # TODO: Allow GUI to be loaded with command line args, parse into PHIL self.iota_phil = inp.master_phil self.prefs_phil = None self.target_phil = None self.term_file = None # Create some defaults on startup # Figure out temp folder self.gparams = self.iota_phil.extract() tmp_folder = '/tmp/{}_{}'.format(user, pid) self.gparams.advanced.temporary_output_folder = tmp_folder self.iota_phil = self.iota_phil.format(python_object=self.gparams) # Menu bar menubar = wx.MenuBar() # Status bar self.sb = self.CreateStatusBar() # Help menu item with the about dialog m_help = wx.Menu() m_file = wx.Menu() self.mb_load_script = m_file.Append(wx.ID_OPEN, '&Load Script...') self.mb_save_script = m_file.Append(wx.ID_SAVE, '&Save Script...') m_file.AppendSeparator() self.mb_reset = m_file.Append(wx.ID_ANY, '&Reset Settings') self.mb_about = m_help.Append(wx.ID_ANY, '&About') menubar.Append(m_file, '&File') menubar.Append(m_help, '&Help') self.SetMenuBar(menubar) self.main_sizer = wx.BoxSizer(wx.VERTICAL) self.SetSizer(self.main_sizer) # Toolbar self.toolbar = self.CreateToolBar(style=wx.TB_3DBUTTONS | wx.TB_TEXT) quit_bmp = bitmaps.fetch_icon_bitmap('actions', 'exit') self.tb_btn_quit = self.toolbar.AddLabelTool(wx.ID_EXIT, label='Quit', bitmap=quit_bmp, shortHelp='Quit', longHelp='Quit IOTA') pref_bmp = bitmaps.fetch_icon_bitmap('apps', 'advancedsettings') self.tb_btn_prefs = self.toolbar.AddLabelTool( wx.ID_ANY, label='Preferences', bitmap=pref_bmp, shortHelp='Preferences', longHelp='IOTA Preferences') self.toolbar.AddSeparator() load_bmp = bitmaps.fetch_icon_bitmap('actions', 'open') self.tb_btn_load = self.toolbar.AddLabelTool( wx.ID_ANY, label='Load Script', bitmap=load_bmp, shortHelp='Load Script', longHelp='Load IOTA Script') save_bmp = bitmaps.fetch_icon_bitmap('actions', 'save') self.tb_btn_save = self.toolbar.AddLabelTool( wx.ID_ANY, label='Save Script', bitmap=save_bmp, shortHelp='Save Script', longHelp='Save IOTA Script') reset_bmp = bitmaps.fetch_icon_bitmap('actions', 'reload') self.tb_btn_reset = self.toolbar.AddLabelTool( wx.ID_ANY, label='Reset', bitmap=reset_bmp, shortHelp='Reset Settings', longHelp='Reset IOTA settings with defaults') self.toolbar.AddSeparator() analyze_bmp = bitmaps.fetch_icon_bitmap('mimetypes', 'text-x-generic-2') self.tb_btn_analysis = self.toolbar.AddLabelTool( wx.ID_ANY, label='Recover', bitmap=analyze_bmp, shortHelp='Recover', longHelp='Recover run, show statistics and restart if aborted ') run_bmp = bitmaps.fetch_icon_bitmap('actions', 'run') self.tb_btn_run = self.toolbar.AddLabelTool( wx.ID_ANY, label='Run', bitmap=run_bmp, shortHelp='Run', longHelp='Run all stages of refinement') # Test buttons for test windows - comment out when not needed # self.toolbar.AddSeparator() test_bmp = bitmaps.fetch_icon_bitmap('actions', 'utilities') self.tb_btn_test = self.toolbar.AddLabelTool(wx.ID_ANY, label='Test', bitmap=test_bmp) self.Bind(wx.EVT_TOOL, self.onRun, self.tb_btn_test) self.toolbar.RemoveTool(self.tb_btn_test.GetId()) # These buttons will be disabled until input path is provided self.toolbar.EnableTool(self.tb_btn_run.GetId(), False) self.toolbar.Realize() # Instantiate windows self.input_window = frm.InputWindow(self, phil=self.iota_phil) # Single input window self.main_sizer.Add(self.input_window, 1, flag=wx.ALL | wx.EXPAND, border=10) self.main_sizer.Add((-1, 20)) # button bindings self.Bind(wx.EVT_TOOL, self.onQuit, self.tb_btn_quit) self.Bind(wx.EVT_TOOL, self.onPreferences, self.tb_btn_prefs) self.Bind(wx.EVT_TOOL, self.onRun, self.tb_btn_run) self.Bind(wx.EVT_TOOL, self.onRecovery, self.tb_btn_analysis) self.Bind(wx.EVT_TOOL, self.onLoadScript, self.tb_btn_load) self.Bind(wx.EVT_TOOL, self.onOutputScript, self.tb_btn_save) self.Bind(wx.EVT_TOOL, self.onReset, self.tb_btn_reset) # Menubar button bindings self.Bind(wx.EVT_MENU, self.OnAboutBox, self.mb_about) self.Bind(wx.EVT_MENU, self.onOutputScript, self.mb_save_script) self.Bind(wx.EVT_MENU, self.onLoadScript, self.mb_load_script) self.Bind(wx.EVT_MENU, self.onReset, self.mb_reset) # Bindings to Input Window self.Bind(wx.EVT_BUTTON, self.onImportOptions, self.input_window.opt_btn_import) self.Bind(wx.EVT_BUTTON, self.onProcessOptions, self.input_window.opt_btn_process) self.Bind(wx.EVT_BUTTON, self.onAnalysisOptions, self.input_window.opt_btn_analysis) # File list control bindings self.Bind(ulc.EVT_LIST_INSERT_ITEM, self.onItemInserted, self.input_window.input) def read_command_line_options(self): help_message = '''This command will run the IOTA GUI ''' self.args, self.phil_args = parse_command_args('').parse_known_args() if self.args.path is not None and len(self.args.path) > 0: for carg in self.args.path: if os.path.exists(carg): if os.path.isfile(carg) and os.path.basename( carg).endswith('.param'): self.load_script(filepath=carg, update_input_window=False) else: self.input_window.input.add_item(os.path.abspath(carg)) if self.args.watch > 0: self.gparams.advanced.monitor_mode = True self.gparams.advanced.monitor_mode_timeout = True self.gparams.advanced.monitor_mode_timeout_length = self.args.watch[ 0] if self.args.random > 0: self.gparams.advanced.random_sample.flag_on = True self.gparams.advanced.random_sample.number = self.args.random[0] if self.args.tmp is not None: self.gparams.advanced.temporary_output_folder = self.args.tmp[0] if self.args.nproc is not None: self.gparams.n_processors = self.args.nproc self.iota_phil = self.iota_phil.format(python_object=self.gparams) # Parse in-line params into phil argument_interpreter = argint(master_phil=self.iota_phil) consume = [] for arg in self.phil_args: try: command_line_params = argument_interpreter.process(arg=arg) self.iota_phil = self.iota_phil.fetch(sources=[ command_line_params, ]) consume.append(arg) except Sorry, e: pass for item in consume: self.phil_args.remove(item) if len(self.phil_args) > 0: raise Sorry("Not all arguments processed, remaining: {}".format( self.phil_args)) self.gparams = self.iota_phil.extract() self.update_input_window()
def set_default_model_type(self, model_type): if (model_type not in self._possible_model_types): raise Sorry( 'Unrecognized model type, "%s," possible choices are %s.' % (model_type, ', '.join(self._possible_model_types))) self._default_model_type = model_type
def process_input_array(self, arr): array = arr.deep_copy() work_array = arr multiplicities = None try: if self.merge_equivalents : array, multiplicities, merge = MergeData(array, self.settings.show_anomalous_pairs) settings = self.settings data = array.data() #import code, traceback; code.interact(local=locals(), banner="".join( traceback.format_stack(limit=10) ) ) self.missing_set = oop.null() #if (array.is_xray_intensity_array()): # data.set_selected(data < 0, flex.double(data.size(), 0.)) if (array.is_unique_set_under_symmetry()) and (settings.map_to_asu): array = array.map_to_asu() if (multiplicities is not None): multiplicities = multiplicities.map_to_asu() if (settings.d_min is not None): array = array.resolution_filter(d_min=settings.d_min) if (multiplicities is not None): multiplicities = multiplicities.resolution_filter( d_min=settings.d_min) self.filtered_array = array.deep_copy() if (settings.expand_anomalous): if not array.is_unique_set_under_symmetry(): raise Sorry("Error! Cannot generate bijvoet mates of unmerged reflections.") array = array.generate_bijvoet_mates() original_symmetry = array.crystal_symmetry() if (multiplicities is not None): multiplicities = multiplicities.generate_bijvoet_mates() if (self.settings.show_missing): self.missing_set = array.complete_set().lone_set(array) if self.settings.show_anomalous_pairs: self.missing_set = self.missing_set.select( self.missing_set.centric_flags().data(), negate=True) if (settings.expand_to_p1): if not array.is_unique_set_under_symmetry(): raise Sorry("Error! Cannot expand unmerged reflections to P1.") original_symmetry = array.crystal_symmetry() array = array.expand_to_p1().customized_copy( crystal_symmetry=original_symmetry) #array = array.niggli_cell().expand_to_p1() #self.missing_set = self.missing_set.niggli_cell().expand_to_p1() self.missing_set = self.missing_set.expand_to_p1().customized_copy( crystal_symmetry=original_symmetry) if (multiplicities is not None): multiplicities = multiplicities.expand_to_p1().customized_copy( crystal_symmetry=original_symmetry) data = array.data() self.r_free_mode = False self.phases = flex.double(data.size(), float('nan')) self.radians = flex.double(data.size(), float('nan')) self.ampl = flex.double(data.size(), float('nan')) self.sigmas = None if isinstance(data, flex.bool): self.r_free_mode = True data_as_float = flex.double(data.size(), 0.0) data_as_float.set_selected(data==True, flex.double(data.size(), 1.0)) data = data_as_float self.data = data #.deep_copy() else : if isinstance(data, flex.double): self.data = data #.deep_copy() elif isinstance(data, flex.complex_double): self.data = data #.deep_copy() self.ampl = flex.abs(data) self.phases = flex.arg(data) * 180.0/math.pi # purge nan values from array to avoid crash in fmod_positive() #b = flex.bool([bool(math.isnan(e)) for e in self.phases]) b = graphics_utils.IsNansArray( self.phases ) # replace the nan values with an arbitrary float value self.phases = self.phases.set_selected(b, 42.4242) # Cast negative degrees to equivalent positive degrees self.phases = flex.fmod_positive(self.phases, 360.0) self.radians = flex.arg(data) # replace the nan values with an arbitrary float value self.radians = self.radians.set_selected(b, 0.424242) elif hasattr(array.data(), "as_double"): self.data = data else: raise RuntimeError("Unexpected data type: %r" % data) if (settings.show_data_over_sigma): if (array.sigmas() is None): raise Sorry("sigmas not defined.") sigmas = array.sigmas() non_zero_sel = sigmas != 0 array = array.select(non_zero_sel) array = array.customized_copy(data=array.data()/array.sigmas()) self.data = array.data() if (multiplicities is not None): multiplicities = multiplicities.select(non_zero_sel) if array.sigmas() is not None: self.sigmas = array.sigmas() else: self.sigmas = None work_array = array except Exception as e: print(to_str(e) + "".join(traceback.format_stack(limit=10))) raise e return None, None work_array.set_info(arr.info() ) multiplicities = multiplicities return work_array, multiplicities
def main(self): # FIXME import simulation code import cPickle as pickle import math from dials.util.command_line import Importer from dials.algorithms.integration import ReflectionPredictor from libtbx.utils import Sorry # Parse the command line params, options, args = self.parser.parse_args() importer = Importer(args) if len(importer.imagesets) == 0 and len(importer.crystals) == 0: self.config().print_help() return if len(importer.imagesets) != 1: raise Sorry('need 1 sweep: %d given' % len(importer.imagesets)) if len(importer.crystals) != 1: raise Sorry('need 1 crystal: %d given' % len(importer.crystals)) sweep = importer.imagesets[0] crystal = importer.crystals[0] # generate predictions for possible reflections => generate a # reflection list predict = ReflectionPredictor() predicted = predict(sweep, crystal) # sort with James's reflection table: should this not go somewhere central? from dials.scratch.jmp.container.reflection_table import ReflectionTable # calculate shoebox sizes: take parameters from params & transform # from reciprocal space to image space to decide how big a shoe box to use table = ReflectionTable() table['miller_index'] = predicted.miller_index() indexer = table.index_map('miller_index') candidates = [] unique = sorted(indexer) for h, k, l in unique: try: for _h in h - 1, h + 1: if not indexer[(_h, k, l)]: raise ValueError('missing') for _k in k - 1, k + 1: if not indexer[(h, _k, l)]: raise ValueError('missing') for _l in l - 1, l + 1: if not indexer[(h, k, _l)]: raise ValueError('missing') candidates.append((h, k, l)) except ValueError: continue from dials.algorithms.simulation.utils import build_prediction_matrix from dials.algorithms.simulation.generate_test_reflections import \ master_phil from libtbx.phil import command_line cmd = command_line.argument_interpreter(master_params=master_phil) working_phil = cmd.process_and_fetch(args=args[2:]) params = working_phil.extract() node_size = params.rs_node_size window_size = params.rs_window_size reference = params.integrated_data_file scale = params.integrated_data_file_scale if reference: counts_database = {} from iotbx import mtz m = mtz.object(reference) mi = m.extract_miller_indices() i = m.extract_reals('IMEAN').data s = m.space_group().build_derived_point_group() for j in range(len(mi)): for op in s.all_ops(): hkl = tuple(map(int, op * mi[j])) counts = max(0, int(math.floor(i[j] * scale))) counts_database[hkl] = counts counts_database[(-hkl[0], -hkl[1], -hkl[2])] = counts else: def constant_factory(value): import itertools return itertools.repeat(value).next from collections import defaultdict counts_database = defaultdict(constant_factory(params.counts)) from dials.model.data import ReflectionList useful = ReflectionList() d_matrices = [] for h, k, l in candidates: hkl = predicted[indexer[(h, k, l)][0]] _x = hkl.image_coord_px[0] _y = hkl.image_coord_px[1] _z = hkl.frame_number # build prediction matrix mhkl = predicted[indexer[(h - 1, k, l)][0]] phkl = predicted[indexer[(h + 1, k, l)][0]] hmkl = predicted[indexer[(h, k - 1, l)][0]] hpkl = predicted[indexer[(h, k + 1, l)][0]] hkml = predicted[indexer[(h, k, l - 1)][0]] hkpl = predicted[indexer[(h, k, l + 1)][0]] d = build_prediction_matrix(hkl, mhkl, phkl, hmkl, hpkl, hkml, hkpl) d_matrices.append(d) # construct the shoebox parameters: outline the ellipsoid x, y, z = [], [], [] for dh in (1, 0, 0), (0, 1, 0), (0, 0, 1): dxyz = -1 * window_size * d * dh x.append(dxyz[0] + _x) y.append(dxyz[1] + _y) z.append(dxyz[2] + _z) dxyz = window_size * d * dh x.append(dxyz[0] + _x) y.append(dxyz[1] + _y) z.append(dxyz[2] + _z) hkl.bounding_box = (int(math.floor(min(x))), int(math.floor(max(x)) + 1), int(math.floor(min(y))), int(math.floor(max(y)) + 1), int(math.floor(min(z))), int(math.floor(max(z)) + 1)) try: counts = counts_database[hkl.miller_index] useful.append(hkl) except KeyError: continue from dials.algorithms import shoebox shoebox.allocate(useful) from dials.util.command_line import ProgressBar p = ProgressBar(title='Generating shoeboxes') # now for each reflection perform the simulation for j, refl in enumerate(useful): p.update(j * 100.0 / len(useful)) d = d_matrices[j] from scitbx.random import variate, normal_distribution g = variate(normal_distribution(mean=0, sigma=node_size)) counts = counts_database[refl.miller_index] dhs = g(counts) dks = g(counts) dls = g(counts) self.map_to_image_space(refl, d, dhs, dks, dls) p.finished('Generated %d shoeboxes' % len(useful)) # now for each reflection add background from dials.algorithms.simulation.generate_test_reflections import \ random_background_plane p = ProgressBar(title='Generating background') for j, refl in enumerate(useful): p.update(j * 100.0 / len(useful)) if params.background: random_background_plane(refl.shoebox, params.background, 0.0, 0.0, 0.0) else: random_background_plane(refl.shoebox, params.background_a, params.background_b, params.background_c, params.background_d) p.finished('Generated %d backgrounds' % len(useful)) if params.output.all: pickle.dump(useful, open(params.output.all, 'w')) return