def run(args, command_name="mmtbx.csv_to_mtz"): if (len(args) == 0): args = ["--help"] try: command_line = (iotbx_option_parser( usage="%s [reflection_csv_file] [options]" % command_name, description='Example: %s 1m5u-sf.csv --use_model=1m5u.pdb' % command_name ).enable_symmetry_comprehensive().option( None, "--output_file", action="store", default=False, type="string", help="Output mtz file name." ).option( None, "--use_model", action="store", default=False, type="string", help= "Use PDB model to make better guess about reflection data type.") ).process(args=args) except Exception, e: if (str(e) != "0"): print str(e) sys.exit(0)
def process_zeolite_atlas(args): command_line = (iotbx_option_parser() .option(None, "--tag", action="store", type="string") .option(None, "--verbose", action="store_true") ).process(args=args) atlas_file = libtbx.env.find_in_repositories( relative_path="phenix_regression/misc/strudat_zeolite_atlas", test=os.path.isfile) if (atlas_file is None): print "Skipping process_zeolite_atlas(): input file not available" return if (command_line.options.verbose): out = sys.stdout else: out = StringIO() all_entries = strudat.read_all_entries(open(atlas_file)) for i,entry in enumerate(all_entries.entries): structure = entry.as_xray_structure() if (command_line.options.tag is not None): if (command_line.options.tag != entry.tag): continue print >> out, "strudat tag:", entry.tag exercise(structure=structure, out=out) exercise_shake_sites_in_place(structure=structure)
def exercise(args, distance_cutoff=3.5, max_shell=5): command_line = (iotbx_option_parser() .option(None, "--tag", action="store", type="string") .option(None, "--full", action="store_true") .option(None, "--verbose", action="store_true") ).process(args=args) co = command_line.options atlas_file = libtbx.env.find_in_repositories( relative_path="phenix_regression/misc/strudat_zeolite_atlas", test=os.path.isfile) if (atlas_file is None): print "Skipping exercise(): input file not available" return all_entries = strudat.read_all_entries(open(atlas_file)) for i,entry in enumerate(all_entries.entries): structure = entry.as_xray_structure() if (co.tag is not None): if (co.tag != entry.tag): continue elif (not (co.full or i % 20 == 0)): continue if (co.verbose): print "strudat tag:", entry.tag exercise_simple( structure, distance_cutoff, max_shell, co.verbose) exercise_shell_asu_tables(structure, co.verbose)
def process_zeolite_atlas(args): command_line = (iotbx_option_parser().option( None, "--tag", action="store", type="string").option(None, "--verbose", action="store_true")).process(args=args) atlas_file = libtbx.env.find_in_repositories( relative_path="phenix_regression/misc/strudat_zeolite_atlas", test=os.path.isfile) if (atlas_file is None): print("Skipping process_zeolite_atlas(): input file not available") return if (command_line.options.verbose): out = sys.stdout else: out = StringIO() with open(atlas_file) as f: all_entries = strudat.read_all_entries(f) for i, entry in enumerate(all_entries.entries): structure = entry.as_xray_structure() if (command_line.options.tag is not None): if (command_line.options.tag != entry.tag): continue print("strudat tag:", entry.tag, file=out) exercise(structure=structure, out=out) exercise_shake_sites_in_place(structure=structure)
def run(args, command_name = "phenix.cif_as_mtz", out=sys.stdout, return_as_miller_arrays=False): if (len(args) == 0): args = ["--help"] try: command_line = (iotbx_option_parser( usage="%s [reflection_cif_file] [options]" % command_name, description='Example: %s r1o9ksf.ent --symmetry=pdb1o9k.ent'%command_name) .enable_symmetry_comprehensive() .option(None, "--output_file_name", action="store", default=False, type="string", help="Output mtz file name.") .option(None, "--wavelength_id", action="store", default=None, type="int", help="Extract data set with given wavelength_id.") .option(None, "--crystal_id", action="store", default=None, type="int", help="Extract data set with given crystal_id.") .option(None, "--output_r_free_label", action="store", default="R-free-flags", type="string", help="MTZ column label to use for R-free flags (default: R-free-flags)") .option(None, "--merge", action="store_true", help="Merge non-unique data where present.") .option(None, "--incompatible_flags_to_work_set", action="store_true", help="When merging place reflections with incompatible flags into the " "working set.") .option(None, "--remove_systematic_absences", action="store_true", help="Remove systematic absent reflections.") .option(None, "--map_to_asu", action="store_true", help="Map to asymmetric unit.") .option("--show_details_if_error", action="store_true", help="Show data details for some errors.") .option("--show_log", action="store_true", help="Show some output.") .option("--ignore_bad_sigmas", action="store_true", help="Set sigmas to None instead of raising an error when bad sigmas " "are present.") .option("--extend_flags", action="store_true", help="Extend R-free flags to cover all reflections if necessary.") ).process(args=args) except Exception, e: if(str(e) != "0"): print str(e) sys.exit(0)
def run(args, command_name="phenix.tls"): if (len(args) == 0): args = ["--help"] usage_fmt = "%s pdb_file [parameters: file or command line string]" des_fmt = "Example: %s model.pdb fit_tls_to.selection='%s' fit_tls_to.selection='%s'" command_line = (iotbx_option_parser( usage=usage_fmt % command_name, description=banner).option( "--show_defaults", action="store_true", help="Do not output to the screen (except errors).").option( "--silent", action="store_true", help="Suppress output to the screen.")).process(args=args) # log = sys.stdout if (not command_line.options.silent): utils.print_header("TLS tools", out=log) if (command_line.options.show_defaults): master_params.show(out=log) print >> log return if (not command_line.options.silent): print >> log, banner # processed_args = utils.process_command_line_args( args=command_line.args, master_params=master_params, log=log) reflection_files = processed_args.reflection_files if (processed_args.crystal_symmetry is None): raise Sorry("No crystal symmetry found.") if (len(processed_args.pdb_file_names) == 0): raise Sorry("No PDB file found.") params = processed_args.params if (not command_line.options.silent): utils.print_header("Input parameters", out=log) params.show(out=log) params = params.extract() # if (processed_args.crystal_symmetry.unit_cell() is None or processed_args.crystal_symmetry.space_group() is None): raise Sorry("No CRYST1 record found.") pdb_combined = iotbx.pdb.combine_unique_pdb_files( file_names=processed_args.pdb_file_names) pdb_combined.report_non_unique(out=log) if (len(pdb_combined.unique_file_names) == 0): raise Sorry("No coordinate file given.") raw_records = pdb_combined.raw_records try: pdb_inp = iotbx.pdb.input(source_info=None, lines=flex.std_string(raw_records)) except ValueError, e: raise Sorry("Model format (PDB or mmCIF) error:\n%s" % str(e))
def run(args, command_name = "phenix.nm"): if(len(args) == 0): args = ["--help"] usage_fmt = "%s pdb_file [parameters: file or command line string]" des_fmt = "Example: %s model.pdb fit_tls_to.selection='%s' fit_tls_to.selection='%s'" command_line = (iotbx_option_parser( usage = usage_fmt % command_name, description = banner) .option("--show_defaults", action="store_true", help="Do not output to the screen (except errors).") .option("--silent", action="store_true", help="Suppress output to the screen.") .option("--use_elbow", action="store_false", help="Use eLBOW if there are unknown ligands in PDB file (available in PHENIX only).") ).process(args=args)
def run(args, command_name = "mmtbx.csv_to_mtz"): if (len(args) == 0): args = ["--help"] try: command_line = (iotbx_option_parser( usage="%s [reflection_csv_file] [options]" % command_name, description='Example: %s 1m5u-sf.csv --use_model=1m5u.pdb'%command_name) .enable_symmetry_comprehensive() .option(None, "--output_file", action="store", default=False, type="string", help="Output mtz file name.") .option(None, "--use_model", action="store", default=False, type="string", help="Use PDB model to make better guess about reflection data type.") ).process(args=args) except Exception, e: if(str(e) != "0"): print str(e) sys.exit(0)
def exercise(args, distance_cutoff=3.5, max_shell=5): command_line = (iotbx_option_parser().option( None, "--tag", action="store", type="string").option(None, "--full", action="store_true").option( None, "--verbose", action="store_true")).process(args=args) co = command_line.options atlas_file = libtbx.env.find_in_repositories( relative_path="phenix_regression/misc/strudat_zeolite_atlas", test=os.path.isfile) if (atlas_file is None): print "Skipping exercise(): input file not available" return all_entries = strudat.read_all_entries(open(atlas_file)) for i, entry in enumerate(all_entries.entries): structure = entry.as_xray_structure() if (co.tag is not None): if (co.tag != entry.tag): continue elif (not (co.full or i % 20 == 0)): continue if (co.verbose): print "strudat tag:", entry.tag exercise_simple(structure, distance_cutoff, max_shell, co.verbose) exercise_shell_asu_tables(structure, co.verbose)
def run(args, command_name="phenix.cif_as_mtz", out=sys.stdout, return_as_miller_arrays=False): if (len(args) == 0): args = ["--help"] try: command_line = (iotbx_option_parser( usage="%s [reflection_cif_file] [options]" % command_name, description='Example: %s r1o9ksf.ent --symmetry=pdb1o9k.ent' % command_name ).enable_symmetry_comprehensive().option( None, "--output_file_name", action="store", default=False, type="string", help="Output mtz file name." ).option( None, "--wavelength_id", action="store", default=None, type="int", help="Extract data set with given wavelength_id." ).option( None, "--crystal_id", action="store", default=None, type="int", help="Extract data set with given crystal_id." ).option( None, "--output_r_free_label", action="store", default="R-free-flags", type="string", help= "MTZ column label to use for R-free flags (default: R-free-flags)" ).option( None, "--merge", action="store_true", help="Merge non-unique data where present." ).option( None, "--incompatible_flags_to_work_set", action="store_true", help= "When merging place reflections with incompatible flags into the " "working set." ).option( None, "--remove_systematic_absences", action="store_true", help="Remove systematic absent reflections." ).option( None, "--map_to_asu", action="store_true", help="Map to asymmetric unit." ).option( "--show_details_if_error", action="store_true", help="Show data details for some errors." ).option( "--show_log", action="store_true", help="Show some output." ).option( "--ignore_bad_sigmas", action="store_true", help= "Set sigmas to None instead of raising an error when bad sigmas " "are present." ).option( "--extend_flags", action="store_true", help="Extend R-free flags to cover all reflections if necessary.") ).process(args=args) except Exception, e: if (str(e) != "0"): print str(e) sys.exit(0)
def run(args, command_name="phenix.fobs_minus_fobs_map", log=None): if (len(args) == 0): args = ["--help"] examples = """Examples: phenix.fobs_minus_fobs_map f_obs_1_file=data1.mtz f_obs_2_file=data2.sca \ f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 model.pdb phenix.fobs_minus_fobs_map f_obs_1_file=data.mtz f_obs_2_file=data.mtz \ f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 phase_source=model.pdb \ high_res=2.0 sigma_cutoff=2 scattering_table=neutron""" command_line = (iotbx_option_parser( usage="%s [options]" % command_name, description=examples).option("--silent", action="store_true", help="Suppress output to the screen."). enable_symmetry_comprehensive()).process(args=args) # if (log is None): log = sys.stdout if (not command_line.options.silent): utils.print_header("phenix.fobs_minus_fobs_map", out=log) print("Command line arguments: ", file=log) print(args, file=log) print(file=log) # processed_args = utils.process_command_line_args( args=command_line.args, cmd_cs=command_line.symmetry, master_params=fo_minus_fo_master_params(), absolute_angle_tolerance=5, absolute_length_tolerance=1, log=log, suppress_symmetry_related_errors=True) working_phil = processed_args.params if (not command_line.options.silent): print("*** Parameters:", file=log) working_phil.show(out=log) print(file=log) params = working_phil.extract() consensus_symmetry = None if (params.ignore_non_isomorphous_unit_cells): if (None in [ params.f_obs_1_file_name, params.f_obs_2_file_name, params.phase_source ]): raise Sorry( "The file parameters (f_obs_1_file_name, f_obs_2_file_name, " + "phase_source) must be specified explicitly when " + "ignore_non_isomorphous_unit_cells=True.") symm_manager = iotbx.symmetry.manager() pdb_in = iotbx.file_reader.any_file(params.phase_source, force_type="pdb") symm_manager.process_pdb_file(pdb_in) hkl_in_1 = iotbx.file_reader.any_file(params.f_obs_1_file_name, force_type="hkl") sg_err_1, uc_err_1 = symm_manager.process_reflections_file(hkl_in_1) hkl_in_2 = iotbx.file_reader.any_file(params.f_obs_2_file_name, force_type="hkl") sg_err_2, uc_err_2 = symm_manager.process_reflections_file(hkl_in_2) out = StringIO() symm_manager.show(out=out) if (sg_err_1) or (sg_err_2): raise Sorry(( "Incompatible space groups in input files:\n%s\nAll files " + "must have the same point group (and ideally the same space group). " + "Please note that any symmetry information in the PDB file will be " + "used first.") % out.getvalue()) elif (uc_err_1) or (uc_err_2): libtbx.call_back( message="warn", data= ("Crystal symmetry mismatch:\n%s\nCalculations will continue " + "using the symmetry in the PDB file (or if not available, the " + "first reflection file), but the maps should be treated with " + "extreme suspicion.") % out.getvalue()) crystal_symmetry = symm_manager.as_symmetry_object() else: processed_args = utils.process_command_line_args( args=command_line.args, cmd_cs=command_line.symmetry, master_params=fo_minus_fo_master_params(), suppress_symmetry_related_errors=False, absolute_angle_tolerance=5, absolute_length_tolerance=1, log=StringIO()) crystal_symmetry = processed_args.crystal_symmetry # pdb_file_names = processed_args.pdb_file_names if (len(processed_args.pdb_file_names) == 0): if (params.phase_source is not None): pdb_file_names = [params.phase_source] else: raise Sorry("No PDB file found.") # Extaract Fobs1, Fobs2 f_obss = [] if (len(processed_args.reflection_files) == 2): for reflection_file in processed_args.reflection_files: reflection_file_server = reflection_file_utils.reflection_file_server( crystal_symmetry=crystal_symmetry, force_symmetry=True, reflection_files=[reflection_file], err=null_out()) # XXX UGLY !!! try: parameters = utils.data_and_flags_master_params().extract() if (params.f_obs_1_label is not None): parameters.labels = [params.f_obs_1_label] determine_data_and_flags_result = utils.determine_data_and_flags( reflection_file_server=reflection_file_server, keep_going=True, parameters=parameters, log=null_out()) except: # intentional parameters = utils.data_and_flags_master_params().extract() if (params.f_obs_2_label is not None): parameters.labels = [params.f_obs_2_label] determine_data_and_flags_result = utils.determine_data_and_flags( reflection_file_server=reflection_file_server, keep_going=True, parameters=parameters, log=null_out()) f_obss.append(determine_data_and_flags_result.f_obs) else: if ([params.f_obs_1_file_name, params.f_obs_2_file_name].count(None) == 2): raise Sorry("No reflection data file found.") for file_name, label in zip( [params.f_obs_1_file_name, params.f_obs_2_file_name], [params.f_obs_1_label, params.f_obs_2_label]): reflection_file = reflection_file_reader.any_reflection_file( file_name=file_name, ensure_read_access=False) reflection_file_server = reflection_file_utils.reflection_file_server( crystal_symmetry=crystal_symmetry, force_symmetry=True, reflection_files=[reflection_file], err=null_out()) parameters = utils.data_and_flags_master_params().extract() if (label is not None): parameters.labels = [label] determine_data_and_flags_result = utils.determine_data_and_flags( reflection_file_server=reflection_file_server, parameters=parameters, keep_going=True, log=null_out()) f_obss.append(determine_data_and_flags_result.f_obs) if (len(f_obss) != 2): raise Sorry(" ".join(errors)) if (not command_line.options.silent): for ifobs, fobs in enumerate(f_obss): print("*** Summary for data set %d:" % ifobs, file=log) fobs.show_comprehensive_summary(f=log) print(file=log) pdb_combined = combine_unique_pdb_files(file_names=pdb_file_names) pdb_combined.report_non_unique(out=log) if (len(pdb_combined.unique_file_names) == 0): raise Sorry("No coordinate file given.") # raw_recs = flex.std_string() for rec in pdb_combined.raw_records: if (rec.upper().count("CRYST1") == 0): raw_recs.append(rec) raw_recs.append( iotbx.pdb.format_cryst1_record(crystal_symmetry=crystal_symmetry)) # pdb_in = iotbx.pdb.input(source_info=None, lines=raw_recs) model = mmtbx.model.manager(model_input=pdb_in) d_min = min(f_obss[0].d_min(), f_obss[1].d_min()) model.setup_scattering_dictionaries( scattering_table=params.scattering_table, d_min=d_min) xray_structure = model.get_xray_structure() hierarchy = model.get_hierarchy() # omit_sel = flex.bool(hierarchy.atoms_size(), False) if (params.advanced.omit_selection is not None): print("Will omit selection from phasing model:", file=log) print(" " + params.advanced.omit_selection, file=log) omit_sel = hierarchy.atom_selection_cache().selection( params.advanced.omit_selection) print("%d atoms selected for removal" % omit_sel.count(True), file=log) del hierarchy xray_structure = xray_structure.select(~omit_sel) if (not command_line.options.silent): print("*** Model summary:", file=log) xray_structure.show_summary(f=log) print(file=log) info0 = f_obss[0].info() info1 = f_obss[1].info() f_obss[0] = f_obss[0].resolution_filter( d_min=params.high_resolution, d_max=params.low_resolution).set_info(info0) f_obss[1] = f_obss[1].resolution_filter( d_min=params.high_resolution, d_max=params.low_resolution).set_info(info1) if (params.sigma_cutoff is not None): for i in [0, 1]: if (f_obss[i].sigmas() is not None): sel = f_obss[i].data( ) > f_obss[i].sigmas() * params.sigma_cutoff f_obss[i] = f_obss[i].select(sel).set_info(info0) for k, f_obs in enumerate(f_obss): if (f_obs.indices().size() == 0): raise Sorry( "No data left in array %d (labels=%s) after filtering!" % (k + 1, f_obs.info().label_string())) output_file_name = params.output_file if (output_file_name is None) and (params.file_name_prefix is not None): output_file_name = "%s_%s.mtz" % (params.file_name_prefix, params.job_id) output_files = compute_fo_minus_fo_map( data_arrays=f_obss, xray_structure=xray_structure, log=log, silent=command_line.options.silent, output_file=output_file_name, peak_search=params.find_peaks_holes, map_cutoff=params.map_cutoff, peak_search_params=params.peak_search, multiscale=params.advanced.multiscale, anomalous=params.advanced.anomalous).file_names return output_files
def run(args, command_name = "phenix.fobs_minus_fobs_map", log=None): if(len(args) == 0): args = ["--help"] examples = """Examples: phenix.fobs_minus_fobs_map f_obs_1_file=data1.mtz f_obs_2_file=data2.sca \ f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 model.pdb phenix.fobs_minus_fobs_map f_obs_1_file=data.mtz f_obs_2_file=data.mtz \ f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 phase_source=model.pdb \ high_res=2.0 sigma_cutoff=2 scattering_table=neutron""" command_line = (iotbx_option_parser( usage="%s [options]" % command_name, description=examples) .option("--silent", action="store_true", help="Suppress output to the screen.") .enable_symmetry_comprehensive() ).process(args=args) # if (log is None) : log = sys.stdout if(not command_line.options.silent): utils.print_header("phenix.fobs_minus_fobs_map", out = log) print >> log, "Command line arguments: " print >> log, args print >> log # processed_args = utils.process_command_line_args( args=command_line.args, cmd_cs=command_line.symmetry, master_params=fo_minus_fo_master_params(), absolute_angle_tolerance=5, absolute_length_tolerance=1, log=log, suppress_symmetry_related_errors=True) working_phil = processed_args.params if(not command_line.options.silent): print >> log, "*** Parameters:" working_phil.show(out = log) print >> log params = working_phil.extract() consensus_symmetry = None if (params.ignore_non_isomorphous_unit_cells) : if (None in [params.f_obs_1_file_name, params.f_obs_2_file_name, params.phase_source]): raise Sorry("The file parameters (f_obs_1_file_name, f_obs_2_file_name, "+ "phase_source) must be specified explicitly when "+ "ignore_non_isomorphous_unit_cells=True.") symm_manager = iotbx.symmetry.manager() pdb_in = iotbx.file_reader.any_file(params.phase_source, force_type="pdb") symm_manager.process_pdb_file(pdb_in) hkl_in_1 = iotbx.file_reader.any_file(params.f_obs_1_file_name, force_type="hkl") sg_err_1, uc_err_1 = symm_manager.process_reflections_file(hkl_in_1) hkl_in_2 = iotbx.file_reader.any_file(params.f_obs_2_file_name, force_type="hkl") sg_err_2, uc_err_2 = symm_manager.process_reflections_file(hkl_in_2) out = StringIO() symm_manager.show(out=out) if (sg_err_1) or (sg_err_2) : raise Sorry(("Incompatible space groups in input files:\n%s\nAll files "+ "must have the same point group (and ideally the same space group). "+ "Please note that any symmetry information in the PDB file will be "+ "used first.") % out.getvalue()) elif (uc_err_1) or (uc_err_2) : libtbx.call_back(message="warn", data=("Crystal symmetry mismatch:\n%s\nCalculations will continue "+ "using the symmetry in the PDB file (or if not available, the "+ "first reflection file), but the maps should be treated with "+ "extreme suspicion.") % out.getvalue()) crystal_symmetry = symm_manager.as_symmetry_object() else : processed_args = utils.process_command_line_args( args=command_line.args, cmd_cs=command_line.symmetry, master_params=fo_minus_fo_master_params(), absolute_angle_tolerance=5, absolute_length_tolerance=1, log=StringIO()) crystal_symmetry = processed_args.crystal_symmetry # pdb_file_names = processed_args.pdb_file_names if(len(processed_args.pdb_file_names) == 0): if(params.phase_source is not None): pdb_file_names = [params.phase_source] else: raise Sorry("No PDB file found.") # Extaract Fobs1, Fobs2 f_obss = [] if(len(processed_args.reflection_files)==2): for reflection_file in processed_args.reflection_files: reflection_file_server = reflection_file_utils.reflection_file_server( crystal_symmetry = crystal_symmetry, force_symmetry = True, reflection_files = [reflection_file], err = null_out()) # XXX UGLY !!! try: parameters = utils.data_and_flags_master_params().extract() if(params.f_obs_1_label is not None): parameters.labels = [params.f_obs_1_label] determine_data_and_flags_result = utils.determine_data_and_flags( reflection_file_server = reflection_file_server, keep_going = True, parameters = parameters, log = null_out()) except: # intentional parameters = utils.data_and_flags_master_params().extract() if(params.f_obs_2_label is not None): parameters.labels = [params.f_obs_2_label] determine_data_and_flags_result = utils.determine_data_and_flags( reflection_file_server = reflection_file_server, keep_going = True, parameters = parameters, log = null_out()) f_obss.append(determine_data_and_flags_result.f_obs) else: if([params.f_obs_1_file_name,params.f_obs_2_file_name].count(None)==2): raise Sorry("No reflection data file found.") for file_name, label in zip([params.f_obs_1_file_name,params.f_obs_2_file_name], [params.f_obs_1_label,params.f_obs_2_label]): reflection_file = reflection_file_reader.any_reflection_file( file_name = file_name, ensure_read_access = False) reflection_file_server = reflection_file_utils.reflection_file_server( crystal_symmetry = crystal_symmetry, force_symmetry = True, reflection_files = [reflection_file], err = null_out()) parameters = utils.data_and_flags_master_params().extract() if(label is not None): parameters.labels = [label] determine_data_and_flags_result = utils.determine_data_and_flags( reflection_file_server = reflection_file_server, parameters = parameters, keep_going = True, log = null_out()) f_obss.append(determine_data_and_flags_result.f_obs) if(len(f_obss)!=2): raise Sorry(" ".join(errors)) if(not command_line.options.silent): for ifobs, fobs in enumerate(f_obss): print >> log, "*** Summary for data set %d:"%ifobs fobs.show_comprehensive_summary(f = log) print >> log pdb_combined = combine_unique_pdb_files(file_names = pdb_file_names) pdb_combined.report_non_unique(out = log) if(len(pdb_combined.unique_file_names) == 0): raise Sorry("No coordinate file given.") # raw_recs = flex.std_string() for rec in pdb_combined.raw_records: if(rec.upper().count("CRYST1")==0): raw_recs.append(rec) raw_recs.append(iotbx.pdb.format_cryst1_record( crystal_symmetry = crystal_symmetry)) # pdb_in = iotbx.pdb.input(source_info = None, lines = raw_recs) hierarchy = pdb_in.construct_hierarchy() omit_sel = flex.bool(hierarchy.atoms_size(), False) if (params.advanced.omit_selection is not None) : print >> log, "Will omit selection from phasing model:" print >> log, " " + params.advanced.omit_selection omit_sel = hierarchy.atom_selection_cache().selection( params.advanced.omit_selection) print >> log, "%d atoms selected for removal" % omit_sel.count(True) xray_structure = pdb_in.xray_structure_simple() xray_structure = xray_structure.select(~omit_sel) if(not command_line.options.silent): print >> log, "*** Model summary:" xray_structure.show_summary(f = log) print >> log info0 = f_obss[0].info() info1 = f_obss[1].info() f_obss[0] = f_obss[0].resolution_filter(d_min = params.high_resolution, d_max = params.low_resolution).set_info(info0) f_obss[1] = f_obss[1].resolution_filter(d_min = params.high_resolution, d_max = params.low_resolution).set_info(info1) if(params.sigma_cutoff is not None): for i in [0,1]: if(f_obss[i].sigmas() is not None): sel = f_obss[i].data() > f_obss[i].sigmas()*params.sigma_cutoff f_obss[i] = f_obss[i].select(sel).set_info(info0) for k, f_obs in enumerate(f_obss) : if (f_obs.indices().size() == 0) : raise Sorry("No data left in array %d (labels=%s) after filtering!" % (k+1, f_obs.info().label_string())) output_file_name = params.output_file if (output_file_name is None) and (params.file_name_prefix is not None) : output_file_name = "%s_%s.mtz" % (params.file_name_prefix, params.job_id) output_files = compute_fo_minus_fo_map( data_arrays = f_obss, xray_structure = xray_structure, log = log, silent = command_line.options.silent, output_file = output_file_name, peak_search=params.find_peaks_holes, map_cutoff=params.map_cutoff, peak_search_params=params.peak_search, multiscale=params.advanced.multiscale, anomalous=params.advanced.anomalous).file_names return output_files
def run(args, command_name): from iotbx.option_parser import option_parser as iotbx_option_parser import libtbx.utils show_times = libtbx.utils.show_times(time_start="now") command_line = (iotbx_option_parser( usage=command_name+" [options] [cod_id...]") .enable_chunk(easy_all=True) .enable_multiprocessing() .option(None, "--parse_only", action="store_true") .option(None, "--cif_only", action="store_true") .option(None, "--hkl_only", action="store_true") .option("-v", "--verbose", action="store_true") ).process(args=args) if (command_line.run_multiprocessing_chunks_if_applicable( command_call=[command_name, __file__])): show_times() return co = command_line.options cod_ids = command_line.args assert [co.cif_only, co.hkl_only].count(True) <= 1 if co.cif_only: ext = "cif" elif co.hkl_only: ext = "hkl" else: ext = None verbose = co.verbose parse_only = co.parse_only # cod_hkl_cif = cod_tools.build_hkl_cif(cod_ids=cod_ids, ext=ext) cod_hkl_cif.show_summary() hkl_files = cod_hkl_cif.hkl cif_files = cod_hkl_cif.cif # n_total = 0 # parsing_errors = {} build_errors = {} ignored_errors = {} skipped = set() # files_to_parse = [] files_to_parse.extend(hkl_files.values()) files_to_parse.extend(cif_files.values()) for i, path in enumerate(files_to_parse): n_total += 1 if (i % command_line.chunk.n != command_line.chunk.i): continue try: cod_id = os.path.basename(path) cif_obj = iotbx.cif.reader(file_path=path) if parse_only: continue skip_file = False for cif_block in cif_obj.model().values(): value = cif_block.get("_cod_error_flag") keys = set(cif_block.keys()) if (value in ["errors", "retracted"]): skip_file = True skipped.add(cod_id) if verbose: print "SKIPPING: _cod_error_flag %s: %s" % (value, cod_id) elif (len(set([ "_space_group_symop_ssg_operation_algebraic", "_space_group_ssg_name"]).intersection(keys)) != 0): skipped.add(cod_id) if verbose: print "SKIPPING: COD entry with super-space group:", cod_id elif (len(set([ "_refln_index_m", "_refln_index_m_1"]).intersection(keys)) != 0): if verbose: print "SKIPPING: COD entry with _refln_index_m:", cod_id skipped.add(cod_id) if skip_file: continue if path.endswith('.cif'): cif_obj.build_crystal_structures() elif path.endswith('.hkl'): cif_obj.build_miller_arrays() else: iotbx.cif.cctbx_data_structures_from_cif(cif_model=cif_obj.model()) except KeyboardInterrupt: print "CAUGHT EXCEPTION: KeyboardInterrupt" return except CifBuilderError, e: e_str = str(e) if not verbose and ( e_str.startswith("No atomic coordinates could be found") or e_str.startswith( "No symmetry instructions could be extracted from the cif block")): ignored_errors.setdefault(cod_id, e_str) continue sys.stdout.flush() print >> sys.stderr, \ "CAUGHT EXCEPTION: %s: %s: %s" % (command_name, cod_id, str(e)) if verbose: traceback.print_exc() print >> sys.stderr build_errors.setdefault(cod_id, e_str) sys.stderr.flush() except CifParserError, e: sys.stdout.flush() e_str = str(e) parsing_errors.setdefault(cod_id, e_str) print >> sys.stderr, \ "PARSING ERROR: %s: %s: %s" % (command_name, cod_id, e_str) if verbose: traceback.print_exc() print >> sys.stderr sys.stderr.flush()
def run(args, command_name = "mmtbx.csv_to_mtz"): if (len(args) == 0): args = ["--help"] try: command_line = (iotbx_option_parser( usage="%s [reflection_csv_file] [options]" % command_name, description='Example: %s 1m5u-sf.csv --use_model=1m5u.pdb'%command_name) .enable_symmetry_comprehensive() .option(None, "--output_file", action="store", default=False, type="string", help="Output mtz file name.") .option(None, "--use_model", action="store", default=False, type="string", help="Use PDB model to make better guess about reflection data type.") ).process(args=args) except Exception as e: if(str(e) != "0"): print(str(e)) sys.exit(0) crystal_symmetry = command_line.symmetry if(command_line.symmetry.unit_cell() is None or command_line.symmetry.space_group_info() is None): if(command_line.options.use_model): crystal_symmetry = crystal_symmetry_from_pdb.extract_from( file_name=command_line.options.use_model) if(crystal_symmetry.unit_cell() is None or crystal_symmetry.space_group_info() is None): raise Sorry( "Crystal symmetry is not defined. Please use the --symmetry option.\n" "Type %s without arguments to see more options."%command_name) if(len(command_line.args) > 1): print("%d arguments are given from the command line:"% \ len(command_line.args), command_line.args) raise Sorry("Please specify one reflection csv file.") file_name = command_line.args[0] if(not os.path.isfile(file_name)): raise Sorry("File is not found: %s"%file_name) data = flex.double() sigmas = flex.double() flags = flex.int() column_ids, columns = parse_csv_file(file_name=file_name) data_label_root = column_ids[3] ms = miller.set(crystal_symmetry, flex.miller_index(columns[0])) for d in columns[1]: data.append(float(d)) for sig in columns[2]: sigmas.append(float(sig)) for flag in columns[3]: flags.append(int(flag)) assert len(data) == len(sigmas) assert len(flags) == len(data) ma = miller.array(ms, data, sigmas) if data_label_root.startswith('F'): ma.set_observation_type_xray_amplitude() elif data_label_root.startswith('I'): ma.set_observation_type_xray_intensity() else: ma.set_observation_type_xray_amplitude() flags_ma = miller.set( crystal_symmetry = crystal_symmetry, indices = ma.indices()).array(data = flags) mtz_dataset = ma.as_mtz_dataset( column_root_label = data_label_root) mtz_dataset.add_miller_array( miller_array = flags_ma, column_root_label = "R-free-flags") mtz_object = mtz_dataset.mtz_object() mtz_object.write(file_name = command_line.options.output_file)
def run(args, command_name="phenix.cif_as_mtz", out=sys.stdout, return_as_miller_arrays=False): if (len(args) == 0): args = ["--help"] try: command_line = (iotbx_option_parser( usage="%s [reflection_cif_file] [options]" % command_name, description='Example: %s r1o9ksf.ent --symmetry=pdb1o9k.ent' % command_name ).enable_symmetry_comprehensive().option( None, "--output_file_name", action="store", default=False, type="string", help="Output mtz file name." ).option( None, "--wavelength_id", action="store", default=None, type="int", help="Extract data set with given wavelength_id." ).option( None, "--crystal_id", action="store", default=None, type="int", help="Extract data set with given crystal_id." ).option( None, "--output_r_free_label", action="store", default="R-free-flags", type="string", help= "MTZ column label to use for R-free flags (default: R-free-flags)" ).option( None, "--merge", action="store_true", help="Merge non-unique data where present." ).option( None, "--incompatible_flags_to_work_set", action="store_true", help= "When merging place reflections with incompatible flags into the " "working set." ).option( None, "--remove_systematic_absences", action="store_true", help="Remove systematic absent reflections." ).option( None, "--map_to_asu", action="store_true", help="Map to asymmetric unit." ).option( "--show_details_if_error", action="store_true", help="Show data details for some errors." ).option( "--show_log", action="store_true", help="Show some output." ).option( "--ignore_bad_sigmas", action="store_true", help= "Set sigmas to None instead of raising an error when bad sigmas " "are present." ).option( "--extend_flags", action="store_true", help="Extend R-free flags to cover all reflections if necessary.") ).process(args=args) except Exception as e: if (str(e) != "0"): print(str(e)) sys.exit(0) crystal_symmetry = command_line.symmetry if (len(command_line.args) > 1): print("%d arguments are given from the command line:"% \ len(command_line.args), command_line.args, file=out) raise Sorry("Please specify one reflection cif file.") file_name = command_line.args[0] if (not os.path.isfile(file_name)): raise Sorry("File is not found: %s" % file_name) output_r_free_label = command_line.options.output_r_free_label if ((not output_r_free_label[0] in string.ascii_uppercase) or (re.search(r"[^a-zA-Z0-9_\-]", output_r_free_label))): raise Sorry(( "%s is not a suitable column label. MTZ format requires " + "an uppercase letter as the first character, and only alphanumeric " + "characters or hyphens in the rest of the string.") % output_r_free_label) result = process_files( file_name=file_name, crystal_symmetry=crystal_symmetry, output_file_name=command_line.options.output_file_name, wavelength_id=command_line.options.wavelength_id, crystal_id=command_line.options.crystal_id, show_details_if_error=command_line.options.show_details_if_error, output_r_free_label=command_line.options.output_r_free_label, merge_non_unique_under_symmetry=command_line.options.merge, map_to_asu=command_line.options.map_to_asu, remove_systematic_absences=command_line.options. remove_systematic_absences, incompatible_flags_to_work_set=command_line.options. incompatible_flags_to_work_set, return_as_miller_arrays=return_as_miller_arrays, ignore_bad_sigmas=command_line.options.ignore_bad_sigmas, extend_flags=command_line.options.extend_flags, log=out) if return_as_miller_arrays: return result
def run(args, command_name): from iotbx.option_parser import option_parser as iotbx_option_parser import libtbx.utils show_times = libtbx.utils.show_times(time_start="now") command_line = (iotbx_option_parser( usage=command_name + " [options] [cod_id...]").enable_chunk( easy_all=True).enable_multiprocessing().option( None, "--parse_only", action="store_true").option( None, "--cif_only", action="store_true").option( None, "--hkl_only", action="store_true").option( "-v", "--verbose", action="store_true")).process(args=args) if (command_line.run_multiprocessing_chunks_if_applicable( command_call=[command_name, __file__])): show_times() return co = command_line.options cod_ids = command_line.args assert [co.cif_only, co.hkl_only].count(True) <= 1 if co.cif_only: ext = "cif" elif co.hkl_only: ext = "hkl" else: ext = None verbose = co.verbose parse_only = co.parse_only # cod_hkl_cif = cod_tools.build_hkl_cif(cod_ids=cod_ids, ext=ext) cod_hkl_cif.show_summary() hkl_files = cod_hkl_cif.hkl cif_files = cod_hkl_cif.cif # n_total = 0 # parsing_errors = {} build_errors = {} ignored_errors = {} skipped = set() # files_to_parse = [] files_to_parse.extend(hkl_files.values()) files_to_parse.extend(cif_files.values()) for i, path in enumerate(files_to_parse): n_total += 1 if (i % command_line.chunk.n != command_line.chunk.i): continue try: cod_id = os.path.basename(path) cif_obj = iotbx.cif.reader(file_path=path) if parse_only: continue skip_file = False for cif_block in cif_obj.model().values(): value = cif_block.get("_cod_error_flag") keys = set(cif_block.keys()) if (value in ["errors", "retracted"]): skip_file = True skipped.add(cod_id) if verbose: print("SKIPPING: _cod_error_flag %s: %s" % (value, cod_id)) elif (len( set([ "_space_group_symop_ssg_operation_algebraic", "_space_group_ssg_name" ]).intersection(keys)) != 0): skipped.add(cod_id) if verbose: print("SKIPPING: COD entry with super-space group:", cod_id) elif (len( set(["_refln_index_m", "_refln_index_m_1" ]).intersection(keys)) != 0): if verbose: print("SKIPPING: COD entry with _refln_index_m:", cod_id) skipped.add(cod_id) if skip_file: continue if path.endswith('.cif'): cif_obj.build_crystal_structures() elif path.endswith('.hkl'): cif_obj.build_miller_arrays() else: iotbx.cif.cctbx_data_structures_from_cif( cif_model=cif_obj.model()) except KeyboardInterrupt: print("CAUGHT EXCEPTION: KeyboardInterrupt") return except CifBuilderError as e: e_str = str(e) if not verbose and (e_str.startswith( "No atomic coordinates could be found" ) or e_str.startswith( "No symmetry instructions could be extracted from the cif block" )): ignored_errors.setdefault(cod_id, e_str) continue sys.stdout.flush() print("CAUGHT EXCEPTION: %s: %s: %s" % (command_name, cod_id, str(e)), file=sys.stderr) if verbose: traceback.print_exc() print(file=sys.stderr) build_errors.setdefault(cod_id, e_str) sys.stderr.flush() except CifParserError as e: sys.stdout.flush() e_str = str(e) parsing_errors.setdefault(cod_id, e_str) print("PARSING ERROR: %s: %s: %s" % (command_name, cod_id, e_str), file=sys.stderr) if verbose: traceback.print_exc() print(file=sys.stderr) sys.stderr.flush() except Exception as e: sys.stdout.flush() e_str = str(e) build_errors.setdefault(cod_id, e_str) print("CAUGHT EXCEPTION: %s: %s: %s" % (command_name, cod_id, e_str), file=sys.stderr) if verbose: traceback.print_exc() print(file=sys.stderr) sys.stderr.flush() print() print("Number successfully parsed: %i/%i" \ % (n_total-len(parsing_errors),n_total)) if not parse_only: print("Number skipped:", len(skipped)) print("Number of exceptions caught:", len(build_errors)) print("Number of exceptions ignored:", len(ignored_errors)) print() # show_times() result = group_args(n_hkl=len(hkl_files), n_cif=len(cif_files), n_hkl_cif_pairs=len(cod_hkl_cif.hkl_cif_pairs), parsing_errors=parsing_errors, build_errors=build_errors, ignored_errors=ignored_errors, skipped=skipped) easy_pickle.dump("result_%03i.pickle" % command_line.chunk.i, result) print()
def run(self, args, command_name, out=sys.stdout): command_line = (iotbx_option_parser( usage="%s [options]" % command_name, description='Example: %s data.mtz data.mtz ref_model.pdb'%command_name) .option(None, "--show_defaults", action="store_true", help="Show list of parameters.") ).process(args=args) cif_file = None processed_args = utils.process_command_line_args( args = args, log = sys.stdout, master_params = master_phil) params = processed_args.params if(params is None): params = master_phil self.params = params.extract().ensemble_probability pdb_file_names = processed_args.pdb_file_names if len(pdb_file_names) != 1 : raise Sorry("Only one PDB structure may be used") pdb_file = file_reader.any_file(pdb_file_names[0]) self.log = multi_out() self.log.register(label="stdout", file_object=sys.stdout) self.log.register( label="log_buffer", file_object=StringIO(), atexit_send_to=None) sys.stderr = self.log log_file = open(pdb_file_names[0].split('/')[-1].replace('.pdb','') + '_pensemble.log', "w") self.log.replace_stringio( old_label="log_buffer", new_label="log", new_file_object=log_file) utils.print_header(command_name, out = self.log) params.show(out = self.log) # f_obs = None r_free_flags = None reflection_files = processed_args.reflection_files if self.params.fobs_vs_fcalc_post_nll: if len(reflection_files) == 0: raise Sorry("Fobs from input MTZ required for fobs_vs_fcalc_post_nll") if len(reflection_files) > 0: crystal_symmetry = processed_args.crystal_symmetry print >> self.log, 'Reflection file : ', processed_args.reflection_file_names[0] utils.print_header("Model and data statistics", out = self.log) rfs = reflection_file_server( crystal_symmetry = crystal_symmetry, reflection_files = processed_args.reflection_files, log = self.log) parameters = utils.data_and_flags_master_params().extract() determine_data_and_flags_result = utils.determine_data_and_flags( reflection_file_server = rfs, parameters = parameters, data_parameter_scope = "refinement.input.xray_data", flags_parameter_scope = "refinement.input.xray_data.r_free_flags", data_description = "X-ray data", keep_going = True, log = self.log) f_obs = determine_data_and_flags_result.f_obs number_of_reflections = f_obs.indices().size() r_free_flags = determine_data_and_flags_result.r_free_flags test_flag_value = determine_data_and_flags_result.test_flag_value if(r_free_flags is None): r_free_flags=f_obs.array(data=flex.bool(f_obs.data().size(), False)) # process PDB pdb_file.assert_file_type("pdb") # pdb_in = hierarchy.input(file_name=pdb_file.file_name) ens_pdb_hierarchy = pdb_in.construct_hierarchy() ens_pdb_hierarchy.atoms().reset_i_seq() ens_pdb_xrs_s = pdb_in.input.xray_structures_simple() number_structures = len(ens_pdb_xrs_s) print >> self.log, 'Number of structure in ensemble : ', number_structures # Calculate sigmas from input map only if self.params.assign_sigma_from_map and self.params.ensemble_sigma_map_input is not None: # process MTZ input_file = file_reader.any_file(self.params.ensemble_sigma_map_input) if input_file.file_type == "hkl" : if input_file.file_object.file_type() != "ccp4_mtz" : raise Sorry("Only MTZ format accepted for map input") else: mtz_file = input_file else: raise Sorry("Only MTZ format accepted for map input") miller_arrays = mtz_file.file_server.miller_arrays map_coeffs_1 = miller_arrays[0] # xrs_list = [] for n, ens_pdb_xrs in enumerate(ens_pdb_xrs_s): # get sigma levels from ensemble fc for each structure xrs = get_map_sigma(ens_pdb_hierarchy = ens_pdb_hierarchy, ens_pdb_xrs = ens_pdb_xrs, map_coeffs_1 = map_coeffs_1, residue_detail = self.params.residue_detail, ignore_hd = self.params.ignore_hd, log = self.log) xrs_list.append(xrs) # write ensemble pdb file, occupancies as sigma level filename = pdb_file_names[0].split('/')[-1].replace('.pdb','') + '_vs_' + self.params.ensemble_sigma_map_input.replace('.mtz','') + '_pensemble.pdb' write_ensemble_pdb(filename = filename, xrs_list = xrs_list, ens_pdb_hierarchy = ens_pdb_hierarchy ) # Do full analysis vs Fobs else: model_map_coeffs = [] fmodel = None # Get <fcalc> for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s): ens_pdb_xrs.set_occupancies(1.0) if model == 0: # If mtz not supplied get fobs from xray structure... # Use input Fobs for scoring against nll if self.params.fobs_vs_fcalc_post_nll: dummy_fobs = f_obs else: if f_obs == None: if self.params.fcalc_high_resolution == None: raise Sorry("Please supply high resolution limit or input mtz file.") dummy_dmin = self.params.fcalc_high_resolution dummy_dmax = self.params.fcalc_low_resolution else: print >> self.log, 'Supplied mtz used to determine high and low resolution cuttoffs' dummy_dmax, dummy_dmin = f_obs.d_max_min() # dummy_fobs = abs(ens_pdb_xrs.structure_factors(d_min = dummy_dmin).f_calc()) dummy_fobs.set_observation_type_xray_amplitude() # If mtz supplied, free flags are over written to prevent array size error r_free_flags = dummy_fobs.array(data=flex.bool(dummy_fobs.data().size(),False)) # fmodel = utils.fmodel_simple( scattering_table = "wk1995", xray_structures = [ens_pdb_xrs], f_obs = dummy_fobs, target_name = 'ls', bulk_solvent_and_scaling = False, r_free_flags = r_free_flags ) f_calc_ave = fmodel.f_calc().array(data = fmodel.f_calc().data()*0).deep_copy() # XXX Important to ensure scale is identical for each model and <model> fmodel.set_scale_switch = 1.0 f_calc_ave_total = fmodel.f_calc().data().deep_copy() else: fmodel.update_xray_structure(xray_structure = ens_pdb_xrs, update_f_calc = True, update_f_mask = False) f_calc_ave_total += fmodel.f_calc().data().deep_copy() print >> self.log, 'Model :', model+1 print >> self.log, "\nStructure vs real Fobs (no bulk solvent or scaling)" print >> self.log, 'Rwork : %5.4f '%fmodel.r_work() print >> self.log, 'Rfree : %5.4f '%fmodel.r_free() print >> self.log, 'K1 : %5.4f '%fmodel.scale_k1() fcalc_edm = fmodel.electron_density_map() fcalc_map_coeffs = fcalc_edm.map_coefficients(map_type = 'Fc') fcalc_mtz_dataset = fcalc_map_coeffs.as_mtz_dataset(column_root_label ='Fc') if self.params.output_model_and_model_ave_mtz: fcalc_mtz_dataset.mtz_object().write(file_name = str(model+1)+"_Fc.mtz") model_map_coeffs.append(fcalc_map_coeffs.deep_copy()) fmodel.update(f_calc = f_calc_ave.array(f_calc_ave_total / number_structures)) print >> self.log, "\nEnsemble vs real Fobs (no bulk solvent or scaling)" print >> self.log, 'Rwork : %5.4f '%fmodel.r_work() print >> self.log, 'Rfree : %5.4f '%fmodel.r_free() print >> self.log, 'K1 : %5.4f '%fmodel.scale_k1() # Get <Fcalc> map fcalc_ave_edm = fmodel.electron_density_map() fcalc_ave_map_coeffs = fcalc_ave_edm.map_coefficients(map_type = 'Fc').deep_copy() fcalc_ave_mtz_dataset = fcalc_ave_map_coeffs.as_mtz_dataset(column_root_label ='Fc') if self.params.output_model_and_model_ave_mtz: fcalc_ave_mtz_dataset.mtz_object().write(file_name = "aveFc.mtz") fcalc_ave_map_coeffs = fcalc_ave_map_coeffs.fft_map() fcalc_ave_map_coeffs.apply_volume_scaling() fcalc_ave_map_data = fcalc_ave_map_coeffs.real_map_unpadded() fcalc_ave_map_stats = maptbx.statistics(fcalc_ave_map_data) print >> self.log, "<Fcalc> Map Stats :" fcalc_ave_map_stats.show_summary(f = self.log) offset = fcalc_ave_map_stats.min() model_neg_ll = [] number_previous_scatters = 0 # Run through structure list again and get probability xrs_list = [] for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s): if self.params.verbose: print >> self.log, '\n\nModel : ', model+1 # Get model atom sigmas vs Fcalc fcalc_map = model_map_coeffs[model].fft_map() fcalc_map.apply_volume_scaling() fcalc_map_data = fcalc_map.real_map_unpadded() fcalc_map_stats = maptbx.statistics(fcalc_map_data) if self.params.verbose: print >> self.log, "Fcalc map stats :" fcalc_map_stats.show_summary(f = self.log) xrs = get_map_sigma(ens_pdb_hierarchy = ens_pdb_hierarchy, ens_pdb_xrs = ens_pdb_xrs, fft_map_1 = fcalc_map, model_i = model, residue_detail = self.params.residue_detail, ignore_hd = self.params.ignore_hd, number_previous_scatters = number_previous_scatters, log = self.log) fcalc_sigmas = xrs.scatterers().extract_occupancies() del fcalc_map # Get model atom sigmas vs <Fcalc> xrs = get_map_sigma(ens_pdb_hierarchy = ens_pdb_hierarchy, ens_pdb_xrs = ens_pdb_xrs, fft_map_1 = fcalc_ave_map_coeffs, model_i = model, residue_detail = self.params.residue_detail, ignore_hd = self.params.ignore_hd, number_previous_scatters = number_previous_scatters, log = self.log) ### For testing other residue averaging options #print xrs.residue_selections fcalc_ave_sigmas = xrs.scatterers().extract_occupancies() # Probability of model given <model> prob = fcalc_ave_sigmas / fcalc_sigmas # XXX debug option if False: for n,p in enumerate(prob): print >> self.log, ' {0:5d} {1:5.3f}'.format(n,p) # Set probabilty between 0 and 1 # XXX Make Histogram / more stats prob_lss_zero = flex.bool(prob <= 0) prob_grt_one = flex.bool(prob > 1) prob.set_selected(prob_lss_zero, 0.001) prob.set_selected(prob_grt_one, 1.0) xrs.set_occupancies(prob) xrs_list.append(xrs) sum_neg_ll = sum(-flex.log(prob)) model_neg_ll.append((sum_neg_ll, model)) if self.params.verbose: print >> self.log, 'Model probability stats :' print >> self.log, prob.min_max_mean().show() print >> self.log, ' Count < 0.0 : ', prob_lss_zero.count(True) print >> self.log, ' Count > 1.0 : ', prob_grt_one.count(True) # For averaging by residue number_previous_scatters += ens_pdb_xrs.sites_cart().size() # write ensemble pdb file, occupancies as sigma level write_ensemble_pdb(filename = pdb_file_names[0].split('/')[-1].replace('.pdb','') + '_pensemble.pdb', xrs_list = xrs_list, ens_pdb_hierarchy = ens_pdb_hierarchy ) # XXX Test ordering models by nll # XXX Test removing nth percentile atoms if self.params.sort_ensemble_by_nll_score or self.params.fobs_vs_fcalc_post_nll: for percentile in [1.0,0.975,0.95,0.9,0.8,0.6,0.2]: model_neg_ll = sorted(model_neg_ll) f_calc_ave_total_reordered = None print_list = [] for i_neg_ll in model_neg_ll: xrs = xrs_list[i_neg_ll[1]] nll_occ = xrs.scatterers().extract_occupancies() # Set q=0 nth percentile atoms sorted_nll_occ = sorted(nll_occ, reverse=True) number_atoms = len(sorted_nll_occ) percentile_prob_cutoff = sorted_nll_occ[int(number_atoms * percentile)-1] cutoff_selections = flex.bool(nll_occ < percentile_prob_cutoff) cutoff_nll_occ = flex.double(nll_occ.size(), 1.0).set_selected(cutoff_selections, 0.0) #XXX Debug if False: print '\nDebug' for x in xrange(len(cutoff_selections)): print cutoff_selections[x], nll_occ[x], cutoff_nll_occ[x] print percentile print percentile_prob_cutoff print cutoff_selections.count(True) print cutoff_selections.size() print cutoff_nll_occ.count(0.0) print 'Count q = 1 : ', cutoff_nll_occ.count(1.0) print 'Count scatterers size : ', cutoff_nll_occ.size() xrs.set_occupancies(cutoff_nll_occ) fmodel.update_xray_structure(xray_structure = xrs, update_f_calc = True, update_f_mask = True) if f_calc_ave_total_reordered == None: f_calc_ave_total_reordered = fmodel.f_calc().data().deep_copy() f_mask_ave_total_reordered = fmodel.f_masks()[0].data().deep_copy() cntr = 1 else: f_calc_ave_total_reordered += fmodel.f_calc().data().deep_copy() f_mask_ave_total_reordered += fmodel.f_masks()[0].data().deep_copy() cntr+=1 fmodel.update(f_calc = f_calc_ave.array(f_calc_ave_total_reordered / cntr).deep_copy(), f_mask = f_calc_ave.array(f_mask_ave_total_reordered / cntr).deep_copy() ) # Update solvent and scale # XXX Will need to apply_back_trace on latest version fmodel.set_scale_switch = 0 fmodel.update_all_scales() # Reset occ for outout xrs.set_occupancies(nll_occ) # k1 updated vs Fobs if self.params.fobs_vs_fcalc_post_nll: print_list.append([cntr, i_neg_ll[0], i_neg_ll[1], fmodel.r_work(), fmodel.r_free()]) # Order models by nll and print summary print >> self.log, '\nModels ranked by nll <Fcalc> R-factors recalculated' print >> self.log, 'Percentile cutoff : {0:5.3f}'.format(percentile) xrs_list_sorted_nll = [] print >> self.log, ' | NLL <Rw> <Rf> Ens Model' for info in print_list: print >> self.log, ' {0:4d} | {1:8.1f} {2:8.4f} {3:8.4f} {4:12d}'.format( info[0], info[1], info[3], info[4], info[2]+1, ) xrs_list_sorted_nll.append(xrs_list[info[2]]) # Output nll ordered ensemble write_ensemble_pdb(filename = 'nll_ordered_' + pdb_file_names[0].split('/')[-1].replace('.pdb','') + '_pensemble.pdb', xrs_list = xrs_list_sorted_nll, ens_pdb_hierarchy = ens_pdb_hierarchy )
def run(args): from iotbx.option_parser import option_parser as iotbx_option_parser import libtbx.utils show_times = libtbx.utils.show_times(time_start="now") command_call = ["iotbx.python", __file__] command_line = (iotbx_option_parser( usage=" ".join(command_call) + " [options] directory|file...") .enable_chunk(easy_all=True) .enable_multiprocessing() ).process(args=args, min_nargs=1) if (command_line.run_multiprocessing_chunks_if_applicable( command_call=command_call)): show_times() return co = command_line.options # print "TIME BEGIN cod_refine:", date_and_time() print # master_phil = get_master_phil() argument_interpreter = master_phil.command_line_argument_interpreter() phil_objects = [] remaining_args = [] for arg in command_line.args: if (arg.find("=") >= 0): phil_objects.append(argument_interpreter.process(arg=arg)) else: remaining_args.append(arg) work_phil = master_phil.fetch(sources=phil_objects) work_phil.show() print params = work_phil.extract() # qi_dict = {} all_pickles = [] for arg in remaining_args: if (op.isdir(arg)): for node in sorted(os.listdir(arg)): if (node.endswith(".pickle")): all_pickles.append(op.join(arg, node)) elif (node.startswith("qi_") and len(node) == 10): qi = open(op.join(arg, node)).read().splitlines() if (len(qi) == 1): cod_id = node[3:] quick_info = eval(qi[0]) assert cod_id not in qi_dict qi_dict[cod_id] = quick_info elif (op.isfile(arg)): all_pickles.append(arg) else: raise RuntimeError("Not a file or directory: %s" % arg) print "Number of pickle files:", len(all_pickles) print "Number of quick_infos:", len(qi_dict) sort_choice = params.sorting_of_pickle_files if (len(qi_dict) != 0 and sort_choice is not None): print "Sorting pickle files by n_atoms * n_refl:", sort_choice assert sort_choice in ["down", "up"] def sort_pickle_files(): if (sort_choice == "down"): i_sign = -1 else: i_sign = 1 buffer = [] for i,path in enumerate(all_pickles): cod_id = op.basename(path).split(".",1)[0] qi = qi_dict.get(cod_id) if (qi is None): nn = 2**31 else: nn = qi[0] * qi[1] * qi[2] buffer.append((nn, i_sign*i, path)) buffer.sort() if (i_sign < 0): buffer.reverse() result = [] for elem in buffer: result.append(elem[-1]) return result all_pickles = sort_pickle_files() print # rss = params.random_subset.size if (rss is not None and rss > 0): seed = params.random_subset.seed print "Selecting subset of %d pickle files using random seed %d" % ( rss, seed) mt = flex.mersenne_twister(seed=seed) perm = mt.random_permutation(size=len(all_pickles))[:rss] flags = flex.bool(len(all_pickles), False).set_selected(perm, True) all_pickles = flex.select(all_pickles, permutation=flags.iselection()) print # from libtbx.path import makedirs_race if (params.wdir_root is not None): makedirs_race(path=params.wdir_root) if (params.pickle_refined_dir is not None): makedirs_race(path=params.pickle_refined_dir) # n_caught = 0 for i_pickle,pickle_file_name in enumerate(all_pickles): if (i_pickle % command_line.chunk.n != command_line.chunk.i): continue tm = user_plus_sys_time() try: process(params, pickle_file_name) except KeyboardInterrupt: print >> sys.stderr, "CAUGHT EXCEPTION: KeyboardInterrupt" traceback.print_exc() print >> sys.stderr sys.stderr.flush() return except Exception: sys.stdout.flush() print >> sys.stderr, "CAUGHT EXCEPTION: %s" % pickle_file_name traceback.print_exc() print >> sys.stderr sys.stderr.flush() n_caught += 1 else: print "done_with: %s (%.2f seconds)" % (pickle_file_name, tm.elapsed()) print sys.stdout.flush() print print "Number of exceptions caught:", n_caught # show_times() print print "TIME END cod_refine:", date_and_time()
def run(args, command_name): from iotbx.option_parser import option_parser as iotbx_option_parser from libtbx import easy_pickle import libtbx.utils show_times = libtbx.utils.show_times(time_start="now") command_line = ( iotbx_option_parser(usage=command_name + " [options] [cod_id...]") .enable_chunk(easy_all=True) .enable_multiprocessing() .option(None, "--max_atoms", type="int", default=99, metavar="INT") .option(None, "--min_distance", type="float", default=0.5, metavar="FLOAT") .option(None, "--i_obs_i_calc_plot", action="store_true", default=False) .option(None, "--f_obs_f_calc_plot", action="store_true", default=False) .option(None, "--max_fan_outlier_fraction", type="float", default=0.05, metavar="FLOAT") .option(None, "--fan_offset_low", type="float", default=0.05, metavar="FLOAT") .option(None, "--fan_offset_high", type="float", default=0.10, metavar="FLOAT") .option(None, "--min_f_obs_f_calc_correlation", type="float", default=0.99, metavar="FLOAT") .option(None, "--at_least_one_special_position", action="store_true", default=False) .option(None, "--pickle_dir", type="str", default="cod_ma_xs", metavar="PATH") ).process(args=args) if command_line.run_multiprocessing_chunks_if_applicable(command_call=[command_name]): show_times() return co = command_line.options # from iotbx.cif import cod_tools hkl_cif = cod_tools.build_hkl_cif(cod_ids=command_line.args) hkl_cif.show_summary() # pickle_dir = co.pickle_dir if co.at_least_one_special_position: pickle_dir += "_special" if not op.isdir(pickle_dir): from libtbx.path import makedirs_race makedirs_race(path=pickle_dir) n_caught = 0 for i_pair, pair in enumerate(hkl_cif.hkl_cif_pairs.values()): cod_id = op.basename(pair[0])[:-4] if i_pair % command_line.chunk.n != command_line.chunk.i: continue try: cd = cod_data(cod_id=cod_id, hkl_cif_pair=pair) except KeyboardInterrupt: print "CAUGHT EXCEPTION: KeyboardInterrupt" return except Exception, e: sys.stdout.flush() print >> sys.stderr, "CAUGHT EXCEPTION: %s: %s: %s" % (command_name, cod_id, str(e)) traceback.print_exc() print >> sys.stderr sys.stderr.flush() n_caught += 1 else: if cd.is_useful(co): easy_pickle.dump( file_name="%s/%s.pickle" % (pickle_dir, cod_id), obj=(cd.c_obs, cd.xray_structure, cd.edge_list) ) print >> open("%s/qi_%s" % (pickle_dir, cod_id), "w"), cd.quick_info() else: print "filtering out:", cod_id print "done_with:", cod_id print
def run(args): from iotbx.option_parser import option_parser as iotbx_option_parser import libtbx.utils show_times = libtbx.utils.show_times(time_start="now") command_call = ["iotbx.python", __file__] command_line = (iotbx_option_parser( usage=" ".join(command_call) + " [options] directory|file...").enable_chunk( easy_all=True).enable_multiprocessing()).process(args=args, min_nargs=1) if (command_line.run_multiprocessing_chunks_if_applicable( command_call=command_call)): show_times() return co = command_line.options # print "TIME BEGIN cod_refine:", date_and_time() print # master_phil = get_master_phil() argument_interpreter = master_phil.command_line_argument_interpreter() phil_objects = [] remaining_args = [] for arg in command_line.args: if (arg.find("=") >= 0): phil_objects.append(argument_interpreter.process(arg=arg)) else: remaining_args.append(arg) work_phil = master_phil.fetch(sources=phil_objects) work_phil.show() print params = work_phil.extract() # qi_dict = {} all_pickles = [] for arg in remaining_args: if (op.isdir(arg)): for node in sorted(os.listdir(arg)): if (node.endswith(".pickle")): all_pickles.append(op.join(arg, node)) elif (node.startswith("qi_") and len(node) == 10): qi = open(op.join(arg, node)).read().splitlines() if (len(qi) == 1): cod_id = node[3:] quick_info = eval(qi[0]) assert cod_id not in qi_dict qi_dict[cod_id] = quick_info elif (op.isfile(arg)): all_pickles.append(arg) else: raise RuntimeError("Not a file or directory: %s" % arg) print "Number of pickle files:", len(all_pickles) print "Number of quick_infos:", len(qi_dict) sort_choice = params.sorting_of_pickle_files if (len(qi_dict) != 0 and sort_choice is not None): print "Sorting pickle files by n_atoms * n_refl:", sort_choice assert sort_choice in ["down", "up"] def sort_pickle_files(): if (sort_choice == "down"): i_sign = -1 else: i_sign = 1 buffer = [] for i, path in enumerate(all_pickles): cod_id = op.basename(path).split(".", 1)[0] qi = qi_dict.get(cod_id) if (qi is None): nn = 2**31 else: nn = qi[0] * qi[1] * qi[2] buffer.append((nn, i_sign * i, path)) buffer.sort() if (i_sign < 0): buffer.reverse() result = [] for elem in buffer: result.append(elem[-1]) return result all_pickles = sort_pickle_files() print # rss = params.random_subset.size if (rss is not None and rss > 0): seed = params.random_subset.seed print "Selecting subset of %d pickle files using random seed %d" % ( rss, seed) mt = flex.mersenne_twister(seed=seed) perm = mt.random_permutation(size=len(all_pickles))[:rss] flags = flex.bool(len(all_pickles), False).set_selected(perm, True) all_pickles = flex.select(all_pickles, permutation=flags.iselection()) print # from libtbx.path import makedirs_race if (params.wdir_root is not None): makedirs_race(path=params.wdir_root) if (params.pickle_refined_dir is not None): makedirs_race(path=params.pickle_refined_dir) # n_caught = 0 for i_pickle, pickle_file_name in enumerate(all_pickles): if (i_pickle % command_line.chunk.n != command_line.chunk.i): continue tm = user_plus_sys_time() try: process(params, pickle_file_name) except KeyboardInterrupt: print >> sys.stderr, "CAUGHT EXCEPTION: KeyboardInterrupt" traceback.print_exc() print >> sys.stderr sys.stderr.flush() return except Exception: sys.stdout.flush() print >> sys.stderr, "CAUGHT EXCEPTION: %s" % pickle_file_name traceback.print_exc() print >> sys.stderr sys.stderr.flush() n_caught += 1 else: print "done_with: %s (%.2f seconds)" % (pickle_file_name, tm.elapsed()) print sys.stdout.flush() print print "Number of exceptions caught:", n_caught # show_times() print print "TIME END cod_refine:", date_and_time()
def run(args, command_call, command_line_add_options=None): pdb_mirror_pdb = os.environ.get("PDB_MIRROR_PDB") assert pdb_mirror_pdb is None or op.isdir(pdb_mirror_pdb) from iotbx.option_parser import option_parser as iotbx_option_parser import libtbx.utils show_times = libtbx.utils.show_times(time_start="now") command_line = (iotbx_option_parser( usage=" ".join(command_call) + " [options] file|directory...").enable_chunk(easy_all=True). enable_multiprocessing().call_with_self_as_first_argument( callable=command_line_add_options)).process(args=args) if (command_line.run_multiprocessing_chunks_if_applicable( command_call=command_call)): show_times() return command_line, null_generator() # from libtbx.str_utils import show_string ca = command_line.args if (len(ca) == 0 and pdb_mirror_pdb is not None): ca = [pdb_mirror_pdb] file_infos = [] for arg in ca: if (op.isfile(arg)): bn = op.basename(arg) if (bn.startswith("pdb_codes_")): assert pdb_mirror_pdb is not None for i_line, line in enumerate(open(arg).read().splitlines()): flds = line.split(None, 1) if (len(flds) == 0): raise RuntimeError( "Error interpreting pdb_codes file:\n" " %s" " line number: %d" " line: %s" % (show_string(arg), i_line + 1, show_string(line))) pdb_code = flds[0] atom_selection_string = None if (len(flds) > 1): atom_selection_string = flds[1] file_name = op.join(pdb_mirror_pdb, pdb_code[1:3], "pdb%s.ent.gz" % pdb_code) file_infos.append( file_info(name=file_name, atom_selection_string=atom_selection_string)) elif (bn.startswith("file_names_")): for file_name in open(arg).read().splitlines(): file_infos.append(file_info(name=file_name)) else: file_infos.append(file_info(name=arg)) elif (op.isdir(arg)): file_name_index = op.join(arg, "INDEX") if (op.isfile(file_name_index)): for relative_path in open(file_name_index).read().splitlines(): file_name = op.join(arg, relative_path) file_infos.append(file_info(name=file_name)) else: prev_len_file_infos = len(file_infos) for relative_path in os.listdir(arg): if (relative_path.endswith((".ent", ".ent.gz", "ent.Z", ".pdb", ".pdb.gz", "pdb.Z"))): file_name = op.join(arg, relative_path) file_infos.append(file_info(name=file_name)) if (len(file_infos) == prev_len_file_infos): raise RuntimeError( "No INDEX file and no pdb files found in directory: %s" % show_string(arg)) else: raise RuntimeError("Not a file or directory: %s" % show_string(arg)) # return command_line, pdb_inp_generator(file_infos=file_infos, chunk_n=command_line.chunk.n, chunk_i=command_line.chunk.i)
def run(args, command_name="phenix.tls"): if (len(args) == 0): args = ["--help"] usage_fmt = "%s pdb_file [parameters: file or command line string]" des_fmt = "Example: %s model.pdb fit_tls_to.selection='%s' fit_tls_to.selection='%s'" command_line = (iotbx_option_parser( usage=usage_fmt % command_name, description=banner).option( "--show_defaults", action="store_true", help="Do not output to the screen (except errors).").option( "--silent", action="store_true", help="Suppress output to the screen.")).process(args=args) # log = sys.stdout if (not command_line.options.silent): utils.print_header("TLS tools", out=log) if (command_line.options.show_defaults): master_params.show(out=log) print(file=log) return if (not command_line.options.silent): print(banner, file=log) # processed_args = utils.process_command_line_args( args=command_line.args, master_params=master_params, log=log) reflection_files = processed_args.reflection_files if (processed_args.crystal_symmetry is None): raise Sorry("No crystal symmetry found.") if (len(processed_args.pdb_file_names) == 0): raise Sorry("No PDB file found.") params = processed_args.params if (not command_line.options.silent): utils.print_header("Input parameters", out=log) params.show(out=log) params = params.extract() # if (processed_args.crystal_symmetry.unit_cell() is None or processed_args.crystal_symmetry.space_group() is None): raise Sorry("No CRYST1 record found.") pdb_combined = iotbx.pdb.combine_unique_pdb_files( file_names=processed_args.pdb_file_names) pdb_combined.report_non_unique(out=log) if (len(pdb_combined.unique_file_names) == 0): raise Sorry("No coordinate file given.") raw_records = pdb_combined.raw_records try: pdb_inp = iotbx.pdb.input(source_info=None, lines=flex.std_string(raw_records)) except ValueError as e: raise Sorry("Model format (PDB or mmCIF) error:\n%s" % str(e)) model = mmtbx.model.manager( model_input=pdb_inp, restraint_objects=processed_args.cif_objects, crystal_symmetry=processed_args.crystal_symmetry, log=log) if (not command_line.options.silent): utils.print_header("TLS groups from PDB file header", out=log) pdb_inp_tls = mmtbx.tls.tools.tls_from_pdb_inp( remark_3_records=model._model_input.extract_remark_iii_records(3), pdb_hierarchy=model.get_hierarchy()) # tls_groups = [] if (pdb_inp_tls.tls_present): if (pdb_inp_tls.error_string is not None): raise Sorry(pdb_inp_tls.error_string) pdb_tls = mmtbx.tls.tools.extract_tls_from_pdb(pdb_inp_tls=pdb_inp_tls, model=model) tls_groups = pdb_tls.pdb_inp_tls.tls_params # tls_selections_strings = [] # if (len(tls_groups) == 0 and not command_line.options.silent): print("No TLS groups found in PDB file header.", file=log) else: for i_seq, tls_group in enumerate(tls_groups): tls_selections_strings.append(tls_group.selection_string) if (not command_line.options.silent): print("TLS group %d: %s" % (i_seq + 1, tls_group.selection_string), file=log) mmtbx.tls.tools.show_tls_one_group(tlso=tls_group, out=log) print(file=log) # if (len(tls_selections_strings) > 0 and len(params.selection) > 0): raise Sorry( "Two TLS selection sources found: PDB file header and parameters.") if (len(params.selection) > 0): tls_selections_strings = params.selection if ([params.combine_tls, params.extract_tls].count(True) > 1): raise Sorry( "Cannot simultaneously pereform: combine_tls and extract_tls") if ([params.combine_tls, params.extract_tls].count(True) > 0): if (len(tls_selections_strings) == 0): raise Sorry("No TLS selections found.") # if (len(tls_selections_strings)): if (not command_line.options.silent): utils.print_header("TLS groups selections", out=log) selections = utils.get_atom_selections( model=model, selection_strings=tls_selections_strings) if (not command_line.options.silent): print("Number of TLS groups: ", len(selections), file=log) print("Number of atoms: %d" % model.get_number_of_atoms(), file=log) n_atoms_in_tls = 0 for sel_a in selections: n_atoms_in_tls += sel_a.size() if (not command_line.options.silent): print("Number of atoms in TLS groups: %d" % n_atoms_in_tls, file=log) print(file=log) assert len(tls_selections_strings) == len(selections) if (not command_line.options.silent): for sel_a, sel_s in zip(selections, tls_selections_strings): print("Selection string:\n%s" % sel_s, file=log) print("selects %d atoms." % sel_a.size(), file=log) print(file=log) print("Ready-to-use in phenix.refine:\n", file=log) for sel_a, sel_s in zip(selections, tls_selections_strings): print(sel_s, file=log) # ofn = params.output_file_name if (ofn is None): ofn = os.path.splitext( os.path.basename(processed_args.pdb_file_names[0]))[0] if (len(processed_args.pdb_file_names) > 1): ofn = ofn + "_el_al" if (params.combine_tls): ofn = ofn + "_combine_tls.pdb" elif (params.extract_tls): ofn = ofn + "_extract_tls.pdb" else: ofn = None if (ofn is not None): ofo = open(ofn, "w") # if (params.extract_tls): utils.print_header( "Fit TLS matrices to B-factors of selected sets of atoms", out=log) tlsos = mmtbx.tls.tools.generate_tlsos( selections=selections, xray_structure=model.get_xray_structure(), value=0.0) for rt, rl, rs in [[1, 0, 1], [1, 1, 1], [0, 1, 1], [1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 1, 1], [0, 0, 1]] * 10: tlsos = mmtbx.tls.tools.tls_from_uanisos( xray_structure=model.get_xray_structure(), selections=selections, tlsos_initial=tlsos, number_of_macro_cycles=10, max_iterations=100, refine_T=rt, refine_L=rl, refine_S=rs, enforce_positive_definite_TL=params. enforce_positive_definite_TL, verbose=-1, out=log) mmtbx.tls.tools.show_tls(tlsos=tlsos, out=log) u_cart_from_tls = mmtbx.tls.tools.u_cart_from_tls( sites_cart=model.get_sites_cart(), selections=selections, tlsos=tlsos) unit_cell = model.get_xray_structure().unit_cell() for i_seq, sc in enumerate(model.get_xray_structure().scatterers()): if (u_cart_from_tls[i_seq] != (0, 0, 0, 0, 0, 0)): u_star_tls = adptbx.u_cart_as_u_star( unit_cell, tuple(u_cart_from_tls[i_seq])) sc.u_star = tuple( flex.double(sc.u_star) - flex.double(u_star_tls)) for sel in selections: model.get_xray_structure().convert_to_isotropic(selection=sel) mmtbx.tls.tools.remark_3_tls(tlsos=tlsos, selection_strings=tls_selections_strings, out=ofo) # if (params.combine_tls): utils.print_header("Combine B_tls with B_residual", out=log) mmtbx.tls.tools.combine_tls_and_u_local( xray_structure=model.get_xray_structure(), tls_selections=selections, tls_groups=tls_groups) print("All done.", file=log) # if (ofn is not None): utils.print_header("Write output PDB file %s" % ofn, out=log) model.set_sites_cart_from_xrs() pdb_str = model.model_as_pdb() ofo.write(pdb_str) ofo.close() print("All done.", file=log)
def run(args): from iotbx.option_parser import option_parser as iotbx_option_parser import libtbx.utils show_times = libtbx.utils.show_times(time_start="now") command_call = ["iotbx.python", __file__] command_line = (iotbx_option_parser( usage=" ".join(command_call) + " [options] directory|file...") .enable_chunk(easy_all=True) .enable_multiprocessing() ).process(args=args, min_nargs=1) if (command_line.run_multiprocessing_chunks_if_applicable( command_call=command_call)): show_times() return co = command_line.options # print "TIME BEGIN pdb_dev:", date_and_time() print libtbx.utils.host_and_user().show() print sys.stdout.flush() # from cctbx.omz import cod_refine master_phil = cod_refine.get_master_phil( max_atoms=None, f_calc_options_algorithm="direct *fft", bulk_solvent_correction=True) argument_interpreter = master_phil.command_line_argument_interpreter() phil_objects = [] remaining_args = [] for arg in command_line.args: if (arg.find("=") >= 0): phil_objects.append(argument_interpreter.process(arg=arg)) else: remaining_args.append(arg) work_phil = master_phil.fetch(sources=phil_objects) work_phil.show() print params = work_phil.extract() # mtz_pdb_pairs = [] arg_iter = iter(remaining_args) pdb_v3_mirror_dir = os.environ.get("PDB_MIRROR_PDB") assert pdb_v3_mirror_dir is None or op.isdir(pdb_v3_mirror_dir) cci_pdbmtz_path = os.environ.get("CCI_PDBMTZ") assert cci_pdbmtz_path is None or op.isdir(cci_pdbmtz_path) for arg in arg_iter: def get_next(expected_exts): def raise_bad_file(what, fn=None): msg = "%s file name (%s expected)" % (what, " or ".join(expected_exts)) if (fn is None): msg += "." else: msg += ": " + show_string(fn) raise RuntimeError(msg) try: arg = arg_iter.next() except StopIteration: raise_bad_file("Missing") if (not arg.endswith(tuple(expected_exts))): raise_bad_file("Unexpected", arg) return arg if (op.isfile(arg) and arg.endswith((".mtz", ".pdb", ".ent"))): if (arg.endswith(".mtz")): fn_mtz = arg fn_pdb = get_next([".pdb", ".ent"]) else: fn_pdb = arg fn_mtz = get_next([".mtz"]) else: fn_mtz = arg+".mtz" def raise_mtz_but_no_pdb(): raise RuntimeError( "MTZ file found but no PDB file: %s" % show_string(fn_mtz)) if (op.isfile(fn_mtz)): for ext in [".pdb", ".ent"]: fn_pdb = arg+ext if (op.isfile(fn_pdb)): break else: raise_mtz_but_no_pdb() else: fn_mtz = op.join(cci_pdbmtz_path, arg+".mtz") if (not op.isfile(fn_mtz)): raise RuntimeError( "MTZ file not found: %s" % show_string(fn_mtz)) fn_pdb = op.join(pdb_v3_mirror_dir, arg[1:3], "pdb"+arg+".ent.gz") if (not op.isfile(fn_pdb)): raise_mtz_but_no_pdb() mtz_pdb_pairs.append((fn_mtz, fn_pdb)) # n_caught = 0 for i_pair,mtz_pdb_pair in enumerate(mtz_pdb_pairs): if (i_pair % command_line.chunk.n != command_line.chunk.i): continue tm = user_plus_sys_time() try: process(params, mtz_pdb_pair) except KeyboardInterrupt: print >> sys.stderr, "CAUGHT EXCEPTION: KeyboardInterrupt" traceback.print_exc() print >> sys.stderr sys.stderr.flush() return except Exception: sys.stdout.flush() print >> sys.stderr, "CAUGHT EXCEPTION: %s" % ", ".join(mtz_pdb_pair) traceback.print_exc() print >> sys.stderr sys.stderr.flush() n_caught += 1 else: print "done_with: %s, %s (%.2f seconds)" % ( mtz_pdb_pair + (tm.elapsed(),)) print sys.stdout.flush() print print "Number of exceptions caught:", n_caught # show_times() print print "TIME END pdb_dev:", date_and_time() sys.stdout.flush()
def run(args, command_name = "phenix.tls"): if(len(args) == 0): args = ["--help"] usage_fmt = "%s pdb_file [parameters: file or command line string]" des_fmt = "Example: %s model.pdb fit_tls_to.selection='%s' fit_tls_to.selection='%s'" command_line = (iotbx_option_parser( usage = usage_fmt % command_name, description = banner) .option("--show_defaults", action="store_true", help="Do not output to the screen (except errors).") .option("--silent", action="store_true", help="Suppress output to the screen.") ).process(args=args) # log = sys.stdout if(not command_line.options.silent): utils.print_header("TLS tools", out = log) if(command_line.options.show_defaults): master_params.show(out = log) print >> log return if(not command_line.options.silent): print >> log, banner # processed_args = utils.process_command_line_args(args = command_line.args, master_params = master_params, log = log) reflection_files = processed_args.reflection_files if(processed_args.crystal_symmetry is None): raise Sorry("No crystal symmetry found.") if(len(processed_args.pdb_file_names) == 0): raise Sorry("No PDB file found.") params = processed_args.params if(not command_line.options.silent): utils.print_header("Input parameters", out = log) params.show(out = log) params = params.extract() # if(processed_args.crystal_symmetry.unit_cell() is None or processed_args.crystal_symmetry.space_group() is None): raise Sorry("No CRYST1 record found.") mmtbx_pdb_file = utils.pdb_file( pdb_file_names = processed_args.pdb_file_names, cif_objects = processed_args.cif_objects, crystal_symmetry = processed_args.crystal_symmetry, log = log) # if(not command_line.options.silent): utils.print_header("TLS groups from PDB file header", out = log) pdb_inp_tls = mmtbx.tls.tools.tls_from_pdb_inp( remark_3_records = mmtbx_pdb_file.pdb_inp.extract_remark_iii_records(3), pdb_hierarchy = mmtbx_pdb_file.pdb_inp.construct_hierarchy()) # tls_groups = [] if(pdb_inp_tls.tls_present): if(pdb_inp_tls.error_string is not None): raise Sorry(pdb_inp_tls.error_string) mmtbx_pdb_file.set_ppf() xray_structure = get_xrs_helper(mmtbx_pdb_file = mmtbx_pdb_file, log = log, silent = command_line.options.silent) pdb_tls = mmtbx.tls.tools.extract_tls_from_pdb( pdb_inp_tls = pdb_inp_tls, all_chain_proxies = mmtbx_pdb_file.processed_pdb_file.all_chain_proxies, xray_structure = xray_structure) tls_groups = pdb_tls.pdb_inp_tls.tls_params # tls_selections_strings = [] # if(len(tls_groups) == 0 and not command_line.options.silent): print >> log, "No TLS groups found in PDB file header." else: for i_seq, tls_group in enumerate(tls_groups): tls_selections_strings.append(tls_group.selection_string) if(not command_line.options.silent): print >> log, "TLS group %d: %s" % (i_seq+1, tls_group.selection_string) mmtbx.tls.tools.show_tls_one_group(tlso = tls_group, out = log) print >> log # if(len(tls_selections_strings) > 0 and len(params.selection) > 0): raise Sorry("Two TLS selection sources found: PDB file header and parameters.") if(len(params.selection) > 0): tls_selections_strings = params.selection xray_structure = get_xrs_helper(mmtbx_pdb_file = mmtbx_pdb_file, log = log, silent = command_line.options.silent) if([params.combine_tls, params.extract_tls].count(True) > 1): raise Sorry("Cannot simultaneously pereform: combine_tls and extract_tls") if([params.combine_tls, params.extract_tls].count(True) > 0): if(len(tls_selections_strings)==0): raise Sorry("No TLS selections found.") # if(len(tls_selections_strings)): if(not command_line.options.silent): utils.print_header("TLS groups selections", out = log) selections = utils.get_atom_selections( all_chain_proxies = mmtbx_pdb_file.processed_pdb_file.all_chain_proxies, selection_strings = tls_selections_strings, xray_structure = xray_structure) if(not command_line.options.silent): print >> log, "Number of TLS groups: ", len(selections) print >> log, "Number of atoms: %d" % xray_structure.scatterers().size() n_atoms_in_tls = 0 for sel_a in selections: n_atoms_in_tls += sel_a.size() if(not command_line.options.silent): print >> log, "Number of atoms in TLS groups: %d" % n_atoms_in_tls print >> log assert len(tls_selections_strings) == len(selections) if(not command_line.options.silent): for sel_a, sel_s in zip(selections,tls_selections_strings): print >> log, "Selection string:\n%s" % sel_s print >> log, "selects %d atoms." % sel_a.size() print >> log print >> log, "Ready-to-use in phenix.refine:\n" for sel_a, sel_s in zip(selections,tls_selections_strings): print >> log, sel_s # ofn = params.output_file_name if(ofn is None): ofn = os.path.splitext(os.path.basename(processed_args.pdb_file_names[0]))[0] if(len(processed_args.pdb_file_names) > 1): ofn = ofn+"_el_al" if(params.combine_tls): ofn = ofn+"_combine_tls.pdb" elif(params.extract_tls): ofn = ofn+"_extract_tls.pdb" else: ofn = None if(ofn is not None): ofo = open(ofn, "w") # if(params.extract_tls): utils.print_header( "Fit TLS matrices to B-factors of selected sets of atoms", out = log) tlsos = mmtbx.tls.tools.generate_tlsos( selections = selections, xray_structure = xray_structure, value = 0.0) for rt,rl,rs in [[1,0,1],[1,1,1],[0,1,1], [1,0,0],[0,1,0],[0,0,1],[1,1,1], [0,0,1]]*10: tlsos = mmtbx.tls.tools.tls_from_uanisos( xray_structure = xray_structure, selections = selections, tlsos_initial = tlsos, number_of_macro_cycles = 10, max_iterations = 100, refine_T = rt, refine_L = rl, refine_S = rs, enforce_positive_definite_TL = params.enforce_positive_definite_TL, verbose = -1, out = log) mmtbx.tls.tools.show_tls(tlsos = tlsos, out = log) u_cart_from_tls = mmtbx.tls.tools.u_cart_from_tls( sites_cart = xray_structure.sites_cart(), selections = selections, tlsos = tlsos) unit_cell = xray_structure.unit_cell() for i_seq, sc in enumerate(xray_structure.scatterers()): if(u_cart_from_tls[i_seq] != (0,0,0,0,0,0)): u_star_tls = adptbx.u_cart_as_u_star(unit_cell, tuple(u_cart_from_tls[i_seq])) sc.u_star = tuple(flex.double(sc.u_star) - flex.double(u_star_tls)) for sel in selections: xray_structure.convert_to_isotropic(selection = sel) mmtbx.tls.tools.remark_3_tls(tlsos = tlsos, selection_strings = tls_selections_strings, out = ofo) # if(params.combine_tls): utils.print_header("Combine B_tls with B_residual", out = log) mmtbx.tls.tools.combine_tls_and_u_local(xray_structure = xray_structure, tls_selections = selections, tls_groups = tls_groups) print >> log, "All done." # if(ofn is not None): utils.print_header("Write output PDB file %s"%ofn, out = log) utils.write_pdb_file( xray_structure = xray_structure, pdb_hierarchy = mmtbx_pdb_file.processed_pdb_file.all_chain_proxies.pdb_hierarchy, out = ofo) ofo.close() print >> log, "All done."
def run(args): from iotbx.option_parser import option_parser as iotbx_option_parser import libtbx.utils show_times = libtbx.utils.show_times(time_start="now") command_call = ["iotbx.python", __file__] command_line = (iotbx_option_parser( usage=" ".join(command_call) + " [options] directory|file...").enable_chunk( easy_all=True).enable_multiprocessing()).process(args=args, min_nargs=1) if (command_line.run_multiprocessing_chunks_if_applicable( command_call=command_call)): show_times() return co = command_line.options # print("TIME BEGIN pdb_dev:", date_and_time()) print() libtbx.utils.host_and_user().show() print() sys.stdout.flush() # from cctbx.omz import cod_refine master_phil = cod_refine.get_master_phil( max_atoms=None, f_calc_options_algorithm="direct *fft", bulk_solvent_correction=True) argument_interpreter = master_phil.command_line_argument_interpreter() phil_objects = [] remaining_args = [] for arg in command_line.args: if (arg.find("=") >= 0): phil_objects.append(argument_interpreter.process(arg=arg)) else: remaining_args.append(arg) work_phil = master_phil.fetch(sources=phil_objects) work_phil.show() print() params = work_phil.extract() # mtz_pdb_pairs = [] arg_iter = iter(remaining_args) pdb_v3_mirror_dir = os.environ.get("PDB_MIRROR_PDB") assert pdb_v3_mirror_dir is None or op.isdir(pdb_v3_mirror_dir) cci_pdbmtz_path = os.environ.get("CCI_PDBMTZ") assert cci_pdbmtz_path is None or op.isdir(cci_pdbmtz_path) for arg in arg_iter: def get_next(expected_exts): def raise_bad_file(what, fn=None): msg = "%s file name (%s expected)" % ( what, " or ".join(expected_exts)) if (fn is None): msg += "." else: msg += ": " + show_string(fn) raise RuntimeError(msg) try: arg = next(arg_iter) except StopIteration: raise_bad_file("Missing") if (not arg.endswith(tuple(expected_exts))): raise_bad_file("Unexpected", arg) return arg if (op.isfile(arg) and arg.endswith((".mtz", ".pdb", ".ent"))): if (arg.endswith(".mtz")): fn_mtz = arg fn_pdb = get_next([".pdb", ".ent"]) else: fn_pdb = arg fn_mtz = get_next([".mtz"]) else: fn_mtz = arg + ".mtz" def raise_mtz_but_no_pdb(): raise RuntimeError("MTZ file found but no PDB file: %s" % show_string(fn_mtz)) if (op.isfile(fn_mtz)): for ext in [".pdb", ".ent"]: fn_pdb = arg + ext if (op.isfile(fn_pdb)): break else: raise_mtz_but_no_pdb() else: fn_mtz = op.join(cci_pdbmtz_path, arg + ".mtz") if (not op.isfile(fn_mtz)): raise RuntimeError("MTZ file not found: %s" % show_string(fn_mtz)) fn_pdb = op.join(pdb_v3_mirror_dir, arg[1:3], "pdb" + arg + ".ent.gz") if (not op.isfile(fn_pdb)): raise_mtz_but_no_pdb() mtz_pdb_pairs.append((fn_mtz, fn_pdb)) # n_caught = 0 for i_pair, mtz_pdb_pair in enumerate(mtz_pdb_pairs): if (i_pair % command_line.chunk.n != command_line.chunk.i): continue tm = user_plus_sys_time() try: process(params, mtz_pdb_pair) except KeyboardInterrupt: print("CAUGHT EXCEPTION: KeyboardInterrupt", file=sys.stderr) traceback.print_exc() print(file=sys.stderr) sys.stderr.flush() return except Exception: sys.stdout.flush() print("CAUGHT EXCEPTION: %s" % ", ".join(mtz_pdb_pair), file=sys.stderr) traceback.print_exc() print(file=sys.stderr) sys.stderr.flush() n_caught += 1 else: print("done_with: %s, %s (%.2f seconds)" % (mtz_pdb_pair + (tm.elapsed(), ))) print() sys.stdout.flush() print() print("Number of exceptions caught:", n_caught) # show_times() print() print("TIME END pdb_dev:", date_and_time()) sys.stdout.flush()
def run(self, args, command_name, out=sys.stdout): command_line = (iotbx_option_parser( usage="%s [options]" % command_name, description='Example: %s data.mtz data.mtz ref_model.pdb' % command_name).option( None, "--show_defaults", action="store_true", help="Show list of parameters.")).process(args=args) cif_file = None processed_args = utils.process_command_line_args( args=args, log=sys.stdout, master_params=master_phil) params = processed_args.params if (params is None): params = master_phil self.params = params.extract().ensemble_probability pdb_file_names = processed_args.pdb_file_names if len(pdb_file_names) != 1: raise Sorry("Only one PDB structure may be used") pdb_file = file_reader.any_file(pdb_file_names[0]) self.log = multi_out() self.log.register(label="stdout", file_object=sys.stdout) self.log.register(label="log_buffer", file_object=StringIO(), atexit_send_to=None) sys.stderr = self.log log_file = open( pdb_file_names[0].split('/')[-1].replace('.pdb', '') + '_pensemble.log', "w") self.log.replace_stringio(old_label="log_buffer", new_label="log", new_file_object=log_file) utils.print_header(command_name, out=self.log) params.show(out=self.log) # f_obs = None r_free_flags = None reflection_files = processed_args.reflection_files if self.params.fobs_vs_fcalc_post_nll: if len(reflection_files) == 0: raise Sorry( "Fobs from input MTZ required for fobs_vs_fcalc_post_nll") if len(reflection_files) > 0: crystal_symmetry = processed_args.crystal_symmetry print('Reflection file : ', processed_args.reflection_file_names[0], file=self.log) utils.print_header("Model and data statistics", out=self.log) rfs = reflection_file_server( crystal_symmetry=crystal_symmetry, reflection_files=processed_args.reflection_files, log=self.log) parameters = extract_xtal_data.data_and_flags_master_params( ).extract() determine_data_and_flags_result = extract_xtal_data.run( reflection_file_server=rfs, parameters=parameters, data_parameter_scope="refinement.input.xray_data", flags_parameter_scope="refinement.input.xray_data.r_free_flags", data_description="X-ray data", keep_going=True, log=self.log) f_obs = determine_data_and_flags_result.f_obs number_of_reflections = f_obs.indices().size() r_free_flags = determine_data_and_flags_result.r_free_flags test_flag_value = determine_data_and_flags_result.test_flag_value if (r_free_flags is None): r_free_flags = f_obs.array( data=flex.bool(f_obs.data().size(), False)) # process PDB pdb_file.assert_file_type("pdb") # pdb_in = hierarchy.input(file_name=pdb_file.file_name) ens_pdb_hierarchy = pdb_in.construct_hierarchy() ens_pdb_hierarchy.atoms().reset_i_seq() ens_pdb_xrs_s = pdb_in.input.xray_structures_simple() number_structures = len(ens_pdb_xrs_s) print('Number of structure in ensemble : ', number_structures, file=self.log) # Calculate sigmas from input map only if self.params.assign_sigma_from_map and self.params.ensemble_sigma_map_input is not None: # process MTZ input_file = file_reader.any_file( self.params.ensemble_sigma_map_input) if input_file.file_type == "hkl": if input_file.file_object.file_type() != "ccp4_mtz": raise Sorry("Only MTZ format accepted for map input") else: mtz_file = input_file else: raise Sorry("Only MTZ format accepted for map input") miller_arrays = mtz_file.file_server.miller_arrays map_coeffs_1 = miller_arrays[0] # xrs_list = [] for n, ens_pdb_xrs in enumerate(ens_pdb_xrs_s): # get sigma levels from ensemble fc for each structure xrs = get_map_sigma(ens_pdb_hierarchy=ens_pdb_hierarchy, ens_pdb_xrs=ens_pdb_xrs, map_coeffs_1=map_coeffs_1, residue_detail=self.params.residue_detail, ignore_hd=self.params.ignore_hd, log=self.log) xrs_list.append(xrs) # write ensemble pdb file, occupancies as sigma level filename = pdb_file_names[0].split('/')[-1].replace( '.pdb', '') + '_vs_' + self.params.ensemble_sigma_map_input.replace( '.mtz', '') + '_pensemble.pdb' write_ensemble_pdb(filename=filename, xrs_list=xrs_list, ens_pdb_hierarchy=ens_pdb_hierarchy) # Do full analysis vs Fobs else: model_map_coeffs = [] fmodel = None # Get <fcalc> for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s): ens_pdb_xrs.set_occupancies(1.0) if model == 0: # If mtz not supplied get fobs from xray structure... # Use input Fobs for scoring against nll if self.params.fobs_vs_fcalc_post_nll: dummy_fobs = f_obs else: if f_obs == None: if self.params.fcalc_high_resolution == None: raise Sorry( "Please supply high resolution limit or input mtz file." ) dummy_dmin = self.params.fcalc_high_resolution dummy_dmax = self.params.fcalc_low_resolution else: print( 'Supplied mtz used to determine high and low resolution cuttoffs', file=self.log) dummy_dmax, dummy_dmin = f_obs.d_max_min() # dummy_fobs = abs( ens_pdb_xrs.structure_factors( d_min=dummy_dmin).f_calc()) dummy_fobs.set_observation_type_xray_amplitude() # If mtz supplied, free flags are over written to prevent array size error r_free_flags = dummy_fobs.array( data=flex.bool(dummy_fobs.data().size(), False)) # fmodel = utils.fmodel_simple( scattering_table="wk1995", xray_structures=[ens_pdb_xrs], f_obs=dummy_fobs, target_name='ls', bulk_solvent_and_scaling=False, r_free_flags=r_free_flags) f_calc_ave = fmodel.f_calc().array( data=fmodel.f_calc().data() * 0).deep_copy() # XXX Important to ensure scale is identical for each model and <model> fmodel.set_scale_switch = 1.0 f_calc_ave_total = fmodel.f_calc().data().deep_copy() else: fmodel.update_xray_structure(xray_structure=ens_pdb_xrs, update_f_calc=True, update_f_mask=False) f_calc_ave_total += fmodel.f_calc().data().deep_copy() print('Model :', model + 1, file=self.log) print("\nStructure vs real Fobs (no bulk solvent or scaling)", file=self.log) print('Rwork : %5.4f ' % fmodel.r_work(), file=self.log) print('Rfree : %5.4f ' % fmodel.r_free(), file=self.log) print('K1 : %5.4f ' % fmodel.scale_k1(), file=self.log) fcalc_edm = fmodel.electron_density_map() fcalc_map_coeffs = fcalc_edm.map_coefficients(map_type='Fc') fcalc_mtz_dataset = fcalc_map_coeffs.as_mtz_dataset( column_root_label='Fc') if self.params.output_model_and_model_ave_mtz: fcalc_mtz_dataset.mtz_object().write( file_name=str(model + 1) + "_Fc.mtz") model_map_coeffs.append(fcalc_map_coeffs.deep_copy()) fmodel.update(f_calc=f_calc_ave.array(f_calc_ave_total / number_structures)) print("\nEnsemble vs real Fobs (no bulk solvent or scaling)", file=self.log) print('Rwork : %5.4f ' % fmodel.r_work(), file=self.log) print('Rfree : %5.4f ' % fmodel.r_free(), file=self.log) print('K1 : %5.4f ' % fmodel.scale_k1(), file=self.log) # Get <Fcalc> map fcalc_ave_edm = fmodel.electron_density_map() fcalc_ave_map_coeffs = fcalc_ave_edm.map_coefficients( map_type='Fc').deep_copy() fcalc_ave_mtz_dataset = fcalc_ave_map_coeffs.as_mtz_dataset( column_root_label='Fc') if self.params.output_model_and_model_ave_mtz: fcalc_ave_mtz_dataset.mtz_object().write(file_name="aveFc.mtz") fcalc_ave_map_coeffs = fcalc_ave_map_coeffs.fft_map() fcalc_ave_map_coeffs.apply_volume_scaling() fcalc_ave_map_data = fcalc_ave_map_coeffs.real_map_unpadded() fcalc_ave_map_stats = maptbx.statistics(fcalc_ave_map_data) print("<Fcalc> Map Stats :", file=self.log) fcalc_ave_map_stats.show_summary(f=self.log) offset = fcalc_ave_map_stats.min() model_neg_ll = [] number_previous_scatters = 0 # Run through structure list again and get probability xrs_list = [] for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s): if self.params.verbose: print('\n\nModel : ', model + 1, file=self.log) # Get model atom sigmas vs Fcalc fcalc_map = model_map_coeffs[model].fft_map() fcalc_map.apply_volume_scaling() fcalc_map_data = fcalc_map.real_map_unpadded() fcalc_map_stats = maptbx.statistics(fcalc_map_data) if self.params.verbose: print("Fcalc map stats :", file=self.log) fcalc_map_stats.show_summary(f=self.log) xrs = get_map_sigma( ens_pdb_hierarchy=ens_pdb_hierarchy, ens_pdb_xrs=ens_pdb_xrs, fft_map_1=fcalc_map, model_i=model, residue_detail=self.params.residue_detail, ignore_hd=self.params.ignore_hd, number_previous_scatters=number_previous_scatters, log=self.log) fcalc_sigmas = xrs.scatterers().extract_occupancies() del fcalc_map # Get model atom sigmas vs <Fcalc> xrs = get_map_sigma( ens_pdb_hierarchy=ens_pdb_hierarchy, ens_pdb_xrs=ens_pdb_xrs, fft_map_1=fcalc_ave_map_coeffs, model_i=model, residue_detail=self.params.residue_detail, ignore_hd=self.params.ignore_hd, number_previous_scatters=number_previous_scatters, log=self.log) ### For testing other residue averaging options #print xrs.residue_selections fcalc_ave_sigmas = xrs.scatterers().extract_occupancies() # Probability of model given <model> prob = fcalc_ave_sigmas / fcalc_sigmas # XXX debug option if False: for n, p in enumerate(prob): print(' {0:5d} {1:5.3f}'.format(n, p), file=self.log) # Set probabilty between 0 and 1 # XXX Make Histogram / more stats prob_lss_zero = flex.bool(prob <= 0) prob_grt_one = flex.bool(prob > 1) prob.set_selected(prob_lss_zero, 0.001) prob.set_selected(prob_grt_one, 1.0) xrs.set_occupancies(prob) xrs_list.append(xrs) sum_neg_ll = sum(-flex.log(prob)) model_neg_ll.append((sum_neg_ll, model)) if self.params.verbose: print('Model probability stats :', file=self.log) print(prob.min_max_mean().show(), file=self.log) print(' Count < 0.0 : ', prob_lss_zero.count(True), file=self.log) print(' Count > 1.0 : ', prob_grt_one.count(True), file=self.log) # For averaging by residue number_previous_scatters += ens_pdb_xrs.sites_cart().size() # write ensemble pdb file, occupancies as sigma level write_ensemble_pdb( filename=pdb_file_names[0].split('/')[-1].replace('.pdb', '') + '_pensemble.pdb', xrs_list=xrs_list, ens_pdb_hierarchy=ens_pdb_hierarchy) # XXX Test ordering models by nll # XXX Test removing nth percentile atoms if self.params.sort_ensemble_by_nll_score or self.params.fobs_vs_fcalc_post_nll: for percentile in [1.0, 0.975, 0.95, 0.9, 0.8, 0.6, 0.2]: model_neg_ll = sorted(model_neg_ll) f_calc_ave_total_reordered = None print_list = [] for i_neg_ll in model_neg_ll: xrs = xrs_list[i_neg_ll[1]] nll_occ = xrs.scatterers().extract_occupancies() # Set q=0 nth percentile atoms sorted_nll_occ = sorted(nll_occ, reverse=True) number_atoms = len(sorted_nll_occ) percentile_prob_cutoff = sorted_nll_occ[ int(number_atoms * percentile) - 1] cutoff_selections = flex.bool( nll_occ < percentile_prob_cutoff) cutoff_nll_occ = flex.double(nll_occ.size(), 1.0).set_selected( cutoff_selections, 0.0) #XXX Debug if False: print('\nDebug') for x in range(len(cutoff_selections)): print(cutoff_selections[x], nll_occ[x], cutoff_nll_occ[x]) print(percentile) print(percentile_prob_cutoff) print(cutoff_selections.count(True)) print(cutoff_selections.size()) print(cutoff_nll_occ.count(0.0)) print('Count q = 1 : ', cutoff_nll_occ.count(1.0)) print('Count scatterers size : ', cutoff_nll_occ.size()) xrs.set_occupancies(cutoff_nll_occ) fmodel.update_xray_structure(xray_structure=xrs, update_f_calc=True, update_f_mask=True) if f_calc_ave_total_reordered == None: f_calc_ave_total_reordered = fmodel.f_calc().data( ).deep_copy() f_mask_ave_total_reordered = fmodel.f_masks( )[0].data().deep_copy() cntr = 1 else: f_calc_ave_total_reordered += fmodel.f_calc().data( ).deep_copy() f_mask_ave_total_reordered += fmodel.f_masks( )[0].data().deep_copy() cntr += 1 fmodel.update( f_calc=f_calc_ave.array( f_calc_ave_total_reordered / cntr).deep_copy(), f_mask=f_calc_ave.array( f_mask_ave_total_reordered / cntr).deep_copy()) # Update solvent and scale # XXX Will need to apply_back_trace on latest version fmodel.set_scale_switch = 0 fmodel.update_all_scales() # Reset occ for outout xrs.set_occupancies(nll_occ) # k1 updated vs Fobs if self.params.fobs_vs_fcalc_post_nll: print_list.append([ cntr, i_neg_ll[0], i_neg_ll[1], fmodel.r_work(), fmodel.r_free() ]) # Order models by nll and print summary print( '\nModels ranked by nll <Fcalc> R-factors recalculated', file=self.log) print('Percentile cutoff : {0:5.3f}'.format(percentile), file=self.log) xrs_list_sorted_nll = [] print(' | NLL <Rw> <Rf> Ens Model', file=self.log) for info in print_list: print(' {0:4d} | {1:8.1f} {2:8.4f} {3:8.4f} {4:12d}'. format( info[0], info[1], info[3], info[4], info[2] + 1, ), file=self.log) xrs_list_sorted_nll.append(xrs_list[info[2]]) # Output nll ordered ensemble write_ensemble_pdb( filename='nll_ordered_' + pdb_file_names[0].split('/')[-1].replace('.pdb', '') + '_pensemble.pdb', xrs_list=xrs_list_sorted_nll, ens_pdb_hierarchy=ens_pdb_hierarchy)
def run(args, command_name): from iotbx.option_parser import option_parser as iotbx_option_parser from libtbx import easy_pickle import libtbx.utils show_times = libtbx.utils.show_times(time_start="now") command_line = (iotbx_option_parser( usage=command_name+" [options] [cod_id...]") .enable_chunk(easy_all=True) .enable_multiprocessing() .option(None, "--max_atoms", type="int", default=99, metavar="INT") .option(None, "--min_distance", type="float", default=0.5, metavar="FLOAT") .option(None, "--i_obs_i_calc_plot", action="store_true", default=False) .option(None, "--f_obs_f_calc_plot", action="store_true", default=False) .option(None, "--max_fan_outlier_fraction", type="float", default=0.05, metavar="FLOAT") .option(None, "--fan_offset_low", type="float", default=0.05, metavar="FLOAT") .option(None, "--fan_offset_high", type="float", default=0.10, metavar="FLOAT") .option(None, "--min_f_obs_f_calc_correlation", type="float", default=0.99, metavar="FLOAT") .option(None, "--at_least_one_special_position", action="store_true", default=False) .option(None, "--pickle_dir", type="str", default="cod_ma_xs", metavar="PATH") ).process(args=args) if (command_line.run_multiprocessing_chunks_if_applicable( command_call=[command_name])): show_times() return co = command_line.options # from iotbx.cif import cod_tools hkl_cif = cod_tools.build_hkl_cif(cod_ids=command_line.args) hkl_cif.show_summary() # pickle_dir = co.pickle_dir if (co.at_least_one_special_position): pickle_dir += "_special" if (not op.isdir(pickle_dir)): from libtbx.path import makedirs_race makedirs_race(path=pickle_dir) n_caught = 0 for i_pair,pair in enumerate(hkl_cif.hkl_cif_pairs.values()): cod_id = op.basename(pair[0])[:-4] if (i_pair % command_line.chunk.n != command_line.chunk.i): continue try: cd = cod_data(cod_id=cod_id, hkl_cif_pair=pair) except KeyboardInterrupt: print("CAUGHT EXCEPTION: KeyboardInterrupt") return except Exception as e: sys.stdout.flush() print("CAUGHT EXCEPTION: %s: %s: %s" % (command_name, cod_id, str(e)), file=sys.stderr) traceback.print_exc() print(file=sys.stderr) sys.stderr.flush() n_caught += 1 else: if (cd.is_useful(co)): easy_pickle.dump( file_name="%s/%s.pickle" % (pickle_dir, cod_id), obj=(cd.c_obs, cd.xray_structure, cd.edge_list)) print(cd.quick_info(), file=open("%s/qi_%s" % (pickle_dir, cod_id), "w")) else: print("filtering out:", cod_id) print("done_with:", cod_id) print() print() print("Number of exceptions caught:", n_caught) # show_times()