def run(): data_dir = os.path.dirname(os.path.abspath(__file__)) data_mtz = os.path.join(data_dir, 'data', 'phaser_1.mtz') params = ed.data_and_flags_master_params().extract() params.file_name = data_mtz params.labels=['FP,SIGFP'] phases_params = ed.experimental_phases_master_params().extract() phases_params.labels="HLA,HLB,HLC,HLD" reflection_files = [ iotbx.reflection_file_reader.any_reflection_file(data_mtz)] server = iotbx.reflection_file_utils.reflection_file_server( crystal_symmetry = None, reflection_files = reflection_files, err = sys.stdout) fo = ed.run( reflection_file_server = server, parameters = params, experimental_phases_params = phases_params, extract_r_free_flags = False).f_obs assert cctbx.miller.array(fo, cctbx.miller.array)
def get_data(pdbf, mtzf): pdb_inp = iotbx.pdb.input(file_name=pdbf) xrs = pdb_inp.xray_structure_simple() # selection = xrs.scatterers().extract_occupancies() > 0 xrs = xrs.select(selection) selection = ~xrs.hd_selection() xrs = xrs.select(selection) # #xrs.switch_to_neutron_scattering_dictionary() # reflection_file = reflection_file_reader.any_reflection_file( file_name=mtzf, ensure_read_access=False) rfs = reflection_file_utils.reflection_file_server( crystal_symmetry=xrs.crystal_symmetry(), force_symmetry=True, reflection_files=[reflection_file], err=null_out()) determine_data_and_flags_result = extract_xtal_data.run( reflection_file_server=rfs, keep_going=True, extract_r_free_flags=True, force_non_anomalous=True, allow_mismatch_flags=True) f_obs = determine_data_and_flags_result.f_obs r_free_flags = determine_data_and_flags_result.r_free_flags fmodel = mmtbx.f_model.manager(f_obs=f_obs, r_free_flags=r_free_flags, xray_structure=xrs) fmodel.update_all_scales(remove_outliers=True, apply_scale_k1_to_f_obs=True) # Skip rw = fmodel.r_work() rf = fmodel.r_free() if (fmodel.f_obs().d_min() > 3): return None if (fmodel.f_obs().completeness() < 0.95): return None if (rw > 0.25): return None if (rf <= rw): return None if (abs(rf - rw) * 100 < 2): return None if (fmodel.f_obs().resolution_filter(d_min=8).completeness() < 0.95): return None # def f_obs(): return fmodel.f_obs() def r_free_flags(): return fmodel.r_free_flags() def f_calc(): return fmodel.f_calc() def flags(): return fmodel.r_free_flags() return group_args(f_obs=f_obs, r_free_flags=r_free_flags, xray_structure=fmodel.xray_structure, f_calc=f_calc, flags=flags)
def get_f_obs_and_flags(reflection_file_name, crystal_symmetry, f_obs_label=None, r_free_flags_label=None, log=None): reflection_files = [] reflection_files.append( reflection_file_reader.any_reflection_file( file_name=reflection_file_name, ensure_read_access=False)) reflection_file_server = reflection_file_utils.reflection_file_server( crystal_symmetry=crystal_symmetry, force_symmetry=True, reflection_files=reflection_files, err=log) parameters = extract_xtal_data.data_and_flags_master_params().extract() if (f_obs_label is not None): parameters.labels = command_line.options.f_obs_label if (r_free_flags_label is not None): parameters.r_free_flags.label = command_line.options.r_free_flags_label determine_data_and_flags_result = extract_xtal_data.run( reflection_file_server=reflection_file_server, parameters=parameters, keep_going=True) f_obs = determine_data_and_flags_result.f_obs r_free_flags = determine_data_and_flags_result.r_free_flags return f_obs, r_free_flags
def get_fobs_rfree(self, crystal_symmetry): f_obs, r_free_flags = None, None rfs = self.data_manager.get_reflection_file_server( filenames=self.data_manager.get_miller_array_names(), crystal_symmetry=crystal_symmetry, logger=null_out()) parameters = extract_xtal_data.data_and_flags_master_params().extract() if (self.params.data_labels is not None): parameters.labels = self.params.data_labels if (self.params.r_free_flags_labels is not None): parameters.r_free_flags.label = self.params.r_free_flags_labels determined_data_and_flags = extract_xtal_data.run( reflection_file_server=rfs, parameters=parameters, keep_going=True, working_point_group=crystal_symmetry.space_group( ).build_derived_point_group()) f_obs = determined_data_and_flags.f_obs r_free_flags = determined_data_and_flags.r_free_flags assert (f_obs is not None) if (self.params.data_labels is None): self.params.data_labels = f_obs.info().label_string() if (r_free_flags is not None): self.params.r_free_flags_labels = r_free_flags.info().label_string( ) return f_obs, r_free_flags
def run(processed_args, params): if (params.scattering_table not in ["n_gaussian", "wk1995", "it1992", "neutron"]): raise Sorry("Incorrect scattering_table.") crystal_symmetry = None crystal_symmetries = [] for f in [str(params.pdb_file_name), str(params.reflection_file_name)]: cs = crystal_symmetry_from_any.extract_from(f) if (cs is not None): crystal_symmetries.append(cs) if (len(crystal_symmetries) == 1): crystal_symmetry = crystal_symmetries[0] elif (len(crystal_symmetries) == 0): raise Sorry("No crystal symmetry found.") else: if (not crystal_symmetries[0].is_similar_symmetry( crystal_symmetries[1])): raise Sorry("Crystal symmetry mismatch between different files.") crystal_symmetry = crystal_symmetries[0] f_obs = None r_free_flags = None if (params.reflection_file_name is not None): reflection_file = reflection_file_reader.any_reflection_file( file_name=params.reflection_file_name, ensure_read_access=True) rfs = reflection_file_server(crystal_symmetry=crystal_symmetry, reflection_files=[reflection_file]) parameters = extract_xtal_data.data_and_flags_master_params().extract() if (params.data_labels is not None): parameters.labels = [processed_args.data_labels] determine_data_and_flags_result = extract_xtal_data.run( reflection_file_server=rfs, parameters=parameters, keep_going=True, log=StringIO()) f_obs = determine_data_and_flags_result.f_obs r_free_flags = determine_data_and_flags_result.r_free_flags if (r_free_flags is None): r_free_flags = f_obs.array( data=flex.bool(f_obs.data().size(), False)) test_flag_value = None xray_structure = iotbx.pdb.input( file_name=params.pdb_file_name).xray_structure_simple() xray_structure_da = None if (params.external_da_pdb_file_name is not None): xray_structure_da = iotbx.pdb.input( file_name=params.external_da_pdb_file_name).xray_structure_simple( ) if (f_obs is not None): f_obs = f_obs.resolution_filter(d_min=params.high_resolution, d_max=params.low_resolution) r_free_flags = r_free_flags.resolution_filter( d_min=params.high_resolution, d_max=params.low_resolution) # assert params.mode in ["build", "build_and_refine"] grow_density(f_obs=f_obs, r_free_flags=r_free_flags, xray_structure=xray_structure, xray_structure_da=xray_structure_da, params=params)
def run(args, external_params=None, out=sys.stdout): from mmtbx.refinement import select_best_starting_model from iotbx.file_reader import any_file import iotbx.phil cmdline = iotbx.phil.process_command_line_with_files( args=args, master_phil=master_params(), pdb_file_def="input.model", reflection_file_def="input.xray_data.file_name", usage_string="""\ mmtbx.select_best_starting_model data.mtz model1.pdb model2.pdb [...] Given experimental data and a set of models, determines whether any of the models can be refined directly (i.e. if isomorphous), and picks the one with the best R-free below a specified cutoff. Will optionally perform rigid-body refinement on suitable models if requested.""") params = cmdline.work.extract() validate_params(params) hkl_in = any_file(params.input.xray_data.file_name) data_and_flags = extract_xtal_data.run( reflection_file_server=hkl_in.file_server, parameters=params.input.xray_data, data_parameter_scope="input.data", flags_parameter_scope="input.data.r_free_flags", log=out) model_data = [] for file_name in params.input.model: model_in = cmdline.get_file(file_name=file_name, force_type="pdb").file_object pdb_hierarchy = model_in.hierarchy xray_structure = model_in.xray_structure_simple() model_data.append((pdb_hierarchy, xray_structure)) # we can optionally pass a parameter block from elsewhere (e.g. phenix ligand # pipeline) and just use this run() method to load files params_ = external_params if (params_ is None): params_ = params result = select_best_starting_model.select_model( model_names=params.input.model, model_data=model_data, f_obs=data_and_flags.f_obs, r_free_flags=data_and_flags.r_free_flags, params=params_, nproc=params.nproc, skip_twin_detection=params.input.skip_twin_detection, log=out) if result.success() and params.output.write_files: result.save_best_model(file_name=params.output.model_file_name) print("", file=out) print("Wrote best model to %s" % params.output.model_file_name, file=out) result.save_updated_data(file_name=params.output.data_file_name) print("Wrote updated data to %s" % params.output.data_file_name, file=out) return result
def get_fmodel( self, array_type, crystal_symmetry=None, parameters=None, # XXX Replace with what DataManager uses experimental_phases_params=None, # XXX Need to be part of 'parameters' scattering_table=None # XXX Make part of parameters? ): # if (array_type == "x_ray"): assert scattering_table in ["wk1995", "it1992", "n_gaussian"] elif (array_type == "neutron"): assert scattering_table == "neutron" elif (array_type == "electron"): assert scattering_table == "electron" # Gather models of apropriate type models = [] for filename in self.get_model_names(model_type=array_type): models.append(self.get_model(filename)) if (len(models) == 0): raise Sorry("No model of '%s' type found to make fmodel." % array_type) if (len(models) > 1): raise Sorry("More than one model of '%s' type found." % array_type) model = models[0] # Get reflection file server rfs = self.get_reflection_file_server(array_type=array_type) # Resolve symmetry issues (nplace) self._resolve_symmetry_conflicts(params=crystal_symmetry, model=model, reflection_file_server=rfs) # Get reflection data data = extract_xtal_data.run( reflection_file_server=rfs, parameters=parameters, experimental_phases_params=experimental_phases_params, working_point_group=model.crystal_symmetry().space_group( ).build_derived_point_group(), remark_r_free_flags_md5_hexdigest=model. get_header_r_free_flags_md5_hexdigest()).result() if (len(data.err) > 0): raise Sorry("\n".join(data.err)) if (data.f_obs is None): raise Sorry("Diffraction date are not available to make fmodel.") # Setup scattering table of xray_structure model.setup_scattering_dictionaries(scattering_table=scattering_table, d_min=data.f_obs.d_min()) # Create and return fmodel fmodel = mmtbx.f_model.manager( f_obs=data.f_obs, r_free_flags=data.r_free_flags, abcd=data.experimental_phases, xray_structure=model.get_xray_structure(), origin=data.mtz_object) return fmodel
def get_map_from_hkl(hkl_file_object, params, xrs, log): print("Processing input hkl file...", file=log) crystal_symmetry = hkl_file_object.crystal_symmetry() rfs = reflection_file_utils.reflection_file_server( crystal_symmetry=crystal_symmetry, force_symmetry=True, reflection_files=[hkl_file_object.file_content], err=StringIO()) parameters = extract_xtal_data.data_and_flags_master_params().extract() if (params.data_labels is not None): parameters.labels = params.data_labels if (params.r_free_flags_labels is not None): parameters.r_free_flags.label = params.r_free_flags_labels determined_data_and_flags = extract_xtal_data.run( reflection_file_server=rfs, parameters=parameters, keep_going=True, working_point_group=crystal_symmetry.space_group( ).build_derived_point_group(), log=StringIO(), symmetry_safety_check=True) f_obs = determined_data_and_flags.f_obs if (params.data_labels is None): params.data_labels = f_obs.info().label_string() r_free_flags = determined_data_and_flags.r_free_flags assert f_obs is not None print("Input data:", file=log) print(" Iobs or Fobs:", f_obs.info().labels, file=log) if (r_free_flags is not None): print(" Free-R flags:", r_free_flags.info().labels, file=log) params.r_free_flags_labels = r_free_flags.info().label_string() else: print(" Free-R flags: Not present", file=log) fmodel = mmtbx.f_model.manager(f_obs=f_obs, r_free_flags=r_free_flags, xray_structure=xrs) fmodel.update_all_scales() fft_map = fmodel.electron_density_map().fft_map( resolution_factor=0.25, map_type="2mFo-DFc", use_all_data=False) # Exclude free reflections fft_map.apply_sigma_scaling() map_data = fft_map.real_map_unpadded(in_place=False) if params.debug: fft_map.as_xplor_map(file_name="%s_21.map" % params.output_prefix) iotbx.mrcfile.write_ccp4_map( file_name="%s_21.ccp4" % params.output_prefix, unit_cell=crystal_symmetry.unit_cell(), space_group=crystal_symmetry.space_group(), map_data=map_data, labels=flex.std_string([""])) return map_data, crystal_symmetry
def get_fobs_rfree(self, crystal_symmetry): rfs = self.data_manager.get_reflection_file_server( filenames=[self.data_manager.get_default_miller_array_name()], crystal_symmetry=crystal_symmetry, logger=null_out()) parameters = extract_xtal_data.data_and_flags_master_params().extract() determined_data_and_flags = extract_xtal_data.run( reflection_file_server=rfs, parameters=parameters, keep_going=True, working_point_group=crystal_symmetry.space_group( ).build_derived_point_group()) f_obs = determined_data_and_flags.f_obs r_free_flags = determined_data_and_flags.r_free_flags return f_obs, r_free_flags
def extract_data_and_flags(params, crystal_symmetry=None): data_and_flags = None if (params.reflection_file_name is not None): reflection_file = reflection_file_reader.any_reflection_file( file_name=params.reflection_file_name) reflection_file_server = reflection_file_utils.reflection_file_server( crystal_symmetry=crystal_symmetry, force_symmetry=True, reflection_files=[reflection_file]) parameters = extract_xtal_data.data_and_flags_master_params().extract() parameters.force_anomalous_flag_to_be_equal_to = False if (params.data_labels is not None): parameters.labels = [params.data_labels] if (params.high_resolution is not None): parameters.high_resolution = params.high_resolution if (params.low_resolution is not None): parameters.low_resolution = params.low_resolution data_and_flags = extract_xtal_data.run( reflection_file_server=reflection_file_server, parameters=parameters, extract_r_free_flags=False) #XXX return data_and_flags
def run(self, args, command_name, out=sys.stdout): command_line = (iotbx_option_parser( usage="%s [options]" % command_name, description='Example: %s data.mtz data.mtz ref_model.pdb' % command_name).option( None, "--show_defaults", action="store_true", help="Show list of parameters.")).process(args=args) cif_file = None processed_args = utils.process_command_line_args( args=args, log=sys.stdout, master_params=master_phil) params = processed_args.params if (params is None): params = master_phil self.params = params.extract().ensemble_probability pdb_file_names = processed_args.pdb_file_names if len(pdb_file_names) != 1: raise Sorry("Only one PDB structure may be used") pdb_file = file_reader.any_file(pdb_file_names[0]) self.log = multi_out() self.log.register(label="stdout", file_object=sys.stdout) self.log.register(label="log_buffer", file_object=StringIO(), atexit_send_to=None) sys.stderr = self.log log_file = open( pdb_file_names[0].split('/')[-1].replace('.pdb', '') + '_pensemble.log', "w") self.log.replace_stringio(old_label="log_buffer", new_label="log", new_file_object=log_file) utils.print_header(command_name, out=self.log) params.show(out=self.log) # f_obs = None r_free_flags = None reflection_files = processed_args.reflection_files if self.params.fobs_vs_fcalc_post_nll: if len(reflection_files) == 0: raise Sorry( "Fobs from input MTZ required for fobs_vs_fcalc_post_nll") if len(reflection_files) > 0: crystal_symmetry = processed_args.crystal_symmetry print('Reflection file : ', processed_args.reflection_file_names[0], file=self.log) utils.print_header("Model and data statistics", out=self.log) rfs = reflection_file_server( crystal_symmetry=crystal_symmetry, reflection_files=processed_args.reflection_files, log=self.log) parameters = extract_xtal_data.data_and_flags_master_params( ).extract() determine_data_and_flags_result = extract_xtal_data.run( reflection_file_server=rfs, parameters=parameters, data_parameter_scope="refinement.input.xray_data", flags_parameter_scope="refinement.input.xray_data.r_free_flags", data_description="X-ray data", keep_going=True, log=self.log) f_obs = determine_data_and_flags_result.f_obs number_of_reflections = f_obs.indices().size() r_free_flags = determine_data_and_flags_result.r_free_flags test_flag_value = determine_data_and_flags_result.test_flag_value if (r_free_flags is None): r_free_flags = f_obs.array( data=flex.bool(f_obs.data().size(), False)) # process PDB pdb_file.assert_file_type("pdb") # pdb_in = hierarchy.input(file_name=pdb_file.file_name) ens_pdb_hierarchy = pdb_in.construct_hierarchy() ens_pdb_hierarchy.atoms().reset_i_seq() ens_pdb_xrs_s = pdb_in.input.xray_structures_simple() number_structures = len(ens_pdb_xrs_s) print('Number of structure in ensemble : ', number_structures, file=self.log) # Calculate sigmas from input map only if self.params.assign_sigma_from_map and self.params.ensemble_sigma_map_input is not None: # process MTZ input_file = file_reader.any_file( self.params.ensemble_sigma_map_input) if input_file.file_type == "hkl": if input_file.file_object.file_type() != "ccp4_mtz": raise Sorry("Only MTZ format accepted for map input") else: mtz_file = input_file else: raise Sorry("Only MTZ format accepted for map input") miller_arrays = mtz_file.file_server.miller_arrays map_coeffs_1 = miller_arrays[0] # xrs_list = [] for n, ens_pdb_xrs in enumerate(ens_pdb_xrs_s): # get sigma levels from ensemble fc for each structure xrs = get_map_sigma(ens_pdb_hierarchy=ens_pdb_hierarchy, ens_pdb_xrs=ens_pdb_xrs, map_coeffs_1=map_coeffs_1, residue_detail=self.params.residue_detail, ignore_hd=self.params.ignore_hd, log=self.log) xrs_list.append(xrs) # write ensemble pdb file, occupancies as sigma level filename = pdb_file_names[0].split('/')[-1].replace( '.pdb', '') + '_vs_' + self.params.ensemble_sigma_map_input.replace( '.mtz', '') + '_pensemble.pdb' write_ensemble_pdb(filename=filename, xrs_list=xrs_list, ens_pdb_hierarchy=ens_pdb_hierarchy) # Do full analysis vs Fobs else: model_map_coeffs = [] fmodel = None # Get <fcalc> for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s): ens_pdb_xrs.set_occupancies(1.0) if model == 0: # If mtz not supplied get fobs from xray structure... # Use input Fobs for scoring against nll if self.params.fobs_vs_fcalc_post_nll: dummy_fobs = f_obs else: if f_obs == None: if self.params.fcalc_high_resolution == None: raise Sorry( "Please supply high resolution limit or input mtz file." ) dummy_dmin = self.params.fcalc_high_resolution dummy_dmax = self.params.fcalc_low_resolution else: print( 'Supplied mtz used to determine high and low resolution cuttoffs', file=self.log) dummy_dmax, dummy_dmin = f_obs.d_max_min() # dummy_fobs = abs( ens_pdb_xrs.structure_factors( d_min=dummy_dmin).f_calc()) dummy_fobs.set_observation_type_xray_amplitude() # If mtz supplied, free flags are over written to prevent array size error r_free_flags = dummy_fobs.array( data=flex.bool(dummy_fobs.data().size(), False)) # fmodel = utils.fmodel_simple( scattering_table="wk1995", xray_structures=[ens_pdb_xrs], f_obs=dummy_fobs, target_name='ls', bulk_solvent_and_scaling=False, r_free_flags=r_free_flags) f_calc_ave = fmodel.f_calc().array( data=fmodel.f_calc().data() * 0).deep_copy() # XXX Important to ensure scale is identical for each model and <model> fmodel.set_scale_switch = 1.0 f_calc_ave_total = fmodel.f_calc().data().deep_copy() else: fmodel.update_xray_structure(xray_structure=ens_pdb_xrs, update_f_calc=True, update_f_mask=False) f_calc_ave_total += fmodel.f_calc().data().deep_copy() print('Model :', model + 1, file=self.log) print("\nStructure vs real Fobs (no bulk solvent or scaling)", file=self.log) print('Rwork : %5.4f ' % fmodel.r_work(), file=self.log) print('Rfree : %5.4f ' % fmodel.r_free(), file=self.log) print('K1 : %5.4f ' % fmodel.scale_k1(), file=self.log) fcalc_edm = fmodel.electron_density_map() fcalc_map_coeffs = fcalc_edm.map_coefficients(map_type='Fc') fcalc_mtz_dataset = fcalc_map_coeffs.as_mtz_dataset( column_root_label='Fc') if self.params.output_model_and_model_ave_mtz: fcalc_mtz_dataset.mtz_object().write( file_name=str(model + 1) + "_Fc.mtz") model_map_coeffs.append(fcalc_map_coeffs.deep_copy()) fmodel.update(f_calc=f_calc_ave.array(f_calc_ave_total / number_structures)) print("\nEnsemble vs real Fobs (no bulk solvent or scaling)", file=self.log) print('Rwork : %5.4f ' % fmodel.r_work(), file=self.log) print('Rfree : %5.4f ' % fmodel.r_free(), file=self.log) print('K1 : %5.4f ' % fmodel.scale_k1(), file=self.log) # Get <Fcalc> map fcalc_ave_edm = fmodel.electron_density_map() fcalc_ave_map_coeffs = fcalc_ave_edm.map_coefficients( map_type='Fc').deep_copy() fcalc_ave_mtz_dataset = fcalc_ave_map_coeffs.as_mtz_dataset( column_root_label='Fc') if self.params.output_model_and_model_ave_mtz: fcalc_ave_mtz_dataset.mtz_object().write(file_name="aveFc.mtz") fcalc_ave_map_coeffs = fcalc_ave_map_coeffs.fft_map() fcalc_ave_map_coeffs.apply_volume_scaling() fcalc_ave_map_data = fcalc_ave_map_coeffs.real_map_unpadded() fcalc_ave_map_stats = maptbx.statistics(fcalc_ave_map_data) print("<Fcalc> Map Stats :", file=self.log) fcalc_ave_map_stats.show_summary(f=self.log) offset = fcalc_ave_map_stats.min() model_neg_ll = [] number_previous_scatters = 0 # Run through structure list again and get probability xrs_list = [] for model, ens_pdb_xrs in enumerate(ens_pdb_xrs_s): if self.params.verbose: print('\n\nModel : ', model + 1, file=self.log) # Get model atom sigmas vs Fcalc fcalc_map = model_map_coeffs[model].fft_map() fcalc_map.apply_volume_scaling() fcalc_map_data = fcalc_map.real_map_unpadded() fcalc_map_stats = maptbx.statistics(fcalc_map_data) if self.params.verbose: print("Fcalc map stats :", file=self.log) fcalc_map_stats.show_summary(f=self.log) xrs = get_map_sigma( ens_pdb_hierarchy=ens_pdb_hierarchy, ens_pdb_xrs=ens_pdb_xrs, fft_map_1=fcalc_map, model_i=model, residue_detail=self.params.residue_detail, ignore_hd=self.params.ignore_hd, number_previous_scatters=number_previous_scatters, log=self.log) fcalc_sigmas = xrs.scatterers().extract_occupancies() del fcalc_map # Get model atom sigmas vs <Fcalc> xrs = get_map_sigma( ens_pdb_hierarchy=ens_pdb_hierarchy, ens_pdb_xrs=ens_pdb_xrs, fft_map_1=fcalc_ave_map_coeffs, model_i=model, residue_detail=self.params.residue_detail, ignore_hd=self.params.ignore_hd, number_previous_scatters=number_previous_scatters, log=self.log) ### For testing other residue averaging options #print xrs.residue_selections fcalc_ave_sigmas = xrs.scatterers().extract_occupancies() # Probability of model given <model> prob = fcalc_ave_sigmas / fcalc_sigmas # XXX debug option if False: for n, p in enumerate(prob): print(' {0:5d} {1:5.3f}'.format(n, p), file=self.log) # Set probabilty between 0 and 1 # XXX Make Histogram / more stats prob_lss_zero = flex.bool(prob <= 0) prob_grt_one = flex.bool(prob > 1) prob.set_selected(prob_lss_zero, 0.001) prob.set_selected(prob_grt_one, 1.0) xrs.set_occupancies(prob) xrs_list.append(xrs) sum_neg_ll = sum(-flex.log(prob)) model_neg_ll.append((sum_neg_ll, model)) if self.params.verbose: print('Model probability stats :', file=self.log) print(prob.min_max_mean().show(), file=self.log) print(' Count < 0.0 : ', prob_lss_zero.count(True), file=self.log) print(' Count > 1.0 : ', prob_grt_one.count(True), file=self.log) # For averaging by residue number_previous_scatters += ens_pdb_xrs.sites_cart().size() # write ensemble pdb file, occupancies as sigma level write_ensemble_pdb( filename=pdb_file_names[0].split('/')[-1].replace('.pdb', '') + '_pensemble.pdb', xrs_list=xrs_list, ens_pdb_hierarchy=ens_pdb_hierarchy) # XXX Test ordering models by nll # XXX Test removing nth percentile atoms if self.params.sort_ensemble_by_nll_score or self.params.fobs_vs_fcalc_post_nll: for percentile in [1.0, 0.975, 0.95, 0.9, 0.8, 0.6, 0.2]: model_neg_ll = sorted(model_neg_ll) f_calc_ave_total_reordered = None print_list = [] for i_neg_ll in model_neg_ll: xrs = xrs_list[i_neg_ll[1]] nll_occ = xrs.scatterers().extract_occupancies() # Set q=0 nth percentile atoms sorted_nll_occ = sorted(nll_occ, reverse=True) number_atoms = len(sorted_nll_occ) percentile_prob_cutoff = sorted_nll_occ[ int(number_atoms * percentile) - 1] cutoff_selections = flex.bool( nll_occ < percentile_prob_cutoff) cutoff_nll_occ = flex.double(nll_occ.size(), 1.0).set_selected( cutoff_selections, 0.0) #XXX Debug if False: print('\nDebug') for x in range(len(cutoff_selections)): print(cutoff_selections[x], nll_occ[x], cutoff_nll_occ[x]) print(percentile) print(percentile_prob_cutoff) print(cutoff_selections.count(True)) print(cutoff_selections.size()) print(cutoff_nll_occ.count(0.0)) print('Count q = 1 : ', cutoff_nll_occ.count(1.0)) print('Count scatterers size : ', cutoff_nll_occ.size()) xrs.set_occupancies(cutoff_nll_occ) fmodel.update_xray_structure(xray_structure=xrs, update_f_calc=True, update_f_mask=True) if f_calc_ave_total_reordered == None: f_calc_ave_total_reordered = fmodel.f_calc().data( ).deep_copy() f_mask_ave_total_reordered = fmodel.f_masks( )[0].data().deep_copy() cntr = 1 else: f_calc_ave_total_reordered += fmodel.f_calc().data( ).deep_copy() f_mask_ave_total_reordered += fmodel.f_masks( )[0].data().deep_copy() cntr += 1 fmodel.update( f_calc=f_calc_ave.array( f_calc_ave_total_reordered / cntr).deep_copy(), f_mask=f_calc_ave.array( f_mask_ave_total_reordered / cntr).deep_copy()) # Update solvent and scale # XXX Will need to apply_back_trace on latest version fmodel.set_scale_switch = 0 fmodel.update_all_scales() # Reset occ for outout xrs.set_occupancies(nll_occ) # k1 updated vs Fobs if self.params.fobs_vs_fcalc_post_nll: print_list.append([ cntr, i_neg_ll[0], i_neg_ll[1], fmodel.r_work(), fmodel.r_free() ]) # Order models by nll and print summary print( '\nModels ranked by nll <Fcalc> R-factors recalculated', file=self.log) print('Percentile cutoff : {0:5.3f}'.format(percentile), file=self.log) xrs_list_sorted_nll = [] print(' | NLL <Rw> <Rf> Ens Model', file=self.log) for info in print_list: print(' {0:4d} | {1:8.1f} {2:8.4f} {3:8.4f} {4:12d}'. format( info[0], info[1], info[3], info[4], info[2] + 1, ), file=self.log) xrs_list_sorted_nll.append(xrs_list[info[2]]) # Output nll ordered ensemble write_ensemble_pdb( filename='nll_ordered_' + pdb_file_names[0].split('/')[-1].replace('.pdb', '') + '_pensemble.pdb', xrs_list=xrs_list_sorted_nll, ens_pdb_hierarchy=ens_pdb_hierarchy)
def run(args, command_name="phenix.twin_map_utils"): log=sys.stdout params=None if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args): print_help(command_name=command_name) else: log = multi_out() if (not "--quiet" in args): log.register(label="stdout", file_object=sys.stdout) string_buffer = StringIO() string_buffer_plots = StringIO() log.register(label="log_buffer", file_object=string_buffer) phil_objects = [] argument_interpreter = master_params.command_line_argument_interpreter( home_scope="map_coefs") for arg in args: command_line_params = None arg_is_processed = False # is it a file? if (os.path.isfile(arg)): ## is this a file name? # check if it is a phil file try: command_line_params = iotbx.phil.parse(file_name=arg) if command_line_params is not None: phil_objects.append(command_line_params) arg_is_processed = True except KeyboardInterrupt: raise except Exception : pass else: try: command_line_params = argument_interpreter.process(arg=arg) if command_line_params is not None: phil_objects.append(command_line_params) arg_is_processed = True except KeyboardInterrupt: raise except Exception : pass if not arg_is_processed: print("##----------------------------------------------##", file=log) print("## Unknown file or keyword:", arg, file=log) print("##----------------------------------------------##", file=log) print(file=log) raise Sorry("Unknown file or keyword: %s" % arg) effective_params = master_params.fetch(sources=phil_objects) params = effective_params.extract() """ new_params = master_params.format(python_object=params) new_params.show(out=log) """ # now get the unit cell from the pdb file hkl_xs = crystal_symmetry_from_any.extract_from( file_name=params.twin_utils.input.xray_data.file_name) pdb_xs = crystal_symmetry_from_any.extract_from( file_name=params.twin_utils.input.model.file_name) phil_xs=None if ([params.twin_utils.input.unit_cell, params.twin_utils.input.space_group]).count(None)<2: phil_xs = crystal.symmetry( unit_cell=params.twin_utils.input.unit_cell, space_group_info=params.twin_utils.input.space_group ) combined_xs = crystal.select_crystal_symmetry( None,phil_xs, [pdb_xs],[hkl_xs]) # inject the unit cell and symmetry in the phil scope please params.twin_utils.input.unit_cell = combined_xs.unit_cell() params.twin_utils.input.space_group = \ sgtbx.space_group_info( group = combined_xs.space_group() ) new_params = master_params.format(python_object=params) new_params.show(out=log) if params.twin_utils.input.unit_cell is None: raise Sorry("unit cell not specified") if params.twin_utils.input.space_group is None: raise Sorry("space group not specified") if params.twin_utils.input.xray_data.file_name is None: raise Sorry("Xray data not specified") if params.twin_utils.input.model.file_name is None: raise Sorry("pdb file with model not specified") #----------------------------------------------------------- # # step 1: read in the reflection file # phil_xs = crystal.symmetry( unit_cell=params.twin_utils.input.unit_cell, space_group_info=params.twin_utils.input.space_group ) xray_data_server = reflection_file_utils.reflection_file_server( crystal_symmetry = phil_xs, force_symmetry = True, reflection_files=[]) miller_array = None free_flags = None tmp_params = extract_xtal_data.data_and_flags_master_params().extract() # insert proper values please tmp_params.file_name = params.twin_utils.input.xray_data.file_name tmp_params.labels = params.twin_utils.input.xray_data.obs_labels tmp_params.r_free_flags.file_name=params.twin_utils.input.xray_data.file_name tmp_params.r_free_flags.label=params.twin_utils.input.xray_data.free_flag tmp_object = extract_xtal_data.run( reflection_file_server = xray_data_server, parameters = tmp_params, log=log ) miller_array = tmp_object.extract_data() if miller_array.is_xray_intensity_array(): miller_array = miller_array.f_sq_as_f() assert miller_array.is_xray_amplitude_array() free_flags = tmp_object.extract_flags(data = miller_array) print(file=log) print("Attempting to extract Free R flags", file=log) free_flags = free_flags.customized_copy( data = flex.bool( free_flags.data()==1 ) ) if free_flags is None: free_flags = miller_array.generate_r_free_flags(use_lattice_symmetry=True) assert miller_array.observation_type() is not None print(file=log) print("Summary info of observed data", file=log) print("=============================", file=log) miller_array.show_summary(f=log) print(file=log) if miller_array.indices().size() == 0: raise Sorry("No data available") #---------------------------------------------------------------- # Step 2: get an xray structure from the PDB file # model = pdb.input(file_name=params.twin_utils.input.model.file_name).xray_structure_simple( crystal_symmetry=phil_xs) print("Atomic model summary", file=log) print("====================", file=log) model.show_summary(f=log) print(file=log) #---------------------------------------------------------------- # step 3: get the twin laws for this xs twin_laws = twin_analyses.twin_laws( miller_array, lattice_symmetry_max_delta=\ params.twin_utils.parameters.twinning.max_delta, out=log) print(file=log) print("Preliminary data analyses", file=log) print("==========================", file=log) twin_laws.show(out=log) #--------- # step 3: # make twin model managers for all twin laws print(file=log) print("Overall and bulk solvent scale paranmeters and twin fraction estimation", file=log) print("=======================================================================", file=log) twin_models = [] operator_count = 0 if params.twin_utils.parameters.twinning.twin_law is not None: tmp_law = sgtbx.rt_mx( params.twin_utils.parameters.twinning.twin_law ) tmp_law = twin_analyses.twin_law(tmp_law,None,None,None,None,None) twin_laws.operators = [ tmp_law ] for twin_law in twin_laws.operators: operator_count += 1 operator_hkl = sgtbx.change_of_basis_op( twin_law.operator ).as_hkl() twin_model = twin_f_model.twin_model_manager( f_obs=miller_array, r_free_flags = free_flags, xray_structure=model, twin_law = twin_law.operator, detwin_mode = params.twin_utils.parameters.twinning.detwin_mode, out=log) print("--- bulk solvent scaling ---", file=log) twin_model.update_all_scales() twin_model.r_values() twin_model.target() twin_model.show_k_sol_b_sol_b_cart_target() twin_model.show_essential() wfofc = twin_model.map_coefficients(map_type="mFo-DFc" ) wtfofc = twin_model.map_coefficients(map_type="2mFo-DFc" ) grad = twin_model.map_coefficients(map_type="gradient" ) mtz_dataset = wtfofc.as_mtz_dataset( column_root_label="FWT") mtz_dataset = mtz_dataset.add_miller_array( miller_array = wfofc, column_root_label = "DFWT" ) mtz_dataset = mtz_dataset.add_miller_array( miller_array = grad, column_root_label = "GRAD" ) name = params.twin_utils.output.map_coeffs_root+"_"+str(operator_count)+".mtz" print(file=log) print("Writing %s for twin law %s"%(name,operator_hkl), file=log) print(file=log) mtz_dataset.mtz_object().write( file_name=name) if params.twin_utils.output.obs_and_calc is not None: # i want also a Fobs and Fmodel combined dataset please mtz_dataset = miller_array.as_mtz_dataset( column_root_label="FOBS") mtz_dataset = mtz_dataset.add_miller_array( miller_array = twin_model.f_model(), column_root_label="FMODEL") name = params.twin_utils.output.obs_and_calc mtz_dataset.mtz_object().write( file_name=name) if len(twin_laws.operators)==0: print(file=log) print("No twin laws were found", file=log) print("Performing maximum likelihood based bulk solvent scaling", file=log) f_model_object = f_model.manager( f_obs = miller_array, r_free_flags = free_flags, xray_structure = model ) f_model_object.update_all_scales(log=log) tfofc = f_model_object.map_coefficients(map_type="2mFobs-DFmodel") fofc = f_model_object.map_coefficients(map_type="mFobs-DFmodel") mtz_dataset = tfofc.as_mtz_dataset( column_root_label="FWT") mtz_dataset = mtz_dataset.add_miller_array( miller_array = fofc, column_root_label = "DELFWT" ) name = params.twin_utils.output.map_coeffs_root+"_ML.mtz" mtz_dataset.mtz_object().write( file_name=name) if params.twin_utils.output.obs_and_calc is not None: # i want also a Fobs and Fmodel combined dataset please mtz_dataset = miller_array.as_mtz_dataset( column_root_label="FOBS") mtz_dataset = mtz_dataset.add_miller_array( miller_array = f_model_object.f_model(), column_root_label="FMODEL") name = params.twin_utils.output.obs_and_calc mtz_dataset.mtz_object().write( file_name=name) print(file=log) print(file=log) print("All done \n", file=log) logfile = open(params.twin_utils.output.logfile,'w') print(string_buffer.getvalue(), file=logfile) print(file=log)
def run(args, out=None, log=sys.stdout): if (len(args) == 0) or (args == ["--help"]): print(msg, file=log) defaults(log=log, silent=False) return parsed = defaults(log=log, silent=True) # processed_args = utils.process_command_line_args(args=args, log=log, master_params=parsed) params = processed_args.params.extract() # reflection_files = processed_args.reflection_files if (len(reflection_files) == 0): raise Sorry("No reflection file found.") crystal_symmetry = processed_args.crystal_symmetry if (crystal_symmetry is None): raise Sorry("No crystal symmetry found.") if (len(processed_args.pdb_file_names) == 0): raise Sorry("No PDB file found.") pdb_file_names = processed_args.pdb_file_names # rfs = reflection_file_server(crystal_symmetry=crystal_symmetry, reflection_files=reflection_files) parameters = extract_xtal_data.data_and_flags_master_params().extract() if (params.f_obs_label is not None): parameters.labels = params.f_obs_label if (params.r_free_flags_label is not None): parameters.r_free_flags.label = params.r_free_flags_label if (params.high_resolution is not None): parameters.high_resolution = params.high_resolution determine_data_and_flags_result = extract_xtal_data.run( reflection_file_server=rfs, parameters=parameters, data_parameter_scope="refinement.input.xray_data", flags_parameter_scope="refinement.input.xray_data.r_free_flags", data_description="X-ray data", keep_going=True, log=StringIO()) f_obs = determine_data_and_flags_result.f_obs # Data show_header(l="Data:", log=log) f_obs.show_comprehensive_summary(prefix=" ", f=log) # R-free-flags show_header(l="R-free-flags:", log=log) r_free_flags = determine_data_and_flags_result.r_free_flags test_flag_value = determine_data_and_flags_result.test_flag_value if (r_free_flags is None): r_free_flags = f_obs.array(data=flex.bool(f_obs.data().size(), False)) test_flag_value = None print(" not available", file=log) else: print(" flag value:", test_flag_value, file=log) # Model pdb_combined = iotbx.pdb.combine_unique_pdb_files( file_names=processed_args.pdb_file_names) pdb_combined.report_non_unique(out=log) if (len(pdb_combined.unique_file_names) == 0): raise Sorry("No coordinate file given.") raw_records = pdb_combined.raw_records try: pdb_inp = iotbx.pdb.input(source_info=None, lines=flex.std_string(raw_records)) except ValueError as e: raise Sorry("Model format (PDB or mmCIF) error:\n%s" % str(e)) model = mmtbx.model.manager(model_input=pdb_inp, crystal_symmetry=crystal_symmetry, log=StringIO()) # scattering_table = params.scattering_table exptl_method = pdb_inp.get_experiment_type() if (exptl_method is not None) and ("NEUTRON" in exptl_method): scattering_table = "neutron" model.setup_scattering_dictionaries(scattering_table=scattering_table, d_min=f_obs.d_min()) # # Model vs data # show_header(l="Model vs Data:", log=log) fmodel = mmtbx.f_model.manager(xray_structure=model.get_xray_structure(), f_obs=f_obs, r_free_flags=r_free_flags, twin_law=params.twin_law) fmodel.update_all_scales(update_f_part1=True) fmodel.show(log=log, show_header=False, show_approx=False) print(" r_work: %6.4f" % fmodel.r_work(), file=log) if (test_flag_value is not None): print(" r_free: %6.4f" % fmodel.r_free(), file=log) else: print(" r_free: None", file=log) print(file=log) n_outl = f_obs.data().size() - fmodel.f_obs().data().size() print(" Number of F-obs outliers:", n_outl, file=log) # # Extract information from PDB file header and output (if any) # pub_r_work = None pub_r_free = None pub_high = None pub_low = None pub_sigma = None pub_program_name = None pub_solv_cont = None pub_matthews = None published_results = pdb_inp.get_r_rfree_sigma(file_name=pdb_file_names[0]) if (published_results is not None): pub_r_work = published_results.r_work pub_r_free = published_results.r_free pub_high = published_results.high pub_low = published_results.low pub_sigma = published_results.sigma pub_program_name = pdb_inp.get_program_name() pub_solv_cont = pdb_inp.get_solvent_content() pub_matthews = pdb_inp.get_matthews_coeff() # show_header(l="Information extracted from PDB file header:", log=log) print(" program_name : %-s" % format_value("%s", pub_program_name), file=log) print(" year : %-s" % format_value("%s", pdb_inp.extract_header_year()), file=log) print(" r_work : %-s" % format_value("%s", pub_r_work), file=log) print(" r_free : %-s" % format_value("%s", pub_r_free), file=log) print(" high_resolution : %-s" % format_value("%s", pub_high), file=log) print(" low_resolution : %-s" % format_value("%s", pub_low), file=log) print(" sigma_cutoff : %-s" % format_value("%s", pub_sigma), file=log) print(" matthews_coeff : %-s" % format_value("%s", pub_matthews), file=log) print(" solvent_cont : %-s" % format_value("%s", pub_solv_cont), file=log) if (exptl_method is not None): print(" exptl_method : %-s" % format_value("%s", exptl_method), file=log) # # Recompute R-factors using published cutoffs fmodel_cut = fmodel tmp_sel = flex.bool(fmodel.f_obs().data().size(), True) if (pub_sigma is not None and fmodel.f_obs().sigmas() is not None): tmp_sel &= fmodel.f_obs().data() > fmodel.f_obs().sigmas() * pub_sigma if (pub_high is not None and abs(pub_high - fmodel.f_obs().d_min()) > 0.03): tmp_sel &= fmodel.f_obs().d_spacings().data() > pub_high if (pub_low is not None and abs(pub_low - fmodel.f_obs().d_max_min()[0]) > 0.03): tmp_sel &= fmodel.f_obs().d_spacings().data() < pub_low if (tmp_sel.count(True) != tmp_sel.size() and tmp_sel.count(True) > 0): show_header(l="After applying resolution and sigma cutoffs:", log=log) fmodel = mmtbx.f_model.manager( xray_structure=model.get_xray_structure(), f_obs=fmodel.f_obs().select(tmp_sel), r_free_flags=fmodel.r_free_flags().select(tmp_sel), twin_law=params.twin_law) fmodel.update_all_scales(update_f_part1=True) fmodel.show(log=log, show_header=False, show_approx=False) print(" r_work: %6.4f" % fmodel.r_work(), file=log) if (test_flag_value is not None): print(" r_free: %6.4f" % fmodel.r_free(), file=log) else: print(" r_free: None", file=log) print(file=log) n_outl = f_obs.data().size() - fmodel.f_obs().data().size() print(" Number of F-obs outliers:", n_outl, file=log)
def run(args, log = sys.stdout): if(len(args)==0): print(legend, file=log) defaults(log=log) return # parsed = defaults(log=log) processed_args = mmtbx.utils.process_command_line_args(args = args, log = sys.stdout, master_params = parsed) params = processed_args.params.extract() reflection_files = processed_args.reflection_files if(len(reflection_files) == 0): if (params.hkl_file is None): raise Sorry("No reflection file found.") else : hkl_in = file_reader.any_file(params.hkl_file, force_type="hkl") hkl_in.assert_file_type("hkl") reflection_files = [ hkl_in.file_object ] crystal_symmetry = processed_args.crystal_symmetry if(crystal_symmetry is None): if (params.space_group is not None) and (params.unit_cell is not None): from cctbx import crystal crystal_symmetry = crystal.symmetry( space_group_info=params.space_group, unit_cell=params.unit_cell) else : raise Sorry("No crystal symmetry found.") if(len(processed_args.pdb_file_names) == 0): if (params.pdb_file is None): raise Sorry("No model file found.") else : pdb_file_names = [ params.pdb_file ] else : pdb_file_names = processed_args.pdb_file_names # rfs = reflection_file_utils.reflection_file_server( crystal_symmetry = crystal_symmetry, force_symmetry = True, reflection_files = reflection_files, err = StringIO()) parameters = extract_xtal_data.data_and_flags_master_params().extract() parameters.labels = params.f_obs_label parameters.r_free_flags.label = params.r_free_flags_label determine_data_and_flags_result = extract_xtal_data.run( reflection_file_server = rfs, parameters = parameters, keep_going = True, log = StringIO()) f_obs = determine_data_and_flags_result.f_obs print("Input data:") print(" Iobs or Fobs:", f_obs.info().labels) r_free_flags = determine_data_and_flags_result.r_free_flags print(" Free-R flags:", r_free_flags.info().labels) # experimental_phases = determine_data_and_flags_result.experimental_phases # if(r_free_flags is None): r_free_flags=f_obs.array(data=flex.bool(f_obs.data().size(), False)) # pdb_inp = mmtbx.utils.pdb_inp_from_multiple_files(pdb_file_names, log=sys.stdout) model = mmtbx.model.manager( model_input = pdb_inp, crystal_symmetry = crystal_symmetry, log = sys.stdout) if(model.get_number_of_models()>1): #XXX support multi-models raise Sorry("Multiple model file not supported in this tool.") # XXX Twining not supported xray_structure = model.get_xray_structure() if (not xray_structure.unit_cell().is_similar_to(f_obs.unit_cell())): raise Sorry("The unit cells in the model and reflections files are not "+ "isomorphous.") print("Input model:") print(" number of atoms:", xray_structure.scatterers().size()) fmodel = mmtbx.f_model.manager( xray_structure = xray_structure, r_free_flags = r_free_flags, f_obs = f_obs, abcd = experimental_phases) fmodel.update_all_scales( update_f_part1 = True, remove_outliers = params.remove_f_obs_outliers, bulk_solvent_and_scaling = params.bulk_solvent_and_scaling) print("Overall statistics:") fmodel.info().show_all() # print("Output data:") if(params.output_file_name is not None): output_file_name = params.output_file_name else: pdb_file_bn = os.path.basename(pdb_file_names[0]) hkl_file_bn = os.path.basename(reflection_files[0].file_name()) try: pdb_file_prefix = pdb_file_bn[:pdb_file_bn.index(".")] except ValueError: pdb_file_prefix = pdb_file_bn try: hkl_file_prefix = hkl_file_bn[:hkl_file_bn.index(".")] except ValueError: hkl_file_prefix = hkl_file_bn output_file_name = "%s_%s.mtz"%(pdb_file_prefix, hkl_file_prefix) print(" file name:", output_file_name) print(" to see the contnt of %s:"%output_file_name) print(" phenix.mtz.dump %s"%output_file_name) out = open(output_file_name,"w") fmodel.export(out = out) out.close() print("All done.") return output_file_name
def run(args, log=sys.stdout, as_gui_program=False): if (len(args) == 0): parsed = defaults(log=log) parsed.show(prefix=" ", out=log) return command_line = (option_parser().enable_symmetry_comprehensive().option( "-q", "--quiet", action="store_true", default=False, help="suppress output").option("--output_plots", action="store_true", default=False)).process(args=args) parsed = defaults(log=log) processed_args = mmtbx.utils.process_command_line_args( args=command_line.args, cmd_cs=command_line.symmetry, master_params=parsed, log=log, suppress_symmetry_related_errors=True) processed_args.params.show(out=log) params = processed_args.params.extract().density_modification output_plots = command_line.options.output_plots crystal_symmetry = crystal.symmetry( unit_cell=params.input.unit_cell, space_group_info=params.input.space_group) reflection_files = {} for rfn in (params.input.reflection_data.file_name, params.input.experimental_phases.file_name, params.input.map_coefficients.file_name): if os.path.isfile(str(rfn)) and rfn not in reflection_files: reflection_files.setdefault( rfn, iotbx.reflection_file_reader.any_reflection_file( file_name=rfn, ensure_read_access=False)) # TODO is reflection_files a dict ? server = iotbx.reflection_file_utils.reflection_file_server( crystal_symmetry=crystal_symmetry, reflection_files=list(reflection_files.values())) o = extract_xtal_data.run( server, parameters=params.input.reflection_data, experimental_phases_params=params.input.experimental_phases, extract_r_free_flags=False) fo = o.f_obs hl_coeffs = o.experimental_phases if params.input.map_coefficients.file_name is not None: map_coeffs = server.get_phases_deg( file_name=params.input.map_coefficients.file_name, labels=params.input.map_coefficients.labels, convert_to_phases_if_necessary=False, original_phase_units=None, parameter_scope="", parameter_name="labels").map_to_asu() else: map_coeffs = None ncs_object = None if params.input.ncs_file_name is not None: ncs_object = ncs.ncs() ncs_object.read_ncs(params.input.ncs_file_name) ncs_object.display_all(log=log) fo = fo.map_to_asu() hl_coeffs = hl_coeffs.map_to_asu() fo = fo.eliminate_sys_absent().average_bijvoet_mates() hl_coeffs = hl_coeffs.eliminate_sys_absent().average_bijvoet_mates() model_map = None model_map_coeffs = None if len(processed_args.pdb_file_names): pdb_inp = mmtbx.utils.pdb_inp_from_multiple_files( pdb_files=processed_args.pdb_file_names, log=log) xs = pdb_inp.xray_structure_simple() fo_, hl_ = fo, hl_coeffs if params.change_basis_to_niggli_cell: change_of_basis_op = xs.change_of_basis_op_to_niggli_cell() xs = xs.change_basis(change_of_basis_op) fo_ = fo_.change_basis(change_of_basis_op).map_to_asu() hl_ = hl_.change_basis(change_of_basis_op).map_to_asu() #fo_, hl_ = fo_.common_sets(hl_) fmodel_refined = mmtbx.utils.fmodel_simple( f_obs=fo_, scattering_table= "wk1995", #XXX pva: 1) neutrons? 2) move up as a parameter. xray_structures=[xs], bulk_solvent_correction=True, anisotropic_scaling=True, r_free_flags=fo_.array(data=flex.bool(fo_.size(), False))) fmodel_refined.update(abcd=hl_) master_phil = mmtbx.maps.map_and_map_coeff_master_params() map_params = master_phil.fetch( iotbx.phil.parse("""\ map_coefficients { map_type = 2mFo-DFc isotropize = True } """)).extract().map_coefficients[0] model_map_coeffs = mmtbx.maps.map_coefficients_from_fmodel( fmodel=fmodel_refined, params=map_params) model_map = model_map_coeffs.fft_map( resolution_factor=params.grid_resolution_factor).real_map_unpadded( ) import time t0 = time.time() dm = density_modify(params, fo, hl_coeffs, ncs_object=ncs_object, map_coeffs=map_coeffs, model_map_coeffs=model_map_coeffs, log=log, as_gui_program=as_gui_program) time_dm = time.time() - t0 print("Time taken for density modification: %.2fs" % time_dm, file=log) # run cns if 0: from cctbx.development import cns_density_modification cns_result = cns_density_modification.run(params, fo, hl_coeffs) print(cns_result.modified_map.all()) print(dm.map.all()) dm_map_coeffs = dm.map_coeffs_in_original_setting from cctbx import maptbx, miller crystal_gridding = maptbx.crystal_gridding( dm_map_coeffs.unit_cell(), space_group_info=dm_map_coeffs.space_group().info(), pre_determined_n_real=cns_result.modified_map.all()) dm_map = miller.fft_map(crystal_gridding, dm_map_coeffs).apply_sigma_scaling() corr = flex.linear_correlation(cns_result.modified_map.as_1d(), dm_map.real_map_unpadded().as_1d()) print("CNS dm/mmtbx dm correlation:") corr.show_summary() if dm.model_map_coeffs is not None: model_map = miller.fft_map( crystal_gridding, dm.miller_array_in_original_setting( dm.model_map_coeffs)).apply_sigma_scaling() corr = flex.linear_correlation( cns_result.modified_map.as_1d(), model_map.real_map_unpadded().as_1d()) print("CNS dm/model correlation:") corr.show_summary() if output_plots: plots_to_make = ( "fom", "skewness", "r1_factor", "r1_factor_fom", "mean_solvent_density", "mean_protein_density", "f000_over_v", "k_flip", "rms_solvent_density", "rms_protein_density", "standard_deviation_local_rms", "mean_delta_phi", "mean_delta_phi_initial", ) from matplotlib.backends.backend_pdf import PdfPages from libtbx import pyplot stats = dm.get_stats() pdf = PdfPages("density_modification.pdf") if len(dm.correlation_coeffs) > 1: if 0: start_coeffs, model_coeffs = dm.map_coeffs_start.common_sets( model_map_coeffs) model_phases = model_coeffs.phases(deg=True).data() exptl_phases = nearest_phase( model_phases, start_coeffs.phases(deg=True).data(), deg=True) corr = flex.linear_correlation(exptl_phases, model_phases) corr.show_summary() fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.set_title("phases start") ax.set_xlabel("Experimental phases") ax.set_ylabel("Phases from refined model") ax.scatter(exptl_phases, model_phases, marker="x", s=10) pdf.savefig(fig) # dm_coeffs, model_coeffs = dm.map_coeffs.common_sets( model_map_coeffs) model_phases = model_coeffs.phases(deg=True).data() dm_phases = nearest_phase(model_phases, dm_coeffs.phases(deg=True).data(), deg=True) corr = flex.linear_correlation(dm_phases, model_phases) corr.show_summary() fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.set_title("phases dm") ax.set_xlabel("Phases from density modification") ax.set_ylabel("Phases from refined model") ax.scatter(dm_phases, model_phases, marker="x", s=10) pdf.savefig(fig) # data = dm.correlation_coeffs fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.set_title("correlation coefficient") ax.plot(list(range(1, dm.i_cycle + 2)), data) pdf.savefig(fig) # data = dm.mean_phase_errors fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.set_title("Mean effective phase errors") ax.plot(list(range(1, dm.i_cycle + 2)), data) pdf.savefig(fig) for plot in plots_to_make: data = [ getattr(stats.get_cycle_stats(i), plot) for i in range(1, dm.i_cycle + 2) ] fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.set_title(plot.replace("_", " ")) ax.plot(list(range(1, dm.i_cycle + 2)), data) pdf.savefig(fig) data = [ stats.get_cycle_stats(i).rms_solvent_density / stats.get_cycle_stats(i).rms_protein_density for i in range(1, dm.i_cycle + 2) ] fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.set_title("RMS solvent/protein density ratio") ax.plot(list(range(1, dm.i_cycle + 2)), data) pdf.savefig(fig) pdf.close() dm_map_coeffs = dm.map_coeffs_in_original_setting dm_hl_coeffs = dm.hl_coeffs_in_original_setting # output map if requested map_params = params.output.map if map_params.file_name is not None: fft_map = dm_map_coeffs.fft_map( resolution_factor=params.grid_resolution_factor) if map_params.scale == "sigma": fft_map.apply_sigma_scaling() else: fft_map.apply_volume_scaling() gridding_first = gridding_last = None title_lines = [] if map_params.format == "xplor": fft_map.as_xplor_map(file_name=map_params.file_name, title_lines=title_lines, gridding_first=gridding_first, gridding_last=gridding_last) else: fft_map.as_ccp4_map(file_name=map_params.file_name, gridding_first=gridding_first, gridding_last=gridding_last, labels=title_lines) # output map coefficients if requested mtz_params = params.output.mtz # Decide if we are going to actually write the mtz if mtz_params.file_name is not None: orig_fom, final_fom = dm.start_and_end_fom() if mtz_params.skip_output_if_worse and final_fom < orig_fom: ok_to_write_mtz = False print( "Not writing out mtz. Final FOM (%7.3f) worse than start (%7.3f)" % (final_fom, orig_fom)) else: # usual ok_to_write_mtz = True else: ok_to_write_mtz = True if mtz_params.file_name is not None and ok_to_write_mtz: label_decorator = iotbx.mtz.ccp4_label_decorator() fo = dm.miller_array_in_original_setting( dm.f_obs_complete).common_set(dm_map_coeffs) mtz_dataset = fo.as_mtz_dataset(column_root_label="F", label_decorator=label_decorator) mtz_dataset.add_miller_array(dm_map_coeffs, column_root_label="FWT", label_decorator=label_decorator) phase_source = dm.miller_array_in_original_setting( dm.phase_source).common_set(dm_map_coeffs) mtz_dataset.add_miller_array( phase_source.array(data=flex.abs(phase_source.data())), column_root_label="FOM", column_types='W', label_decorator=label_decorator) mtz_dataset.add_miller_array( phase_source.array(data=phase_source.phases(deg=True).data()), column_root_label="PHIB", column_types='P', label_decorator=None) if mtz_params.output_hendrickson_lattman_coefficients: mtz_dataset.add_miller_array(dm_hl_coeffs, column_root_label="HL", label_decorator=label_decorator) mtz_dataset.mtz_object().write(mtz_params.file_name) return result(map_file=map_params.file_name, mtz_file=mtz_params.file_name, stats=dm.get_stats())
def __init__(self, args, master_phil, out=sys.stdout, process_pdb_file=True, require_data=True, create_fmodel=True, prefer_anomalous=None, force_non_anomalous=False, set_wavelength_from_model_header=False, set_inelastic_form_factors=None, usage_string=None, create_log_buffer=False, remove_unknown_scatterers=False, generate_input_phil=False): import mmtbx.monomer_library.pdb_interpretation import mmtbx.monomer_library.server import mmtbx.utils import mmtbx.model from iotbx import crystal_symmetry_from_any import iotbx.phil if generate_input_phil: from six import string_types assert isinstance(master_phil, string_types) master_phil = generate_master_phil_with_inputs( phil_string=master_phil) if isinstance(master_phil, str): master_phil = iotbx.phil.parse(master_phil) if (usage_string is not None): if (len(args) == 0) or ("--help" in args): raise Usage("""%s\n\nFull parameters:\n%s""" % (usage_string, master_phil.as_str(prefix=" "))) if (force_non_anomalous): assert (not prefer_anomalous) assert (set_inelastic_form_factors in [None, "sasaki", "henke"]) self.args = args self.master_phil = master_phil self.processed_pdb_file = self.pdb_inp = None self.pdb_hierarchy = self.xray_structure = None self.geometry = None self.sequence = None self.fmodel = None self.f_obs = None self.r_free_flags = None self.intensity_flag = None self.raw_data = None self.raw_flags = None self.test_flag_value = None self.miller_arrays = None self.hl_coeffs = None self.cif_objects = [] self.log = out if ("--quiet" in args) or ("quiet=True" in args): self.log = null_out() elif create_log_buffer: self.log = multi_out() self.log.register(label="stdout", file_object=out) self.log.register(label="log_buffer", file_object=StringIO()) make_header("Collecting inputs", out=self.log) cmdline = iotbx.phil.process_command_line_with_files( args=args, master_phil=master_phil, pdb_file_def="input.pdb.file_name", reflection_file_def="input.xray_data.file_name", cif_file_def="input.monomers.file_name", seq_file_def="input.sequence") self.working_phil = cmdline.work params = self.working_phil.extract() if len(params.input.pdb.file_name) == 0: raise Sorry("At least one PDB file is required as input.") self.cif_file_names = params.input.monomers.file_name self.pdb_file_names = params.input.pdb.file_name # SYMMETRY HANDLING - PDB FILES self.crystal_symmetry = pdb_symm = None for pdb_file_name in params.input.pdb.file_name: pdb_symm = crystal_symmetry_from_any.extract_from(pdb_file_name) if (pdb_symm is not None): break # DATA INPUT data_and_flags = hkl_symm = hkl_in = None if (params.input.xray_data.file_name is None): if (require_data): raise Sorry( "At least one reflections file is required as input.") else: # FIXME this may still require that the data file has full crystal # symmetry defined (although for MTZ input this will not be a problem) make_sub_header("Processing X-ray data", out=self.log) hkl_in = file_reader.any_file(params.input.xray_data.file_name) hkl_in.check_file_type("hkl") hkl_server = hkl_in.file_server symm = hkl_server.miller_arrays[0].crystal_symmetry() if ((symm is None) or (symm.space_group() is None) or (symm.unit_cell() is None)): if (pdb_symm is not None): from iotbx.reflection_file_utils import reflection_file_server print( "No symmetry in X-ray data file - using PDB symmetry:", file=self.log) pdb_symm.show_summary(f=out, prefix=" ") hkl_server = reflection_file_server( crystal_symmetry=pdb_symm, reflection_files=[hkl_in.file_object]) else: raise Sorry( "No crystal symmetry information found in input files." ) if (hkl_server is None): hkl_server = hkl_in.file_server try: pp = params.input.experimental_phases except AttributeError: pp = None data_and_flags = extract_xtal_data.run( reflection_file_server=hkl_server, parameters=params.input.xray_data, experimental_phases_params=pp, prefer_anomalous=prefer_anomalous, force_non_anomalous=force_non_anomalous) self.intensity_flag = data_and_flags.f_obs.is_xray_intensity_array( ) self.raw_data = data_and_flags.raw_data self.raw_flags = data_and_flags.raw_flags self.test_flag_value = data_and_flags.test_flag_value self.f_obs = data_and_flags.f_obs self.r_free_flags = data_and_flags.r_free_flags self.miller_arrays = hkl_in.file_server.miller_arrays self.hl_coeffs = None target_name = "ml" if (data_and_flags.experimental_phases is not None): target_name = "mlhl" self.hl_coeffs = data_and_flags.experimental_phases hkl_symm = self.raw_data.crystal_symmetry() if len(self.cif_file_names) > 0: for file_name in self.cif_file_names: cif_obj = mmtbx.monomer_library.server.read_cif( file_name=file_name) self.cif_objects.append((file_name, cif_obj)) # SYMMETRY HANDLING - COMBINED if (hkl_symm is not None): use_symmetry = hkl_symm # check for weird crystal symmetry # modified from mmtbx.command_line.secondary_structure_restraints # plan to centralize functionality in another location # ------------------------------------------------------------------------- cs = pdb_symm corrupted_cs = False if cs is not None: if [cs.unit_cell(), cs.space_group()].count(None) > 0: corrupted_cs = True cs = None elif cs.unit_cell().volume() < 10: corrupted_cs = True cs = None if cs is None: if corrupted_cs: print("Symmetry information is corrupted,", end=' ', file=out) else: print("Symmetry information was not found,", end=' ', file=out) if (hkl_symm is not None): print("using symmetry from data.", file=out) cs = hkl_symm else: print("putting molecule in P1 box.", file=out) pdb_combined = iotbx.pdb.combine_unique_pdb_files( file_names=self.pdb_file_names) pdb_structure = iotbx.pdb.input(source_info=None, lines=flex.std_string( pdb_combined.raw_records)) atoms = pdb_structure.atoms() box = uctbx.non_crystallographic_unit_cell_with_the_sites_in_its_center( sites_cart=atoms.extract_xyz(), buffer_layer=3) atoms.set_xyz(new_xyz=box.sites_cart) cs = box.crystal_symmetry() pdb_symm = cs # ------------------------------------------------------------------------- from iotbx.symmetry import combine_model_and_data_symmetry self.crystal_symmetry = combine_model_and_data_symmetry( model_symmetry=pdb_symm, data_symmetry=hkl_symm) if (self.crystal_symmetry is not None) and (self.f_obs is not None): self.f_obs = self.f_obs.customized_copy( crystal_symmetry=self.crystal_symmetry).eliminate_sys_absent( ).set_info(self.f_obs.info()) self.r_free_flags = self.r_free_flags.customized_copy( crystal_symmetry=self.crystal_symmetry).eliminate_sys_absent( ).set_info(self.r_free_flags.info()) # PDB INPUT self.unknown_residues_flag = False self.unknown_residues_error_message = False pdb_combined = mmtbx.utils.combine_unique_pdb_files( file_names=params.input.pdb.file_name, ) pdb_combined.report_non_unique(out=self.log) pdb_raw_records = pdb_combined.raw_records try: self.pdb_inp = iotbx.pdb.input( source_info=None, lines=flex.std_string(pdb_raw_records)) except ValueError as e: raise Sorry("Model format (PDB or mmCIF) error:\n%s" % str(e)) if (remove_unknown_scatterers): h = self.pdb_inp.construct_hierarchy() known_sel = h.atom_selection_cache().selection("not element X") if known_sel.count(False) > 0: self.pdb_inp = iotbx.pdb.input( source_info=None, lines=h.select(known_sel).as_pdb_string()) model_params = mmtbx.model.manager.get_default_pdb_interpretation_params( ) pdb_interp_params = getattr(params, "pdb_interpretation", None) if pdb_interp_params is None: pdb_interp_params = iotbx.phil.parse( input_string=mmtbx.monomer_library.pdb_interpretation. grand_master_phil_str, process_includes=True).extract() pdb_interp_params = pdb_interp_params.pdb_interpretation model_params.pdb_interpretation = pdb_interp_params stop_for_unknowns = getattr(pdb_interp_params, "stop_for_unknowns", False) or remove_unknown_scatterers if not process_pdb_file: stop_for_unknowns = True and not remove_unknown_scatterers self.model = mmtbx.model.manager( model_input=self.pdb_inp, crystal_symmetry=self.crystal_symmetry, restraint_objects=self.cif_objects, stop_for_unknowns=stop_for_unknowns, log=self.log) if process_pdb_file: make_sub_header("Processing PDB file(s)", out=self.log) self.model.process(pdb_interpretation_params=model_params, make_restraints=True) full_grm = self.model.get_restraints_manager() self.geometry = full_grm.geometry self.processed_pdb_file = self.model._processed_pdb_file # to remove later XXX self.xray_structure = self.model.get_xray_structure() self.pdb_hierarchy = self.model.get_hierarchy() self.pdb_hierarchy.atoms().reset_i_seq() # wavelength if (params.input.energy is not None): if (params.input.wavelength is not None): raise Sorry("Both wavelength and energy have been specified!") params.input.wavelength = 12398.424468024265 / params.input.energy if (set_wavelength_from_model_header and params.input.wavelength is None): wavelength = self.pdb_inp.extract_wavelength() if (wavelength is not None): print("", file=self.log) print("Using wavelength = %g from PDB header" % wavelength, file=self.log) params.input.wavelength = wavelength # set scattering table if (data_and_flags is not None): self.model.setup_scattering_dictionaries( scattering_table=params.input.scattering_table, d_min=self.f_obs.d_min(), log=self.log, set_inelastic_form_factors=set_inelastic_form_factors, iff_wavelength=params.input.wavelength) self.xray_structure.show_summary(f=self.log) # FMODEL SETUP if (create_fmodel) and (data_and_flags is not None): make_sub_header("F(model) initialization", out=self.log) skip_twin_detection = getattr(params.input, "skip_twin_detection", True) twin_law = getattr(params.input, "twin_law", None) if (twin_law is Auto): if (self.hl_coeffs is not None): raise Sorry( "Automatic twin law determination not supported when " + "experimental phases are used.") elif (not skip_twin_detection): twin_law = Auto if (twin_law is Auto): print("Twinning will be detected automatically.", file=self.log) self.fmodel = mmtbx.utils.fmodel_simple( xray_structures=[self.xray_structure], scattering_table=params.input.scattering_table, f_obs=self.f_obs, r_free_flags=self.r_free_flags, skip_twin_detection=skip_twin_detection, target_name=target_name, log=self.log) else: if ((twin_law is not None) and (self.hl_coeffs is not None)): raise Sorry( "Automatic twin law determination not supported when " + "experimental phases are used.") self.fmodel = mmtbx.utils.fmodel_manager( f_obs=self.f_obs, xray_structure=self.xray_structure, r_free_flags=self.r_free_flags, twin_law=params.input.twin_law, hl_coeff=self.hl_coeffs, target_name=target_name) self.fmodel.update_all_scales(params=None, log=self.log, optimize_mask=True, show=True) self.fmodel.info().show_rfactors_targets_scales_overall( out=self.log) # SEQUENCE if (params.input.sequence is not None): seq_file = file_reader.any_file(params.input.sequence, force_type="seq", raise_sorry_if_errors=True) self.sequence = seq_file.file_object # UNMERGED DATA self.unmerged_i_obs = None if hasattr(params.input, "unmerged_data"): if (params.input.unmerged_data.file_name is not None): self.unmerged_i_obs = load_and_validate_unmerged_data( f_obs=self.f_obs, file_name=params.input.unmerged_data.file_name, data_labels=params.input.unmerged_data.labels, log=self.log) self.params = params print("", file=self.log) print("End of input processing", file=self.log)
def run(args, log=sys.stdout, use_output_directory=True, suppress_fmodel_output=False): print(legend, file=log) print("-" * 79, file=log) master_params = mmtbx.maps.maps_including_IO_master_params() if (len(args) == 0 or (len(args) == 1 and args[0] == "NO_PARAMETER_FILE")): if (not (len(args) == 1 and args[0] == "NO_PARAMETER_FILE")): parameter_file_name = "maps.params" print( "Creating parameter file '%s' in the following directory:\n%s" % (parameter_file_name, os.path.abspath('.')), file=log) if (os.path.isfile(parameter_file_name)): msg = "File '%s' exists already. Re-name it or move and run the command again." raise Sorry(msg % parameter_file_name) pfo = open(parameter_file_name, "w") else: pfo = log print("\nAll phenix.maps parameters::\n", file=pfo) master_params = master_params.fetch(iotbx.phil.parse(default_params)) master_params.show(out=pfo, prefix=" ", expert_level=1) return processed_args = mmtbx.utils.process_command_line_args( args=args, log=log, master_params=master_params) working_phil = processed_args.params params = working_phil.extract() fmodel_data_file_format = params.maps.output.fmodel_data_file_format if (len(params.maps.map_coefficients) == 0) and (len(params.maps.map) == 0): print("No map input specified - using default map types", file=log) working_phil = master_params.fetch( sources=[working_phil, iotbx.phil.parse(default_params)]) params = working_phil.extract() # XXX BUG - the extra fetch will always set fmodel_data_file_format to # mtz; this is probaby a low-level phil problem if (fmodel_data_file_format is None) or (suppress_fmodel_output): params.maps.output.fmodel_data_file_format = None analyze_input_params(params=params) have_phil_file_input = len(processed_args.phil_file_names) > 0 if (len(processed_args.pdb_file_names) > 1): raise Sorry("Only one model file is allowed as input.") if ((params.maps.input.pdb_file_name is None) and (len(processed_args.pdb_file_names) == 1)): params.maps.input.pdb_file_name = processed_args.pdb_file_names[0] if (not os.path.isfile(str(params.maps.input.pdb_file_name))): raise Sorry( "model file is not given: maps.input.pdb_file_name=%s is not a file"%\ str(params.maps.input.pdb_file_name)) if ((params.maps.input.reflection_data.file_name is None) and (params.maps.input.reflection_data.r_free_flags.file_name is None) and (len(processed_args.reflection_file_names) == 1)): params.maps.input.reflection_data.file_name = \ processed_args.reflection_file_names[0] print("FORMAT:", params.maps.output.fmodel_data_file_format, file=log) working_phil = master_params.format(python_object=params) print("-" * 79, file=log) print("\nParameters to compute maps::\n", file=log) working_phil.show(out=log, prefix=" ") pdb_inp = iotbx.pdb.input(file_name=params.maps.input.pdb_file_name) # get all crystal symmetries cs_from_coordinate_files = [pdb_inp.crystal_symmetry_from_cryst1()] cs_from_reflection_files = [] for rfn in [ params.maps.input.reflection_data.file_name, params.maps.input.reflection_data.r_free_flags.file_name ]: if (os.path.isfile(str(rfn))): try: cs_from_reflection_files.append( crystal_symmetry_from_any.extract_from(rfn)) except KeyboardInterrupt: raise except RuntimeError: pass crystal_symmetry = None try: crystal_symmetry = crystal.select_crystal_symmetry( from_coordinate_files=cs_from_coordinate_files, from_reflection_files=cs_from_reflection_files) except AssertionError as e: if ("No unit cell and symmetry information supplied" in str(e)): raise Sorry( "Missing or incomplete symmetry information. This program " + "will only work with reflection file formats that contain both " + "unit cell and space group records, such as MTZ files.") # reflection_files = [] reflection_file_names = [] for rfn in [ params.maps.input.reflection_data.file_name, params.maps.input.reflection_data.r_free_flags.file_name ]: if (os.path.isfile(str(rfn))) and (not rfn in reflection_file_names): reflection_files.append( reflection_file_reader.any_reflection_file( file_name=rfn, ensure_read_access=False)) reflection_file_names.append(rfn) reflection_file_server = reflection_file_utils.reflection_file_server( crystal_symmetry=crystal_symmetry, force_symmetry=True, reflection_files=reflection_files, #[], err=log) # reflection_data_master_params = extract_xtal_data.data_and_flags_master_params( master_scope_name="reflection_data") reflection_data_input_params = processed_args.params.get( "maps.input.reflection_data") reflection_data_params = reflection_data_master_params.fetch( reflection_data_input_params).extract().reflection_data # determine_data_and_flags_result = extract_xtal_data.run( reflection_file_server=reflection_file_server, parameters=reflection_data_params, keep_going=True) f_obs = determine_data_and_flags_result.f_obs r_free_flags = determine_data_and_flags_result.r_free_flags test_flag_value = determine_data_and_flags_result.test_flag_value if (r_free_flags is None): r_free_flags = f_obs.array(data=flex.bool(f_obs.data().size(), False)) test_flag_value = None print("-" * 79, file=log) print("\nInput model file:", params.maps.input.pdb_file_name, file=log) pdb_hierarchy = pdb_inp.construct_hierarchy(set_atom_i_seq=True) atom_selection_manager = pdb_hierarchy.atom_selection_cache() xray_structure = pdb_hierarchy.extract_xray_structure( crystal_symmetry=crystal_symmetry) # apply omit selection if (params.maps.omit.selection is not None): omit_selection = atom_selection_manager.selection( string=params.maps.omit.selection) keep_selection = ~omit_selection xray_structure = xray_structure.select(selection=keep_selection) pdb_hierarchy = pdb_hierarchy.select(keep_selection) atom_selection_manager = pdb_hierarchy.atom_selection_cache() # mmtbx.utils.setup_scattering_dictionaries( scattering_table=params.maps.scattering_table, xray_structure=xray_structure, d_min=f_obs.d_min(), log=log) if (params.maps.wavelength is not None): if (params.maps.scattering_table == "neutron"): raise Sorry( "Wavelength parameter not supported when the neutron " + "scattering table is used.") xray_structure.set_inelastic_form_factors( photon=params.maps.wavelength, table="sasaki") xray_structure.show_summary(f=log, prefix=" ") print("-" * 79, file=log) print("Bulk solvent correction and anisotropic scaling:", file=log) fmodel = mmtbx.utils.fmodel_simple( xray_structures=[xray_structure], scattering_table=params.maps.scattering_table, f_obs=f_obs, r_free_flags=r_free_flags, outliers_rejection=params.maps.input.reflection_data. outliers_rejection, skip_twin_detection=params.maps.skip_twin_detection, bulk_solvent_correction=params.maps.bulk_solvent_correction, anisotropic_scaling=params.maps.anisotropic_scaling) fmodel_info = fmodel.info() fmodel_info.show_rfactors_targets_scales_overall(out=log) print("-" * 79, file=log) print("Compute maps.", file=log) # XXX if run from the Phenix GUI, the output directory parameter is actually # one level up from the current directory, and use_output_directory=False if (params.maps.output.directory is not None) and (use_output_directory): assert os.path.isdir(params.maps.output.directory) output_dir = params.maps.output.directory else: output_dir = os.getcwd() if params.maps.output.prefix is not None: file_name_base = os.path.join( output_dir, os.path.basename(params.maps.output.prefix)) else: file_name_base = params.maps.input.pdb_file_name if (file_name_base.count(".") > 0): file_name_base = file_name_base[:file_name_base.index(".")] xplor_maps = mmtbx.maps.compute_xplor_maps( fmodel=fmodel, params=params.maps.map, atom_selection_manager=atom_selection_manager, file_name_prefix=None, file_name_base=file_name_base, pdb_hierarchy=pdb_hierarchy) cmo = mmtbx.maps.compute_map_coefficients( fmodel=fmodel, params=params.maps.map_coefficients, pdb_hierarchy=pdb_hierarchy, log=log) map_coeff_file_name = file_name_base + "_map_coeffs.mtz" r_free_flags_output = None if (params.maps.output.include_r_free_flags): r_free_flags_output = fmodel.r_free_flags().average_bijvoet_mates() write_mtz_file_result = cmo.write_mtz_file( file_name=map_coeff_file_name, r_free_flags=r_free_flags_output) if (params.maps.output.fmodel_data_file_format is not None): fmodel_file_name = file_name_base + "_fmodel." + \ params.maps.output.fmodel_data_file_format print("Writing fmodel arrays (Fobs, Fcalc, m, ...) to %s file."%\ fmodel_file_name, file=log) fmodel_file_object = open(fmodel_file_name, "w") fmodel.export(out=fmodel_file_object, format=params.maps.output.fmodel_data_file_format) fmodel_file_object.close() print("All done.", file=log) if (write_mtz_file_result): print("Map coefficients: %s" % map_coeff_file_name, file=log) for file_name in xplor_maps: print("CCP4 or XPLOR map: %s" % file_name, file=log) print("-" * 79, file=log) return (map_coeff_file_name, xplor_maps)
def run(args, log): cmdline = mmtbx.utils.process_command_line_args( args = args, master_params = get_master_phil()) if(len(args)==0 or '--help' in args): get_help() return elif('--defaults' in args or '--show' in args): print_legend_and_usage(log) return elif('--version' in args): print(__version__) return print("Running refinement", file=log) cmdline.params.show(out=log, prefix=" ") params = cmdline.params.extract() if len(cmdline.pdb_file_names)==0: raise Sorry('PDB model files not provided and/or not found') model = qr.process_model_file( pdb_file_name = cmdline.pdb_file_names[0], cif_objects = cmdline.cif_objects, crystal_symmetry = cmdline.crystal_symmetry, scattering_table = params.scattering_table) map_data = None fmodel = None if(params.refine.mode=="opt" or params.refine.mode=='gtest'): if (len(cmdline.reflection_files)>0 or cmdline.ccp4_map is not None): print("WARNING: data files not used in optimization or gradient test! ", file=log) elif(len(cmdline.reflection_files)>0 and params.refine.mode=="refine"): # Read reflection data rfs = reflection_file_server( crystal_symmetry = cmdline.crystal_symmetry, reflection_files = cmdline.reflection_files) determine_data_and_flags_result = extract_xtal_data.run( reflection_file_server = rfs, keep_going = True) f_obs = determine_data_and_flags_result.f_obs number_of_reflections = f_obs.indices().size() r_free_flags = determine_data_and_flags_result.r_free_flags test_flag_value = determine_data_and_flags_result.test_flag_value if(r_free_flags is None): r_free_flags=f_obs.generate_r_free_flags() print("WARNING: no free-R flags available in inputs. ", file=log) fmodel = mmtbx.f_model.manager( f_obs = f_obs, r_free_flags = r_free_flags, xray_structure = model.get_xray_structure(), target_name = params.refine.refinement_target_name) if(params.refine.update_all_scales): fmodel.update_all_scales(remove_outliers=False) fmodel.show(show_header=False, show_approx=False) print("Initial r_work=%6.4f r_free=%6.4f" % (fmodel.r_work(), fmodel.r_free()), file=log) elif(cmdline.ccp4_map is not None and params.refine.mode=="refine"): # Read map map_data = cmdline.ccp4_map.map_data() # Normalize map map_data = map_data - flex.mean(map_data) sd = map_data.sample_standard_deviation() assert sd != 0 map_data = map_data/sd else: raise Sorry("Refinement requested (refine.mode==refine) but no data provided.") log.flush() qr.run( model = model, # XXX This is not mmtbx.model.manager !!! (see above). fmodel = fmodel, map_data = map_data, params = params, rst_file = params.rst_file, prefix = os.path.basename(cmdline.pdb_file_names[0])[:-4], log = log)
def run(args, command_name = "phenix.fobs_minus_fobs_map", log=None): if(len(args) == 0): args = ["--help"] examples = """Examples: phenix.fobs_minus_fobs_map f_obs_1_file=data1.mtz f_obs_2_file=data2.sca \ f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 model.pdb phenix.fobs_minus_fobs_map f_obs_1_file=data.mtz f_obs_2_file=data.mtz \ f_obs_1_label=FOBS1 f_obs_2_label=FOBS2 phase_source=model.pdb \ high_res=2.0 sigma_cutoff=2 scattering_table=neutron""" command_line = (iotbx_option_parser( usage="%s [options]" % command_name, description=examples) .option("--silent", action="store_true", help="Suppress output to the screen.") .enable_symmetry_comprehensive() ).process(args=args) # if (log is None): log = sys.stdout if(not command_line.options.silent): utils.print_header("phenix.fobs_minus_fobs_map", out = log) print("Command line arguments: ", file=log) print(args, file=log) print(file=log) # processed_args = utils.process_command_line_args( args=command_line.args, cmd_cs=command_line.symmetry, master_params=fo_minus_fo_master_params(), absolute_angle_tolerance=5, absolute_length_tolerance=1, log=log, suppress_symmetry_related_errors=True) working_phil = processed_args.params if(not command_line.options.silent): print("*** Parameters:", file=log) working_phil.show(out = log) print(file=log) params = working_phil.extract() consensus_symmetry = None if (params.ignore_non_isomorphous_unit_cells): if (None in [params.f_obs_1_file_name, params.f_obs_2_file_name, params.phase_source]): raise Sorry("The file parameters (f_obs_1_file_name, f_obs_2_file_name, "+ "phase_source) must be specified explicitly when "+ "ignore_non_isomorphous_unit_cells=True.") symm_manager = iotbx.symmetry.manager() pdb_in = iotbx.file_reader.any_file(params.phase_source, force_type="pdb") symm_manager.process_pdb_file(pdb_in) hkl_in_1 = iotbx.file_reader.any_file(params.f_obs_1_file_name, force_type="hkl") sg_err_1, uc_err_1 = symm_manager.process_reflections_file(hkl_in_1) hkl_in_2 = iotbx.file_reader.any_file(params.f_obs_2_file_name, force_type="hkl") sg_err_2, uc_err_2 = symm_manager.process_reflections_file(hkl_in_2) out = StringIO() symm_manager.show(out=out) if (sg_err_1) or (sg_err_2): raise Sorry(("Incompatible space groups in input files:\n%s\nAll files "+ "must have the same point group (and ideally the same space group). "+ "Please note that any symmetry information in the PDB file will be "+ "used first.") % out.getvalue()) elif (uc_err_1) or (uc_err_2): libtbx.call_back(message="warn", data=("Crystal symmetry mismatch:\n%s\nCalculations will continue "+ "using the symmetry in the PDB file (or if not available, the "+ "first reflection file), but the maps should be treated with "+ "extreme suspicion.") % out.getvalue()) crystal_symmetry = symm_manager.as_symmetry_object() else : processed_args = utils.process_command_line_args( args=command_line.args, cmd_cs=command_line.symmetry, master_params=fo_minus_fo_master_params(), suppress_symmetry_related_errors = False, absolute_angle_tolerance=5, absolute_length_tolerance=1, log=StringIO()) crystal_symmetry = processed_args.crystal_symmetry # pdb_file_names = processed_args.pdb_file_names if(len(processed_args.pdb_file_names) == 0): if(params.phase_source is not None): pdb_file_names = [params.phase_source] else: raise Sorry("No PDB file found.") # Extaract Fobs1, Fobs2 f_obss = [] if(len(processed_args.reflection_files)==2): for reflection_file in processed_args.reflection_files: reflection_file_server = reflection_file_utils.reflection_file_server( crystal_symmetry = crystal_symmetry, force_symmetry = True, reflection_files = [reflection_file], err = null_out()) # XXX UGLY !!! try: parameters = extract_xtal_data.data_and_flags_master_params().extract() if(params.f_obs_1_label is not None): parameters.labels = [params.f_obs_1_label] determine_data_and_flags_result = extract_xtal_data.run( reflection_file_server = reflection_file_server, keep_going = True, parameters = parameters) except: # intentional parameters = extract_xtal_data.data_and_flags_master_params().extract() if(params.f_obs_2_label is not None): parameters.labels = [params.f_obs_2_label] determine_data_and_flags_result = extract_xtal_data.run( reflection_file_server = reflection_file_server, keep_going = True, parameters = parameters) f_obss.append(determine_data_and_flags_result.f_obs) else: if([params.f_obs_1_file_name,params.f_obs_2_file_name].count(None)==2): raise Sorry("No reflection data file found.") for file_name, label in zip([params.f_obs_1_file_name,params.f_obs_2_file_name], [params.f_obs_1_label,params.f_obs_2_label]): reflection_file = reflection_file_reader.any_reflection_file( file_name = file_name, ensure_read_access = False) reflection_file_server = reflection_file_utils.reflection_file_server( crystal_symmetry = crystal_symmetry, force_symmetry = True, reflection_files = [reflection_file], err = null_out()) parameters = extract_xtal_data.data_and_flags_master_params().extract() if(label is not None): parameters.labels = [label] determine_data_and_flags_result = extract_xtal_data.run( reflection_file_server = reflection_file_server, parameters = parameters, keep_going = True) f_obss.append(determine_data_and_flags_result.f_obs) if(len(f_obss)!=2): raise Sorry(" ".join(errors)) if(not command_line.options.silent): for ifobs, fobs in enumerate(f_obss): print("*** Summary for data set %d:"%ifobs, file=log) fobs.show_comprehensive_summary(f = log) print(file=log) pdb_combined = combine_unique_pdb_files(file_names = pdb_file_names) pdb_combined.report_non_unique(out = log) if(len(pdb_combined.unique_file_names) == 0): raise Sorry("No coordinate file given.") # raw_recs = flex.std_string() for rec in pdb_combined.raw_records: if(rec.upper().count("CRYST1")==0): raw_recs.append(rec) raw_recs.append(iotbx.pdb.format_cryst1_record( crystal_symmetry = crystal_symmetry)) # pdb_in = iotbx.pdb.input(source_info = None, lines = raw_recs) model = mmtbx.model.manager(model_input = pdb_in) d_min = min(f_obss[0].d_min(), f_obss[1].d_min()) model.setup_scattering_dictionaries( scattering_table = params.scattering_table, d_min = d_min) xray_structure = model.get_xray_structure() hierarchy = model.get_hierarchy() # omit_sel = flex.bool(hierarchy.atoms_size(), False) if (params.advanced.omit_selection is not None): print("Will omit selection from phasing model:", file=log) print(" " + params.advanced.omit_selection, file=log) omit_sel = hierarchy.atom_selection_cache().selection( params.advanced.omit_selection) print("%d atoms selected for removal" % omit_sel.count(True), file=log) del hierarchy xray_structure = xray_structure.select(~omit_sel) if(not command_line.options.silent): print("*** Model summary:", file=log) xray_structure.show_summary(f = log) print(file=log) info0 = f_obss[0].info() info1 = f_obss[1].info() f_obss[0] = f_obss[0].resolution_filter(d_min = params.high_resolution, d_max = params.low_resolution).set_info(info0) f_obss[1] = f_obss[1].resolution_filter(d_min = params.high_resolution, d_max = params.low_resolution).set_info(info1) if(params.sigma_cutoff is not None): for i in [0,1]: if(f_obss[i].sigmas() is not None): sel = f_obss[i].data() > f_obss[i].sigmas()*params.sigma_cutoff f_obss[i] = f_obss[i].select(sel).set_info(info0) for k, f_obs in enumerate(f_obss): if (f_obs.indices().size() == 0): raise Sorry("No data left in array %d (labels=%s) after filtering!" % (k+1, f_obs.info().label_string())) output_file_name = params.output_file if (output_file_name is None) and (params.file_name_prefix is not None): output_file_name = "%s_%s.mtz" % (params.file_name_prefix, params.job_id) output_files = compute_fo_minus_fo_map( data_arrays = f_obss, xray_structure = xray_structure, log = log, silent = command_line.options.silent, output_file = output_file_name, peak_search=params.find_peaks_holes, map_cutoff=params.map_cutoff, peak_search_params=params.peak_search, multiscale=params.advanced.multiscale, anomalous=params.advanced.anomalous).file_names return output_files