def run(args, command_name="phenix.remove_outliers"): if (len(args) == 0 or "--help" in args or "--h" in args or "-h" in args): print_help(command_name=command_name) else: log = multi_out() plot_out = None if (not "--quiet" in args): log.register(label="stdout", file_object=sys.stdout) string_buffer = StringIO() string_buffer_plots = StringIO() log.register(label="log_buffer", file_object=string_buffer) phil_objects = [] argument_interpreter = master_params.command_line_argument_interpreter( home_scope="outlier_detection") for arg in args: command_line_params = None arg_is_processed = False # is it a file? if arg == "--quiet": arg_is_processed = True if (os.path.isfile(arg)): ## is this a file name? # check if it is a phil file try: command_line_params = iotbx.phil.parse(file_name=arg) if command_line_params is not None: phil_objects.append(command_line_params) arg_is_processed = True except KeyboardInterrupt: raise except Exception: pass else: try: command_line_params = argument_interpreter.process(arg=arg) if command_line_params is not None: phil_objects.append(command_line_params) arg_is_processed = True except KeyboardInterrupt: raise except Exception: pass if not arg_is_processed: print >> log, "##----------------------------------------------##" print >> log, "## Unknown file or keyword:", arg print >> log, "##----------------------------------------------##" print >> log raise Sorry("Unknown file or keyword: %s" % arg) effective_params = master_params.fetch(sources=phil_objects) params = effective_params.extract() if not os.path.exists(params.outlier_utils.input.xray_data.file_name): raise Sorry("File %s can not be found" % (params.outlier_utils.input.xray_data.file_name)) if params.outlier_utils.input.model.file_name is not None: if not os.path.exists(params.outlier_utils.input.model.file_name): raise Sorry("File %s can not be found" % (params.outlier_utils.input.model.file_name)) # now get the unit cell from the pdb file hkl_xs = None if params.outlier_utils.input.xray_data.file_name is not None: hkl_xs = crystal_symmetry_from_any.extract_from( file_name=params.outlier_utils.input.xray_data.file_name) pdb_xs = None if params.outlier_utils.input.model.file_name is not None: pdb_xs = crystal_symmetry_from_any.extract_from( file_name=params.outlier_utils.input.model.file_name) phil_xs = crystal.symmetry( unit_cell=params.outlier_utils.input.unit_cell, space_group_info=params.outlier_utils.input.space_group) phil_xs.show_summary() hkl_xs.show_summary() combined_xs = crystal.select_crystal_symmetry(None, phil_xs, [pdb_xs], [hkl_xs]) # inject the unit cell and symmetry in the phil scope please params.outlier_utils.input.unit_cell = combined_xs.unit_cell() params.outlier_utils.input.space_group = \ sgtbx.space_group_info( group = combined_xs.space_group() ) new_params = master_params.format(python_object=params) new_params.show(out=log) if params.outlier_utils.input.unit_cell is None: raise Sorry("unit cell not specified") if params.outlier_utils.input.space_group is None: raise Sorry("space group not specified") if params.outlier_utils.input.xray_data.file_name is None: raise Sorry("Xray data not specified") if params.outlier_utils.input.model.file_name is None: print "PDB file not specified. Basic wilson outlier rejections only." #----------------------------------------------------------- # # step 1: read in the reflection file # phil_xs = crystal.symmetry( unit_cell=params.outlier_utils.input.unit_cell, space_group_info=params.outlier_utils.input.space_group) xray_data_server = reflection_file_utils.reflection_file_server( crystal_symmetry=phil_xs, force_symmetry=True, reflection_files=[]) miller_array = None miller_array = xray_data_server.get_xray_data( file_name=params.outlier_utils.input.xray_data.file_name, labels=params.outlier_utils.input.xray_data.obs_labels, ignore_all_zeros=True, parameter_scope='outlier_utils.input.xray_data', parameter_name='obs_labels') info = miller_array.info() miller_array = miller_array.map_to_asu() miller_array = miller_array.select(miller_array.indices() != (0, 0, 0)) #we have to check if the sigma's make any sense at all if not miller_array.sigmas_are_sensible(): miller_array = miller_array.customized_copy( data=miller_array.data(), sigmas=None).set_observation_type(miller_array) miller_array = miller_array.select(miller_array.data() > 0) if miller_array.sigmas() is not None: miller_array = miller_array.select(miller_array.sigmas() > 0) if (miller_array.is_xray_intensity_array()): miller_array = miller_array.f_sq_as_f() elif (miller_array.is_complex_array()): miller_array = abs(miller_array) miller_array.set_info(info) merged_anomalous = False if miller_array.anomalous_flag(): miller_array = miller_array.average_bijvoet_mates( ).set_observation_type(miller_array) merged_anomalous = True miller_array = miller_array.map_to_asu() # get the free reflections please free_flags = None if params.outlier_utils.input.xray_data.free_flags is None: free_flags = miller_array.generate_r_free_flags( fraction=params.outlier_utils.\ additional_parameters.free_flag_generation.fraction, max_free=params.outlier_utils.\ additional_parameters.free_flag_generation.max_number, lattice_symmetry_max_delta=params.outlier_utils.\ additional_parameters.free_flag_generation.lattice_symmetry_max_delta, use_lattice_symmetry=params.outlier_utils.\ additional_parameters.free_flag_generation.use_lattice_symmetry ) else: free_flags = xray_data_server.get_xray_data( file_name=params.outlier_utils.input.xray_data.file_name, labels=params.outlier_utils.input.xray_data.free_flags, ignore_all_zeros=True, parameter_scope='outlier_utils.input.xray_data', parameter_name='free_flags') if free_flags.anomalous_flag(): free_flags = free_flags.average_bijvoet_mates() merged_anomalous = True free_flags = free_flags.customized_copy(data=flex.bool( free_flags.data() == 1)) free_flags = free_flags.map_to_asu() free_flags = free_flags.common_set(miller_array) print >> log print >> log, "Summary info of observed data" print >> log, "=============================" miller_array.show_summary(f=log) if merged_anomalous: print >> log, "For outlier detection purposes, the Bijvoet pairs have been merged." print >> log print >> log, "Constructing an outlier manager" print >> log, "===============================" print >> log outlier_manager = outlier_rejection.outlier_manager(miller_array, free_flags, out=log) basic_array = None extreme_array = None model_based_array = None basic_array = outlier_manager.basic_wilson_outliers( p_basic_wilson = params.outlier_utils.outlier_detection.\ parameters.basic_wilson.level, return_data = True) extreme_array = outlier_manager.extreme_wilson_outliers( p_extreme_wilson = params.outlier_utils.outlier_detection.parameters.\ extreme_wilson.level, return_data = True) beamstop_array = outlier_manager.beamstop_shadow_outliers( level = params.outlier_utils.outlier_detection.parameters.\ beamstop.level, d_min = params.outlier_utils.outlier_detection.parameters.\ beamstop.d_min, return_data=True) #---------------------------------------------------------------- # Step 2: get an xray structure from the PDB file # if params.outlier_utils.input.model.file_name is not None: model = pdb.input(file_name=params.outlier_utils.input.model. file_name).xray_structure_simple( crystal_symmetry=phil_xs) print >> log, "Atomic model summary" print >> log, "====================" model.show_summary(f=log) print >> log # please make an f_model object for bulk solvent scaling etc etc f_model_object = f_model.manager(f_obs=miller_array, r_free_flags=free_flags, xray_structure=model) print >> log, "Bulk solvent scaling of the data" print >> log, "================================" print >> log, "Maximum likelihood bulk solvent scaling." print >> log f_model_object.update_all_scales(log=log, remove_outliers=False) plot_out = StringIO() model_based_array = outlier_manager.model_based_outliers( f_model_object.f_model(), level=params.outlier_utils.outlier_detection.parameters. model_based.level, return_data=True, plot_out=plot_out) #check what needs to be put out please if params.outlier_utils.output.hklout is not None: if params.outlier_utils.outlier_detection.protocol == "model": if params.outlier_utils.input.model.file_name == None: print >> log, "Model based rejections requested. No model was supplied." print >> log, "Switching to writing out rejections based on extreme value Wilson statistics." params.outlier_utils.outlier_detection.protocol = "extreme" output_array = None print >> log if params.outlier_utils.outlier_detection.protocol == "basic": print >> log, "Non-outliers found by the basic wilson statistics" print >> log, "protocol will be written out." output_array = basic_array new_set_of_free_flags = free_flags.common_set(basic_array) if params.outlier_utils.outlier_detection.protocol == "extreme": print >> log, "Non-outliers found by the extreme value wilson statistics" print >> log, "protocol will be written out." output_array = extreme_array new_set_of_free_flags = free_flags.common_set(extreme_array) if params.outlier_utils.outlier_detection.protocol == "model": print >> log, "Non-outliers found by the model based" print >> log, "protocol will be written out to the file:" print >> log, params.outlier_utils.output.hklout print >> log output_array = model_based_array new_set_of_free_flags = free_flags.common_set( model_based_array) if params.outlier_utils.outlier_detection.protocol == "beamstop": print >> log, "Outliers found for the beamstop shadow" print >> log, "problems detection protocol will be written to the file:" print >> log, params.outlier_utils.output.hklout print >> log output_array = model_based_array new_set_of_free_flags = free_flags.common_set( model_based_array) mtz_dataset = output_array.as_mtz_dataset(column_root_label="FOBS") mtz_dataset = mtz_dataset.add_miller_array( miller_array=new_set_of_free_flags, column_root_label="Free_R_Flag") mtz_dataset.mtz_object().write( file_name=params.outlier_utils.output.hklout) if (params.outlier_utils.output.logfile is not None): final_log = StringIO() print >> final_log, string_buffer.getvalue() print >> final_log if plot_out is not None: print >> final_log, plot_out.getvalue() outfile = open(params.outlier_utils.output.logfile, 'w') outfile.write(final_log.getvalue()) print >> log print >> log, "A logfile named %s was created." % ( params.outlier_utils.output.logfile) print >> log, "This logfile contains the screen output and" print >> log, "(possibly) some ccp4 style loggraph plots"
def __init__(self, miller_array, parameters, out=None, n_residues=100, n_bases=0): self.params=parameters self.miller_array=miller_array.deep_copy().set_observation_type( miller_array).merge_equivalents().array() self.out = out if self.out is None: self.out = sys.stdout if self.out == "silent": self.out = null_out() self.no_aniso_array = self.miller_array if self.params.aniso.action == "remove_aniso": # first perfom aniso scaling aniso_scale_and_b = absolute_scaling.ml_aniso_absolute_scaling( miller_array = self.miller_array, n_residues = n_residues, n_bases = n_bases) aniso_scale_and_b.p_scale = 0 # set the p_scale back to 0! aniso_scale_and_b.show(out=out) # now do aniso correction please self.aniso_p_scale = aniso_scale_and_b.p_scale self.aniso_u_star = aniso_scale_and_b.u_star self.aniso_b_cart = aniso_scale_and_b.b_cart if self.params.aniso.final_b == "eigen_min": b_use=aniso_scale_and_b.eigen_values[2] elif self.params.aniso.final_b == "eigen_mean" : b_use=flex.mean(aniso_scale_and_b.eigen_values) elif self.params.aniso.final_b == "user_b_iso": assert self.params.aniso.b_iso is not None b_use=self.params.aniso.b_iso else: b_use = 30 b_cart_aniso_removed = [ -b_use, -b_use, -b_use, 0, 0, 0] u_star_aniso_removed = adptbx.u_cart_as_u_star( miller_array.unit_cell(), adptbx.b_as_u( b_cart_aniso_removed ) ) ## I do things in two steps, but can easely be done in 1 step ## just for clarity, thats all. self.no_aniso_array = absolute_scaling.anisotropic_correction( self.miller_array,0.0,aniso_scale_and_b.u_star ) self.no_aniso_array = absolute_scaling.anisotropic_correction( self.no_aniso_array,0.0,u_star_aniso_removed) self.no_aniso_array = self.no_aniso_array.set_observation_type( miller_array ) # that is done now, now we can do outlier detection if desired outlier_manager = outlier_rejection.outlier_manager( self.no_aniso_array, None, out=self.out) self.new_miller_array = self.no_aniso_array if self.params.outlier.action == "basic": print >> self.out, "Non-outliers found by the basic wilson statistics" print >> self.out, "protocol will be written out." basic_array = outlier_manager.basic_wilson_outliers( p_basic_wilson = self.params.outlier.parameters.basic_wilson.level, return_data = True) self.new_miller_array = basic_array if self.params.outlier.action == "extreme": print >> self.out, "Non-outliers found by the extreme value wilson statistics" print >> self.out, "protocol will be written out." extreme_array = outlier_manager.extreme_wilson_outliers( p_extreme_wilson = self.params.outlier.parameters.extreme_wilson.level, return_data = True) self.new_miller_array = extreme_array if self.params.outlier.action == "beamstop": print >> self.out, "Outliers found for the beamstop shadow" print >> self.out, "problems detection protocol will be written out." beamstop_array = outlier_manager.beamstop_shadow_outliers( level = self.params.outlier.parameters.beamstop.level, d_min = self.params.outlier.parameters.beamstop.d_min, return_data=True) self.new_miller_array = beamstop_array if self.params.outlier.action == "None": self.new_miller_array = self.no_aniso_array # now we can twin or detwin the data if needed self.final_array = self.new_miller_array if self.params.symmetry.action == "twin": alpha = self.params.symmetry.twinning_parameters.fraction if (alpha is None): raise Sorry("Twin fraction not specified, not twinning data") elif not (0 <= alpha <= 0.5): raise Sorry("Twin fraction must be between 0 and 0.5.") print >> self.out print >> self.out, "Twinning given data" print >> self.out, "-------------------" print >> self.out print >> self.out, "Artifically twinning the data with fraction %3.2f" %\ alpha self.final_array = self.new_miller_array.twin_data( twin_law = self.params.symmetry.twinning_parameters.twin_law, alpha=alpha).as_intensity_array() elif (self.params.symmetry.action == "detwin"): twin_law = self.params.symmetry.twinning_parameters.twin_law alpha = self.params.symmetry.twinning_parameters.fraction if (alpha is None): raise Sorry("Twin fraction not specified, not detwinning data") elif not (0 <= alpha <= 0.5): raise Sorry("Twin fraction must be between 0 and 0.5.") print >> self.out, """ Attempting to detwin data ------------------------- Detwinning data with: - twin law: %s - twin fraciton: %.2f BE WARNED! DETWINNING OF DATA DOES NOT SOLVE YOUR TWINNING PROBLEM! PREFERABLY, REFINEMENT SHOULD BE CARRIED OUT AGAINST ORIGINAL DATA ONLY USING A TWIN SPECIFIC TARGET FUNCTION! """ % (twin_law, alpha) self.final_array = self.new_miller_array.detwin_data( twin_law=twin_law, alpha=alpha).as_intensity_array() assert self.final_array is not None
def run(args, command_name="phenix.remove_outliers"): if (len(args)==0 or "--help" in args or "--h" in args or "-h" in args): print_help(command_name=command_name) else: log = multi_out() plot_out = None if (not "--quiet" in args): log.register(label="stdout", file_object=sys.stdout) string_buffer = StringIO() string_buffer_plots = StringIO() log.register(label="log_buffer", file_object=string_buffer) phil_objects = [] argument_interpreter = master_params.command_line_argument_interpreter( home_scope="outlier_detection") for arg in args: command_line_params = None arg_is_processed = False # is it a file? if arg=="--quiet": arg_is_processed = True if (os.path.isfile(arg)): ## is this a file name? # check if it is a phil file try: command_line_params = iotbx.phil.parse(file_name=arg) if command_line_params is not None: phil_objects.append(command_line_params) arg_is_processed = True except KeyboardInterrupt: raise except Exception : pass else: try: command_line_params = argument_interpreter.process(arg=arg) if command_line_params is not None: phil_objects.append(command_line_params) arg_is_processed = True except KeyboardInterrupt: raise except Exception : pass if not arg_is_processed: print >> log, "##----------------------------------------------##" print >> log, "## Unknown file or keyword:", arg print >> log, "##----------------------------------------------##" print >> log raise Sorry("Unknown file or keyword: %s" % arg) effective_params = master_params.fetch(sources=phil_objects) params = effective_params.extract() if not os.path.exists( params.outlier_utils.input.xray_data.file_name ) : raise Sorry("File %s can not be found"%(params.outlier_utils.input.xray_data.file_name) ) if params.outlier_utils.input.model.file_name is not None: if not os.path.exists( params.outlier_utils.input.model.file_name ): raise Sorry("File %s can not be found"%(params.outlier_utils.input.model.file_name) ) # now get the unit cell from the pdb file hkl_xs = None if params.outlier_utils.input.xray_data.file_name is not None: hkl_xs = crystal_symmetry_from_any.extract_from( file_name=params.outlier_utils.input.xray_data.file_name) pdb_xs = None if params.outlier_utils.input.model.file_name is not None: pdb_xs = crystal_symmetry_from_any.extract_from( file_name=params.outlier_utils.input.model.file_name) phil_xs = crystal.symmetry( unit_cell=params.outlier_utils.input.unit_cell, space_group_info=params.outlier_utils.input.space_group ) phil_xs.show_summary() hkl_xs.show_summary() combined_xs = crystal.select_crystal_symmetry( None,phil_xs, [pdb_xs],[hkl_xs]) # inject the unit cell and symmetry in the phil scope please params.outlier_utils.input.unit_cell = combined_xs.unit_cell() params.outlier_utils.input.space_group = \ sgtbx.space_group_info( group = combined_xs.space_group() ) new_params = master_params.format(python_object=params) new_params.show(out=log) if params.outlier_utils.input.unit_cell is None: raise Sorry("unit cell not specified") if params.outlier_utils.input.space_group is None: raise Sorry("space group not specified") if params.outlier_utils.input.xray_data.file_name is None: raise Sorry("Xray data not specified") if params.outlier_utils.input.model.file_name is None: print "PDB file not specified. Basic wilson outlier rejections only." #----------------------------------------------------------- # # step 1: read in the reflection file # phil_xs = crystal.symmetry( unit_cell=params.outlier_utils.input.unit_cell, space_group_info=params.outlier_utils.input.space_group ) xray_data_server = reflection_file_utils.reflection_file_server( crystal_symmetry = phil_xs, force_symmetry = True, reflection_files=[]) miller_array = None miller_array = xray_data_server.get_xray_data( file_name = params.outlier_utils.input.xray_data.file_name, labels = params.outlier_utils.input.xray_data.obs_labels, ignore_all_zeros = True, parameter_scope = 'outlier_utils.input.xray_data', parameter_name = 'obs_labels' ) info = miller_array.info() miller_array = miller_array.map_to_asu() miller_array = miller_array.select( miller_array.indices() != (0,0,0)) #we have to check if the sigma's make any sense at all if not miller_array.sigmas_are_sensible(): miller_array = miller_array.customized_copy( data = miller_array.data(), sigmas=None).set_observation_type(miller_array) miller_array = miller_array.select( miller_array.data() > 0 ) if miller_array.sigmas() is not None: miller_array = miller_array.select( miller_array.sigmas() > 0 ) if (miller_array.is_xray_intensity_array()): miller_array = miller_array.f_sq_as_f() elif (miller_array.is_complex_array()): miller_array = abs(miller_array) miller_array.set_info(info) merged_anomalous=False if miller_array.anomalous_flag(): miller_array = miller_array.average_bijvoet_mates().set_observation_type( miller_array ) merged_anomalous=True miller_array = miller_array.map_to_asu() # get the free reflections please free_flags = None if params.outlier_utils.input.xray_data.free_flags is None: free_flags = miller_array.generate_r_free_flags( fraction=params.outlier_utils.\ additional_parameters.free_flag_generation.fraction, max_free=params.outlier_utils.\ additional_parameters.free_flag_generation.max_number, lattice_symmetry_max_delta=params.outlier_utils.\ additional_parameters.free_flag_generation.lattice_symmetry_max_delta, use_lattice_symmetry=params.outlier_utils.\ additional_parameters.free_flag_generation.use_lattice_symmetry ) else: free_flags = xray_data_server.get_xray_data( file_name = params.outlier_utils.input.xray_data.file_name, labels = params.outlier_utils.input.xray_data.free_flags, ignore_all_zeros = True, parameter_scope = 'outlier_utils.input.xray_data', parameter_name = 'free_flags' ) if free_flags.anomalous_flag(): free_flags = free_flags.average_bijvoet_mates() merged_anomalous=True free_flags = free_flags.customized_copy( data = flex.bool( free_flags.data() == 1 )) free_flags = free_flags.map_to_asu() free_flags = free_flags.common_set( miller_array ) print >> log print >> log, "Summary info of observed data" print >> log, "=============================" miller_array.show_summary(f=log) if merged_anomalous: print >> log, "For outlier detection purposes, the Bijvoet pairs have been merged." print >> log print >> log, "Constructing an outlier manager" print >> log, "===============================" print >> log outlier_manager = outlier_rejection.outlier_manager( miller_array, free_flags, out=log) basic_array = None extreme_array = None model_based_array = None basic_array = outlier_manager.basic_wilson_outliers( p_basic_wilson = params.outlier_utils.outlier_detection.\ parameters.basic_wilson.level, return_data = True) extreme_array = outlier_manager.extreme_wilson_outliers( p_extreme_wilson = params.outlier_utils.outlier_detection.parameters.\ extreme_wilson.level, return_data = True) beamstop_array = outlier_manager.beamstop_shadow_outliers( level = params.outlier_utils.outlier_detection.parameters.\ beamstop.level, d_min = params.outlier_utils.outlier_detection.parameters.\ beamstop.d_min, return_data=True) #---------------------------------------------------------------- # Step 2: get an xray structure from the PDB file # if params.outlier_utils.input.model.file_name is not None: model = pdb.input(file_name=params.outlier_utils.input.model.file_name).xray_structure_simple( crystal_symmetry=phil_xs) print >> log, "Atomic model summary" print >> log, "====================" model.show_summary(f=log) print >> log # please make an f_model object for bulk solvent scaling etc etc f_model_object = f_model.manager( f_obs = miller_array, r_free_flags = free_flags, xray_structure = model ) print >> log, "Bulk solvent scaling of the data" print >> log, "================================" print >> log, "Maximum likelihood bulk solvent scaling." print >> log f_model_object.update_solvent_and_scale(out=log) plot_out = StringIO() model_based_array = outlier_manager.model_based_outliers( f_model_object.f_model(), level=params.outlier_utils.outlier_detection.parameters.model_based.level, return_data=True, plot_out=plot_out) #check what needs to be put out please if params.outlier_utils.output.hklout is not None: if params.outlier_utils.outlier_detection.protocol == "model": if params.outlier_utils.input.model.file_name == None: print >> log, "Model based rejections requested. No model was supplied." print >> log, "Switching to writing out rejections based on extreme value Wilson statistics." params.outlier_utils.outlier_detection.protocol="extreme" output_array = None print >> log if params.outlier_utils.outlier_detection.protocol == "basic": print >> log, "Non-outliers found by the basic wilson statistics" print >> log, "protocol will be written out." output_array = basic_array new_set_of_free_flags = free_flags.common_set( basic_array ) if params.outlier_utils.outlier_detection.protocol == "extreme": print >> log, "Non-outliers found by the extreme value wilson statistics" print >> log, "protocol will be written out." output_array = extreme_array new_set_of_free_flags = free_flags.common_set( extreme_array ) if params.outlier_utils.outlier_detection.protocol == "model": print >> log, "Non-outliers found by the model based" print >> log, "protocol will be written out to the file:" print >> log, params.outlier_utils.output.hklout print >> log output_array = model_based_array new_set_of_free_flags = free_flags.common_set( model_based_array ) if params.outlier_utils.outlier_detection.protocol == "beamstop": print >> log, "Outliers found for the beamstop shadow" print >> log, "problems detection protocol will be written to the file:" print >> log, params.outlier_utils.output.hklout print >> log output_array = model_based_array new_set_of_free_flags = free_flags.common_set( model_based_array ) mtz_dataset = output_array.as_mtz_dataset( column_root_label="FOBS") mtz_dataset = mtz_dataset.add_miller_array( miller_array = new_set_of_free_flags, column_root_label = "Free_R_Flag" ) mtz_dataset.mtz_object().write( file_name=params.outlier_utils.output.hklout) if (params.outlier_utils.output.logfile is not None): final_log = StringIO() print >> final_log, string_buffer.getvalue() print >> final_log if plot_out is not None: print >> final_log, plot_out.getvalue() outfile = open( params.outlier_utils.output.logfile, 'w' ) outfile.write( final_log.getvalue() ) print >> log print >> log, "A logfile named %s was created."%( params.outlier_utils.output.logfile) print >> log, "This logfile contains the screen output and" print >> log, "(possibly) some ccp4 style loggraph plots"
def exercise( d_min=3.5, k_sol=0.3, b_sol=60.0, b_cart=[0, 0, 0, 0, 0, 0], anomalous_flag=False, scattering_table="it1992", space_group_info=None, ): space_groups = [str(space_group_info)] for sg in space_groups: ### get random structure xray_structure = random_structure.xray_structure( space_group_info=sgtbx.space_group_info(sg), elements=(("O", "C", "N") * 50), volume_per_atom=100, min_distance=1.5, general_positions_only=True, random_u_iso=True, ) xray_structure.scattering_type_registry(table=scattering_table) ### Get FOBS for scale in [0.0001, 1.0, 1000.0]: dummy = abs(xray_structure.structure_factors(d_min=d_min, anomalous_flag=anomalous_flag).f_calc()) flags = dummy.generate_r_free_flags(fraction=0.1, max_free=99999999) fmodel = mmtbx.f_model.manager( xray_structure=xray_structure, r_free_flags=flags, target_name="ls_wunit_k1", f_obs=dummy, k_sol=k_sol, b_sol=b_sol, b_cart=b_cart, ) fmodel.update_xray_structure(xray_structure=xray_structure, update_f_calc=True, update_f_mask=True) f_obs = abs(fmodel.f_model()) f_obs = f_obs.array(data=f_obs.data() * scale) f_obs.set_observation_type(observation_type=observation_types.amplitude()) ### look at non-model based outliers detection om = outlier_rejection.outlier_manager(miller_obs=f_obs, r_free_flags=flags, out="silent") tmp1 = om.basic_wilson_outliers() tmp2 = om.extreme_wilson_outliers() tmp3 = om.beamstop_shadow_outliers() # start loop over distorted models for error in [0.0, 0.8]: for fraction in [0.0, 0.5]: # get distorted model xrs_dc = xray_structure.deep_copy_scatterers() sel = xrs_dc.random_remove_sites_selection(fraction=fraction) xrs_dc = xrs_dc.select(sel) xrs_dc.shake_sites_in_place(rms_difference=error) xrs_dc.scattering_type_registry(table=scattering_table) for k_sol in [0.50]: for b_sol in [60.0]: fmodel = mmtbx.f_model.manager( xray_structure=xrs_dc, r_free_flags=flags, target_name="ls_wunit_k1", f_obs=f_obs, k_sol=k_sol, b_sol=b_sol, b_cart=b_cart, ) a, b = fmodel.alpha_beta() o_sel = om.model_based_outliers(f_model=fmodel.f_model()) n_out = o_sel.data().count(False) assert n_out < 5
def __init__(self, miller_array, parameters, out=None, n_residues=100, n_bases=0): self.params=parameters self.miller_array=miller_array.deep_copy().set_observation_type( miller_array).merge_equivalents().array() self.out = out if self.out is None: self.out = sys.stdout if self.out == "silent": self.out = null_out() self.no_aniso_array = self.miller_array if self.params.aniso.action == "remove_aniso": # first perfom aniso scaling aniso_scale_and_b = absolute_scaling.ml_aniso_absolute_scaling( miller_array = self.miller_array, n_residues = n_residues, n_bases = n_bases) aniso_scale_and_b.p_scale = 0 # set the p_scale back to 0! aniso_scale_and_b.show(out=out) # now do aniso correction please self.aniso_p_scale = aniso_scale_and_b.p_scale self.aniso_u_star = aniso_scale_and_b.u_star self.aniso_b_cart = aniso_scale_and_b.b_cart if self.params.aniso.final_b == "eigen_min": b_use=aniso_scale_and_b.eigen_values[2] elif self.params.aniso.final_b == "eigen_mean" : b_use=flex.mean(aniso_scale_and_b.eigen_values) elif self.params.aniso.final_b == "user_b_iso": assert self.params.aniso.b_iso is not None b_use=self.params.aniso.b_iso else: b_use = 30 b_cart_aniso_removed = [ -b_use, -b_use, -b_use, 0, 0, 0] u_star_aniso_removed = adptbx.u_cart_as_u_star( miller_array.unit_cell(), adptbx.b_as_u( b_cart_aniso_removed ) ) ## I do things in two steps, but can easely be done in 1 step ## just for clarity, thats all. self.no_aniso_array = absolute_scaling.anisotropic_correction( self.miller_array,0.0,aniso_scale_and_b.u_star ) self.no_aniso_array = absolute_scaling.anisotropic_correction( self.no_aniso_array,0.0,u_star_aniso_removed) self.no_aniso_array = self.no_aniso_array.set_observation_type( miller_array ) # that is done now, now we can do outlier detection if desired outlier_manager = outlier_rejection.outlier_manager( self.no_aniso_array, None, out=self.out) self.new_miller_array = self.no_aniso_array if self.params.outlier.action == "basic": print >> self.out, "Non-outliers found by the basic wilson statistics" print >> self.out, "protocol will be written out." basic_array = outlier_manager.basic_wilson_outliers( p_basic_wilson = self.params.outlier.parameters.basic_wilson.level, return_data = True) self.new_miller_array = basic_array if self.params.outlier.action == "extreme": print >> self.out, "Non-outliers found by the extreme value wilson statistics" print >> self.out, "protocol will be written out." extreme_array = outlier_manager.extreme_wilson_outliers( p_extreme_wilson = self.params.outlier.parameters.extreme_wilson.level, return_data = True) self.new_miller_array = extreme_array if self.params.outlier.action == "beamstop": print >> self.out, "Outliers found for the beamstop shadow" print >> self.out, "problems detection protocol will be written out." beamstop_array = outlier_manager.beamstop_shadow_outliers( level = self.params.outlier.parameters.beamstop.level, d_min = self.params.outlier.parameters.beamstop.d_min, return_data=True) self.new_miller_array = beamstop_array if self.params.outlier.action == "None": self.new_miller_array = self.no_aniso_array # now we can twin or detwin the data if needed self.final_array = self.new_miller_array if self.params.symmetry.action == "twin": alpha = self.params.symmetry.twinning_parameters.fraction if (alpha is None) : raise Sorry("Twin fraction not specified, not twinning data") elif not (0 <= alpha <= 0.5): raise Sorry("Twin fraction must be between 0 and 0.5.") print >> self.out print >> self.out, "Twinning given data" print >> self.out, "-------------------" print >> self.out print >> self.out, "Artifically twinning the data with fraction %3.2f" %\ alpha self.final_array = self.new_miller_array.twin_data( twin_law = self.params.symmetry.twinning_parameters.twin_law, alpha=alpha).as_intensity_array() elif (self.params.symmetry.action == "detwin") : twin_law = self.params.symmetry.twinning_parameters.twin_law alpha = self.params.symmetry.twinning_parameters.fraction if (alpha is None) : raise Sorry("Twin fraction not specified, not detwinning data") elif not (0 <= alpha <= 0.5): raise Sorry("Twin fraction must be between 0 and 0.5.") print >> self.out, """ Attempting to detwin data ------------------------- Detwinning data with: - twin law: %s - twin fraciton: %.2f BE WARNED! DETWINNING OF DATA DOES NOT SOLVE YOUR TWINNING PROBLEM! PREFERABLY, REFINEMENT SHOULD BE CARRIED OUT AGAINST ORIGINAL DATA ONLY USING A TWIN SPECIFIC TARGET FUNCTION! """ % (twin_law, alpha) self.final_array = self.new_miller_array.detwin_data( twin_law=twin_law, alpha=alpha).as_intensity_array() assert self.final_array is not None
def exercise(d_min = 3.5, k_sol = 0.3, b_sol = 60.0, b_cart = [0,0,0,0,0,0], anomalous_flag = False, scattering_table = "it1992", space_group_info = None): space_groups = [ str(space_group_info) ] for sg in space_groups: ### get random structure xray_structure = random_structure.xray_structure( space_group_info = sgtbx.space_group_info(sg), elements = (("O","C","N")*50), volume_per_atom = 100, min_distance = 1.5, general_positions_only = True, random_u_iso = True) xray_structure.scattering_type_registry(table = scattering_table) ### Get FOBS for scale in [0.0001, 1.0, 1000.0]: dummy = abs(xray_structure.structure_factors( d_min = d_min, anomalous_flag = anomalous_flag).f_calc()) flags = dummy.generate_r_free_flags(fraction = 0.1, max_free = 99999999) fmodel = mmtbx.f_model.manager(xray_structure = xray_structure, r_free_flags = flags, target_name = "ls_wunit_k1", f_obs = dummy, k_sol = k_sol, b_sol = b_sol, b_cart = b_cart) fmodel.update_xray_structure(xray_structure = xray_structure, update_f_calc = True, update_f_mask = True) f_obs = abs(fmodel.f_model()) f_obs = f_obs.array(data = f_obs.data()*scale) f_obs.set_observation_type(observation_type = observation_types.amplitude()) ### look at non-model based outliers detection om = outlier_rejection.outlier_manager(miller_obs = f_obs, r_free_flags = flags, out = "silent") tmp1 = om.basic_wilson_outliers() tmp2 = om.extreme_wilson_outliers() tmp3 = om.beamstop_shadow_outliers() # start loop over distorted models for error in [0.0, 0.8]: for fraction in [0.0,0.5]: # get distorted model xrs_dc = xray_structure.deep_copy_scatterers() sel = xrs_dc.random_remove_sites_selection(fraction = fraction) xrs_dc = xrs_dc.select(sel) xrs_dc.shake_sites_in_place(rms_difference=error) xrs_dc.scattering_type_registry(table = scattering_table) for k_sol in [0.50,]: for b_sol in [60.,]: fmodel = mmtbx.f_model.manager( xray_structure = xrs_dc, r_free_flags = flags, target_name = "ls_wunit_k1", f_obs = f_obs, k_sol = k_sol, b_sol = b_sol, b_cart = b_cart) a,b = fmodel.alpha_beta() o_sel = om.model_based_outliers(f_model = fmodel.f_model()) n_out = o_sel.data().count(False) assert (n_out < 5)