def compute_functional_and_gradients(O): if (O.number_of_function_evaluations == 0): O.number_of_function_evaluations += 1 return O.f_start, O.g_start O.number_of_function_evaluations += 1 O.sites_cart_residue = flex.vec3_double(O.x) rs_f = maptbx.real_space_target_simple( unit_cell=O.unit_cell, density_map=O.density_map, sites_cart=O.sites_cart_residue, selection=flex.bool(O.sites_cart_residue.size(),True)) O.real_space_target = rs_f rs_g = maptbx.real_space_gradients_simple( unit_cell=O.unit_cell, density_map=O.density_map, sites_cart=O.sites_cart_residue, delta=O.real_space_gradients_delta, selection=flex.bool(O.sites_cart_residue.size(),True)) O.rs_f = rs_f rs_f *= -O.real_space_target_weight rs_g *= -O.real_space_target_weight if (O.geometry_restraints_manager is None): f = rs_f g = rs_g else: O.sites_cart_all.set_selected(O.residue_i_seqs, O.sites_cart_residue) gr_e = O.geometry_restraints_manager.energies_sites( sites_cart=O.sites_cart_all, compute_gradients=True) f = rs_f + gr_e.target g = rs_g + gr_e.gradients.select(indices=O.residue_i_seqs) return f, g.as_double()
def _select(self, x, selection): try: lx = len(x) except Exception: lx = x.size() if(lx == 0): return x if(self.is_bool(x)): x = x.select(selection) elif(self.is_size_t(x[0])): x_new = [] for i_seq, item in enumerate(x): val = flex.bool(selection.size(), item).select(selection).iselection() if(val.size() > 0): x_new.append(val) x = x_new elif(self.is_size_t(x[0][0])): x_new = [] for item1 in x: tmp = [] for item2 in item1: if(self.is_size_t(item2)): v=flex.bool(selection.size(),item2).select(selection).iselection() if(v.size() > 0): tmp.append(v) else: raise RuntimeError("Bad selection array type.") if(len(tmp) > 0): x_new.append(tmp) x = x_new else: raise RuntimeError("Bad selection array type.") return x
def add_new_solvent(self): b_solv = self.params.b_iso new_scatterers = flex.xray_scatterer( self.sites.size(), xray.scatterer(occupancy = self.params.occupancy, b = b_solv, scattering_type = self.params.scattering_type, label = 'HOH')) new_scatterers.set_sites(self.sites) solvent_xray_structure = xray.structure( special_position_settings = self.model.xray_structure, scatterers = new_scatterers) xrs_sol = self.model.xray_structure.select(self.model.solvent_selection()) xrs_mac = self.model.xray_structure.select(~self.model.solvent_selection()) xrs_sol = xrs_sol.concatenate(other = solvent_xray_structure) sol_sel = flex.bool(xrs_mac.scatterers().size(), False) sol_sel.extend( flex.bool(xrs_sol.scatterers().size(), True) ) self.model.add_solvent( solvent_xray_structure = solvent_xray_structure, residue_name = self.params.output_residue_name, atom_name = self.params.output_atom_name, chain_id = self.params.output_chain_id, refine_occupancies = self.params.refine_occupancies, refine_adp = self.params.new_solvent) self.fmodel.update_xray_structure( xray_structure = self.model.xray_structure, update_f_calc = False)
def test_minimum_multiplicity_selection(): # some groups of indices with increasing multiplicity hkl = flex.miller_index( [(0,0,1), (0,0,2), (0,0,2), (0,0,3), (0,0,3), (0,0,3), (0,0,4), (0,0,4), (0,0,4), (0,0,4), (0,0,5), (0,0,5), (0,0,5), (0,0,5), (0,0,5)]) # test the various possible selections with multiplicity from 1 to 6 sel = minimum_multiplicity_selection(hkl, 1) assert sel.count(True) == len(hkl) sel = minimum_multiplicity_selection(hkl, 2) assert (sel == flex.bool([False,] + [True] * 14)).all_eq(True) sel = minimum_multiplicity_selection(hkl, 3) assert (sel == flex.bool([False] * 3 + [True] * 12)).all_eq(True) sel = minimum_multiplicity_selection(hkl, 4) assert (sel == flex.bool([False] * 6 + [True] * 9)).all_eq(True) sel = minimum_multiplicity_selection(hkl, 5) assert (sel == flex.bool([False] * 10 + [True] * 5)).all_eq(True) sel = minimum_multiplicity_selection(hkl, 6) assert sel.count(False) == len(hkl) print "OK"
def exercise_expand(): sg = sgtbx.space_group("P 41 (1,-1,0)") h = flex.miller_index(((3,1,-2), (1,-2,0))) assert tuple(sg.is_centric(h)) == (0, 1) p1 = miller.expand_to_p1_iselection( space_group=sg, anomalous_flag=False, indices=h, build_iselection=False) p1_i0 = ((-3,-1,2), (-1, 3,2),(3,1,2),(1,-3,2),(1,-2, 0),(2,1,0)) assert tuple(p1.indices) == p1_i0 assert p1.iselection.size() == 0 p1 = miller.expand_to_p1_iselection( space_group=sg, anomalous_flag=True, indices=h, build_iselection=False) assert tuple(p1.indices) \ == ((3,1,-2), (1,-3,-2), (-3,-1,-2), (-1,3,-2), (1,-2,0), (-2,-1,0), (-1,2,0), (2,1,0)) p1 = miller.expand_to_p1_iselection( space_group=sg, anomalous_flag=False, indices=h, build_iselection=True) assert tuple(p1.indices) == p1_i0 assert tuple(p1.iselection) == (0,0,0,0,1,1) a = flex.double((1,2)) p = flex.double((10,90)) p1 = miller.expand_to_p1_phases( space_group=sg, anomalous_flag=False, indices=h, data=p, deg=True) assert approx_equal(tuple(p1.data), (-10,110,110,-10, 90,30)) p1 = miller.expand_to_p1_phases( space_group=sg, anomalous_flag=True, indices=h, data=p, deg=True) assert approx_equal(tuple(p1.data), (10,-110,-110,10, 90,-30,-90,30)) p = flex.double([x * math.pi/180 for x in p]) v = [x * math.pi/180 for x in p1.data] p1 = miller.expand_to_p1_phases( space_group=sg, anomalous_flag=True, indices=h, data=p, deg=False) assert approx_equal(tuple(p1.data), v) f = flex.polar(a, p) p1 = miller.expand_to_p1_complex( space_group=sg, anomalous_flag=True, indices=h, data=f) assert approx_equal(tuple(flex.abs(p1.data)), (1,1,1,1,2,2,2,2)) assert approx_equal(tuple(flex.arg(p1.data)), v) hl = flex.hendrickson_lattman([(1,2,3,4), (5,6,7,8)]) p1 = miller.expand_to_p1_hendrickson_lattman( space_group=sg, anomalous_flag=True, indices=h, data=hl) assert approx_equal(p1.data, [ [1,2,3,4], [1.232051,-1.866025,-4.964102,0.5980762], [1.232051,-1.866025,-4.964102,0.5980762], [1,2,3,4], [5,6,7,8], [2.696152,-7.330127,-10.4282,2.062178], [-5,-6,7,8], [7.696152,-1.330127,3.428203,-10.06218]]) b = flex.bool([True,False]) p1 = miller.expand_to_p1_iselection( space_group=sg, anomalous_flag=True, indices=h, build_iselection=True) assert b.select(p1.iselection).all_eq( flex.bool([True, True, True, True, False, False, False, False])) i = flex.int([13,17]) p1 = miller.expand_to_p1_iselection( space_group=sg, anomalous_flag=True, indices=h, build_iselection=True) assert i.select(p1.iselection).all_eq(flex.int([13,13,13,13,17,17,17,17])) # assert approx_equal(miller.statistical_mean(sg, False, h, a), 4/3.) assert approx_equal(miller.statistical_mean(sg, True, h, a), 3/2.)
def exclude_outliers_from_reference_restraints_selection( pdb_hierarchy, restraints_selection): from mmtbx.validation.ramalyze import ramalyze # the import below is SLOW!!! from mmtbx.rotamer.rotamer_eval import RotamerEval assert restraints_selection is not None # ramachandran plot outliers rama_outlier_selection = ramalyze(pdb_hierarchy=pdb_hierarchy, outliers_only=False).outlier_selection() rama_outlier_selection = flex.bool(restraints_selection.size(), rama_outlier_selection) # rotamer outliers rota_outlier_selection = flex.size_t() rotamer_manager = RotamerEval() # SLOW!!! for model in pdb_hierarchy.models(): for chain in model.chains(): for residue_group in chain.residue_groups(): conformers = residue_group.conformers() if(len(conformers)>1): continue for conformer in residue_group.conformers(): residue = conformer.only_residue() if(rotamer_manager.evaluate_residue(residue)=="OUTLIER"): rota_outlier_selection.extend(residue.atoms().extract_i_seq()) rota_outlier_selection = flex.bool(restraints_selection.size(), rota_outlier_selection) outlier_selection = rama_outlier_selection | rota_outlier_selection return restraints_selection & (~outlier_selection)
def rebatch(hklin, hklout, first_batch=None, include_range=None, exclude_range=None): """Need to implement: include batch range, exclude batches, add N to batches, start batches at N.""" if include_range is None: include_range = [] if exclude_range is None: exclude_range = [] assert not (len(include_range) and len(exclude_range)) assert not (len(include_range) and first_batch) assert not (len(exclude_range) and first_batch) mtz_obj = mtz.object(file_name=hklin) batch_column = None batch_dataset = None for crystal in mtz_obj.crystals(): for dataset in crystal.datasets(): for column in dataset.columns(): if column.label() == "BATCH": batch_column = column batch_dataset = dataset if not batch_column: raise RuntimeError, "no BATCH column found in %s" % hklin batch_column_values = batch_column.extract_values(not_a_number_substitute=-1) valid = flex.bool() if exclude_range: exclude_sel = flex.bool(batch_column_values.size(), False) for (start, end) in exclude_range: exclude_sel.set_selected((batch_column_values >= start) & (batch_column_values <= end), True) mtz_obj.delete_reflections(exclude_sel.iselection()) elif include_range: exclude_sel = flex.bool(batch_column_values.size(), True) for (start, end) in include_range: exclude_sel.set_selected((batch_column_values >= start) & (batch_column_values <= end), False) mtz_obj.delete_reflections(exclude_sel.iselection()) # modify batch columns, and also the batch headers elif first_batch is not None: offset = first_batch - min(batch_column_values) batch_column_values = batch_column_values + offset for batch in mtz_obj.batches(): batch.set_num(int(batch.num() + offset)) # done modifying batch_column.set_values(values=batch_column_values, selection_valid=valid) # and write this lot out as hklout mtz_obj.write(file_name=hklout)
def include_residue_selection(selection, residue_iselection): size = selection.size() selection_ = selection.deep_copy() selection_ = selection_.iselection() selection_.extend(residue_iselection) new_sel = flex.bool(size, selection_) rs = flex.bool(size, residue_iselection).select(new_sel).iselection() return new_sel, rs
def e_pot(O, sites_moved): if ( O.last_sites_moved is None or O.last_sites_moved.id() is not sites_moved.id()): O.last_sites_moved = sites_moved sites_cart = sites_moved # if (O.reduced_geo_manager is None): flags = None if (O.orca_experiments): flags = cctbx.geometry_restraints.flags.flags( nonbonded=False, default=True) else: # computing nonbonded interactions only with geo_manager, # contributions from other restraints below with reduced_geo_manager flags = cctbx.geometry_restraints.flags.flags( nonbonded=True, default=False) geo_energies = O.geo_manager.energies_sites( sites_cart=sites_cart, flags=flags, custom_nonbonded_function=O.custom_nonbonded_function, compute_gradients=True) if (0): # XXX orca_experiments print "geo_energies:" geo_energies.show() if (0): # XXX orca_experiments O.geo_manager.show_sorted(site_labels=O.site_labels) O.f = geo_energies.target O.g = geo_energies.gradients if (O.reduced_geo_manager is not None): reduced_geo_energies = O.reduced_geo_manager.energies_sites( sites_cart=sites_cart, compute_gradients=True) O.f += reduced_geo_energies.target O.g += reduced_geo_energies.gradients O.last_grms = group_args(geo=flex.mean_sq(O.g.as_double())**0.5) # if (O.density_map is not None): rs_f = maptbx.real_space_target_simple( unit_cell=O.geo_manager.crystal_symmetry.unit_cell(), density_map=O.density_map, sites_cart=sites_cart, selection=flex.bool(sites_cart.size(),True)) rs_g = maptbx.real_space_gradients_simple( unit_cell=O.geo_manager.crystal_symmetry.unit_cell(), density_map=O.density_map, sites_cart=sites_cart, delta=O.real_space_gradients_delta, selection=flex.bool(sites_cart.size(),True)) rs_f *= -O.real_space_target_weight rs_g *= -O.real_space_target_weight O.f += rs_f O.g += rs_g O.last_grms.real = flex.mean_sq(rs_g.as_double())**0.5 O.last_grms.real_or_xray = "real" # O.last_grms.total = flex.mean_sq(O.g.as_double())**0.5 return O.f
def inflate(self, sites_individual = None, sites_torsion_angles = None, sites_rigid_body = None, adp_individual_iso = None, adp_individual_aniso = None, adp_group = None, group_h = None, adp_tls = None, s_occupancies = None, occupancies_group = None, size_all = None): # XXX group_anomalous selection should be added if(sites_individual is not None and self.sites_individual is not None): assert self.is_bool(sites_individual) self.sites_individual.extend(sites_individual) if( sites_torsion_angles is not None and self.sites_torsion_angles is not None): assert self.is_bool(sites_torsion_angles) self.sites_torsion_angles.extend(sites_torsion_angles) if(adp_individual_iso is not None): assert self.is_bool(adp_individual_iso) if(self.adp_individual_iso is None): assert size_all is not None self.adp_individual_iso = flex.bool(size_all, False) # inflate existing iso flags if present elif (self.adp_individual_iso.size() < size_all) : n_new = size_all - self.adp_individual_iso.size() self.adp_individual_iso.extend(flex.bool(n_new, False)) self.adp_individual_iso.extend(adp_individual_iso) if(adp_individual_aniso is not None): assert self.is_bool(adp_individual_aniso) if(self.adp_individual_aniso is None): assert size_all is not None self.adp_individual_aniso = flex.bool(size_all, False) # inflate existing aniso flags if present elif (self.adp_individual_aniso.size() < size_all) : n_new = size_all - self.adp_individual_aniso.size() self.adp_individual_aniso.extend(flex.bool(n_new, False)) self.adp_individual_aniso.extend(adp_individual_aniso) if(sites_rigid_body is not None): assert hasattr(sites_rigid_body, 'count') self.sites_rigid_body.extend(sites_rigid_body) if(adp_group is not None): assert hasattr(adp_group, 'count') self.adp_group.extend(adp_group) if(group_h is not None): assert hasattr(group_h, 'count') self.group_h.extend(group_h) if(adp_tls is not None): assert hasattr(adp_tls, 'count') self.adp_tls.extend(adp_tls) if(s_occupancies is not None): assert hasattr(s_occupancies, 'count') if(self.s_occupancies is not None): self.s_occupancies.extend(s_occupancies) self.check_all() return self
def move_solvent_to_the_end_of_atom_list(self): solsel = flex.bool(self.model.solvent_selection().count(False), False) solsel.extend(flex.bool(self.model.solvent_selection().count(True), True)) xrs_sol = self.model.xray_structure.select(self.model.solvent_selection()) if(xrs_sol.hd_selection().count(True) == 0): self.reset_solvent( solvent_selection = solsel, solvent_xray_structure = xrs_sol) self.model.renumber_water() self.fmodel.xray_structure = self.model.xray_structure
def __init__ (self, miller_array, settings, merge=None) : self.miller_array = miller_array self.settings = settings self.merge_equivalents = merge from cctbx import miller from cctbx import crystal from cctbx.array_family import flex self.multiplicities = None self.process_input_array() array = self.work_array uc = array.unit_cell() self.unit_cell = uc self.slice_selection = None self.axis_index = None if (settings.slice_mode) : self.axis_index = ["h","k","l"].index(self.settings.slice_axis) self.slice_selection = miller.simple_slice( indices=array.indices(), slice_axis=self.axis_index, slice_index=settings.slice_index) #if (self.slice_selection.count(True) == 0) : #raise ValueError("No data selected!") index_span = array.index_span() self.d_min = array.d_min() self.hkl_range = index_span.abs_range() self.axes = [ uc.reciprocal_space_vector((self.hkl_range[0],0,0)), uc.reciprocal_space_vector((0,self.hkl_range[1],0)), uc.reciprocal_space_vector((0,0,self.hkl_range[2])) ] self.generate_view_data() if (self.slice_selection is not None) : self.indices = self.work_array.indices().select(self.slice_selection) self.data = self.data.select(self.slice_selection) else : self.indices = array.indices() self.points = uc.reciprocal_space_vector(self.indices) * 100. self.missing_flags = flex.bool(self.radii.size(), False) self.sys_absent_flags = flex.bool(self.radii.size(), False) if (settings.show_missing) : self.generate_missing_reflections() if (settings.show_systematic_absences) and (not settings.show_only_missing): self.generate_systematic_absences() # XXX hack for process_pick_points self.visible_points = flex.bool(self.points.size(), True) n_points = self.points.size() assert (self.colors.size() == n_points) assert (self.indices.size() == n_points) assert (self.radii.size() == n_points) assert (self.missing_flags.size() == n_points) assert (self.sys_absent_flags.size() == n_points) assert (self.data.size() == n_points) self.clear_labels()
def exercise_1(mon_lib_srv, ener_lib): pdb_in = simple_pdb() params = pdb_interpretation.master_params.extract() processed_pdb_file = pdb_interpretation.process( mon_lib_srv=mon_lib_srv, ener_lib=ener_lib, params=params, pdb_inp=pdb_in, log=StringIO()) grm = processed_pdb_file.geometry_restraints_manager() pdb_hierarchy = processed_pdb_file.all_chain_proxies.pdb_hierarchy sites_cart = pdb_hierarchy.atoms().extract_xyz() proxies = reference.add_coordinate_restraints(sites_cart=sites_cart) assert proxies.size() == 29, "expected 29, got %d" % proxies.size() import boost.python ext = boost.python.import_ext("mmtbx_reference_coordinate_ext") grads = flex.vec3_double(sites_cart.size(), (0.0,0.0,0.0)) residual = ext.reference_coordinate_residual_sum( sites_cart=sites_cart, proxies=proxies, gradient_array=grads) assert approx_equal(residual, 0.0) #test selection ca_selection = pdb_hierarchy.get_peptide_c_alpha_selection() ca_sites_cart = sites_cart.select(ca_selection) proxies = reference.add_coordinate_restraints( sites_cart=ca_sites_cart, selection=ca_selection) assert proxies.size() == 3, "expected 3, got %d" % proxies.size() tst_iselection = flex.size_t() for atom in pdb_hierarchy.atoms(): if atom.name == " CA " or atom.name == " N ": tst_iselection.append(atom.i_seq) tst_sites_cart = sites_cart.select(tst_iselection) proxies = reference.add_coordinate_restraints( sites_cart=tst_sites_cart, selection=tst_iselection) assert proxies.size() == 6, "expected 6, got %d" % proxies.size() #test remove selection = flex.bool([False]*29) proxies = proxies.proxy_remove(selection=selection) assert proxies.size() == 6, "expected 6, got %d" % proxies.size() proxies = proxies.proxy_remove(selection=ca_selection) assert proxies.size() == 3, "expected 3, got %d" % proxies.size() selection = flex.bool([True]*29) proxies = proxies.proxy_remove(selection=selection) assert proxies.size() == 0, "expected 0, got %d" % proxies.size()
def generate_torsion_restraints( pdb_hierarchy, sites_cart, selection=None, sigma=2.5, limit=15.0, chi_angles_only=False, top_out_potential=False, origin_id=2): torsion_proxies = geometry_restraints.shared_dihedral_proxy() if pdb_hierarchy.atoms().size() < 4: return torsion_proxies assert not pdb_hierarchy.atoms().extract_i_seq().all_eq(0) bool_pdbh_selection = flex.bool(pdb_hierarchy.atoms_size(), False) if (selection is not None): if (isinstance(selection, flex.bool)): bool_pdbh_selection = selection elif (isinstance(selection, flex.size_t)): bool_pdbh_selection.set_selected(selection, True) if selection is None: bool_pdbh_selection = flex.bool(pdb_hierarchy.atoms_size(), True) actual_selection = bool_pdbh_selection.iselection() assert len(sites_cart) == len(actual_selection) weight = 1.0 / (sigma**2) selection_to_sites_map = get_selection_to_sites_map( sites_cart=sites_cart, selection=actual_selection) residue_torsions = collect_residue_torsion_angles( pdb_hierarchy=pdb_hierarchy, atom_selection=bool_pdbh_selection, chi_angles_only=chi_angles_only) for residue_info in residue_torsions: for chi in residue_info.chis: i_seqs = chi.i_seqs sites = [] for i_seq in i_seqs: sites.append(selection_to_sites_map[i_seq]) di = geometry_restraints.dihedral( sites=sites, angle_ideal=0.0, weight=weight) angle_ideal = di.angle_model dp = geometry_restraints.dihedral_proxy( i_seqs=i_seqs, angle_ideal=angle_ideal, weight=weight, limit=limit, top_out=top_out_potential, origin_id=origin_id) torsion_proxies.append(dp) return torsion_proxies
def add_new_solvent(self): if(self.params.b_iso is None): sol_sel = self.model.solvent_selection() xrs_mac_h = self.model.xray_structure.select(~sol_sel) hd_mac = self.model.xray_structure.hd_selection().select(~sol_sel) xrs_mac = xrs_mac_h.select(~hd_mac) b = xrs_mac.extract_u_iso_or_u_equiv() * math.pi**2*8 b_solv = flex.mean_default(b, None) if(b_solv is not None and b_solv < self.params.b_iso_min or b_solv > self.params.b_iso_max): b_solv = (self.params.b_iso_min + self.params.b_iso_max) / 2. else: b_solv = self.params.b_iso if(self.params.new_solvent == "isotropic"): new_scatterers = flex.xray_scatterer( self.sites.size(), xray.scatterer(occupancy = self.params.occupancy, b = b_solv, scattering_type = self.params.scattering_type)) elif(self.params.new_solvent == "anisotropic"): u_star = adptbx.u_iso_as_u_star(self.model.xray_structure.unit_cell(), adptbx.b_as_u(b_solv)) new_scatterers = flex.xray_scatterer( self.sites.size(), xray.scatterer( occupancy = self.params.occupancy, u = u_star, scattering_type = self.params.scattering_type)) else: raise RuntimeError new_scatterers.set_sites(self.sites) solvent_xray_structure = xray.structure( special_position_settings = self.model.xray_structure, scatterers = new_scatterers) xrs_sol = self.model.xray_structure.select(self.model.solvent_selection()) xrs_mac = self.model.xray_structure.select(~self.model.solvent_selection()) xrs_sol = xrs_sol.concatenate(other = solvent_xray_structure) sol_sel = flex.bool(xrs_mac.scatterers().size(), False) sol_sel.extend( flex.bool(xrs_sol.scatterers().size(), True) ) self.model.add_solvent( solvent_xray_structure = solvent_xray_structure, residue_name = self.params.output_residue_name, atom_name = self.params.output_atom_name, chain_id = self.params.output_chain_id, refine_occupancies = self.params.refine_occupancies, refine_adp = self.params.new_solvent) self.fmodel.update_xray_structure( xray_structure = self.model.xray_structure, update_f_calc = True)
def tst_loglikelihoods(): fobs = flex.double( range(1000) )/200 fcalc = flex.double( [1]*1000 ) sigmas = flex.double( [0]*1000 ) epsilon = flex.double( [1]*1000 ) centric = flex.bool( [False]*1000 ) beta = flex.double( [1]*1000 ) alpha = flex.double( [0.99]*1000 ) tmp_object = scaling.likelihood_ratio_outlier_test( fobs, sigmas, fcalc, epsilon, centric, alpha, beta) cur_lik = tmp_object.log_likelihood() pm_lik = tmp_object.posterior_mode_log_likelihood() mode = tmp_object.posterior_mode() level = 4.5 flags = tmp_object.flag_potential_outliers( 2.0*level ) for fl,pl,l,m,fo in zip(flags,pm_lik,cur_lik,mode,fobs): if pl-l < level*2.0: assert fl else: assert not fl
def apply_tls(xray_structure, params): uc = xray_structure.unit_cell() sg = xray_structure.space_group() selections_1d = flex.bool(xray_structure.scatterers().size(),True) selections = [selections_1d.iselection()] T=random_aniso_adp(space_group=sg, unit_cell=uc, u_scale=params.max_tl, u_min=params.min_tl) L=random_aniso_adp(space_group=sg, unit_cell=uc, u_scale=params.max_tl, u_min=params.min_tl) print " T: %s"%",".join([("%7.3f"%i).strip() for i in T]) print " L: %s"%",".join([("%7.3f"%i).strip() for i in L]) tlsos = mmtbx.tls.tools.generate_tlsos( selections = selections, xray_structure = xray_structure, T=[T], L=[L], S=[[0,0,0,0,0,0,0,0,0]]) u_cart_from_tls = mmtbx.tls.tools.u_cart_from_tls( sites_cart = xray_structure.sites_cart(), selections = selections, tlsos = tlsos) xray_structure.convert_to_anisotropic() u_cart = xray_structure.scatterers().extract_u_cart(uc) utot = u_cart_from_tls+u_cart xray_structure.set_u_cart(u_cart=utot, selection = selections_1d.iselection()) xray_structure.tidy_us() return xray_structure
def run(): pdb_file_name = libtbx.env.find_in_repositories( relative_path="phenix_regression/pdb/adp_restraints.pdb", test=os.path.isfile) processed_pdb_file = pdb_interpretation.process( mon_lib_srv = monomer_library.server.server(), ener_lib = monomer_library.server.ener_lib(), file_name = pdb_file_name) xray_structure = processed_pdb_file.xray_structure() grm = processed_pdb_file.geometry_restraints_manager( plain_pairs_radius = 5.0) grm.pair_proxies(sites_cart=xray_structure.sites_cart()) class parameters: sphere_radius = 1.6 distance_power = 0.0 average_power = 0.0 min_u_sum = 1.e-6 sel = flex.bool(xray_structure.scatterers().size(), True) xray_structure.scatterers().flags_set_grad_u_iso(sel.iselection()) for use_hd in [True, False]: energies_adp = cctbx.adp_restraints.energies_iso( geometry_restraints_manager=grm, xray_structure=xray_structure, use_hd = use_hd, use_u_local_only = False, parameters=parameters) u_iso_restraints = grm.harmonic_restraints( variables = xray_structure.extract_u_iso_or_u_equiv(), type_indices = None, type_weights = 1.0) assert approx_equal(u_iso_restraints.residual_sum, energies_adp.residual_sum) assert approx_equal(u_iso_restraints.gradients, energies_adp.gradients)
def compute_functional_and_gradients(O): if (O.number_of_function_evaluations == 0): O.number_of_function_evaluations += 1 return O.f_start, O.g_start O.number_of_function_evaluations += 1 O.residue_tardy_model.unpack_q(q_packed=O.x) O.sites_cart_residue = O.residue_tardy_model.sites_moved() rs_f = maptbx.real_space_target_simple( unit_cell=O.unit_cell, density_map=O.density_map, sites_cart=O.sites_cart_residue, selection=flex.bool(O.sites_cart_residue.size(),True)) rs_g = real_space_rigid_body_gradients_simple( unit_cell=O.unit_cell, density_map=O.density_map, sites_cart_0=O.sites_cart_residue_0, center_of_mass=O.residue_center_of_mass, q=O.x) O.rs_f = rs_f rs_f *= -O.real_space_target_weight rs_g *= -O.real_space_target_weight if (O.geometry_restraints_manager is None): f = rs_f g = rs_g else: O.sites_cart_all.set_selected(O.residue_i_seqs, O.sites_cart_residue) gr_e = O.geometry_restraints_manager.energies_sites( sites_cart=O.sites_cart_all, compute_gradients=True) O.__d_e_pot_d_sites = gr_e.gradients.select(indices=O.residue_i_seqs) f = rs_f + gr_e.target g = rs_g + O.residue_tardy_model.d_e_pot_d_q_packed() return f, g.as_double()
def add_coordinate_restraints( sites_cart, selection=None, sigma=0.5, limit=1.0, top_out_potential=False): import boost.python ext_rcp = boost.python.import_ext("mmtbx_reference_coordinate_ext") result = ext_rcp.shared_reference_coordinate_proxy() if (selection is not None): if (isinstance(selection, flex.bool)): selection = selection.iselection() if selection is None: selection = flex.bool( len(sites_cart), True).iselection() assert len(sites_cart) == len(selection) weight = 1.0 / (sigma**2) for k, i_seq in enumerate(selection): i_seqs = [i_seq] ref_sites = sites_cart[k] proxy = ext_rcp.reference_coordinate_proxy( i_seqs=i_seqs, ref_sites=ref_sites, weight=weight, limit=limit, top_out=top_out_potential) result.append(proxy) return result
def set_refinable_parameters(xray_structure, parameters, selections, enforce_positivity=False): # XXX PVA: Code below is terribly inefficient and MUST be moved into C++ sz = xray_structure.scatterers().size() i = 0 for sel in selections: # pre-check for positivity begin # spread negative occupancies across i_seqs having positive ones par_all = flex.double() par_neg = flex.double() i_p = i for sel_ in sel: p = parameters[i_p] par_all.append(p) if(p<0): par_neg.append(p) i_p += 1 if(enforce_positivity and par_neg.size()>0): par_all = par_all - flex.min(par_all) fs = flex.sum(par_all) if(fs != 0): par_all = par_all / fs # pre-check for positivity end for j, sel_ in enumerate(sel): sel__b = flex.bool(sz, flex.size_t(sel_)) xray_structure.set_occupancies(par_all[j], sel__b) i+=1
def merge_obs(indices, iobs, sel, symm, anomalous_flag, d_min, d_max): #indices = flex.miller_index(indices) #iobs = flex.double(iobs) if sel is not None and len(sel) > 0: sel = flex.bool(sel) indices = indices.select(sel) iobs = iobs.select(sel) miller_set = miller.set(crystal_symmetry=symm, indices=indices, anomalous_flag=anomalous_flag) array = miller.array(miller_set=miller_set, data=iobs, ).set_observation_type_xray_intensity() array = array.resolution_filter(d_min=d_min, d_max=d_max) ## New way #array = array.map_to_asu() # #merger = yamtbx_dataproc_crystfel_ext.merge_equivalents_crystfel() #merger.add_observations(array.indices(), array.data()) #merger.merge() return array.merge_equivalents(algorithm="crystfel") # if sigmas is None, merge_equivalents_real() is used which simply averages.
def compute_functional_and_gradients(self): if (self.number_of_function_evaluations == 0): self.number_of_function_evaluations += 1 return self.f_start, self.g_start self.number_of_function_evaluations += 1 self.residue_tardy_model.unpack_q(q_packed=self.x) self.sites_cart_residue = self.residue_tardy_model.sites_moved() if(self.states_collector is not None): self.states_collector.add(sites_cart = self.sites_cart_residue) rs_f = maptbx.real_space_target_simple( unit_cell=self.unit_cell, density_map=self.density_map, sites_cart=self.sites_cart_residue, selection=flex.bool(self.sites_cart_residue.size(),True)) rs_g = real_space_rigid_body_gradients_simple( unit_cell=self.unit_cell, density_map=self.density_map, sites_cart_0=self.sites_cart_residue_0, center_of_mass=self.residue_center_of_mass, q=self.x) self.rs_f = rs_f rs_f *= -self.real_space_target_weight rs_g *= -self.real_space_target_weight if (self.geometry_restraints_manager is None): f = rs_f g = rs_g else: gr_e = self.geometry_restraints_manager.energies_sites( sites_cart=self.sites_cart_residue, flags = self.cctbx_geometry_restraints_flags, compute_gradients=True) self.__d_e_pot_d_sites = gr_e.gradients f = rs_f + gr_e.target g = rs_g + self.residue_tardy_model.d_e_pot_d_q_packed() return f, g.as_double()
def read_frames_updated_detail(self): from cctbx.crystal_orientation import crystal_orientation from xfel.cxi.util import is_odd_numbered frames = {'frame_id': flex.int(), 'wavelength': flex.double(), 'cc': flex.double(), 'G': flex.double(), 'BFACTOR': flex.double(), 'RS': flex.double(), 'odd_numbered': flex.bool(), 'thetax': flex.double(), 'thetay': flex.double(), 'orientation': [], 'unit_cell': [], 'unique_file_name': []} stream = open(self.params.output.prefix + '_frame.db', 'r') for row in stream: items = row.split() CO = crystal_orientation([float(t) for t in items[8:17]], True) unique_file_name = eval(items[19]) frames['frame_id'].append(int(items[0])) frames['wavelength'].append(float(items[1])) frames['cc'].append(float(items[20])) frames['G'].append(float(items[5])) frames['BFACTOR'].append(float(items[6])) frames['RS'].append(float(items[7])) frames['thetax'].append(float(items[17])) frames['thetay'].append(float(items[18])) frames['odd_numbered'].append(is_odd_numbered(unique_file_name)) frames['orientation'].append(CO) frames['unit_cell'].append(CO.unit_cell()) frames['unique_file_name'].append(unique_file_name) stream.close() return frames
def exercise_singular_least_squares(): obs = flex.double([1.234]) weights_2345 = flex.double([2.345]) weights_zero = flex.double([0]) r_free_flags = flex.bool([False]) a = flex.double([0]) b = flex.double([0]) for obs_type in ["F", "I"]: for weights,scale_factor in [ (weights_2345, 3.456), (weights_zero, 0)]: tg = ext.targets_least_squares( compute_scale_using_all_data=False, obs_type=obs_type, obs=obs, weights=weights, r_free_flags=r_free_flags, f_calc=flex.complex_double(a, b), derivatives_depth=2, scale_factor=scale_factor) if (weights is weights_2345): assert approx_equal(tg.scale_factor(), scale_factor) assert list(tg.gradients_work()) == [0j] assert list(tg.hessians_work()) == [(1,1,1)] else: assert tg.scale_factor() is None assert tg.target_work() is None assert tg.target_test() is None assert tg.gradients_work().size() == 0 assert tg.hessians_work().size() == 0
def _compute_gradients(self): return -1.*maptbx.real_space_gradients_simple( unit_cell = self.unit_cell, density_map = self.target_map, sites_cart = self.sites_cart, delta = self.real_space_gradients_delta, selection = flex.bool(self.sites_cart.size(), True))
def plot_samples(O, stage): p = O.params.plot_samples if (stage not in p.stages): return if (p.ix is not None): O.plot_samples_ix(stage, p.ix) elif (p.ix_auto == "all"): for ix in xrange(O.x.size()): O.plot_samples_ix(stage, ix) elif (p.ix_auto == "random"): assert p.ix_random.samples_each_scattering_type is not None assert p.ix_random.samples_each_scattering_type > 0 assert p.ix_random.random_seed is not None mt = flex.mersenne_twister(seed=p.ix_random.random_seed) i_seqs_grouped = O.xray_structure.scatterers() \ .extract_scattering_types().i_seqs_by_value().values() i_seqs_selected = flex.bool(O.x.size(), False) for i_seqs in i_seqs_grouped: ps = i_seqs.size() ss = min(ps, p.ix_random.samples_each_scattering_type) isel = mt.random_selection(population_size=ps, sample_size=ss) i_seqs_selected.set_selected(i_seqs.select(isel), True) for ix,(i_sc,_) in enumerate(O.x_info): if (i_seqs_selected[i_sc]): O.plot_samples_ix(stage, ix) else: raise RuntimeError("Unknown plot_samples.ix_auto = %s" % p.ix_auto)
def local_nikonov_scaling(self,out): print >> out print >> out, "Nikonev based local scaling" print >> out, "Maximum depth : %8i"%(self.max_depth) print >> out, "Target neighbours : %8i"%(self.target_neighbours) print >> out, "neighbourhood sphere : %8i"%(self.sphere) print >> out if self.der_primset.is_xray_intensity_array(): raise Sorry(" For Nikonev target in local scaling, amplitudes must be used") assert (False) self.local_scaler = scaling.local_scaling_nikonov( hkl_master=self.master_set.indices(), hkl_sets=self.nat_primset.indices(), data_set_a=self.nat_primset.data(), data_set_b=self.der_primset.data(), epsilons=self.der_primset.epsilons().data().as_double(), centric=flex.bool(self.der_primset.centric_flags().data()), threshold=self.threshold, space_group=self.nat_primset.space_group(), anomalous_flag=self.nat_primset.anomalous_flag(), radius=self.sphere, depth=self.max_depth, target_ref=self.target_neighbours)
def exercise_miller_array_data_types(): miller_set = crystal.symmetry(unit_cell=(10, 10, 10, 90, 90, 90), space_group_symbol="P1").miller_set( indices=flex.miller_index([(1, 2, 3), (4, 5, 6)]), anomalous_flag=False ) for data in [ flex.bool([False, True]), flex.int([0, 1]), flex.size_t([0, 1]), flex.double([0, 1]), flex.complex_double([0, 1]), ]: miller_array = miller_set.array(data=data) if op.isfile("tmp_iotbx_mtz.mtz"): os.remove("tmp_iotbx_mtz.mtz") assert not op.isfile("tmp_iotbx_mtz.mtz") miller_array.as_mtz_dataset(column_root_label="DATA").mtz_object().write(file_name="tmp_iotbx_mtz.mtz") assert op.isfile("tmp_iotbx_mtz.mtz") mtz_obj = mtz.object(file_name="tmp_iotbx_mtz.mtz") miller_arrays_read_back = mtz_obj.as_miller_arrays() assert len(miller_arrays_read_back) == 1 miller_array_read_back = miller_arrays_read_back[0] assert miller_array_read_back.indices().all_eq(miller_array.indices()) if miller_array.is_integer_array() or miller_array.is_bool_array(): assert miller_array_read_back.data().all_eq(flex.int([0, 1])) elif miller_array.is_real_array(): assert miller_array_read_back.data().all_eq(flex.double([0, 1])) elif miller_array.is_complex_array(): assert miller_array_read_back.data().all_eq(flex.complex_double([0, 1])) else: raise RuntimeError("Programming error.")
def suggest_likely_candidates( self, acceptable_violations = 1e+90 ): used = flex.bool( len(self.sg_choices), False ) order = [] all_done = False count = -1 if (len(self.tuple_score) == 0) : return [] tmp_scores = [] for tt in self.tuple_score: tmp_scores.append( tt[0] ) order = flex.sort_permutation( flex.double( tmp_scores ), False ) sorted_rows = [] max_score = flex.min( flex.double( tmp_scores ) ) for ii in order: sg = self.sg_choices[ii] tmp_n = self.n[ii] tmp_violations = self.violations[ii] tmp_mean_i = self.mean_i[ii] tmp_mean_isigi = self.mean_isigi[ii] tuple_score = self.tuple_score[ii] sorted_rows.append( [str(sg), '%i'%(tmp_n), '%8.2f '%(tmp_mean_i), '%8.2f '%(tmp_mean_isigi), ' %i '%(tuple_score[1]), ' %i '%(tuple_score[2]), ' %8.3e '%((tuple_score[0]-max_score)) ]) return sorted_rows
def process_input_array(self, arr): array = arr.deep_copy() work_array = arr multiplicities = None try: if self.merge_equivalents : array, multiplicities, merge = MergeData(array, self.settings.show_anomalous_pairs) settings = self.settings data = array.data() #import code, traceback; code.interact(local=locals(), banner="".join( traceback.format_stack(limit=10) ) ) self.missing_set = oop.null() #if (array.is_xray_intensity_array()): # data.set_selected(data < 0, flex.double(data.size(), 0.)) if (array.is_unique_set_under_symmetry()) and (settings.map_to_asu): array = array.map_to_asu() if (multiplicities is not None): multiplicities = multiplicities.map_to_asu() if (settings.d_min is not None): array = array.resolution_filter(d_min=settings.d_min) if (multiplicities is not None): multiplicities = multiplicities.resolution_filter( d_min=settings.d_min) self.filtered_array = array.deep_copy() if (settings.expand_anomalous): if not array.is_unique_set_under_symmetry(): raise Sorry("Error! Cannot generate bijvoet mates of unmerged reflections.") array = array.generate_bijvoet_mates() original_symmetry = array.crystal_symmetry() if (multiplicities is not None): multiplicities = multiplicities.generate_bijvoet_mates() if (self.settings.show_missing): self.missing_set = array.complete_set().lone_set(array) if self.settings.show_anomalous_pairs: self.missing_set = self.missing_set.select( self.missing_set.centric_flags().data(), negate=True) if (settings.expand_to_p1): if not array.is_unique_set_under_symmetry(): raise Sorry("Error! Cannot expand unmerged reflections to P1.") original_symmetry = array.crystal_symmetry() array = array.expand_to_p1().customized_copy( crystal_symmetry=original_symmetry) #array = array.niggli_cell().expand_to_p1() #self.missing_set = self.missing_set.niggli_cell().expand_to_p1() self.missing_set = self.missing_set.expand_to_p1().customized_copy( crystal_symmetry=original_symmetry) if (multiplicities is not None): multiplicities = multiplicities.expand_to_p1().customized_copy( crystal_symmetry=original_symmetry) data = array.data() self.r_free_mode = False self.phases = flex.double(data.size(), float('nan')) self.radians = flex.double(data.size(), float('nan')) self.ampl = flex.double(data.size(), float('nan')) self.sigmas = None if isinstance(data, flex.bool): self.r_free_mode = True data_as_float = flex.double(data.size(), 0.0) data_as_float.set_selected(data==True, flex.double(data.size(), 1.0)) data = data_as_float self.data = data #.deep_copy() else : if isinstance(data, flex.double): self.data = data #.deep_copy() elif isinstance(data, flex.complex_double): self.data = data #.deep_copy() self.ampl = flex.abs(data) self.phases = flex.arg(data) * 180.0/math.pi # purge nan values from array to avoid crash in fmod_positive() b = flex.bool([bool(math.isnan(e)) for e in self.phases]) # replace the nan values with an arbitrary float value self.phases = self.phases.set_selected(b, 42.4242) # Cast negative degrees to equivalent positive degrees self.phases = flex.fmod_positive(self.phases, 360.0) self.radians = flex.arg(data) # replace the nan values with an arbitrary float value self.radians = self.radians.set_selected(b, 0.424242) elif hasattr(array.data(), "as_double"): self.data = array.data().as_double() else: raise RuntimeError("Unexpected data type: %r" % data) if (settings.show_data_over_sigma): if (array.sigmas() is None): raise Sorry("sigmas not defined.") sigmas = array.sigmas() non_zero_sel = sigmas != 0 array = array.select(non_zero_sel) array = array.customized_copy(data=array.data()/array.sigmas()) self.data = array.data() if (multiplicities is not None): multiplicities = multiplicities.select(non_zero_sel) if array.sigmas() is not None: self.sigmas = array.sigmas() else: self.sigmas = None work_array = array except Exception as e: print(to_str(e) + "".join(traceback.format_stack(limit=10))) raise e return None, None work_array.set_info(arr.info() ) multiplicities = multiplicities return work_array, multiplicities
def generate_view_data(self): from scitbx.array_family import flex from scitbx import graphics_utils settings = self.settings data_for_colors = data_for_radii = None if not self.fullprocessarray: return data = self.data #self.work_array.data() sigmas = self.sigmas if (isinstance(data, flex.double) and data.all_eq(0)): data = flex.double(data.size(), 1) if ((self.multiplicities is not None) and (settings.scale_colors_multiplicity)): data_for_colors = self.multiplicities.data().as_double() assert data_for_colors.size() == data.size() elif (settings.sqrt_scale_colors) and (isinstance(data, flex.double)): data_for_colors = flex.sqrt(flex.abs(data)) elif isinstance(data, flex.complex_double): data_for_colors = self.radians foms_for_colours = self.foms self.colourlabel = self.miller_array.info().labels[1] elif (settings.sigma_color) and sigmas is not None: data_for_colors = sigmas.as_double() self.colourlabel = self.miller_array.info().labels[1] else : data_for_colors = flex.abs(data.deep_copy()) uc = self.work_array.unit_cell() self.min_dist = min(uc.reciprocal_space_vector((1,1,1))) * self.renderscale min_radius = 0.05 * self.min_dist max_radius = 0.45 * self.min_dist if ((self.multiplicities is not None) and (settings.scale_radii_multiplicity)): data_for_radii = self.multiplicities.data().as_double() if (settings.sigma_radius) and sigmas is not None: data_for_radii = sigmas * self.multiplicities.as_double() assert data_for_radii.size() == data.size() elif (settings.sigma_radius) and sigmas is not None: data_for_radii = sigmas.as_double() else : data_for_radii = nth_power_scale(flex.abs(data.deep_copy()), settings.nth_power_scale_radii) if (settings.slice_mode): data = data.select(self.slice_selection) if (not settings.keep_constant_scale): data_for_radii = data_for_radii.select(self.slice_selection) data_for_colors = data_for_colors.select(self.slice_selection) foms_for_colours = foms_for_colours.select(self.slice_selection) if isinstance(data, flex.complex_double): if self.isUsingFOMs(): colors = graphics_utils.colour_by_phi_FOM(data_for_colors, foms_for_colours) else: colors = graphics_utils.colour_by_phi_FOM(data_for_colors, None) elif (settings.color_scheme in ["rainbow", "heatmap", "redblue"]): colors = graphics_utils.color_by_property( properties=data_for_colors, selection=flex.bool(data_for_colors.size(), True), color_all=False, gradient_type=settings.color_scheme) elif (settings.color_scheme == "grayscale"): colors = graphics_utils.grayscale_by_property( properties=data_for_colors, selection=flex.bool(data_for_colors.size(), True), shade_all=False, invert=settings.black_background) else : if (settings.black_background): base_color = (1.0,1.0,1.0) else : base_color = (0.0,0.0,0.0) colors = flex.vec3_double(data_for_colors.size(), base_color) if (settings.slice_mode) and (settings.keep_constant_scale): colors = colors.select(self.slice_selection) data_for_radii = data_for_radii.select(self.slice_selection) #if (settings.sqrt_scale_radii) and (not settings.scale_radii_multiplicity): # data_for_radii = flex.sqrt(flex.abs(data_for_radii)) if len(data_for_radii): dat2 = flex.abs(flex.double([e for e in data_for_radii if not math.isnan(e)])) # don't divide by 0 if dealing with selection of Rfree array where all values happen to be zero scale = max_radius/(flex.max(dat2) + 0.001) radii = data_for_radii * (self.settings.scale * scale) assert radii.size() == colors.size() else: radii = flex.double() max_radius = 0 self.radii = radii self.max_radius = max_radius self.min_radius = min_radius self.colors = colors if isinstance(data, flex.complex_double): self.foms = foms_for_colours
def get_data_from_xac(params, xac): if xac.endswith(".pkl"): tmp = pickle.load(open(xac)) else: tmp = xds_ascii.XDS_ASCII(xac) sel_remove = flex.bool(tmp.iobs.size(), False) if params.min_peak is not None: sel = tmp.peak < params.min_peak sel_remove |= sel elif params.min_peak_percentile is not None: q = numpy.percentile(tmp.peak, params.min_peak_percentile) print "percentile %.2f %s" % (q, xac) sel = tmp.peak < q sel_remove |= sel if params.skip_rejected: sel_remove |= (tmp.sigma_iobs <= 0) if params.dmin is not None: sel_remove |= ~tmp.as_miller_set().resolution_filter_selection( d_min=params.dmin) if params.correct_peak: sel_remove |= (tmp.peak < 1) # remove PEAK==0 # Remove selected print "DEBUG:: removing %d reflections" % sel_remove.count( True) #sum(sel_remove)# tmp.remove_selection(sel_remove) if not params.skip_rejected: tmp.sigma_iobs = flex.abs(tmp.sigma_iobs) # Correct I,sigI if needed if params.correct_peak: tmp.iobs *= tmp.peak * .01 tmp.sigma_iobs *= tmp.peak * .01 if params.cancel_rlp: tmp.iobs /= tmp.rlp tmp.sigma_iobs /= tmp.rlp if params.polarization.correct: # Only works with single-panel detector!! # Assumes detector fast = (1,0,0), slow = (0,1,0) sin_sq_2theta = tmp.symm.unit_cell().sin_sq_two_theta( tmp.indices, tmp.wavelength) cos_sq_2theta = 1. - sin_sq_2theta sin_theta = tmp.wavelength / tmp.symm.unit_cell().d( tmp.indices) / 2. Eppi = numpy.cross(params.polarization.plane_normal, params.polarization.incident_beam_direction) Eppi /= numpy.linalg.norm(Eppi) S = flex.vec3_double( tmp.xd - tmp.orgx, tmp.yd - tmp.orgy, flex.double(tmp.xd.size(), tmp.distance / tmp.qx)) S /= S.norms() zp = S.dot(Eppi.tolist()) * 2. * sin_theta cosrho = zp / flex.sqrt(sin_sq_2theta) P0 = 0.5 * (1. + cos_sq_2theta) PP = (params.polarization.fraction - 0.5) * (2. * cosrho**2 - 1.) * sin_sq_2theta P = P0 - PP # Apply correction tmp.iobs /= P tmp.sigma_iobs /= P if 0: # debug for x, y, p in zip(tmp.xd, tmp.yd, P): print "pdebug:: %.2f %.2f %.4e" % (x, y, p) return tmp.i_obs().customized_copy(anomalous_flag=params.anomalous_flag, space_group_info=sgtbx.space_group_info( params.space_group))
def get_data_from_dials(params, files): from dials.util.options import Importer, flatten_reflections, flatten_experiments importer = Importer(files, read_experiments=True, read_reflections=True) reflections = flatten_reflections(importer.reflections) experiments = flatten_experiments(importer.experiments) assert len(reflections) == len(experiments) == 1 xs = crystal.symmetry(experiments[0].crystal.get_unit_cell(), space_group=experiments[0].crystal.get_space_group()) tmp = reflections[0].select(reflections[0]["id"] >= 0) assert max(tmp["id"]) == 0 if params.dials_data == "sum": assert "intensity.sum.value" in tmp assert "intensity.sum.variance" in tmp else: assert "intensity.prf.value" in tmp assert "intensity.prf.variance" in tmp intensity_key = "intensity.sum.value" if params.dials_data == "sum" else "intensity.prf.value" variance_key = "intensity.sum.variance" if params.dials_data == "sum" else "intensity.prf.variance" sel_remove = flex.bool(tmp.size(), False) if params.min_peak is not None: sel = tmp["partiality"] < params.min_peak / 100. sel_remove |= sel elif params.min_peak_percentile is not None: q = numpy.percentile(tmp["partiality"], params.min_peak_percentile) print "percentile %.2f %s" % (q * 100., xac) sel = tmp["partiality"] < q sel_remove |= sel if params.skip_rejected: sel_remove |= tmp[variance_key] <= 0 if params.dmin is not None: sel_remove |= xs.unit_cell().d(tmp["miller_index"]) < params.dmin if params.correct_peak: sel_remove |= (tmp["partiality"] < .01) # remove PEAK==0 # Remove selected print "DEBUG:: removing %d reflections" % sel_remove.count( True) #sum(sel_remove)# tmp = tmp.select(~sel_remove) ret = miller.array(miller.set(xs, tmp["miller_index"], params.anomalous_flag), data=tmp[intensity_key], sigmas=flex.sqrt(flex.abs(tmp[variance_key]))) scale = flex.double(ret.size(), 1.) if not params.cancel_rlp and "lp" in tmp: scale *= tmp["lp"] if "dqe" in tmp: scale /= tmp["dqe"] if params.correct_peak: scale *= tmp["partiality"] ret = ret.apply_scaling(factor=scale) if params.cancel_rlp and params.polarization.correct: raise "Not implemented" return ret
def exercise_proxy_show(): if sys.platform.startswith("win") and sys.version_info[:2] < (2,6): # This appears to be a windows-specific bug with string formatting # for python versions prior to 2.6, where the exponent is printed # with 3 digits rather than 2. print "Skipping exercise_proxy_show()" return sites_cart = flex.vec3_double(( (-3.1739,10.8317,7.5653),(-2.5419,9.7567,6.6306), (-3.3369,8.8794,4.5191),(-3.4640,9.9882,5.3896))) site_labels = ("C1", "C2", "O16", "N8") u_cart = flex.sym_mat3_double(( (0.0153,0.0206,0.0234,0.0035,-0.0052,-0.0051), (0.0185,0.0109,0.0206,0.0005,-0.0010,0.0002), (0.0295,0.0203,0.0218,-0.0010,-0.0003,-0.0044), (0.0159,0.0154,0.0206,-0.0003,0.0004,0.0036))) u_iso = flex.double((-1,-1,-1,-1)) use_u_aniso = flex.bool((True,True,True,True)) # proxies = adp_restraints.shared_adp_similarity_proxy() sio = StringIO() proxies.show_sorted( by_value="residual", u_cart=flex.sym_mat3_double(), u_iso=flex.double(), use_u_aniso=flex.bool(), f=sio) assert not show_diff(sio.getvalue(), """\ ADP similarity restraints: 0 """) proxies = adp_restraints.shared_adp_similarity_proxy([ adp_restraints.adp_similarity_proxy(i_seqs=[0,1],weight=25), adp_restraints.adp_similarity_proxy(i_seqs=[2,3],weight=0.3)]) sio = StringIO() proxies.show_sorted( by_value="residual", u_cart=u_cart, u_iso=u_iso, use_u_aniso=use_u_aniso, f=sio, prefix=":") assert not show_diff(sio.getvalue(), """\ :ADP similarity restraints: 2 :Sorted by residual: :scatterers 0 : 1 : delta sigma weight rms_deltas residual : U11 -3.20e-03 2.00e-01 2.50e+01 4.96e-03 5.54e-03 : U22 9.70e-03 2.00e-01 2.50e+01 : U33 2.80e-03 2.00e-01 2.50e+01 : U12 3.00e-03 2.00e-01 2.50e+01 : U13 -4.20e-03 2.00e-01 2.50e+01 : U23 -5.30e-03 2.00e-01 2.50e+01 :scatterers 2 : 3 : delta sigma weight rms_deltas residual : U11 1.36e-02 1.83e+00 3.00e-01 6.15e-03 1.02e-04 : U22 4.90e-03 1.83e+00 3.00e-01 : U33 1.20e-03 1.83e+00 3.00e-01 : U12 -7.00e-04 1.83e+00 3.00e-01 : U13 -7.00e-04 1.83e+00 3.00e-01 : U23 -8.00e-03 1.83e+00 3.00e-01 """) sio = StringIO() proxies.show_sorted( by_value="rms_deltas", site_labels=site_labels, u_cart=u_cart, u_iso=flex.double((0.024,0.031,0.021,0.028)), use_u_aniso=flex.bool((False,False,False,False)), f=sio, prefix="=") assert not show_diff(sio.getvalue(), """\ =ADP similarity restraints: 2 =Sorted by rms_deltas: =scatterers C1 = C2 = delta sigma weight residual = Uiso -7.00e-03 2.00e-01 2.50e+01 1.22e-03 =scatterers O16 = N8 = delta sigma weight residual = Uiso -7.00e-03 1.83e+00 3.00e-01 1.47e-05 """) # proxies = adp_restraints.shared_isotropic_adp_proxy() sio = StringIO() proxies.show_sorted( by_value="residual", u_cart=flex.sym_mat3_double(), u_iso=u_iso, use_u_aniso=use_u_aniso, f=sio) assert not show_diff(sio.getvalue(), """\ Isotropic ADP restraints: 0 """) proxies = adp_restraints.shared_isotropic_adp_proxy([ adp_restraints.isotropic_adp_proxy(i_seqs=(0,),weight=25), adp_restraints.isotropic_adp_proxy(i_seqs=(2,),weight=0.3)]) sio = StringIO() proxies.show_sorted( by_value="residual", site_labels=site_labels, u_cart=u_cart, u_iso=u_iso, use_u_aniso=use_u_aniso, f=sio, prefix=" ") assert not show_diff(sio.getvalue(), """\ Isotropic ADP restraints: 2 Sorted by residual: scatterer C1 delta sigma weight rms_deltas residual U11 -4.47e-03 2.00e-01 2.50e+01 4.27e-03 4.11e-03 U22 8.33e-04 2.00e-01 2.50e+01 U33 3.63e-03 2.00e-01 2.50e+01 U12 3.50e-03 2.00e-01 2.50e+01 U13 -5.20e-03 2.00e-01 2.50e+01 U23 -5.10e-03 2.00e-01 2.50e+01 scatterer O16 delta sigma weight rms_deltas residual U11 5.63e-03 1.83e+00 3.00e-01 3.16e-03 2.69e-05 U22 -3.57e-03 1.83e+00 3.00e-01 U33 -2.07e-03 1.83e+00 3.00e-01 U12 -1.00e-03 1.83e+00 3.00e-01 U13 -3.00e-04 1.83e+00 3.00e-01 U23 -4.40e-03 1.83e+00 3.00e-01 """) sio = StringIO() proxies.show_sorted( by_value="rms_deltas", u_cart=u_cart, u_iso=u_iso, use_u_aniso=use_u_aniso, f=sio, prefix="$") assert not show_diff(sio.getvalue(), """\ $Isotropic ADP restraints: 2 $Sorted by rms_deltas: $scatterer 0 $ delta sigma weight rms_deltas residual $ U11 -4.47e-03 2.00e-01 2.50e+01 4.27e-03 4.11e-03 $ U22 8.33e-04 2.00e-01 2.50e+01 $ U33 3.63e-03 2.00e-01 2.50e+01 $ U12 3.50e-03 2.00e-01 2.50e+01 $ U13 -5.20e-03 2.00e-01 2.50e+01 $ U23 -5.10e-03 2.00e-01 2.50e+01 $scatterer 2 $ delta sigma weight rms_deltas residual $ U11 5.63e-03 1.83e+00 3.00e-01 3.16e-03 2.69e-05 $ U22 -3.57e-03 1.83e+00 3.00e-01 $ U33 -2.07e-03 1.83e+00 3.00e-01 $ U12 -1.00e-03 1.83e+00 3.00e-01 $ U13 -3.00e-04 1.83e+00 3.00e-01 $ U23 -4.40e-03 1.83e+00 3.00e-01 """) # proxies = adp_restraints.shared_rigid_bond_proxy() sio = StringIO() proxies.show_sorted( by_value="residual", sites_cart=flex.vec3_double(), u_cart=flex.sym_mat3_double(), f=sio) assert not show_diff(sio.getvalue(), """\ Rigid bond restraints: 0 """) proxies = adp_restraints.shared_rigid_bond_proxy([ adp_restraints.rigid_bond_proxy(i_seqs=(0,1),weight=25), adp_restraints.rigid_bond_proxy(i_seqs=(0,2),weight=15), adp_restraints.rigid_bond_proxy(i_seqs=(2,3),weight=25), adp_restraints.rigid_bond_proxy(i_seqs=(3,1),weight=30)]) sio = StringIO() proxies.show_sorted( by_value="residual", sites_cart=sites_cart, site_labels=site_labels, u_cart=u_cart, f=sio, prefix="*") assert not show_diff(sio.getvalue(), """\ *Rigid bond restraints: 4 *Sorted by residual: *scatterers O16 * N8 * delta_z sigma weight residual * -3.96e-03 2.00e-01 2.50e+01 3.92e-04 *scatterers C1 * C2 * delta_z sigma weight residual * 1.08e-03 2.00e-01 2.50e+01 2.89e-05 *scatterers C1 * O16 * delta_z sigma weight residual * 4.03e-04 2.58e-01 1.50e+01 2.44e-06 *scatterers N8 * C2 * delta_z sigma weight residual * -1.54e-04 1.83e-01 3.00e+01 7.16e-07 """) sio = StringIO() proxies.show_sorted( by_value="delta", sites_cart=sites_cart, u_cart=u_cart, f=sio, prefix="||", max_items=2) assert not show_diff(sio.getvalue(), """\ ||Rigid bond restraints: 4 ||Sorted by delta: ||scatterers 2 || 3 || delta_z sigma weight residual || -3.96e-03 2.00e-01 2.50e+01 3.92e-04 ||scatterers 0 || 1 || delta_z sigma weight residual || 1.08e-03 2.00e-01 2.50e+01 2.89e-05 ||... (remaining 2 not shown) """)
def test_2(): n_sites = 1000 d_min = 2.0 volume_per_atom = 50 fraction_missing = (0.0, ) scale = 5.0 # create dummy model space_group_info = sgtbx.space_group_info("P212121") structure = random_structure.xray_structure( space_group_info=space_group_info, elements=["N"] * (n_sites), volume_per_atom=volume_per_atom, random_u_iso=False) structure.scattering_type_registry(table="wk1995") f_calc = structure.structure_factors(d_min=d_min, anomalous_flag=False, algorithm="direct").f_calc() f_obs = abs(f_calc) for fm in fraction_missing: # partial model n_keep = int(round(structure.scatterers().size() * (1 - fm))) partial_structure = xray.structure(special_position_settings=structure) partial_structure.add_scatterers(structure.scatterers()[:n_keep]) # fcalc (partial model), fobs (fcalc full model) f_calc_partial = partial_structure.structure_factors( d_min=d_min, anomalous_flag=False, algorithm="direct").f_calc() f_calc = abs(f_calc_partial) # define test set reflections flags = flex.bool(f_calc_partial.indices().size(), False) k = 0 for i in xrange(f_calc_partial.indices().size()): k = k + 1 if (k != 10): flags[i] = False else: k = 0 flags[i] = True # *********************************************************TEST = 1 alpha, beta = maxlik.alpha_beta_est_manager( f_obs=f_obs, f_calc=f_calc, free_reflections_per_bin=f_obs.data().size(), flags=flags, interpolation=False, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.size() == beta.size() assert alpha.size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 1.0, 1.e-2) assert approx_equal(flex.max(alpha.data()), 1.0, 1.e-2) assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2) assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2) alpha, beta = maxlik.alpha_beta_est_manager( f_obs=f_obs, f_calc=f_calc, free_reflections_per_bin=f_obs.data().size(), flags=flags, interpolation=True, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.size() == beta.size() assert alpha.size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 1.0, 1.e-2) assert approx_equal(flex.max(alpha.data()), 1.0, 1.e-2) assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2) assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2) # *********************************************************TEST = 2 alpha, beta = maxlik.alpha_beta_est_manager( f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale), f_calc=f_calc, free_reflections_per_bin=200, flags=flags, interpolation=False, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.size() == beta.size() assert alpha.size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2) assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2) alpha, beta = maxlik.alpha_beta_est_manager( f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale), f_calc=f_calc, free_reflections_per_bin=200, flags=flags, interpolation=True, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.size() == beta.size() assert alpha.size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2) assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2) # *********************************************************TEST = 3 alpha, beta = maxlik.alpha_beta_est_manager( f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale), f_calc=f_calc, free_reflections_per_bin=200, flags=flex.bool(f_obs.data().size(), True), interpolation=False, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.size() == beta.size() assert alpha.size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2) assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2) alpha, beta = maxlik.alpha_beta_est_manager( f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale), f_calc=f_calc, free_reflections_per_bin=200, flags=flex.bool(f_obs.data().size(), True), interpolation=True, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.size() == beta.size() assert alpha.size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2) assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2) # *********************************************************TEST = 4 alpha, beta = maxlik.alpha_beta_est_manager( f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale), f_calc=f_calc, free_reflections_per_bin=200, flags=flex.bool(f_obs.data().size(), False), interpolation=False, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.size() == beta.size() assert alpha.size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2) assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2) alpha, beta = maxlik.alpha_beta_est_manager( f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale), f_calc=f_calc, free_reflections_per_bin=200, flags=flex.bool(f_obs.data().size(), False), interpolation=True, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.size() == beta.size() assert alpha.size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2) assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2) assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
def __init__(self, miller_array, settings, merge=None, foms_array=None, fullprocessarray=True): self.miller_array = miller_array self.renderscale = 100.0 self.foms_workarray = foms_array self.SceneCreated = False self.settings = settings self.merge_equivalents = False if not self.miller_array.is_unique_set_under_symmetry(): self.merge_equivalents = merge from cctbx import crystal from cctbx.array_family import flex self.multiplicities = None self.fomlabel = "" self.foms = flex.double(self.miller_array.size(), float('nan')) self._is_using_foms = False self.fullprocessarray = fullprocessarray if self.miller_array.is_complex_array(): # Colour map coefficient as a circular rainbow with saturation as a function of FOMs # process the foms miller array and store the foms data for later use when computing colours if foms_array: assert ( self.miller_array.size() == foms_array.size() ) self.foms_workarray, dummy = self.process_input_array(foms_array) if not self.foms_workarray: return self.foms = self.foms_workarray.data() self.fomlabel = foms_array.info().label_string() self._is_using_foms = True self.work_array, self.multiplicities = self.process_input_array(self.miller_array) if not self.work_array: return array = self.work_array uc = array.unit_cell() self.unit_cell = uc self.slice_selection = None self.axis_index = None if (settings.slice_mode): self.axis_index = ["h","k","l"].index(self.settings.slice_axis) self.slice_selection = miller.simple_slice( indices=array.indices(), slice_axis=self.axis_index, slice_index=settings.slice_index) #if (self.slice_selection.count(True) == 0): #raise ValueError("No data selected!") index_span = array.index_span() self.colourlabel = self.miller_array.info().labels[0] self.d_min = array.d_min() self.min_dist = 0.0 self.hkl_range = index_span.abs_range() self.axes = [ uc.reciprocal_space_vector((self.hkl_range[0],0,0)), uc.reciprocal_space_vector((0,self.hkl_range[1],0)), uc.reciprocal_space_vector((0,0,self.hkl_range[2])) ] self.generate_view_data() if (self.slice_selection is not None): self.indices = self.work_array.indices().select(self.slice_selection).deep_copy() self.data = self.data.select(self.slice_selection) self.phases = self.phases.select(self.slice_selection) self.radians = self.radians.select(self.slice_selection) self.ampl = self.ampl.select(self.slice_selection) if self.sigmas: self.sigmas = self.sigmas.select(self.slice_selection) if foms_array: self.foms = self.foms.select(self.slice_selection) else : self.indices = array.indices() self.points = uc.reciprocal_space_vector(self.indices) * self.renderscale n_points = self.points.size() if not fullprocessarray: self.radii = flex.double() self.radii = ExtendAnyData(self.radii, n_points) self.colors = flex.vec3_double() self.colors = ExtendAnyData(self.colors, n_points) self.missing_flags = flex.bool(self.radii.size(), False) self.sys_absent_flags = flex.bool(self.radii.size(), False) if (settings.show_missing): self.generate_missing_reflections() if (settings.show_systematic_absences) and (not settings.show_only_missing): self.generate_systematic_absences() n_points = self.points.size() assert (self.colors.size() == n_points) assert (self.indices.size() == n_points) assert (self.radii.size() == n_points) assert (self.missing_flags.size() == n_points) assert (self.sys_absent_flags.size() == n_points) assert (self.data.size() == n_points) assert (self.phases.size() == n_points) assert (self.radians.size() == n_points) assert (self.ampl.size() == n_points) if self.sigmas: assert (self.sigmas.size() == n_points) if foms_array: assert (self.foms.size() == n_points) else: self.foms = flex.double(n_points, float('nan')) self.dres = uc.d(self.indices ) self.clear_labels() self.SceneCreated = True
def run(args, log=sys.stdout, as_gui_program=False): if (len(args) == 0): parsed = defaults(log=log) parsed.show(prefix=" ", out=log) return command_line = (option_parser().enable_symmetry_comprehensive().option( "-q", "--quiet", action="store_true", default=False, help="suppress output").option("--output_plots", action="store_true", default=False)).process(args=args) parsed = defaults(log=log) processed_args = mmtbx.utils.process_command_line_args( args=command_line.args, cmd_cs=command_line.symmetry, master_params=parsed, log=log, suppress_symmetry_related_errors=True) processed_args.params.show(out=log) params = processed_args.params.extract().density_modification output_plots = command_line.options.output_plots crystal_symmetry = crystal.symmetry( unit_cell=params.input.unit_cell, space_group_info=params.input.space_group) reflection_files = {} for rfn in (params.input.reflection_data.file_name, params.input.experimental_phases.file_name, params.input.map_coefficients.file_name): if os.path.isfile(str(rfn)) and rfn not in reflection_files: reflection_files.setdefault( rfn, iotbx.reflection_file_reader.any_reflection_file( file_name=rfn, ensure_read_access=False)) server = iotbx.reflection_file_utils.reflection_file_server( crystal_symmetry=crystal_symmetry, reflection_files=reflection_files.values()) fo = mmtbx.utils.determine_data_and_flags( server, parameters=params.input.reflection_data, extract_r_free_flags=False, log=log).f_obs hl_coeffs = mmtbx.utils.determine_experimental_phases( server, params.input.experimental_phases, log=log, parameter_scope="", working_point_group=None, symmetry_safety_check=True, ignore_all_zeros=True) if params.input.map_coefficients.file_name is not None: map_coeffs = server.get_phases_deg( file_name=params.input.map_coefficients.file_name, labels=params.input.map_coefficients.labels, convert_to_phases_if_necessary=False, original_phase_units=None, parameter_scope="", parameter_name="labels").map_to_asu() else: map_coeffs = None ncs_object = None if params.input.ncs_file_name is not None: ncs_object = ncs.ncs() ncs_object.read_ncs(params.input.ncs_file_name) ncs_object.display_all(log=log) fo = fo.map_to_asu() hl_coeffs = hl_coeffs.map_to_asu() fo = fo.eliminate_sys_absent().average_bijvoet_mates() hl_coeffs = hl_coeffs.eliminate_sys_absent().average_bijvoet_mates() model_map = None model_map_coeffs = None if len(processed_args.pdb_file_names): pdb_file = mmtbx.utils.pdb_file( pdb_file_names=processed_args.pdb_file_names) xs = pdb_file.pdb_inp.xray_structure_simple() fo_, hl_ = fo, hl_coeffs if params.change_basis_to_niggli_cell: change_of_basis_op = xs.change_of_basis_op_to_niggli_cell() xs = xs.change_basis(change_of_basis_op) fo_ = fo_.change_basis(change_of_basis_op).map_to_asu() hl_ = hl_.change_basis(change_of_basis_op).map_to_asu() #fo_, hl_ = fo_.common_sets(hl_) fmodel_refined = mmtbx.utils.fmodel_simple( f_obs=fo_, scattering_table= "wk1995", #XXX pva: 1) neutrons? 2) move up as a parameter. xray_structures=[xs], bulk_solvent_correction=True, anisotropic_scaling=True, r_free_flags=fo_.array(data=flex.bool(fo_.size(), False))) fmodel_refined.update(abcd=hl_) master_phil = mmtbx.maps.map_and_map_coeff_master_params() map_params = master_phil.fetch( iotbx.phil.parse("""\ map_coefficients { map_type = 2mFo-DFc isotropize = True } """)).extract().map_coefficients[0] model_map_coeffs = mmtbx.maps.map_coefficients_from_fmodel( fmodel=fmodel_refined, params=map_params) model_map = model_map_coeffs.fft_map( resolution_factor=params.grid_resolution_factor).real_map_unpadded( ) import time t0 = time.time() dm = density_modify(params, fo, hl_coeffs, ncs_object=ncs_object, map_coeffs=map_coeffs, model_map_coeffs=model_map_coeffs, log=log, as_gui_program=as_gui_program) time_dm = time.time() - t0 print >> log, "Time taken for density modification: %.2fs" % time_dm # run cns if 0: from cctbx.development import cns_density_modification cns_result = cns_density_modification.run(params, fo, hl_coeffs) print cns_result.modified_map.all() print dm.map.all() dm_map_coeffs = dm.map_coeffs_in_original_setting from cctbx import maptbx, miller crystal_gridding = maptbx.crystal_gridding( dm_map_coeffs.unit_cell(), space_group_info=dm_map_coeffs.space_group().info(), pre_determined_n_real=cns_result.modified_map.all()) dm_map = miller.fft_map(crystal_gridding, dm_map_coeffs).apply_sigma_scaling() corr = flex.linear_correlation(cns_result.modified_map.as_1d(), dm_map.real_map_unpadded().as_1d()) print "CNS dm/mmtbx dm correlation:" corr.show_summary() if dm.model_map_coeffs is not None: model_map = miller.fft_map( crystal_gridding, dm.miller_array_in_original_setting( dm.model_map_coeffs)).apply_sigma_scaling() corr = flex.linear_correlation( cns_result.modified_map.as_1d(), model_map.real_map_unpadded().as_1d()) print "CNS dm/model correlation:" corr.show_summary() if output_plots: plots_to_make = ( "fom", "skewness", "r1_factor", "r1_factor_fom", "mean_solvent_density", "mean_protein_density", "f000_over_v", "k_flip", "rms_solvent_density", "rms_protein_density", "standard_deviation_local_rms", "mean_delta_phi", "mean_delta_phi_initial", ) from matplotlib.backends.backend_pdf import PdfPages from libtbx import pyplot stats = dm.get_stats() pdf = PdfPages("density_modification.pdf") if len(dm.correlation_coeffs) > 1: if 0: start_coeffs, model_coeffs = dm.map_coeffs_start.common_sets( model_map_coeffs) model_phases = model_coeffs.phases(deg=True).data() exptl_phases = nearest_phase( model_phases, start_coeffs.phases(deg=True).data(), deg=True) corr = flex.linear_correlation(exptl_phases, model_phases) corr.show_summary() fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.set_title("phases start") ax.set_xlabel("Experimental phases") ax.set_ylabel("Phases from refined model") ax.scatter(exptl_phases, model_phases, marker="x", s=10) pdf.savefig(fig) # dm_coeffs, model_coeffs = dm.map_coeffs.common_sets( model_map_coeffs) model_phases = model_coeffs.phases(deg=True).data() dm_phases = nearest_phase(model_phases, dm_coeffs.phases(deg=True).data(), deg=True) corr = flex.linear_correlation(dm_phases, model_phases) corr.show_summary() fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.set_title("phases dm") ax.set_xlabel("Phases from density modification") ax.set_ylabel("Phases from refined model") ax.scatter(dm_phases, model_phases, marker="x", s=10) pdf.savefig(fig) # data = dm.correlation_coeffs fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.set_title("correlation coefficient") ax.plot(range(1, dm.i_cycle + 2), data) pdf.savefig(fig) # data = dm.mean_phase_errors fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.set_title("Mean effective phase errors") ax.plot(range(1, dm.i_cycle + 2), data) pdf.savefig(fig) for plot in plots_to_make: data = [ getattr(stats.get_cycle_stats(i), plot) for i in range(1, dm.i_cycle + 2) ] fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.set_title(plot.replace("_", " ")) ax.plot(range(1, dm.i_cycle + 2), data) pdf.savefig(fig) data = [ stats.get_cycle_stats(i).rms_solvent_density / stats.get_cycle_stats(i).rms_protein_density for i in range(1, dm.i_cycle + 2) ] fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.set_title("RMS solvent/protein density ratio") ax.plot(range(1, dm.i_cycle + 2), data) pdf.savefig(fig) pdf.close() dm_map_coeffs = dm.map_coeffs_in_original_setting dm_hl_coeffs = dm.hl_coeffs_in_original_setting # output map if requested map_params = params.output.map if map_params.file_name is not None: fft_map = dm_map_coeffs.fft_map( resolution_factor=params.grid_resolution_factor) if map_params.scale == "sigma": fft_map.apply_sigma_scaling() else: fft_map.apply_volume_scaling() gridding_first = gridding_last = None title_lines = [] if map_params.format == "xplor": fft_map.as_xplor_map(file_name=map_params.file_name, title_lines=title_lines, gridding_first=gridding_first, gridding_last=gridding_last) else: fft_map.as_ccp4_map(file_name=map_params.file_name, gridding_first=gridding_first, gridding_last=gridding_last, labels=title_lines) # output map coefficients if requested mtz_params = params.output.mtz # Decide if we are going to actually write the mtz if mtz_params.file_name is not None: orig_fom, final_fom = dm.start_and_end_fom() if mtz_params.skip_output_if_worse and final_fom < orig_fom: ok_to_write_mtz = False print "Not writing out mtz. Final FOM (%7.3f) worse than start (%7.3f)" % ( final_fom, orig_fom) else: # usual ok_to_write_mtz = True else: ok_to_write_mtz = True if mtz_params.file_name is not None and ok_to_write_mtz: label_decorator = iotbx.mtz.ccp4_label_decorator() fo = dm.miller_array_in_original_setting( dm.f_obs_complete).common_set(dm_map_coeffs) mtz_dataset = fo.as_mtz_dataset(column_root_label="F", label_decorator=label_decorator) mtz_dataset.add_miller_array(dm_map_coeffs, column_root_label="FWT", label_decorator=label_decorator) phase_source = dm.miller_array_in_original_setting( dm.phase_source).common_set(dm_map_coeffs) mtz_dataset.add_miller_array( phase_source.array(data=flex.abs(phase_source.data())), column_root_label="FOM", column_types='W', label_decorator=label_decorator) mtz_dataset.add_miller_array( phase_source.array(data=phase_source.phases(deg=True).data()), column_root_label="PHIB", column_types='P', label_decorator=None) if mtz_params.output_hendrickson_lattman_coefficients: mtz_dataset.add_miller_array(dm_hl_coeffs, column_root_label="HL", label_decorator=label_decorator) mtz_dataset.mtz_object().write(mtz_params.file_name) return result(map_file=map_params.file_name, mtz_file=mtz_params.file_name, stats=dm.get_stats())
def exercise_expand(): sg = sgtbx.space_group("P 41 (1,-1,0)") h = flex.miller_index(((3, 1, -2), (1, -2, 0))) assert tuple(sg.is_centric(h)) == (0, 1) p1 = miller.expand_to_p1_iselection(space_group=sg, anomalous_flag=False, indices=h, build_iselection=False) p1_i0 = ((-3, -1, 2), (-1, 3, 2), (3, 1, 2), (1, -3, 2), (1, -2, 0), (2, 1, 0)) assert tuple(p1.indices) == p1_i0 assert p1.iselection.size() == 0 p1 = miller.expand_to_p1_iselection(space_group=sg, anomalous_flag=True, indices=h, build_iselection=False) assert tuple(p1.indices) \ == ((3,1,-2), (1,-3,-2), (-3,-1,-2), (-1,3,-2), (1,-2,0), (-2,-1,0), (-1,2,0), (2,1,0)) p1 = miller.expand_to_p1_iselection(space_group=sg, anomalous_flag=False, indices=h, build_iselection=True) assert tuple(p1.indices) == p1_i0 assert tuple(p1.iselection) == (0, 0, 0, 0, 1, 1) a = flex.double((1, 2)) p = flex.double((10, 90)) p1 = miller.expand_to_p1_phases(space_group=sg, anomalous_flag=False, indices=h, data=p, deg=True) assert approx_equal(tuple(p1.data), (-10, 110, 110, -10, 90, 30)) p1 = miller.expand_to_p1_phases(space_group=sg, anomalous_flag=True, indices=h, data=p, deg=True) assert approx_equal(tuple(p1.data), (10, -110, -110, 10, 90, -30, -90, 30)) p = flex.double([x * math.pi / 180 for x in p]) v = [x * math.pi / 180 for x in p1.data] p1 = miller.expand_to_p1_phases(space_group=sg, anomalous_flag=True, indices=h, data=p, deg=False) assert approx_equal(tuple(p1.data), v) f = flex.polar(a, p) p1 = miller.expand_to_p1_complex(space_group=sg, anomalous_flag=True, indices=h, data=f) assert approx_equal(tuple(flex.abs(p1.data)), (1, 1, 1, 1, 2, 2, 2, 2)) assert approx_equal(tuple(flex.arg(p1.data)), v) hl = flex.hendrickson_lattman([(1, 2, 3, 4), (5, 6, 7, 8)]) p1 = miller.expand_to_p1_hendrickson_lattman(space_group=sg, anomalous_flag=True, indices=h, data=hl) assert approx_equal( p1.data, [[1, 2, 3, 4], [1.232051, -1.866025, -4.964102, 0.5980762], [1.232051, -1.866025, -4.964102, 0.5980762], [1, 2, 3, 4], [5, 6, 7, 8], [2.696152, -7.330127, -10.4282, 2.062178], [-5, -6, 7, 8], [7.696152, -1.330127, 3.428203, -10.06218]]) b = flex.bool([True, False]) p1 = miller.expand_to_p1_iselection(space_group=sg, anomalous_flag=True, indices=h, build_iselection=True) assert b.select(p1.iselection).all_eq( flex.bool([True, True, True, True, False, False, False, False])) i = flex.int([13, 17]) p1 = miller.expand_to_p1_iselection(space_group=sg, anomalous_flag=True, indices=h, build_iselection=True) assert i.select(p1.iselection).all_eq( flex.int([13, 13, 13, 13, 17, 17, 17, 17])) # assert approx_equal(miller.statistical_mean(sg, False, h, a), 4 / 3.) assert approx_equal(miller.statistical_mean(sg, True, h, a), 3 / 2.)
def substitute_ss( model, # changed in place params=None, log=null_out(), reference_map=None, verbose=False): """ Substitute secondary structure elements in real_h hierarchy with ideal ones _in_place_. Returns reference torsion proxies - the only thing that cannot be restored with little effort outside the procedure. """ ss_annotation = model.get_ss_annotation() t0 = time() if model.get_hierarchy().models_size() > 1: raise Sorry("Multi model files are not supported") for m in model.get_hierarchy().models(): for chain in m.chains(): if len(chain.conformers()) > 1: raise Sorry("Alternative conformations are not supported.") processed_params = process_params(params) if not processed_params.enabled: return None if ss_annotation is None: return None ann = ss_annotation if model.ncs_constraints_present(): print("Using master NCS to reduce amount of work", file=log) expected_n_hbonds = 0 for h in ann.helices: expected_n_hbonds += h.get_n_maximum_hbonds() edited_h = model.get_hierarchy().deep_copy() n_atoms_in_real_h = model.get_number_of_atoms() selection_cache = model.get_atom_selection_cache() # check the annotation for correctness (atoms are actually in hierarchy) error_msg = "The following secondary structure annotations result in \n" error_msg += "empty atom selections. They don't match the structre: \n" t1 = time() # Checking for SS selections deleted_annotations = ann.remove_empty_annotations( hierarchy=model.get_hierarchy(), asc=selection_cache) if not deleted_annotations.is_empty(): if processed_params.skip_empty_ss_elements: if len(deleted_annotations.helices) > 0: print("Removing the following helices because there are", file=log) print("no corresponding atoms in the model:", file=log) for h in deleted_annotations.helices: print(h.as_pdb_str(), file=log) error_msg += " %s\n" % h if len(deleted_annotations.sheets) > 0: print("Removing the following sheets because there are", file=log) print("no corresponding atoms in the model:", file=log) for sh in deleted_annotations.sheets: print(sh.as_pdb_str(), file=log) error_msg += " %s\n" % sh.as_pdb_str( strand_id=st.strand_id) else: raise Sorry(error_msg) phil_str = ann.as_restraint_groups() # gathering initial special position atoms special_position_settings = crystal.special_position_settings( crystal_symmetry=model.crystal_symmetry()) site_symmetry_table = \ special_position_settings.site_symmetry_table( sites_cart = model.get_sites_cart(), unconditional_general_position_flags=( model.get_atoms().extract_occ() != 1)) original_spi = site_symmetry_table.special_position_indices() t2 = time() # Actually idelizing SS elements fixed_ss_selection = flex.bool(n_atoms_in_real_h, False) log.write("Replacing ss-elements with ideal ones:\n") log.flush() ss_stats = gather_ss_stats(pdb_h=model.get_hierarchy()) n_idealized_elements = 0 master_bool_sel = model.get_master_selection() if master_bool_sel is None or master_bool_sel.size() == 0: master_bool_sel = flex.bool(model.get_number_of_atoms(), True) elif isinstance(master_bool_sel, flex.size_t): master_bool_sel = flex.bool(model.get_number_of_atoms(), master_bool_sel) assert master_bool_sel.size() == model.get_number_of_atoms() for h in ann.helices: log.write(" %s\n" % h.as_pdb_str()) log.flush() if processed_params.skip_good_ss_elements and ss_element_is_good( ss_stats, ([h], [])): log.write(" skipping, good element.\n") else: selstring = h.as_atom_selections() sel = selection_cache.selection(selstring[0]) isel = sel.iselection() if (master_bool_sel & sel).iselection().size() == 0: log.write(" skipping, not in NCS master.\n") continue n_idealized_elements += 1 log.write(" substitute with idealized one.\n") fixed_ss_selection.set_selected(isel, True) all_bsel = flex.bool(n_atoms_in_real_h, False) all_bsel.set_selected(isel, True) sel_h = model.get_hierarchy().select(all_bsel, copy_atoms=True) ideal_h = get_helix(helix_class=h.helix_class, pdb_hierarchy_template=sel_h, rotamer_manager=model.get_rotamer_manager()) # edited_h.select(all_bsel).atoms().set_xyz(ideal_h.atoms().extract_xyz()) set_xyz_carefully(dest_h=edited_h.select(all_bsel), source_h=ideal_h) # set_xyz_smart(dest_h=edited_h.select(all_bsel), source_h=ideal_h) # does not work here for sh in ann.sheets: s = " %s\n" % sh.as_pdb_str() ss = s.replace("\n", "\n ") log.write(ss[:-2]) log.flush() if processed_params.skip_good_ss_elements and ss_element_is_good( ss_stats, ([], [sh])): log.write(" skipping, good element.\n") else: full_sh_selection = flex.bool(n_atoms_in_real_h, False) for st in sh.strands: selstring = st.as_atom_selections() isel = selection_cache.iselection(selstring) full_sh_selection.set_selected(isel, True) if (master_bool_sel & full_sh_selection).iselection().size() == 0: log.write(" skipping, not in NCS master.\n") continue n_idealized_elements += 1 log.write(" substitute with idealized one.\n") for st in sh.strands: selstring = st.as_atom_selections() isel = selection_cache.iselection(selstring) all_bsel = flex.bool(n_atoms_in_real_h, False) all_bsel.set_selected(isel, True) fixed_ss_selection.set_selected(isel, True) sel_h = model.get_hierarchy().select(all_bsel, copy_atoms=True) ideal_h = secondary_structure_from_sequence( pdb_str=beta_pdb_str, sequence=None, pdb_hierarchy_template=sel_h, rotamer_manager=model.get_rotamer_manager(), ) set_xyz_carefully(edited_h.select(all_bsel), ideal_h) # edited_h.select(all_bsel).atoms().set_xyz(ideal_h.atoms().extract_xyz()) if n_idealized_elements == 0: log.write("Nothing was idealized.\n") # Don't do geometry minimization and stuff if nothing was changed. return None # XXX here we want to adopt new coordinates model.set_sites_cart(sites_cart=edited_h.atoms().extract_xyz()) if model.ncs_constraints_present(): model.set_sites_cart_from_hierarchy(multiply_ncs=True) t3 = time() # pre_result_h = edited_h # pre_result_h.reset_i_seq_if_necessary() bsel = flex.bool(n_atoms_in_real_h, False) helix_selection = flex.bool(n_atoms_in_real_h, False) sheet_selection = flex.bool(n_atoms_in_real_h, False) other_selection = flex.bool(n_atoms_in_real_h, False) ss_for_tors_selection = flex.bool(n_atoms_in_real_h, False) nonss_for_tors_selection = flex.bool(n_atoms_in_real_h, False) # set all CA atoms to True for other_selection #isel = selection_cache.iselection("name ca") isel = selection_cache.iselection("name ca or name n or name o or name c") other_selection.set_selected(isel, True) n_main_chain_atoms = other_selection.count(True) isel = selection_cache.iselection("name ca or name n or name o or name c") nonss_for_tors_selection.set_selected(isel, True) main_chain_selection_prefix = "(name ca or name n or name o or name c) %s" t4 = time() print("Preparing selections...", file=log) log.flush() # Here we are just preparing selections for h in ann.helices: ss_sels = h.as_atom_selections()[0] selstring = main_chain_selection_prefix % ss_sels isel = selection_cache.iselection(selstring) helix_selection.set_selected(isel, True) other_selection.set_selected(isel, False) isel = selection_cache.iselection(selstring) ss_for_tors_selection.set_selected(isel, True) nonss_for_tors_selection.set_selected(isel, False) for sheet in ann.sheets: for ss_sels in sheet.as_atom_selections(): selstring = main_chain_selection_prefix % ss_sels isel = selection_cache.iselection(selstring) sheet_selection.set_selected(isel, True) other_selection.set_selected(isel, False) isel = selection_cache.iselection(selstring) ss_for_tors_selection.set_selected(isel, True) nonss_for_tors_selection.set_selected(isel, False) t5 = time() isel = selection_cache.iselection( "not name ca and not name n and not name o and not name c") other_selection.set_selected(isel, False) helix_sheet_intersection = helix_selection & sheet_selection if helix_sheet_intersection.count(True) > 0: sheet_selection = sheet_selection & ~helix_sheet_intersection assert ((helix_selection | sheet_selection) & other_selection).count(True) == 0 from mmtbx.monomer_library.pdb_interpretation import grand_master_phil_str params_line = grand_master_phil_str params_line += "secondary_structure {%s}" % secondary_structure.sec_str_master_phil_str # print "params_line" # print params_line params = iotbx.phil.parse(input_string=params_line, process_includes=True) #.extract() # This does not work the same way for a strange reason. Need to investigate. # The number of resulting hbonds is different later. # w_params = params.extract() # w_params.pdb_interpretation.secondary_structure.protein.remove_outliers = False # w_params.pdb_interpretation.peptide_link.ramachandran_restraints = True # w_params.pdb_interpretation.c_beta_restraints = True # w_params.pdb_interpretation.secondary_structure.enabled = True # params.format(python_object=w_params) # params.show() # print "="*80 # print "="*80 # print "="*80 grm = model.get_restraints_manager() ssm_log = null_out() if verbose: ssm_log = log ss_params = secondary_structure.sec_str_master_phil.fetch().extract() ss_params.secondary_structure.protein.remove_outliers = False ss_manager = secondary_structure.manager( pdb_hierarchy=model.get_hierarchy(), geometry_restraints_manager=grm.geometry, sec_str_from_pdb_file=ss_annotation, params=ss_params.secondary_structure, mon_lib_srv=None, verbose=-1, log=ssm_log) grm.geometry.set_secondary_structure_restraints( ss_manager=ss_manager, hierarchy=model.get_hierarchy(), log=ssm_log) model.get_hierarchy().reset_i_seq_if_necessary() from mmtbx.geometry_restraints import reference if reference_map is None: if verbose: print("Adding reference coordinate restraints...", file=log) grm.geometry.append_reference_coordinate_restraints_in_place( reference.add_coordinate_restraints( sites_cart=model.get_sites_cart().select(helix_selection), selection=helix_selection, sigma=processed_params.sigma_on_reference_helix)) grm.geometry.append_reference_coordinate_restraints_in_place( reference.add_coordinate_restraints( sites_cart=model.get_sites_cart().select(sheet_selection), selection=sheet_selection, sigma=processed_params.sigma_on_reference_sheet)) grm.geometry.append_reference_coordinate_restraints_in_place( reference.add_coordinate_restraints( sites_cart=model.get_sites_cart().select(other_selection), selection=other_selection, sigma=processed_params.sigma_on_reference_non_ss)) # XXX Somewhere here we actually should check placed side-chains for # clashes because we used ones that were in original model and just moved # them to nearest allowed rotamer. The idealization may affect a lot # the orientation of side chain thus justifying changing rotamer on it # to avoid clashes. if processed_params.fix_rotamer_outliers: print("Fixing/checking rotamers...", file=log) # pre_result_h.write_pdb_file(file_name="before_rotamers.pdb") br_txt = model.model_as_pdb() with open("before_rotamers.pdb", 'w') as f: f.write(br_txt) if (reference_map is None): backbone_sample = False else: backbone_sample = True result = mmtbx.refinement.real_space.fit_residues.run( pdb_hierarchy=model.get_hierarchy(), crystal_symmetry=model.crystal_symmetry(), map_data=reference_map, rotamer_manager=mmtbx.idealized_aa_residues.rotamer_manager.load(), sin_cos_table=scitbx.math.sin_cos_table(n=10000), backbone_sample=backbone_sample, mon_lib_srv=model.get_mon_lib_srv(), log=log) model.set_sites_cart( sites_cart=result.pdb_hierarchy.atoms().extract_xyz(), update_grm=True) if verbose: print("Adding chi torsion restraints...", file=log) # only backbone grm.geometry.add_chi_torsion_restraints_in_place( pdb_hierarchy = model.get_hierarchy(), sites_cart = model.get_sites_cart().\ select(ss_for_tors_selection), selection = ss_for_tors_selection, chi_angles_only = False, sigma = processed_params.sigma_on_torsion_ss) grm.geometry.add_chi_torsion_restraints_in_place( pdb_hierarchy = model.get_hierarchy(), sites_cart = model.get_sites_cart().\ select(nonss_for_tors_selection), selection = nonss_for_tors_selection, chi_angles_only = False, sigma = processed_params.sigma_on_torsion_nonss) # real_h.atoms().set_xyz(pre_result_h.atoms().extract_xyz()) # # Check and correct for special positions # real_h = model.get_hierarchy() # just a shortcut here... special_position_settings = crystal.special_position_settings( crystal_symmetry=model.crystal_symmetry()) site_symmetry_table = \ special_position_settings.site_symmetry_table( sites_cart = model.get_sites_cart(), unconditional_general_position_flags=( model.get_atoms().extract_occ() != 1)) spi = site_symmetry_table.special_position_indices() if spi.size() > 0: print("Moving atoms from special positions:", file=log) for spi_i in spi: if spi_i not in original_spi: new_coords = (real_h.atoms()[spi_i].xyz[0] + 0.2, real_h.atoms()[spi_i].xyz[1] + 0.2, real_h.atoms()[spi_i].xyz[2] + 0.2) print(" ", real_h.atoms()[spi_i].id_str(), end=' ', file=log) print(tuple(real_h.atoms()[spi_i].xyz), "-->", new_coords, file=log) real_h.atoms()[spi_i].set_xyz(new_coords) model.set_sites_cart_from_hierarchy() t9 = time() if processed_params.file_name_before_regularization is not None: grm.geometry.pair_proxies(sites_cart=model.get_sites_cart()) grm.geometry.update_ramachandran_restraints_phi_psi_targets( hierarchy=model.get_hierarchy()) print("Outputting model before regularization %s" % processed_params.file_name_before_regularization, file=log) m_txt = model.model_as_pdb() g_txt = model.restraints_as_geo() with open(processed_params.file_name_before_regularization, 'w') as f: f.write(m_txt) geo_fname = processed_params.file_name_before_regularization[:-4] + '.geo' print("Outputting geo file for regularization %s" % geo_fname, file=log) with open(geo_fname, 'w') as f: f.write(g_txt) #testing number of restraints assert grm.geometry.get_n_den_proxies() == 0 if reference_map is None: assert grm.geometry.get_n_reference_coordinate_proxies() == n_main_chain_atoms, "" +\ "%d %d" % (grm.geometry.get_n_reference_coordinate_proxies(), n_main_chain_atoms) refinement_log = null_out() log.write( "Refining geometry of substituted secondary structure elements...") log.flush() if verbose: refinement_log = log t10 = time() if reference_map is None: minimize_wrapper_for_ramachandran( model=model, original_pdb_h=None, excl_string_selection="", log=refinement_log, number_of_cycles=processed_params.n_iter) else: ref_xrs = model.crystal_symmetry() minimize_wrapper_with_map(model=model, target_map=reference_map, refine_ncs_operators=False, number_of_cycles=processed_params.n_macro, log=log) model.set_sites_cart_from_hierarchy() log.write(" Done\n") log.flush() t11 = time() # print >> log, "Initial checking, init : %.4f" % (t1-t0) # print >> log, "Checking SS : %.4f" % (t2-t1) # print >> log, "Initializing selections : %.4f" % (t4-t3) # print >> log, "Looping for selections : %.4f" % (t5-t4) # print >> log, "Finalizing selections : %.4f" % (t6-t5) # print >> log, "PDB interpretation : %.4f" % (t7-t6) # print >> log, "Get GRM : %.4f" % (t8-t7) # print >> log, "Adding restraints to GRM : %.4f" % (t9-t8) # print >> log, "Running GM : %.4f" % (t11-t10) # print_hbond_proxies(grm.geometry,real_h) grm.geometry.remove_reference_coordinate_restraints_in_place() grm.geometry.remove_chi_torsion_restraints_in_place( nonss_for_tors_selection) return grm.geometry.get_chi_torsion_proxies()
def exercise_merge_equivalents(): i = flex.miller_index( ((1, 2, 3), (1, 2, 3), (3, 0, 3), (3, 0, 3), (3, 0, 3), (1, 1, 2))) d = flex.double((1, 2, 3, 4, 5, 6)) m = miller.ext.merge_equivalents_real(i, d) assert tuple(m.indices) == ((1, 2, 3), (3, 0, 3), (1, 1, 2)) assert approx_equal(m.data, (3 / 2., 4, 6)) assert tuple(m.redundancies) == (2, 3, 1) assert approx_equal(m.r_linear, (1 / 3., 1 / 6., 0)) assert approx_equal(m.r_square, (0.1, 0.04, 0)) assert approx_equal(m.r_int, (1. + 2.) / (3. + 12.)) # s = flex.double((1 / 3., 1 / 2., 1 / 4., 1 / 6., 1 / 3., 1 / 5.)) m = miller.ext.merge_equivalents_obs(i, d, s, sigma_dynamic_range=2e-6) assert tuple(m.indices) == ((1, 2, 3), (3, 0, 3), (1, 1, 2)) assert approx_equal(m.data, (17 / 13., (16 * 3 + 36 * 4 + 9 * 5) / (16 + 36 + 9.), 6)) assert approx_equal( m.sigmas, (math.sqrt(1 / 2. / 2), 0.84077140277 / 3**0.5, 1 / 5.)) assert m.sigma_dynamic_range == 2e-6 assert tuple(m.redundancies) == (2, 3, 1) assert approx_equal(m.r_linear, (1 / 3., 0.1762295, 0)) assert approx_equal(m.r_square, (0.1147929, 0.0407901, 0)) assert approx_equal( m.r_int, (abs(1 - 17 / 13.) + abs(2 - 17 / 13.) + abs(3 - 237 / 61.) + abs(4 - 237 / 61.) + abs(5 - 237 / 61.)) / (1 + 2 + 3 + 4 + 5)) # d = flex.complex_double([ complex(-1.706478, 0.248638), complex(1.097872, -0.983523), complex(0.147183, 2.625064), complex(-0.933310, 2.496886), complex(1.745500, -0.686761), complex(-0.620066, 2.097776) ]) m = miller.ext.merge_equivalents_complex(i, d) assert tuple(m.indices) == ((1, 2, 3), (3, 0, 3), (1, 1, 2)) assert approx_equal(m.data, [ complex(-0.304303, -0.367443), complex(0.319791, 1.478396), complex(-0.620066, 2.097776) ]) assert tuple(m.redundancies) == (2, 3, 1) # d = flex.hendrickson_lattman([(-1.706478, 0.248638, 1.653352, -2.411313), (1.097872, -0.983523, -2.756402, 0.294464), (0.147183, 2.625064, 1.003636, 2.563517), (-0.933310, 2.496886, 2.040418, 0.371885), (1.745500, -0.686761, -2.291345, -2.386650), (-0.620066, 2.097776, 0.099784, 0.268107)]) m = miller.ext.merge_equivalents_hl(i, d) assert tuple(m.indices) == ((1, 2, 3), (3, 0, 3), (1, 1, 2)) assert approx_equal(m.data, [(-0.3043030, -0.3674425, -0.5515250, -1.0584245), (0.3197910, 1.4783963, 0.2509030, 0.1829173), (-0.6200660, 2.0977760, 0.0997840, 0.2681070)]) assert tuple(m.redundancies) == (2, 3, 1) # d = flex.bool((True, True, False, False, False, True)) m = miller.ext.merge_equivalents_exact_bool(i, d) assert tuple(m.indices) == ((1, 2, 3), (3, 0, 3), (1, 1, 2)) assert list(m.data) == [True, False, True] assert tuple(m.redundancies) == (2, 3, 1) d = flex.bool((True, True, False, True, False, True)) try: m = miller.ext.merge_equivalents_exact_bool(i, d) except RuntimeError, e: assert str(e) == "cctbx Error: merge_equivalents_exact:"\ " incompatible flags for hkl = (3, 0, 3)"
def run(args, command_name="mmtbx.model_vs_data", show_geometry_statistics=True, model_size_max_atoms=80000, data_size_max_reflections=1000000, unit_cell_max_dimension=800., return_fmodel_and_pdb=False, out=None, log=sys.stdout): import mmtbx.f_model.f_model_info if (len(args) == 0) or (args == ["--help"]): print >> log, msg defaults(log=log, silent=False) return parsed = defaults(log=log, silent=True) # mvd_obj = mvd() # processed_args = utils.process_command_line_args(args=args, log=log, master_params=parsed) params = processed_args.params.extract() # reflection_files = processed_args.reflection_files if (len(reflection_files) == 0): raise Sorry("No reflection file found.") crystal_symmetry = processed_args.crystal_symmetry if (crystal_symmetry is None): raise Sorry("No crystal symmetry found.") if (len(processed_args.pdb_file_names) == 0): raise Sorry("No PDB file found.") pdb_file_names = processed_args.pdb_file_names # rfs = reflection_file_server(crystal_symmetry=crystal_symmetry, reflection_files=reflection_files) parameters = utils.data_and_flags_master_params().extract() if (params.f_obs_label is not None): parameters.labels = params.f_obs_label if (params.r_free_flags_label is not None): parameters.r_free_flags.label = params.r_free_flags_label if (params.high_resolution is not None): parameters.high_resolution = params.high_resolution determine_data_and_flags_result = utils.determine_data_and_flags( reflection_file_server=rfs, parameters=parameters, data_parameter_scope="refinement.input.xray_data", flags_parameter_scope="refinement.input.xray_data.r_free_flags", data_description="X-ray data", keep_going=True, log=StringIO()) f_obs = determine_data_and_flags_result.f_obs number_of_reflections = f_obs.indices().size() if (params.ignore_giant_models_and_datasets and number_of_reflections > data_size_max_reflections): raise Sorry("Too many reflections: %d" % number_of_reflections) # max_unit_cell_dimension = max(f_obs.unit_cell().parameters()[:3]) if (params.ignore_giant_models_and_datasets and max_unit_cell_dimension > unit_cell_max_dimension): raise Sorry("Too large unit cell (max dimension): %s" % str(max_unit_cell_dimension)) # r_free_flags = determine_data_and_flags_result.r_free_flags test_flag_value = determine_data_and_flags_result.test_flag_value if (r_free_flags is None): r_free_flags = f_obs.array(data=flex.bool(f_obs.data().size(), False)) test_flag_value = None # mmtbx_pdb_file = mmtbx.utils.pdb_file( pdb_file_names=pdb_file_names, cif_objects=processed_args.cif_objects, crystal_symmetry=crystal_symmetry, use_neutron_distances=(params.scattering_table == "neutron"), ignore_unknown_nonbonded_energy_types=not show_geometry_statistics, log=log) mmtbx_pdb_file.set_ppf(stop_if_duplicate_labels=False) processed_pdb_file = mmtbx_pdb_file.processed_pdb_file pdb_raw_records = mmtbx_pdb_file.pdb_raw_records pdb_inp = mmtbx_pdb_file.pdb_inp # # just to avoid going any further with bad PDB file.... pdb_inp.xray_structures_simple() # acp = processed_pdb_file.all_chain_proxies atom_selections = group_args( all=acp.selection(string="all"), macromolecule=acp.selection(string="protein or dna or rna"), solvent=acp.selection(string="water"), # XXX single_atom_residue ligand=acp.selection(string="not (protein or dna or rna or water)"), backbone=acp.selection(string="backbone"), sidechain=acp.selection(string="sidechain")) # scattering_table = params.scattering_table exptl_method = pdb_inp.get_experiment_type() if (exptl_method is not None) and ("NEUTRON" in exptl_method): scattering_table = "neutron" xsfppf = mmtbx.utils.xray_structures_from_processed_pdb_file( processed_pdb_file=processed_pdb_file, scattering_table=scattering_table, d_min=f_obs.d_min()) xray_structures = xsfppf.xray_structures if (0): #XXX normalize occupancies if all models have occ=1 so the total=1 n_models = len(xray_structures) for xrs in xray_structures: occ = xrs.scatterers().extract_occupancies() occ = occ / n_models xrs.set_occupancies(occ) model_selections = xsfppf.model_selections mvd_obj.collect(crystal=group_args( uc=f_obs.unit_cell(), sg=f_obs.crystal_symmetry().space_group_info().symbol_and_number(), n_sym_op=f_obs.crystal_symmetry().space_group_info().type().group( ).order_z(), uc_vol=f_obs.unit_cell().volume())) # hierarchy = pdb_inp.construct_hierarchy() pdb_atoms = hierarchy.atoms() pdb_atoms.reset_i_seq() # # Extract TLS pdb_tls = None pdb_inp_tls = pdb_inp.extract_tls_params(hierarchy) pdb_tls = group_args(pdb_inp_tls=pdb_inp_tls, tls_selections=[], tls_selection_strings=[]) # XXX no TLS + multiple models if (pdb_inp_tls.tls_present and pdb_inp_tls.error_string is None and len(xray_structures) == 1): pdb_tls = mmtbx.tls.tools.extract_tls_from_pdb( pdb_inp_tls=pdb_inp_tls, all_chain_proxies=mmtbx_pdb_file.processed_pdb_file. all_chain_proxies, xray_structure=xsfppf.xray_structure_all) if (len(pdb_tls.tls_selections) == len(pdb_inp_tls.tls_params) and len(pdb_inp_tls.tls_params) > 0): xray_structures = [ utils.extract_tls_and_u_total_from_pdb( f_obs=f_obs, r_free_flags=r_free_flags, xray_structure=xray_structures[ 0], # XXX no TLS + multiple models tls_selections=pdb_tls.tls_selections, tls_groups=pdb_inp_tls.tls_params) ] ########################### geometry_statistics = show_geometry( xray_structures=xray_structures, processed_pdb_file=processed_pdb_file, scattering_table=scattering_table, hierarchy=hierarchy, model_selections=model_selections, show_geometry_statistics=show_geometry_statistics, mvd_obj=mvd_obj, atom_selections=atom_selections) ########################### mp = mmtbx.masks.mask_master_params.extract() f_obs_labels = f_obs.info().label_string() f_obs = f_obs.sort(reverse=True, by_value="packed_indices") r_free_flags = r_free_flags.sort(reverse=True, by_value="packed_indices") fmodel = utils.fmodel_simple( xray_structures=xray_structures, scattering_table=scattering_table, mask_params=mp, f_obs=f_obs, r_free_flags=r_free_flags, skip_twin_detection=params.skip_twin_detection) n_outl = f_obs.data().size() - fmodel.f_obs().data().size() mvd_obj.collect(model_vs_data=show_model_vs_data(fmodel)) # Extract information from PDB file header and output (if any) pub_r_work = None pub_r_free = None pub_high = None pub_low = None pub_sigma = None pub_program_name = None pub_solv_cont = None pub_matthews = None published_results = pdb_inp.get_r_rfree_sigma(file_name=pdb_file_names[0]) if (published_results is not None): pub_r_work = published_results.r_work pub_r_free = published_results.r_free pub_high = published_results.high pub_low = published_results.low pub_sigma = published_results.sigma pub_program_name = pdb_inp.get_program_name() pub_solv_cont = pdb_inp.get_solvent_content() pub_matthews = pdb_inp.get_matthews_coeff() mvd_obj.collect(pdb_header=group_args(program_name=pub_program_name, year=pdb_inp.extract_header_year(), r_work=pub_r_work, r_free=pub_r_free, high_resolution=pub_high, low_resolution=pub_low, sigma_cutoff=pub_sigma, matthews_coeff=pub_matthews, solvent_cont=pub_solv_cont, tls=pdb_tls, exptl_method=exptl_method)) # # Recompute R-factors using published cutoffs fmodel_cut = fmodel tmp_sel = flex.bool(fmodel.f_obs().data().size(), True) if (pub_sigma is not None and fmodel.f_obs().sigmas() is not None): tmp_sel &= fmodel.f_obs().data() > fmodel.f_obs().sigmas() * pub_sigma if (pub_high is not None and abs(pub_high - fmodel.f_obs().d_min()) > 0.03): tmp_sel &= fmodel.f_obs().d_spacings().data() > pub_high if (pub_low is not None and abs(pub_low - fmodel.f_obs().d_max_min()[0]) > 0.03): tmp_sel &= fmodel.f_obs().d_spacings().data() < pub_low if (tmp_sel.count(True) != tmp_sel.size() and tmp_sel.count(True) > 0): fmodel_cut = utils.fmodel_simple( xray_structures=xray_structures, scattering_table=scattering_table, f_obs=fmodel.f_obs().select(tmp_sel), r_free_flags=fmodel.r_free_flags().select(tmp_sel), skip_twin_detection=params.skip_twin_detection) mvd_obj.collect( misc=group_args(r_work_cutoff=fmodel_cut.r_work(), r_free_cutoff=fmodel_cut.r_free(), n_refl_cutoff=fmodel_cut.f_obs().data().size())) mvd_obj.collect(data=show_data(fmodel=fmodel, n_outl=n_outl, test_flag_value=test_flag_value, f_obs_labels=f_obs_labels, fmodel_cut=fmodel_cut)) # CC* and friends cc_star_stats = None if (params.unmerged_data is not None): import mmtbx.validation.experimental import mmtbx.command_line f_obs = fmodel.f_obs().average_bijvoet_mates() unmerged_i_obs = mmtbx.command_line.load_and_validate_unmerged_data( f_obs=f_obs, file_name=params.unmerged_data, data_labels=params.unmerged_labels, log=null_out()) cc_star_stats = mmtbx.validation.experimental.merging_and_model_statistics( f_model=fmodel.f_model().average_bijvoet_mates(), f_obs=f_obs, r_free_flags=fmodel.r_free_flags().average_bijvoet_mates(), unmerged_i_obs=unmerged_i_obs, n_bins=params.n_bins) mvd_obj.show(log=out) if (cc_star_stats is not None): cc_star_stats.show_model_vs_data(out=out, prefix=" ") if return_fmodel_and_pdb: mvd_obj.pdb_file = processed_pdb_file mvd_obj.fmodel = fmodel if (len(params.map) > 0): for map_name_string in params.map: map_type_obj = mmtbx.map_names(map_name_string=map_name_string) map_params = mmtbx.maps.map_and_map_coeff_master_params().fetch( mmtbx.maps.cast_map_coeff_params(map_type_obj)).extract() maps_obj = mmtbx.maps.compute_map_coefficients( fmodel=fmodel_cut, params=map_params.map_coefficients) fn = os.path.basename(processed_args.reflection_file_names[0]) if (fn.count(".")): prefix = fn[:fn.index(".")] else: prefix = fn file_name = prefix + "_%s_map_coeffs.mtz" % map_type_obj.format() maps_obj.write_mtz_file(file_name=file_name) # statistics in bins if (not fmodel.twin): print >> log, "Statistics in resolution bins:" mmtbx.f_model.f_model_info.r_work_and_completeness_in_resolution_bins( fmodel=fmodel, out=log, prefix=" ") # report map cc if (params.comprehensive and not fmodel_cut.twin and fmodel_cut.xray_structure is not None): rsc_params = real_space_correlation.master_params().extract() rsc_params.scattering_table = scattering_table real_space_correlation.simple(fmodel=fmodel_cut, pdb_hierarchy=hierarchy, params=rsc_params, log=log, show_results=True) # if (params.dump_result_object_as_pickle): output_prefixes = [] for op in processed_args.pdb_file_names + processed_args.reflection_file_names: op = os.path.basename(op) try: op = op[:op.index(".")] except Exception: pass if (not op in output_prefixes): output_prefixes.append(op) output_prefix = "_".join(output_prefixes) easy_pickle.dump("%s.pickle" % output_prefix, mvd_obj) return mvd_obj
def deep_copy(self): return self.select(flex.bool(self.miller_array.indices().size(),True))
def deep_copy(self): size = self.pdb_hierarchy.atoms_size() return self.select(selection=flex.bool(size, True))
def master(frame_token, iparams, activity): if activity == "scale": n_batch = 1 frame_objects = frame_token indices = range(0, len(frame_objects), n_batch) for i in indices: i_end = i + n_batch if i + n_batch < len(frame_objects) else len( frame_objects) rankreq = comm.recv(source=MPI.ANY_SOURCE) comm.send((activity, (frame_objects[i:i_end], iparams)), dest=rankreq) elif activity == "pre_merge": frame_objects, avg_mode = frame_token n_batch = int(len(frame_objects) / (size * 3)) if n_batch < 10: n_batch = 10 indices = range(0, len(frame_objects), n_batch) for i in indices: i_end = i + n_batch if i + n_batch < len(frame_objects) else len( frame_objects) rankreq = comm.recv(source=MPI.ANY_SOURCE) comm.send((activity, (frame_objects[i:i_end], iparams, avg_mode)), dest=rankreq) elif activity == "merge": frame_objects, avg_mode = frame_token its = intensities_scaler() cpo = its.combine_pre_merge(frame_objects, iparams) #assign at least 100k reflections at a time n_batch = int(1e5 / (len(cpo[1]) / cpo[0])) if n_batch < 1: n_batch = 1 print "Merging with %d batch size" % (n_batch) indices = range(0, cpo[0], n_batch) for i in indices: rankreq = comm.recv(source=MPI.ANY_SOURCE) i_end = i + n_batch if i + n_batch < cpo[0] else cpo[0] sel = flex.bool([ sel_l and sel_h for sel_l, sel_h in zip(cpo[1] >= i, cpo[1] < i_end) ]) batch_prep = [cpo_elem.select(sel) for cpo_elem in cpo[1:13]] batch_prep.insert(0, i_end - i) batch_prep[1] -= i batch_prep.append(cpo[13]) batch_prep.append(cpo[14]) batch_prep.append(cpo[15].select(sel)) batch_prep.append("") comm.send((activity, (tuple(batch_prep), iparams, avg_mode)), dest=rankreq) elif activity == "postref": frame_files, miller_array_ref, results, avg_mode = frame_token #convert results to a dict obj so that pickle_filename is the key pres_dict = {} for pres in results: if pres: pres_dict[pres.pickle_filename] = pres for i in xrange(len(frame_files)): rankreq = comm.recv(source=MPI.ANY_SOURCE) comm.send((activity, (i, frame_files[i], iparams, miller_array_ref, pres_dict[frame_files[i]] if frame_files[i] in pres_dict else None, avg_mode)), dest=rankreq) print "Master for %s is completed. Time to stop all %d clients" % ( activity, size - 1) # stop clients for rankreq in range(size - 1): rankreq = comm.recv(source=MPI.ANY_SOURCE) comm.send('endrun', dest=rankreq)
def show_geometry(xray_structures, processed_pdb_file, scattering_table, hierarchy, model_selections, show_geometry_statistics, mvd_obj, atom_selections): if (len(xray_structures) > 1): tmp = xray_structures[0] for xi in xray_structures[1:]: tmp = tmp.concatenate(xi) xray_structures = tmp else: xray_structures = xray_structures[0] ## utils.assert_xray_structures_equal(x1=xray_structures, x2=processed_pdb_file.xray_structure(), sites=True, adp=False, occupancies=True) ## hd_sel_all = xray_structures.hd_selection() if (show_geometry_statistics): sctr_keys = \ xray_structures.scattering_type_registry().type_count_dict().keys() has_hd = "H" in sctr_keys or "D" in sctr_keys geometry = processed_pdb_file.geometry_restraints_manager( show_energies=False, plain_pairs_radius=5.0, assume_hydrogens_all_missing=not has_hd) restraints_manager_all = mmtbx.restraints.manager(geometry=geometry, normalization=True) models = hierarchy.models() geometry_statistics = [] n_residues_in_altlocs = None for i_seq, model_selection in enumerate(model_selections): hierarchy_i_seq = pdb.hierarchy.root() hierarchy_i_seq.append_model(models[i_seq].detached_copy()) # overall_counts_i_seq = hierarchy_i_seq.overall_counts() n_residues_in_altlocs = \ overall_counts_i_seq.n_alt_conf_pure + \ overall_counts_i_seq.n_alt_conf_proper + \ overall_counts_i_seq.n_alt_conf_improper # resname_classes = [] for k, v in zip(overall_counts_i_seq.resname_classes.keys(), overall_counts_i_seq.resname_classes.values()): resname_classes.append(" ".join([k.replace("common_", ""), str(v)])) # xray_structure = xray_structures.select(model_selection) assert hierarchy_i_seq.atoms_size() == xray_structure.scatterers( ).size() hd_sel = xray_structure.hd_selection() def select_atom_selections(selection=model_selection, atom_selections=atom_selections): result = group_args() result.all = atom_selections.all.select(selection) result.macromolecule = atom_selections.macromolecule.select( selection) result.solvent = atom_selections.solvent.select(selection) result.ligand = atom_selections.ligand.select(selection) result.backbone = atom_selections.backbone.select(selection) result.sidechain = atom_selections.sidechain.select(selection) return result atom_selections_i_model = select_atom_selections( selection=model_selection, atom_selections=atom_selections) if (hd_sel.count(True) > 0 and scattering_table != "neutron"): xray_structure_stat = show_xray_structure_statistics( xray_structure=xray_structure, hd_sel=hd_sel, atom_selections=atom_selections_i_model) else: xray_structure_stat = show_xray_structure_statistics( xray_structure=xray_structure, atom_selections=atom_selections_i_model) model_statistics_geometry_macromolecule = None model_statistics_geometry_solvent = None model_statistics_geometry_ligand = None model_statistics_geometry_all = None molprobity_stats_i_seq = None rms_b_iso_or_b_equiv_bonded = None if (show_geometry_statistics): # exclude hydrogens if (hd_sel.count(True) > 0 and scattering_table != "neutron"): xray_structure = xray_structure.select(~hd_sel) model_selection = model_selection.select(~hd_sel) hierarchy_i_seq = hierarchy_i_seq.select(~hd_sel) geometry = restraints_manager_all.geometry.select( selection=~hd_sel_all) atom_selections_i_model = select_atom_selections( selection=~hd_sel_all, atom_selections=atom_selections_i_model) model_selection_as_bool = flex.bool( xray_structures.scatterers().size(), model_selection) geometry = restraints_manager_all.geometry.select( selection=model_selection_as_bool) restraints_manager = mmtbx.restraints.manager(geometry=geometry, normalization=True) restraints_manager.geometry.pair_proxies( sites_cart=xray_structure.sites_cart()) ### model_statistics_geometry_all = model_statistics.geometry( pdb_hierarchy=hierarchy_i_seq, molprobity_scores=True, restraints_manager=restraints_manager) # if (atom_selections.macromolecule.count(True) > 0): mac_sel = atom_selections_i_model.macromolecule model_statistics_geometry_macromolecule = model_statistics.geometry( pdb_hierarchy=hierarchy_i_seq.select(mac_sel), molprobity_scores=True, restraints_manager=restraints_manager.select(mac_sel)) # if (atom_selections.solvent.count(True) > 0): sol_sel = atom_selections_i_model.solvent model_statistics_geometry_solvent = model_statistics.geometry( pdb_hierarchy=hierarchy_i_seq.select(sol_sel), molprobity_scores=True, restraints_manager=restraints_manager.select(sol_sel)) # if (atom_selections.ligand.count(True) > 0): lig_sel = atom_selections_i_model.ligand model_statistics_geometry_ligand = model_statistics.geometry( pdb_hierarchy=hierarchy_i_seq.select(lig_sel), molprobity_scores=True, restraints_manager=restraints_manager.select(lig_sel)) ### rms_b_iso_or_b_equiv_bonded = utils.rms_b_iso_or_b_equiv_bonded( restraints_manager=restraints_manager, xray_structure=xray_structure) # molprobity_stats_i_seq = molprobity_stats( model_statistics_geometry=model_statistics_geometry_all, resname_classes=overall_counts_i_seq.resname_classes) geometry_statistics.append( group_args( n_residues_in_altlocs=n_residues_in_altlocs, resname_classes=resname_classes, xray_structure_stat=xray_structure_stat, rms_b_iso_or_b_equiv_bonded=rms_b_iso_or_b_equiv_bonded, geometry_all=model_statistics_geometry_all, geometry_macromolecule=model_statistics_geometry_macromolecule, geometry_solvent=model_statistics_geometry_solvent, geometry_ligand=model_statistics_geometry_ligand, molprobity=molprobity_stats_i_seq)) mvd_obj.collect(models=geometry_statistics) return geometry_statistics
def run_by_params(self, iparams, avg_mode='average'): #read all result pickles try: DIR = iparams.run_no + '/pickles/' pickle_results = [ pickle.load(open(DIR + fname, "rb")) for fname in os.listdir(DIR) ] n_results = len(pickle_results) #for the last cycle merging with weak anomalous flag on #fetch the original observations from the integration pickles if iparams.flag_weak_anomalous and iparams.target_anomalous_flag and avg_mode == 'final': prh = postref_handler() for pres in pickle_results: if pres is not None: observations_pickle = pickle.load( open(pres.pickle_filename, 'rb')) inputs, _ = prh.organize_input( observations_pickle, iparams, avg_mode, pickle_filename=pres.pickle_filename) observations_original, alpha_angle_obs, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs two_theta = observations_original.two_theta( wavelength=pres.wavelength).data() ph = partiality_handler() partiality_set, delta_xy_set, rs_set, rh_set = ph.calc_partiality_anisotropy_set( pres.unit_cell, pres.rotx, pres.roty, observations_original.indices(), pres.ry, pres.rz, pres.r0, pres.re, pres.voigt_nu, two_theta, alpha_angle_obs, pres.wavelength, pres.crystal_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) observations, alt_hkl = prh.get_observations_non_polar( observations_original, pres.pickle_filename, iparams) pres.observations = observations pres.observations_original = observations_original pres.partiality = partiality_set pres.rs_set = rs_set pres.rh_set = rh_set pres.mapped_predictions = mapped_predictions except Exception: print "Error reading input pickles." print "Check if prime.run, prime.genref, or prime.postrefine was called prior to merge." exit() from prime.postrefine import prepare_output prep_output = prepare_output(pickle_results, iparams, avg_mode) txt_merge_mean = 'prime.merge\n' if prep_output is not None: #grab linear lists cn_group, group_id_list, miller_indices_all_sort, miller_indices_ori_all_sort, I_obs_all_sort, \ sigI_obs_all_sort, G_all_sort, B_all_sort, p_all_sort, rs_all_sort, wavelength_all_sort, \ sin_sq_all_sort, SE_all_sort, uc_mean, wavelength_mean, pickle_filename_all_sort, txt_prep_out = prep_output #start averaging from prime.postrefine import calc_avg_I_cpp calc_average_I_result = calc_avg_I_cpp( cn_group, group_id_list, miller_indices_all_sort, miller_indices_ori_all_sort, I_obs_all_sort, sigI_obs_all_sort, G_all_sort, B_all_sort, p_all_sort, rs_all_sort, wavelength_all_sort, sin_sq_all_sort, SE_all_sort, avg_mode, iparams, pickle_filename_all_sort) #grab average results miller_index, I_merge, sigI_merge, stats, I_two_halves_tuple, txt_obs_out, txt_reject_out = calc_average_I_result I_even, I_odd, I_even_h, I_odd_h, I_even_k, I_odd_k, I_even_l, I_odd_l = I_two_halves_tuple #remove stat items with nan r_meas_w_top, r_meas_w_btm, r_meas_top, r_meas_btm, multiplicity = stats sel = flex.bool([math.isnan(r_top) or math.isinf(r_top)\ or math.isnan(r_btm) or math.isinf(r_btm) for r_top, r_btm in zip(r_meas_w_top, r_meas_w_btm)]) inverse_sel = ~sel inverse_isel = inverse_sel.iselection() miller_index = miller_index.select(inverse_isel) I_merge = I_merge.select(inverse_isel) sigI_merge = sigI_merge.select(inverse_isel) I_even = I_even.select(inverse_isel) I_odd = I_odd.select(inverse_isel) I_even_h = I_even_h.select(inverse_isel) I_odd_h = I_odd_h.select(inverse_isel) I_even_k = I_even_k.select(inverse_isel) I_odd_k = I_odd_k.select(inverse_isel) I_even_l = I_even_l.select(inverse_isel) I_odd_l = I_odd_l.select(inverse_isel) stat_all = (r_meas_w_top.select(inverse_isel), r_meas_w_btm.select(inverse_isel), \ r_meas_top.select(inverse_isel), r_meas_btm.select(inverse_isel), multiplicity.select(inverse_isel)) #write out rejected reflections f = open(iparams.run_no + '/rejections.txt', 'w') f.write(txt_reject_out) f.close() #get the latest no. of mtz file and write out the reflections DIR = iparams.run_no + '/mtz/' file_no = len([ int(fname.split('.')[0]) for fname in os.listdir(DIR) if os.path.join(DIR, fname).endswith('mtz') ]) from prime.postrefine import write_output miller_array_ref, txt_merge_mean_table = write_output( miller_index, I_merge, sigI_merge, stat_all, (I_even, I_odd, I_even_h, I_odd_h, I_even_k, I_odd_k, I_even_l, I_odd_l), iparams, uc_mean, wavelength_mean, file_no, avg_mode) txt_merge_mean += txt_merge_mean_table + txt_prep_out print txt_merge_mean f = open(iparams.run_no + '/log.txt', 'a') f.write(txt_merge_mean) f.close()
def run(server_info, inp, status): print "<pre>" if (inp.format == "cns_sdb"): print "Minimum distance between symmetrically equivalent sites:", print float(inp.min_distance_sym_equiv) print structures = inp_as_xray_structures(inp) if (len(structures) == 0): print "No CNS sdb files found!" print print "Note that each file must start with {+ file: some_file_name +}" print "in order to be recognized." print else: if (inp.ucparams is None): inp.ucparams = "" if (inp.sgsymbol is None): inp.sgsymbol = "P1" special_position_settings = io_utils.special_position_settings_from_inp( inp) special_position_settings.show_summary() print "Minimum distance between symmetrically equivalent sites:", print special_position_settings.min_distance_sym_equiv() print structures = [ io_utils.structure_from_inp(inp, status, special_position_settings) ] d_min = float(inp.d_min) print "Minimum d-spacing:", d_min if (d_min <= 0.): raise ValueError, "d-spacing must be greater than zero." print min_peak_distance = float(inp.min_peak_distance) print "Minimum peak distance:", min_peak_distance if (min_peak_distance <= 0.): raise ValueError, "min_peak_distance must be greater than zero." print max_reduced_peaks = int(inp.max_reduced_peaks) print "Maximum number of peaks:", max_reduced_peaks if (max_reduced_peaks <= 0): raise ValueError, "max_reduced_peaks must be greater than zero." print for structure in structures: if (inp.format == "cns_sdb"): structure.show_summary().show_scatterers() print if (structure.scatterers().size() == 0): continue reduced_peaks = phase_o_phrenia.calculate_exp_i_two_phi_peaks( xray_structure=structure, d_min=d_min, min_peak_distance=min_peak_distance, max_reduced_peaks=max_reduced_peaks) print "Actual number of peaks:", len(reduced_peaks) print plot_nx = min(len(reduced_peaks), 60) if (plot_nx > 0): plot_ny = max(10, plot_nx // 3) if (plot_nx != max_reduced_peaks): print "Number of peaks used for plot:", plot_nx print print "Plot of relative peak heights:" print plot = flex.bool(flex.grid(plot_nx, plot_ny)) for i in xrange(plot_nx): height = reduced_peaks[i].height h = int(round(height * plot_ny)) h = max(0, min(plot_ny, h)) for j in xrange(h): plot[(i, j)] = True for j in xrange(plot_ny - 1, -1, -1): line = "" for i in xrange(plot_nx): if (plot[(i, j)]): line += "*" else: line += " " print " |" + line.rstrip() print " -" + "-" * plot_nx print print "Peak list:" print " Relative" print " height Fractional coordinates" for peak in reduced_peaks: print " %5.1f" % (peak.height * 100), " %8.5f %8.5f %8.5f" % peak.site print print "</pre>"
def extract(file_name, crystal_symmetry, wavelength_id, crystal_id, show_details_if_error, output_r_free_label, merge_non_unique_under_symmetry, map_to_asu, remove_systematic_absences, all_miller_arrays=None, incompatible_flags_to_work_set=False, ignore_bad_sigmas=False, extend_flags=False, return_as_miller_arrays=False, log=sys.stdout): import iotbx.cif from cctbx import miller if all_miller_arrays is None: base_array_info = miller.array_info( crystal_symmetry_from_file=crystal_symmetry) all_miller_arrays = iotbx.cif.reader( file_path=file_name).build_miller_arrays( base_array_info=base_array_info) if (len(all_miller_arrays) == 0): raise Sorry( "No data arrays were found in this CIF file. Please make " + "sure that the file contains reflection data, rather than the refined " + "model.") column_labels = set() if (extend_flags): map_to_asu = True # TODO: is all_mille_arrays a dict ? If not change back for (data_name, miller_arrays) in six.iteritems(all_miller_arrays): for ma in miller_arrays.values(): other_symmetry = crystal_symmetry try: crystal_symmetry = other_symmetry.join_symmetry( other_symmetry=ma.crystal_symmetry(), force=True) except AssertionError as e: str_e = str(e) from six.moves import cStringIO as StringIO s = StringIO() if "Space group is incompatible with unit cell parameters." in str_e: other_symmetry.show_summary(f=s) ma.crystal_symmetry().show_summary(f=s) str_e += "\n%s" % (s.getvalue()) raise Sorry(str_e) else: raise if (crystal_symmetry.unit_cell() is None or crystal_symmetry.space_group_info() is None): raise Sorry( "Crystal symmetry is not defined. Please use the --symmetry option." ) mtz_object = iotbx.mtz.object() \ .set_title(title="phenix.cif_as_mtz") \ .set_space_group_info(space_group_info=crystal_symmetry.space_group_info()) unit_cell = crystal_symmetry.unit_cell() mtz_crystals = {} mtz_object.set_hkl_base(unit_cell=unit_cell) from iotbx.reflection_file_utils import cif_status_flags_as_int_r_free_flags # generate list of all reflections (for checking R-free flags) from iotbx.reflection_file_utils import make_joined_set all_arrays = [] for (data_name, miller_arrays) in six.iteritems(all_miller_arrays): for ma in miller_arrays.values(): all_arrays.append(ma) complete_set = make_joined_set(all_arrays) if return_as_miller_arrays: miller_array_list = [] for i, (data_name, miller_arrays) in enumerate(six.iteritems(all_miller_arrays)): for ma in miller_arrays.values(): ma = ma.customized_copy( crystal_symmetry=crystal_symmetry).set_info(ma.info()) labels = ma.info().labels label = get_label(miller_array=ma, output_r_free_label=output_r_free_label) if label is None: print("Can't determine output label for %s - skipping." % \ ma.info().label_string(), file=log) continue elif label.startswith(output_r_free_label): ma, _ = cif_status_flags_as_int_r_free_flags( ma, test_flag_value="f") if isinstance(ma.data(), flex.double): data_int = ma.data().iround() assert data_int.as_double().all_eq(ma.data()) ma = ma.customized_copy(data=data_int).set_info(ma.info()) elif ( (ma.is_xray_amplitude_array() or ma.is_xray_intensity_array()) and isinstance(ma.data(), flex.int)): ma = ma.customized_copy(data=ma.data().as_double()).set_info( ma.info()) crys_id = 0 for l in labels: if 'crystal_id' in l: crys_id = int(l.split('=')[-1]) break if crys_id > 0 and crystal_id is None: label += "%i" % crys_id if crystal_id is not None and crys_id > 0 and crys_id != crystal_id: continue if crys_id not in mtz_crystals: mtz_crystals[crys_id] = (mtz_object.add_crystal( name="crystal_%i" % crys_id, project_name="project", unit_cell=unit_cell), {}) crystal, datasets = mtz_crystals[crys_id] w_id = 0 for l in labels: if 'wavelength_id' in l: w_id = int(l.split('=')[-1]) break if wavelength_id is not None and w_id > 0 and w_id != wavelength_id: continue if w_id > 1 and wavelength_id is None: if (label in column_labels): label += "%i" % w_id #print "label is", label if w_id not in datasets: wavelength = ma.info().wavelength if (wavelength is None): wavelength = 0 datasets[w_id] = crystal.add_dataset(name="dataset", wavelength=wavelength) dataset = datasets[w_id] # if all sigmas for an array are set to zero either raise an error, or set sigmas to None if ma.sigmas() is not None and (ma.sigmas() == 0).count(False) == 0: if ignore_bad_sigmas: print("Warning: bad sigmas, setting sigmas to None.", file=log) ma.set_sigmas(None) else: raise Sorry("""Bad sigmas: all sigmas are equal to zero. Add --ignore_bad_sigmas to command arguments to leave out sigmas from mtz file.""" ) if not ma.is_unique_set_under_symmetry(): if merge_non_unique_under_symmetry: print("Warning: merging non-unique data", file=log) if (label.startswith(output_r_free_label) and incompatible_flags_to_work_set): merging = ma.merge_equivalents( incompatible_flags_replacement=0) if merging.n_incompatible_flags > 0: print("Warning: %i reflections were placed in the working set " \ "because of incompatible flags between equivalents." %( merging.n_incompatible_flags), file=log) else: try: merging = ma.merge_equivalents() except Sorry as e: if ("merge_equivalents_exact: incompatible" in str(e)): raise Sorry( str(e) + " for %s" % ma.info().labels[-1] + "\n" + "Add --incompatible_flags_to_work_set to command line " "arguments to place incompatible flags to working set." ) raise ma = merging.array().customized_copy( crystal_symmetry=ma).set_info(ma.info()) elif return_as_miller_arrays: # allow non-unique set pass else: n_all = ma.indices().size() sel_unique = ma.unique_under_symmetry_selection() sel_dup = ~flex.bool(n_all, sel_unique) n_duplicate = sel_dup.count(True) n_uus = sel_unique.size() msg = ( "Miller indices not unique under symmetry: " + file_name + \ "(%d redundant indices out of %d)" % (n_all-n_uus, n_all) + "Add --merge to command arguments to force merging data.") if (show_details_if_error): print(msg) ma.show_comprehensive_summary(prefix=" ") ma.map_to_asu().sort().show_array(prefix=" ") raise Sorry(msg) if (map_to_asu): ma = ma.map_to_asu().set_info(ma.info()) if (remove_systematic_absences): ma = ma.remove_systematic_absences() if (label.startswith(output_r_free_label) and complete_set is not None): n_missing = len(complete_set.lone_set(other=ma).indices()) if (n_missing > 0): if (extend_flags): from cctbx import r_free_utils # determine flag values fvals = list(set(ma.data())) print("fvals", fvals) fval = None if (len(fvals) == 1): fval = fvals[0] elif (len(fvals) == 2): f1 = (ma.data() == fvals[0]).count(True) / ma.data().size() f2 = (ma.data() == fvals[1]).count(True) / ma.data().size() if (f1 < f2): fval = fvals[0] else: fval = fvals[1] elif (len(fvals) == 0): fval = None else: fval = 0 if (not fval in fvals): raise Sorry( "Cannot determine free-R flag value.") # if (fval is not None): ma = r_free_utils.extend_flags( r_free_flags=ma, test_flag_value=fval, array_label=label, complete_set=complete_set, preserve_input_values=True, allow_uniform_flags=True, log=sys.stdout) else: ma = None else: libtbx.warn(( "%d reflections do not have R-free flags in the " + "array '%s' - this may " + "cause problems if you try to use the MTZ file for refinement " + "or map calculation. We recommend that you extend the flags " + "to cover all reflections (--extend_flags on the command line)." ) % (n_missing, label)) # Get rid of fake (0,0,0) reflection in some CIFs if (ma is not None): ma = ma.select_indices(indices=flex.miller_index( ((0, 0, 0), )), negate=True).set_info(ma.info()) if return_as_miller_arrays: miller_array_list.append(ma) continue # don't make a dataset dec = None if ("FWT" in label): dec = iotbx.mtz.ccp4_label_decorator() column_types = None if ("PHI" in label or "PHWT" in label) and (ma.is_real_array()): column_types = "P" elif (label.startswith("DANO") and ma.is_real_array()): if (ma.sigmas() is not None): column_types = "DQ" else: column_types = "D" label_base = label i = 1 while label in column_labels: label = label_base + "-%i" % (i) i += 1 if (ma is not None): column_labels.add(label) dataset.add_miller_array(ma, column_root_label=label, label_decorator=dec, column_types=column_types) if return_as_miller_arrays: return miller_array_list else: return mtz_object
def __init__(self, fmodel, selections=None, params=None, r_initial=None, t_initial=None, bss=None, log=None, monitors=None): global time_rigid_body_total self.params = params save_original_target_name = fmodel.target_name save_bss_anisotropic_scaling = None if (bss is not None): save_bss_anisotropic_scaling = bss.anisotropic_scaling timer_rigid_body_total = user_plus_sys_time() save_r_work = fmodel.r_work() save_r_free = fmodel.r_free() save_xray_structure = fmodel.xray_structure.deep_copy_scatterers() if (log is None): log = sys.stdout if (selections is None): selections = [] selections.append( flex.bool(fmodel.xray_structure.scatterers().size(), True).iselection()) else: assert len(selections) > 0 fmodel.xray_structure.scatterers().flags_set_grads(state=False) fmodel.xray_structure.scatterers().flags_set_grad_site( iselection=flex.bool(fmodel.xray_structure.scatterers().size(), True).iselection()) self.total_rotation = [] self.total_translation = [] for item in selections: self.total_rotation.append(flex.double(3, 0)) self.total_translation.append(flex.double(3, 0)) if (r_initial is None): r_initial = [] for item in selections: r_initial.append(flex.double(3, 0)) if (t_initial is None): t_initial = [] for item in selections: t_initial.append(flex.double(3, 0)) fmodel_copy = fmodel.deep_copy() if (fmodel_copy.mask_params is not None): fmodel_copy.mask_params.verbose = -1 d_mins, target_names = split_resolution_range( d_spacings=fmodel_copy.f_obs_work().d_spacings().data(), n_bodies=len(selections), target=params.target, target_auto_switch_resolution=params.target_auto_switch_resolution, n_ref_first=params.min_number_of_reflections, multi_body_factor_n_ref_first=params.multi_body_factor, d_low=params.max_low_high_res_limit, d_high=params.high_resolution, number_of_zones=params.number_of_zones, zone_exponent=params.zone_exponent, log=log) print >> log if (fmodel.target_name != target_names[0]): fmodel.update(target_name=target_names[0]) self.show(fmodel=fmodel, r_mat=self.total_rotation, t_vec=self.total_translation, header="Start", out=log) if (params.number_of_zones == 1 or monitors is None): monitors_call_back_handler = None else: monitors_call_back_handler = monitors.call_back_handler if (monitors_call_back_handler is not None): monitors_call_back_handler(monitor=None, model=None, fmodel=fmodel, method="rigid_body") for res, target_name in zip(d_mins, target_names): xrs = fmodel_copy.xray_structure.deep_copy_scatterers() fmodel_copy = fmodel.resolution_filter(d_min=res) if (fmodel_copy.target_name != target_name): fmodel_copy.update(target_name=target_name) d_max_min = fmodel_copy.f_obs_work().d_max_min() line = "Refinement at resolution: "+\ str("%7.2f"%d_max_min[0]).strip() + " - " \ + str("%6.2f"%d_max_min[1]).strip() \ + " target=" + fmodel_copy.target_name print_statistics.make_sub_header(line, out=log) fmodel_copy.update_xray_structure(xray_structure=xrs, update_f_calc=True) rworks = flex.double() if (len(d_mins) == 1): n_rigid_body_minimizer_cycles = 1 else: n_rigid_body_minimizer_cycles = min(int(res), 4) for i_macro_cycle in xrange(n_rigid_body_minimizer_cycles): if (bss is not None and params.bulk_solvent_and_scale): if (fmodel_copy.f_obs().d_min() > 3.0): bss.anisotropic_scaling = False fast = True if (bss.mode == "slow"): fast = False fmodel_copy.update_all_scales(update_f_part1=False, params=bss, fast=fast, log=log, remove_outliers=False, optimize_mask=False, refine_hd_scattering=False) if (fmodel_copy.f_obs().d_min() > 3.0): assert save_bss_anisotropic_scaling is not None bss.anisotropic_scaling = save_bss_anisotropic_scaling bss.minimization_b_cart = save_bss_anisotropic_scaling minimized = rigid_body_minimizer( fmodel=fmodel_copy, selections=selections, r_initial=r_initial, t_initial=t_initial, refine_r=params.refine_rotation, refine_t=params.refine_translation, max_iterations=params.max_iterations, euler_angle_convention=params.euler_angle_convention, lbfgs_maxfev=params. lbfgs_line_search_max_function_evaluations) rotation_matrices = [] translation_vectors = [] for i in xrange(len(selections)): self.total_rotation[i] += flex.double(minimized.r_min[i]) self.total_translation[i] += flex.double( minimized.t_min[i]) rot_obj = scitbx.rigid_body.euler( phi=minimized.r_min[i][0], psi=minimized.r_min[i][1], the=minimized.r_min[i][2], convention=params.euler_angle_convention) rotation_matrices.append(rot_obj.rot_mat()) translation_vectors.append(minimized.t_min[i]) new_xrs = apply_transformation( xray_structure=minimized.fmodel.xray_structure, rotation_matrices=rotation_matrices, translation_vectors=translation_vectors, selections=selections) fmodel_copy.update_xray_structure(xray_structure=new_xrs, update_f_calc=True, update_f_mask=True) rwork = minimized.fmodel.r_work() rfree = minimized.fmodel.r_free() assert approx_equal(rwork, fmodel_copy.r_work()) fmodel.update_xray_structure( xray_structure=fmodel_copy.xray_structure, update_f_calc=True, update_f_mask=True) if (bss is not None and params.bulk_solvent_and_scale): fast = True if (bss.mode == "slow"): fast = False fmodel_copy.update_all_scales(update_f_part1=False, params=bss, fast=fast, log=log, remove_outliers=False, optimize_mask=False, refine_hd_scattering=False) self.show(fmodel=fmodel, r_mat=self.total_rotation, t_vec=self.total_translation, header="Rigid body refinement", out=log) if (monitors_call_back_handler is not None): monitors_call_back_handler(monitor=None, model=None, fmodel=fmodel, method="rigid_body") if (bss is not None and params.bulk_solvent_and_scale): fast = True if (bss.mode == "slow"): fast = False fmodel_copy.update_all_scales(update_f_part1=False, params=bss, fast=fast, log=log, remove_outliers=False, optimize_mask=False, refine_hd_scattering=False) print >> log self.show(fmodel=fmodel, r_mat=self.total_rotation, t_vec=self.total_translation, header="Rigid body end", out=log) print >> log self.evaluate_after_end(fmodel, save_r_work, save_r_free, save_xray_structure, log) self.fmodel = fmodel self.fmodel.update(target_name=save_original_target_name) time_rigid_body_total += timer_rigid_body_total.elapsed()
in str(e)): raise Sorry( str(e) + " for %s" % ma.info().labels[-1] + "\n" + "Add --incompatible_flags_to_work_set to command line " "arguments to place incompatible flags to working set." ) raise ma = merging.array().customized_copy( crystal_symmetry=ma).set_info(ma.info()) elif return_as_miller_arrays: # allow non-unique set pass else: n_all = ma.indices().size() sel_unique = ma.unique_under_symmetry_selection() sel_dup = ~flex.bool(n_all, sel_unique) n_duplicate = sel_dup.count(True) n_uus = sel_unique.size() msg = ( "Miller indices not unique under symmetry: " + file_name + \ "(%d redundant indices out of %d)" % (n_all-n_uus, n_all) + "Add --merge to command arguments to force merging data.") if (show_details_if_error): print msg ma.show_comprehensive_summary(prefix=" ") ma.map_to_asu().sort().show_array(prefix=" ") raise Sorry(msg) if (map_to_asu): ma = ma.map_to_asu().set_info(ma.info()) if (remove_systematic_absences): ma = ma.remove_systematic_absences()
def refine(self, selection, optimize_weight = True, start_trial_weight_value = 50, selection_buffer_radius=5, box_cushion=2, rms_bonds_limit = 0.03, rms_angles_limit = 3.0): if(self.ncs_groups is None or len(self.ncs_groups)==0): # no NCS constraints sites_cart_moving = self.sites_cart selection_within = self.xray_structure.selection_within( radius = selection_buffer_radius, selection = selection) sel = selection.select(selection_within) iselection = flex.size_t() for i, state in enumerate(selection): if state: iselection.append(i) box = utils.extract_box_around_model_and_map( xray_structure = self.xray_structure.select(selection=selection_within), map_data = self.target_map, box_cushion = box_cushion) new_unit_cell = box.xray_structure_box.unit_cell() geo_box = self.geometry_restraints_manager.select(selection_within) geo_box = geo_box.discard_symmetry(new_unit_cell=new_unit_cell) geo_box.shift_sites_cart(box.shift_cart) # disaster happens otherwise map_box = box.map_box sites_cart_box = box.xray_structure_box.sites_cart() rsr_simple_refiner = simple( target_map = map_box, selection = sel, real_space_gradients_delta = self.real_space_gradients_delta, max_iterations = self.max_iterations, geometry_restraints_manager = geo_box, gradients_method = self.gradients_method) real_space_result = refinery( refiner = rsr_simple_refiner, xray_structure = box.xray_structure_box, optimize_weight = optimize_weight, start_trial_weight_value = start_trial_weight_value, rms_bonds_limit = rms_bonds_limit, rms_angles_limit = rms_angles_limit) self.weight_optimal = real_space_result.weight_final sites_cart_box_refined = real_space_result.sites_cart_result shift_back = [-box.shift_cart[i] for i in range(3)] sites_cart_box_refined_shifted_back = \ sites_cart_box_refined + shift_back # Sure + not - ? sites_cart_refined = sites_cart_box_refined_shifted_back.select(sel) sites_cart_moving = sites_cart_moving.set_selected( iselection, sites_cart_refined) self.xray_structure.set_sites_cart(sites_cart_moving) self.sites_cart = self.xray_structure.sites_cart() else: # NCS constraints are present # select on xrs, grm, ncs_groups grm = self.geometry_restraints_manager.select(selection) xrs = self.xray_structure.select(selection) sel = flex.bool(xrs.scatterers().size(), True) size = self.xray_structure.scatterers().size() ncs_groups_ = self.ncs_groups.select(selection=selection) # rsr_simple_refiner = simple( target_map = self.target_map, selection = sel, real_space_gradients_delta = self.real_space_gradients_delta, max_iterations = self.max_iterations, geometry_restraints_manager = grm, ncs_groups = ncs_groups_) real_space_result = refinery( refiner = rsr_simple_refiner, xray_structure = xrs, optimize_weight = optimize_weight, start_trial_weight_value = start_trial_weight_value, rms_bonds_limit = rms_bonds_limit, rms_angles_limit = rms_angles_limit) self.weight_optimal = real_space_result.weight_final
def run(self): ''' Function that places H atoms ''' model_has_bogus_cs = False # TODO temporary fix until the code is moved to model class # check if box cussion of 5 A is enough to prevent symm contacts cs = self.model.crystal_symmetry() if (cs is None) or (cs.unit_cell() is None): self.model = shift_and_box_model(model=self.model) model_has_bogus_cs = True # Remove existing H if requested self.n_H_initial = self.model.get_hd_selection().count(True) if not self.keep_existing_H: self.model = self.model.select(~self.model.get_hd_selection()) t0 = time.time() # Add H atoms and place them at center of coordinates pdb_hierarchy = self.add_missing_H_atoms_at_bogus_position() if print_time: print("add_missing_H_atoms_at_bogus_position:", round(time.time() - t0, 2)) # f = open("intermediate1.pdb","w") # f.write(self.model.model_as_pdb()) # place N-terminal propeller hydrogens for m in pdb_hierarchy.models(): for chain in m.chains(): rgs = chain.residue_groups()[0] for ag in rgs.atom_groups(): if ag.get_atom("H"): ag.remove_atom(ag.get_atom('H')) rc = add_n_terminal_hydrogens_to_residue_group(rgs) # rc is always empty list? pdb_hierarchy.sort_atoms_in_place() pdb_hierarchy.atoms().reset_serial() # f = open("intermediate2.pdb","w") # f.write(self.model.model_as_pdb()) p = mmtbx.model.manager.get_default_pdb_interpretation_params() p.pdb_interpretation.clash_guard.nonbonded_distance_threshold = None p.pdb_interpretation.use_neutron_distances = self.use_neutron_distances p.pdb_interpretation.proceed_with_excessive_length_bonds = True #p.pdb_interpretation.automatic_linking.link_metals = True t0 = time.time() #p.pdb_interpretation.restraints_library.cdl=False # XXX this triggers a bug !=360 ro = self.model.get_restraint_objects() self.model = mmtbx.model.manager( model_input=None, pdb_hierarchy=pdb_hierarchy, stop_for_unknowns=self.stop_for_unknowns, crystal_symmetry=self.model.crystal_symmetry(), restraint_objects=ro, log=null_out()) self.model.process(pdb_interpretation_params=p, make_restraints=True) if print_time: print("get new model obj and grm:", round(time.time() - t0, 2)) #f = open("intermediate3.pdb","w") #f.write(self.model.model_as_pdb()) # Only keep H that have been parameterized in riding H procedure sel_h = self.model.get_hd_selection() if sel_h.count(True) == 0: return # get rid of isolated H atoms. #For example when heavy atom is missing, H needs not to be placed sel_isolated = self.model.isolated_atoms_selection() self.sel_lone_H = sel_h & sel_isolated self.model = self.model.select(~self.sel_lone_H) t0 = time.time() # get riding H manager --> parameterize all H atoms sel_h = self.model.get_hd_selection() self.model.setup_riding_h_manager(use_ideal_dihedral=True) sel_h_in_para = flex.bool( [bool(x) for x in self.model.riding_h_manager.h_parameterization]) sel_h_not_in_para = sel_h_in_para.exclusive_or(sel_h) self.site_labels_no_para = [ atom.id_str().replace('pdb=', '').replace('"', '') for atom in self.model.get_hierarchy().atoms().select(sel_h_not_in_para) ] # self.model = self.model.select(~sel_h_not_in_para) if print_time: print("set up riding H manager and some cleanup:", round(time.time() - t0, 2)) # t0 = time.time() # self.exclude_H_on_disulfides() # #self.exclude_h_on_coordinated_S() # if print_time: # print("find disulfides:", round(time.time()-t0, 2)) # f = open("intermediate4.pdb","w") # f.write(model.model_as_pdb()) if self.validate_e: t0 = time.time() self.validate_electrons() if print_time: print("validate electrons:", round(time.time() - t0, 2)) t0 = time.time() # Reset occupancies, ADPs and idealize H atom positions self.model.reset_adp_for_hydrogens(scale=self.adp_scale) self.model.reset_occupancy_for_hydrogens_simple() self.model.idealize_h_riding() if print_time: print("reset adp, occ; idealize:", round(time.time() - t0, 2)) # t0 = time.time() # self.exclude_h_on_coordinated_S() # if print_time: # print("coordinated S:", round(time.time()-t0, 2)) t0 = time.time() self.exclude_H_on_links() if print_time: print("all links:", round(time.time() - t0, 2)) self.n_H_final = self.model.get_hd_selection().count(True)
def exercise_adp_similarity(): u_cart = ((1,3,2,4,3,6),(2,4,2,6,5,1)) u_iso = (-1,-1) use_u_aniso = (True, True) weight = 1 a = adp_restraints.adp_similarity( u_cart=u_cart, weight=weight) assert approx_equal(a.use_u_aniso, use_u_aniso) assert a.weight == weight assert approx_equal(a.residual(), 68) assert approx_equal(a.gradients2(), ((-2.0, -2.0, 0.0, -8.0, -8.0, 20.0), (2.0, 2.0, -0.0, 8.0, 8.0, -20.0))) assert approx_equal(a.deltas(), (-1.0, -1.0, 0.0, -2.0, -2.0, 5.0)) assert approx_equal(a.rms_deltas(), 2.7487370837451071) # u_cart = ((1,3,2,4,3,6),(-1,-1,-1,-1,-1,-1)) u_iso = (-1,2) use_u_aniso = (True, False) a = adp_restraints.adp_similarity( u_cart[0], u_iso[1], weight=weight) assert approx_equal(a.use_u_aniso, use_u_aniso) assert a.weight == weight assert approx_equal(a.residual(), 124) assert approx_equal(a.gradients2(), ((-2, 2, 0, 16, 12, 24), (2, -2, 0, -16, -12, -24))) assert approx_equal(a.deltas(), (-1, 1, 0, 4, 3, 6)) assert approx_equal(a.rms_deltas(), 3.711842908553348) # i_seqs_aa = (1,2) # () - () i_seqs_ai = (1,0) # () - o i_seqs_ia = (3,2) # o - () i_seqs_ii = (0,3) # o - o p_aa = adp_restraints.adp_similarity_proxy(i_seqs=i_seqs_aa,weight=weight) p_ai = adp_restraints.adp_similarity_proxy(i_seqs=i_seqs_ai,weight=weight) p_ia = adp_restraints.adp_similarity_proxy(i_seqs=i_seqs_ia,weight=weight) p_ii = adp_restraints.adp_similarity_proxy(i_seqs=i_seqs_ii,weight=weight) assert p_aa.i_seqs == i_seqs_aa assert p_aa.weight == weight u_cart = flex.sym_mat3_double(((-1,-1,-1,-1,-1,-1), (1,2,2,4,3,6), (2,4,2,6,5,1), (-1,-1,-1,-1,-1,-1))) u_iso = flex.double((1,-1,-1,2)) use_u_aniso = flex.bool((False, True,True,False)) for p in (p_aa,p_ai,p_ia,p_ii): params = adp_restraint_params(u_cart=u_cart, u_iso=u_iso, use_u_aniso=use_u_aniso) a = adp_restraints.adp_similarity(params, proxy=p) assert approx_equal(a.weight, weight) # gradients_aniso_cart = flex.sym_mat3_double(u_cart.size(), (0,0,0,0,0,0)) gradients_iso = flex.double(u_cart.size(), 0) proxies = adp_restraints.shared_adp_similarity_proxy([p,p]) residuals = adp_restraints.adp_similarity_residuals(params, proxies=proxies) assert approx_equal(residuals, (a.residual(),a.residual())) deltas_rms = adp_restraints.adp_similarity_deltas_rms(params, proxies=proxies) assert approx_equal(deltas_rms, (a.rms_deltas(),a.rms_deltas())) residual_sum = adp_restraints.adp_similarity_residual_sum( params, proxies=proxies, gradients_aniso_cart=gradients_aniso_cart, gradients_iso=gradients_iso) assert approx_equal(residual_sum, 2 * a.residual()) fd_grads_aniso, fd_grads_iso = finite_difference_gradients( restraint_type=adp_restraints.adp_similarity, proxy=p, u_cart=u_cart, u_iso=u_iso, use_u_aniso=use_u_aniso) for g,e in zip(gradients_aniso_cart, fd_grads_aniso): assert approx_equal(g, matrix.col(e)*2) for g,e in zip(gradients_iso, fd_grads_iso): assert approx_equal(g, e*2) # # check frame invariance of residual # u_cart_1 = matrix.sym(sym_mat3=(0.1,0.2,0.05,0.03,0.02,0.01)) u_cart_2 = matrix.sym(sym_mat3=(0.21,0.32,0.11,0.02,0.02,0.07)) u_cart = (u_cart_1.as_sym_mat3(),u_cart_2.as_sym_mat3()) u_iso = (-1, -1) use_u_aniso = (True, True) a = adp_restraints.adp_similarity(u_cart, weight=1) expected_residual = a.residual() gen = flex.mersenne_twister() for i in range(20): R = matrix.rec(gen.random_double_r3_rotation_matrix(),(3,3)) u_cart_1_rot = R * u_cart_1 * R.transpose() u_cart_2_rot = R * u_cart_2 * R.transpose() u_cart = (u_cart_1_rot.as_sym_mat3(),u_cart_2_rot.as_sym_mat3()) a = adp_restraints.adp_similarity(u_cart, weight=1) assert approx_equal(a.residual(), expected_residual)
def __init__(self, **kwargs): group_args.__init__(self, **kwargs) # require Dij, d_c P = Profiler("2. calculate rho density") print "finished Dij, now calculating rho_i, the density" from xfel.clustering import Rodriguez_Laio_clustering_2014 # alternative clustering algorithms: see http://scikit-learn.org/stable/modules/clustering.html # also see https://cran.r-project.org/web/packages/dbscan/vignettes/hdbscan.html # see also https://en.wikipedia.org/wiki/Hausdorff_dimension R = Rodriguez_Laio_clustering_2014(distance_matrix=self.Dij, d_c=self.d_c) self.rho = rho = R.get_rho() ave_rho = flex.mean(rho.as_double()) NN = self.Dij.focus()[0] print "The average rho_i is %5.2f, or %4.1f%%" % (ave_rho, 100 * ave_rho / NN) i_max = flex.max_index(rho) P = Profiler("3.transition") print "the index with the highest density is %d" % (i_max) delta_i_max = flex.max( flex.double([self.Dij[i_max, j] for j in xrange(NN)])) print "delta_i_max", delta_i_max rho_order = flex.sort_permutation(rho, reverse=True) rho_order_list = list(rho_order) P = Profiler("4. delta") self.delta = delta = R.get_delta(rho_order=rho_order, delta_i_max=delta_i_max) P = Profiler("5. find cluster maxima") #---- Now hunting for clusters ---Lot's of room for improvement (or simplification) here!!! cluster_id = flex.int(NN, -1) # default -1 means no cluster delta_order = flex.sort_permutation(delta, reverse=True) N_CLUST = 10 # maximum of 10 points to be considered as possible clusters #MAX_PERCENTILE_DELTA = 0.99 # cluster centers have to be in the top 10% percentile delta MAX_PERCENTILE_RHO = 0.99 # cluster centers have to be in the top 75% percentile rho n_cluster = 0 #max_n_delta = min(N_CLUST, int(MAX_PERCENTILE_DELTA*NN)) for ic in xrange(NN): # test the density, rho item_idx = delta_order[ic] if delta[item_idx] > 100: print "A: iteration", ic, "delta", delta[ item_idx], delta[item_idx] < 0.25 * delta[delta_order[0]] if delta[item_idx] < 0.25 * delta[ delta_order[0]]: # too low (another heuristic!) continue item_rho_order = rho_order_list.index(item_idx) if delta[item_idx] > 100: print "B: iteration", ic, item_rho_order, item_rho_order / NN, MAX_PERCENTILE_RHO if item_rho_order / NN < MAX_PERCENTILE_RHO: cluster_id[item_idx] = n_cluster print ic, item_idx, item_rho_order, cluster_id[item_idx] n_cluster += 1 print "Found %d clusters" % n_cluster for x in xrange(NN): if cluster_id[x] >= 0: print "XC", x, cluster_id[x], rho[x], delta[x] self.cluster_id_maxima = cluster_id.deep_copy() P = Profiler("6. assign all points") R.cluster_assignment(rho_order, cluster_id) self.cluster_id_full = cluster_id.deep_copy() # assign the halos P = Profiler("7. assign halos") halo = flex.bool(NN, False) border = R.get_border(cluster_id=cluster_id) for ic in range(n_cluster ): #loop thru all border regions; find highest density print "cluster", ic, "in border", border.count(True) this_border = (cluster_id == ic) & (border == True) print len(this_border), this_border.count(True) if this_border.count(True) > 0: highest_density = flex.max(rho.select(this_border)) halo_selection = (rho < highest_density) & (this_border == True) if halo_selection.count(True) > 0: cluster_id.set_selected(halo_selection, -1) core_selection = (cluster_id == ic) & ~halo_selection highest_density = flex.max(rho.select(core_selection)) too_sparse = core_selection & ( rho.as_double() < highest_density / 10. ) # another heuristic if too_sparse.count(True) > 0: cluster_id.set_selected(too_sparse, -1) self.cluster_id_final = cluster_id.deep_copy() print "%d in the excluded halo" % ((cluster_id == -1).count(True))
def generate_view_data (self) : from scitbx.array_family import flex from scitbx import graphics_utils settings = self.settings data_for_colors = data_for_radii = None data = self.data #self.work_array.data() if (isinstance(data, flex.double) and data.all_eq(0)): data = flex.double(data.size(), 1) if ((self.multiplicities is not None) and (settings.scale_colors_multiplicity)) : data_for_colors = self.multiplicities.data().as_double() assert data_for_colors.size() == data.size() elif (settings.sqrt_scale_colors) and (isinstance(data, flex.double)) : data_for_colors = flex.sqrt(data) else : data_for_colors = data.deep_copy() if ((self.multiplicities is not None) and (settings.scale_radii_multiplicity)) : #data_for_radii = data.deep_copy() data_for_radii = self.multiplicities.data().as_double() assert data_for_radii.size() == data.size() elif (settings.sqrt_scale_radii) and (isinstance(data, flex.double)) : data_for_radii = flex.sqrt(data) else : data_for_radii = data.deep_copy() if (settings.slice_mode) : data = data.select(self.slice_selection) if (not settings.keep_constant_scale) : data_for_radii = data_for_radii.select(self.slice_selection) data_for_colors = data_for_colors.select(self.slice_selection) if (settings.color_scheme in ["rainbow", "heatmap", "redblue"]) : colors = graphics_utils.color_by_property( properties=data_for_colors, selection=flex.bool(data_for_colors.size(), True), color_all=False, gradient_type=settings.color_scheme) elif (settings.color_scheme == "grayscale") : colors = graphics_utils.grayscale_by_property( properties=data_for_colors, selection=flex.bool(data_for_colors.size(), True), shade_all=False, invert=settings.black_background) else : if (settings.black_background) : base_color = (1.0,1.0,1.0) else : base_color = (0.0,0.0,0.0) colors = flex.vec3_double(data_for_colors.size(), base_color) if (settings.slice_mode) and (settings.keep_constant_scale) : colors = colors.select(self.slice_selection) data_for_radii = data_for_radii.select(self.slice_selection) uc = self.work_array.unit_cell() abc = uc.parameters()[0:3] min_dist = min(uc.reciprocal_space_vector((1,1,1))) min_radius = 0.20 * min_dist max_radius = 40 * min_dist if (settings.sqrt_scale_radii) and (not settings.scale_radii_multiplicity): data_for_radii = flex.sqrt(data_for_radii) if len(data_for_radii): max_value = flex.max(data_for_radii) scale = max_radius / max_value radii = data_for_radii * scale too_small = radii < min_radius if (too_small.count(True) > 0) : radii.set_selected(too_small, flex.double(radii.size(), min_radius)) assert radii.size() == colors.size() else: radii = flex.double() max_radius = 0 self.radii = radii self.max_radius = max_radius self.colors = colors
def test_1(): tstart = time.time() fraction_missing = 0.1 d_min = 1.5 # create dummy model symmetry = crystal.symmetry(unit_cell=(15.67, 25.37, 35.68, 90, 90, 90), space_group_symbol="P 21 21 21") structure = xray.structure(crystal_symmetry=symmetry) for k in xrange(1000): scatterer = xray.scatterer( site=((1. + k * abs(math.sin(k))) / 1000.0, (1. + k * abs(math.cos(k))) / 1000.0, (1. + k) / 1000.0), u=abs(math.cos(k)) * 100. / (8. * math.pi**2), occupancy=1.0, scattering_type="C") structure.add_scatterer(scatterer) # partial model n_keep = int(round(structure.scatterers().size() * (1 - fraction_missing))) partial_structure = xray.structure(special_position_settings=structure) partial_structure.add_scatterers(structure.scatterers()[:n_keep]) # fcalc (partial model), fobs (fcalc full model) f_calc = structure.structure_factors(d_min=d_min, anomalous_flag=False).f_calc() f_calc_partial = partial_structure.structure_factors( d_min=d_min, anomalous_flag=False).f_calc() f_obs = abs(f_calc) f_calc = abs(f_calc_partial) d_star_sq = 1. / flex.pow2(f_obs.d_spacings().data()) assert approx_equal(flex.max(f_calc.data()), 6810.19834824) assert approx_equal(flex.min(f_calc.data()), 0.019589341727) assert approx_equal(flex.mean(f_calc.data()), 76.651506629) assert approx_equal(flex.max(f_obs.data()), 6962.58343229) assert approx_equal(flex.min(f_obs.data()), 0.00111552904935) assert approx_equal(flex.mean(f_obs.data()), 74.5148786464) assert f_obs.size() == f_calc.size() # define test set reflections flags = flex.bool(f_calc_partial.indices().size(), False) k = 0 for i in xrange(f_calc_partial.indices().size()): k = k + 1 if (k != 10): flags[i] = False else: k = 0 flags[i] = True assert flags.count(True) == 250 assert flags.count(False) == 2258 assert flags.size() == 2508 # *********************************************************TEST = 1 alpha, beta = maxlik.alpha_beta_est_manager( f_obs=f_obs, f_calc=f_calc, free_reflections_per_bin=1000, flags=flags, interpolation=False, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.data().size() == beta.data().size() assert alpha.data().size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 0.914152454693) assert approx_equal(flex.max(alpha.data()), 0.914152454693) assert approx_equal(flex.min(beta.data()), 818.503411782) assert approx_equal(flex.max(beta.data()), 818.503411782) # *********************************************************TEST = 2 alpha, beta = maxlik.alpha_beta_est_manager( f_obs=f_obs, f_calc=f_calc, free_reflections_per_bin=50, flags=flags, interpolation=False, epsilons=f_obs.epsilons().data().as_double()).alpha_beta() assert alpha.data().size() == beta.data().size() assert alpha.data().size() == f_obs.size() assert approx_equal(flex.min(alpha.data()), 0.910350007113) assert approx_equal(flex.max(alpha.data()), 1.07104387776) assert approx_equal(flex.min(beta.data()), 21.7374310013) assert approx_equal(flex.max(beta.data()), 4222.81104745) # *********************************************************TEST = 3 alpha, beta = maxlik.alpha_beta_calc(f=f_obs, n_atoms_absent=100, n_atoms_included=900, bf_atoms_absent=25.0, final_error=0.0, absent_atom_type="C").alpha_beta() fom = max_lik.fom_and_phase_error( f_obs=f_obs.data(), f_model=flex.abs(f_calc.data()), alpha=alpha.data(), beta=beta.data(), epsilons=f_obs.epsilons().data().as_double(), centric_flags=f_obs.centric_flags().data()).fom() assert flex.max(fom) <= 1.0 assert flex.min(fom) >= 0.0 assert flex.min(alpha.data()) == flex.max(alpha.data()) == 1.0 assert approx_equal(flex.min(beta.data()), 7.964134920) assert approx_equal(flex.max(beta.data()), 13695.1589364) # *********************************************************TEST = 4 xs = crystal.symmetry((3, 4, 5), "P 2 2 2") mi = flex.miller_index(((1, -2, 3), (0, 0, -4))) ms = miller.set(xs, mi) fc = flex.double((1., 2.)) fo = flex.double((1., 2.)) mso = miller.set(xs, mi) mac = miller.array(ms, fc) mao = miller.array(ms, fo) alp = flex.double(2, 0.0) bet = flex.double(2, 1e+9) fom = max_lik.fom_and_phase_error( f_obs=mao.data(), f_model=mac.data(), alpha=alp, beta=bet, epsilons=mao.epsilons().data().as_double(), centric_flags=mao.centric_flags().data()).fom() assert approx_equal(fom, [0.0, 0.0]) alp = flex.double(2, 1.0) bet = flex.double(2, 1e+9) fom = max_lik.fom_and_phase_error( f_obs=mao.data(), f_model=mac.data(), alpha=alp, beta=bet, epsilons=mao.epsilons().data().as_double(), centric_flags=mao.centric_flags().data()).fom() assert approx_equal(fom, [0.0, 0.0]) alp = flex.double(2, 0.0) bet = flex.double(2, 1e-9) fom = max_lik.fom_and_phase_error( f_obs=mao.data(), f_model=mac.data(), alpha=alp, beta=bet, epsilons=mao.epsilons().data().as_double(), centric_flags=mao.centric_flags().data()).fom() assert approx_equal(fom, [0.0, 0.0]) alp = flex.double(2, 1.0) bet = flex.double(2, 1e-9) fom = max_lik.fom_and_phase_error( f_obs=mao.data(), f_model=mac.data(), alpha=alp, beta=bet, epsilons=mao.epsilons().data().as_double(), centric_flags=mao.centric_flags().data()).fom() assert approx_equal(fom, [1.0, 1.0])
def run(args): import libtbx.load_env usage = "%s [options]" % libtbx.env.dispatcher_name parser = OptionParser(usage=usage, phil=phil_scope, check_format=False, epilog=help_message) params, options, args = parser.parse_args(show_diff_phil=True, return_unhandled=True) assert len(args) == 1 from iotbx.reflection_file_reader import any_reflection_file intensities = None batches = None f = args[0] reader = any_reflection_file(f) arrays = reader.as_miller_arrays(merge_equivalents=False) for ma in arrays: print(ma.info().labels) if ma.info().labels == ["I", "SIGI"]: intensities = ma elif ma.info().labels == ["IMEAN", "SIGIMEAN"]: intensities = ma elif ma.info().labels == ["I(+)", "SIGI(+)", "I(-)", "SIGI(-)"]: intensities = ma elif ma.info().labels == ["BATCH"]: batches = ma assert intensities is not None assert batches is not None if params.d_min is not None: intensities = intensities.resolution_filter( d_min=params.d_min).set_info(intensities.info()) refined = refinery(intensities, batches) refined.plot_scales() scaled_unmerged = refined.apply_scales() scales = refined.get_scales() assert reader.file_type() == "ccp4_mtz" mtz_obj = reader.file_content() mtz_dataset = mtz_obj.crystals()[0].datasets()[0] labels = intensities.info().labels i_column = mtz_dataset.columns()[mtz_dataset.column_labels().index( labels[0])] sigi_column = mtz_dataset.columns()[mtz_dataset.column_labels().index( labels[1])] selection_valid = flex.bool(scaled_unmerged.size(), True) i_column.set_values(scaled_unmerged.data().as_float(), selection_valid=selection_valid) sigi_column.set_values(scaled_unmerged.sigmas().as_float(), selection_valid=selection_valid) mtz_dataset.add_column("SCALEUSED", "R").set_values(scales.as_float()) mtz_out = "scaled_unmerged.mtz" print("Writing scaled unmerged intensities to %s" % mtz_out) mtz_obj.write(mtz_out) return
def add(model, use_neutron_distances=False, adp_scale=1, exclude_water=True, protein_only=False, stop_for_unknowns=True, remove_first=True): if(remove_first): model = model.select(~model.get_hd_selection()) pdb_hierarchy = model.get_hierarchy() mon_lib_srv = model.get_mon_lib_srv() get_class = iotbx.pdb.common_residue_names_get_class """ for pmodel in pdb_hierarchy.models(): for chain in pmodel.chains(): for residue_group in chain.residue_groups(): for conformer in residue_group.conformers(): for residue in conformer.residues(): print list(residue.atoms().extract_name()) """ #XXX This breaks for 1jxt, residue 2, TYR for chain in pdb_hierarchy.only_model().chains(): for rg in chain.residue_groups(): for ag in rg.atom_groups(): #print list(ag.atoms().extract_name()) if(get_class(name=ag.resname) == "common_water"): continue if(protein_only and not ag.resname.strip().upper() in aa_codes): continue actual = [a.name.strip().upper() for a in ag.atoms()] mlq = mon_lib_query(residue=ag.resname, mon_lib_srv=mon_lib_srv) expected_h = [] for k, v in six.iteritems(mlq.atom_dict()): if(v.type_symbol=="H"): expected_h.append(k) missing_h = list(set(expected_h).difference(set(actual))) if 0: print(ag.resname, missing_h) new_xyz = ag.atoms().extract_xyz().mean() hetero = ag.atoms()[0].hetero for mh in missing_h: # TODO: this should be probably in a central place if len(mh) < 4: mh = (' ' + mh).ljust(4) a = (iotbx.pdb.hierarchy.atom() .set_name(new_name=mh) .set_element(new_element="H") .set_xyz(new_xyz=new_xyz) .set_hetero(new_hetero=hetero)) ag.append_atom(a) pdb_hierarchy.atoms().reset_serial() #pdb_hierarchy.sort_atoms_in_place() p = mmtbx.model.manager.get_default_pdb_interpretation_params() p.pdb_interpretation.clash_guard.nonbonded_distance_threshold=None p.pdb_interpretation.use_neutron_distances = use_neutron_distances p.pdb_interpretation.proceed_with_excessive_length_bonds=True #p.pdb_interpretation.restraints_library.cdl=False # XXX this triggers a bug !=360 ro = model.get_restraint_objects() model = mmtbx.model.manager( model_input = None, pdb_hierarchy = pdb_hierarchy, build_grm = True, stop_for_unknowns = stop_for_unknowns, crystal_symmetry = model.crystal_symmetry(), restraint_objects = ro, pdb_interpretation_params = p, log = null_out()) # # Remove lone H # sel_h = model.get_hd_selection() # sel_isolated = model.isolated_atoms_selection() # sel_lone = sel_h & sel_isolated # model = model.select(~sel_lone) # Only keep H which have been parameterized in riding H procedure sel_h = model.get_hd_selection() model.setup_riding_h_manager() sel_h_in_para = flex.bool( [bool(x) for x in model.riding_h_manager.h_parameterization]) sel_h_not_in_para = sel_h_in_para.exclusive_or(sel_h) model = model.select(~sel_h_not_in_para) model = exclude_h_on_SS(model = model) model = exclude_h_on_coordinated_S(model = model) # Reset occupancies, ADPs and idealize model.reset_adp_for_hydrogens(scale = adp_scale) model.reset_occupancy_for_hydrogens_simple() model.idealize_h_riding() return model
def __init__(self, model, reference_hierarchy_list=None, reference_file_list=None, params=None, selection=None, log=None): assert [reference_hierarchy_list, reference_file_list].count(None) == 1 if(log is None): log = sys.stdout # self.model = model self.params = params self.selection = selection self.mon_lib_srv = model.get_mon_lib_srv() self.ener_lib = model.get_ener_lib() self.pdb_hierarchy = model.get_hierarchy() self.pdb_hierarchy.reset_i_seq_if_necessary() sites_cart = self.pdb_hierarchy.atoms().extract_xyz() if self.selection is None: self.selection = flex.bool(len(sites_cart), True) if reference_hierarchy_list is None: reference_hierarchy_list = \ utils.process_reference_files( reference_file_list=reference_file_list, log=log) if reference_file_list is None: reference_file_list = \ ["ref%d" % x for x in range(len(reference_hierarchy_list))] # # this takes 20% of constructor time. self.dihedral_proxies_ref = utils.get_reference_dihedral_proxies( reference_hierarchy_list=reference_hierarchy_list, reference_file_list=reference_file_list, mon_lib_srv=self.mon_lib_srv, ener_lib=self.ener_lib, restraint_objects=model.get_restraint_objects(), monomer_parameters=model.get_monomer_parameters(), log=log) self.i_seq_name_hash = utils.build_name_hash( pdb_hierarchy=self.pdb_hierarchy) #reference model components self.sites_cart_ref = {} self.pdb_hierarchy_ref = {} self.i_seq_name_hash_ref = {} self.reference_dihedral_hash = {} self.reference_file_list = reference_file_list #triage reference model files for file, hierarchy in zip(reference_file_list, reference_hierarchy_list): self.sites_cart_ref[file] = hierarchy.atoms().extract_xyz() self.pdb_hierarchy_ref[file] = hierarchy self.i_seq_name_hash_ref[file] = \ utils.build_name_hash( pdb_hierarchy=hierarchy) self.reference_dihedral_hash[file] = \ self.build_dihedral_hash( dihedral_proxies=self.dihedral_proxies_ref[file], sites_cart=self.sites_cart_ref[file], pdb_hierarchy=hierarchy, include_hydrogens=self.params.hydrogens, include_main_chain=self.params.main_chain, include_side_chain=self.params.side_chain) self.match_map = None self.proxy_map = None self.build_reference_dihedral_proxy_hash() # # This takes 80% of constructor time!!! self.residue_match_hash = {} # {key_model: ('file_name', key_ref)} self.match_map = {} # {'file_name':{i_seq_model:i_seq_ref}} if params.use_starting_model_as_reference: self.get_matching_from_self() else: self.get_matching_from_ncs(log=log) if self.match_map == {}: # making empty container new_ref_dih_proxies = self.reference_dihedral_proxies = \ cctbx.geometry_restraints.shared_dihedral_proxy() else: new_ref_dih_proxies = self.get_reference_dihedral_proxies(model)