def exercise () : g = graphics_utils.make_rainbow_gradient(nbins=21) assert approx_equal(g[0], (0.0, 0.0, 1.0)) assert approx_equal(g[10], (0.0, 1.0, 0.0)) assert approx_equal(g[-1], (1.0, 0.0, 0.0)) sel = flex.bool([True] * 10) sel[5] = False r = graphics_utils.color_rainbow( selection=sel, color_all=False) assert approx_equal(r[0], (0.0,0.0,1.0)) assert approx_equal(r[-1], (1.0,0.0,0.0)) assert approx_equal(r[4], (0.0,1.0,0.0)) r2 = graphics_utils.color_rainbow( selection=sel, color_all=True) assert approx_equal(r2[0], (0.0,0.0,1.0)) assert approx_equal(r2[-1], (1.0,0.0,0.0)) assert approx_equal(r2[6], (2/3.,1.0,0.0)) b = flex.double([4.0,5.2,1.7,6.9,9.5,24.3]) c = graphics_utils.color_by_property( properties=b, selection=flex.bool(b.size(), True)) assert approx_equal(c[2], (0.0,0.0,1.0)) c2 = graphics_utils.scale_selected_colors( input_colors=c, selection=flex.bool(c.size(), True), scale=0.9) assert approx_equal(c2[2], (0.0,0.0,0.9)) c3 = graphics_utils.grayscale_by_property( properties=b, selection=flex.bool(b.size(), True)) assert approx_equal(c3[2], (0.95,0.95,0.95))
def selection_around_to_negate( xray_structure, selection_within_radius, iselection, selection_good=None, iselection_backbone=None, iselection_n_external=None, iselection_c_external=None): # takes ~0.002 seconds if([selection_good,iselection_backbone].count(None)==0): selection_backbone = flex.bool(selection_good.size(), iselection_backbone) selection_good = selection_good.set_selected(selection_backbone, True) sel_around = xray_structure.selection_within( radius = selection_within_radius, selection = flex.bool(xray_structure.scatterers().size(), iselection)) if(selection_good is not None): ssb = flex.bool(selection_good.size(), iselection) sel_around_minus_self = sel_around.set_selected(ssb, False) else: sel_around_minus_self = flex.size_t(tuple( set(sel_around.iselection()).difference(set(iselection)))) if(selection_good is not None): negate_selection = sel_around_minus_self & selection_good else: negate_selection = sel_around_minus_self if(iselection_n_external is not None and iselection_n_external.size()>0): negate_selection[iselection_n_external[0]]=False if(iselection_c_external is not None and iselection_c_external.size()>0): negate_selection[iselection_c_external[0]]=False return negate_selection
def __init__(self,maxima): self.verbose=False #for the next spot, define the universe of possible pixels (working targets) # and the map of which pixels have been visited already (pixel_visited) working_targets = list(xrange(len(maxima))) pixel_visited = flex.bool(len(working_targets),False) self.spots = [] while len(working_targets) > 0: # for this spot, indices of pixels known to be in the spot (pixel_members) # and a stack of indices of pixels still in process of testing for connections # (connection_stack). Pop/push operates on the end of stack pixel_members = [working_targets[0]] connection_stack = [0] assert len(pixel_visited)==len(working_targets) pixel_visited[0]=True while len(connection_stack) > 0: idx_current = connection_stack[-1] for idx_target in xrange(len(working_targets)): if not pixel_visited[idx_target]: distance = math.hypot( maxima[working_targets[idx_current]][0]- maxima[working_targets[idx_target]][0], maxima[working_targets[idx_current]][1]- maxima[working_targets[idx_target]][1]) if distance >= 2.0: continue pixel_visited[idx_target]=True pixel_members.append(working_targets[idx_target]) connection_stack.append(idx_target) if connection_stack[-1] == idx_current: connection_stack.pop() if self.verbose: print "new spot with %d pixels"%len(pixel_members),pixel_members for idx in pixel_members:#[working_targets[i] for i in pixel_members]: working_targets.remove(idx) pixel_visited = flex.bool(len(working_targets),False) self.spots.append( [maxima[i] for i in pixel_members] )
def get_mask(self, goniometer=None): '''Creates a mask merging untrusted pixels with active areas.''' from scitbx.array_family import flex detector_base = self.detectorbase # get effective active area coordinates tile_manager = detector_base.get_tile_manager(detector_base.horizons_phil_cache) tiling = tile_manager.effective_tiling_as_flex_int(reapply_peripheral_margin = True) # get the raw data to get the size of the mask data = self.get_raw_data() if tiling is None or len(tiling) == 0: return None # set the mask to the same dimensions as the data mask = flex.bool(flex.grid(data.focus())) # set active areas to True so they are not masked for i in xrange(len(tiling)//4): x1,y1,x2,y2=tiling[4*i:(4*i)+4] sub_array = flex.bool(flex.grid(x2-x1,y2-y1),True) mask.matrix_paste_block_in_place(sub_array,x1,y1) # create untrusted pixel mask detector = self.get_detector() assert len(detector) == 1 trusted_mask = detector[0].get_trusted_range_mask(data) # returns merged untrusted pixels and active areas using bitwise AND (pixels are accepted # if they are inside of the active areas AND inside of the trusted range) return (mask & trusted_mask,)
def random_selection(n_candidates, n_keep): assert n_keep >= 0 and n_keep <= n_candidates n_discard = n_candidates - n_keep if (n_keep > n_discard): selection = flex.bool(n_candidates, True) if (n_discard > 0): _random_selection_core(selection, n_keep, False) else: selection = flex.bool(n_candidates, False) if (n_keep > 0): _random_selection_core(selection, n_discard, True) return selection
def psana_mask_to_dials_mask(self, psana_mask): if psana_mask.dtype == np.bool: psana_mask = flex.bool(psana_mask) else: psana_mask = flex.bool(psana_mask == 1) assert psana_mask.focus() == (32, 185, 388) dials_mask = [] for i in xrange(32): dials_mask.append(psana_mask[i:i+1,:,:194]) dials_mask[-1].reshape(flex.grid(185,194)) dials_mask.append(psana_mask[i:i+1,:,194:]) dials_mask[-1].reshape(flex.grid(185,194)) return dials_mask
def get_bool_selection_to_keep(big_selection, small_selection): """ given 2 iselections (they are sorted), returns bool selection of size big selection showing what are the matches with small selection. Rather fast algorithm but may be beneficial to transfer to C++ O(n+m), where n,m - sizes of selections """ assert big_selection.size >= small_selection.size() result = flex.bool(big_selection.size(), False) i_in_big = 0 i_in_small = 0 size_small = small_selection.size() size_big = big_selection.size() n_matches = 0 nw = 0 while (i_in_big < size_big) and (i_in_small < size_small): if big_selection[i_in_big] == small_selection[i_in_small]: result[i_in_big] = True i_in_big += 1 i_in_small += 1 n_matches += 1 elif big_selection[i_in_big] > small_selection[i_in_small]: i_in_small += 1 nw += 1 else: i_in_big += 1 # this assert is optional, in general case it is not guaranteed that # all numbers from small selection are present in big selection. assert n_matches == size_small, "%d %d" % (n_matches, size_small) return result
def combine_maps ( map_arrays, omit_groups, background_map_coeffs, resolution_factor, flatten_background=False, sigma_scaling=False, control_map=False) : """ For each box, FFT the corresponding omit map coefficients, extract the omit regions, and copy them to the combined map, using Marat's asymmetric map class to handle symmetry expansion internally. """ assert len(map_arrays) == len(omit_groups) space_group = background_map_coeffs.space_group() fft_map = background_map_coeffs.fft_map( symmetry_flags=maptbx.use_space_group_symmetry, resolution_factor=resolution_factor).apply_volume_scaling() if (sigma_scaling) : fft_map.apply_sigma_scaling() background_map = fft_map.real_map_unpadded() #print "full map:", background_map.focus() if (flatten_background) : sel_all = flex.bool(background_map.as_1d().size(), True) background_map.as_1d().set_selected(sel_all, 0) asym_map = asu_map_ext.asymmetric_map(space_group.type(), background_map) origin = asym_map.data().origin() n_real_all = background_map.focus() n_real_asu = asym_map.data().focus() m_real_asu = asym_map.data().all() #print "ORIGIN:", origin #print "N_REAL:", n_real_asu #print "M_REAL:", m_real_asu if (not control_map) : f2g = maptbx.frac2grid(n_real_all) n = 0 for group, map_coeffs in zip(omit_groups, map_arrays) : space_group = map_coeffs.space_group() omit_fft_map = map_coeffs.fft_map( resolution_factor=resolution_factor, symmetry_flags=maptbx.use_space_group_symmetry).apply_volume_scaling() if (sigma_scaling) : omit_fft_map.apply_sigma_scaling() omit_map = omit_fft_map.real_map_unpadded() omit_asym_map = asu_map_ext.asymmetric_map(space_group.type(), omit_map) assert omit_asym_map.data().focus() == n_real_asu for box in group.boxes : if (control_map) : continue grid_start = tuple(f2g(box.frac_min)) grid_end = grid_max(f2g(box.frac_max), n_real_asu) #print grid_start, grid_end for u in range(grid_start[0], grid_end[0]+1) : for v in range(grid_start[1], grid_end[1]+1) : for w in range(grid_start[2], grid_end[2]+1) : try : asym_map.data()[(u,v,w)] = omit_asym_map.data()[(u,v,w)] except IndexError : raise IndexError("n_real: %s origin: %s index: %s" % (str(n_real_asu), str(origin), str((u,v,w)))) return asym_map.map_for_fft()
def join_selections (sel1, sel2) : intersections = sel1.intersection_i_seqs(sel2) unique_sel = flex.bool(len(sel1), True) unique_sel.set_selected(intersections[0], False) sel1 = sel1.select(unique_sel) sel1.extend(sel2) return flex.sorted(sel1)
def chains_and_atoms(pdb_hierarchy, secondary_structure_selection, out=None) : if (out is None) : out = sys.stdout new_secondary_structure_selection = flex.bool() get_class = iotbx.pdb.common_residue_names_get_class chains_and_residue_selections = [] for model in pdb_hierarchy.models(): for chain in model.chains(): result = [] for rg in chain.residue_groups(): result_ = flex.size_t() is_secondary_structure = False for ag in rg.atom_groups(): #print >> out, ag.resname, get_class(name=ag.resname) if(get_class(name=ag.resname) == "common_amino_acid" or get_class(name=ag.resname) == "common_rna_dna"): for atom in ag.atoms(): result_.append(atom.i_seq) if(not is_secondary_structure): is_secondary_structure = \ secondary_structure_selection[atom.i_seq] new_secondary_structure_selection.append( secondary_structure_selection[atom.i_seq]) if(result_.size()>0): result.append( [result_, is_secondary_structure, rg.resid(), rg.unique_resnames()]) if(len(result)>0): chains_and_residue_selections.append([chain.id, result]) print >> out, "Considering these chains:" for ch in chains_and_residue_selections: print >> out, " chain '%s' (number of residues selected: %d)" % (ch[0], len(ch[1])) return chains_and_residue_selections, new_secondary_structure_selection
def apply_default_filter(database_dict, d_min, max_models_for_default_filter, key = "high_resolution"): database_dict = order_by_value(database_dict = database_dict, key = key) values = flex.double() for v in database_dict[key]: values.append(float(v)) diff = flex.abs(values-d_min) min_val = flex.min(diff) i_min_sel = (diff == min_val).iselection() assert i_min_sel.size() > 0 i_min = i_min_sel[i_min_sel.size()//2] i_l = max(0, i_min-max_models_for_default_filter//2) i_r = min(values.size()-1, i_min+max_models_for_default_filter//2) # print "apply_default_filter:" print " found data points dmin->higher =", abs(i_l-i_min) print " found data points dmin->lower =", abs(i_r-i_min) imm = min(abs(i_l-i_min), abs(i_r-i_min)) i_l, i_r = i_min-imm, i_min+imm if (imm == 0) : if (i_l == 0) : i_r = 100 print " used data points dmin->higher =", 0 print " used data points dmin->lower =", i_r elif (i_l == i_r == len(values) - 1) : i_l -= 100 print " used data points dmin->higher =", i_l print " used data points dmin->lower =", 0 else : print " used data points dmin->higher =", imm print " used data points dmin->lower =", imm # selection = flex.bool(values.size(), False) for i in xrange(i_l,i_r): selection[i] = True return select_dict(database_dict = database_dict, selection = selection)
def filter_histogram_of_key_value(database_dict, key, max_reject_fraction, edge_tolerance_small = 1.e-4, n_slots = 3): values = database_dict[key] new_values = convert_to_numeric(values = values) #print flex.min(new_values), flex.max(new_values) size = new_values.size() #print size if(size == 0): return while True: values = database_dict[key] new_values = convert_to_numeric(values = values) selection = flex.bool(new_values.size(), True) histogram = flex.histogram(data = new_values, n_slots = n_slots) l = histogram.data_min() for i, s in enumerate(histogram.slots()): r = histogram.data_min() + histogram.slot_width() * (i+1) r = r+edge_tolerance_small l = max(0, l-edge_tolerance_small) #print "%8.4f %8.4f %d" % (l, r, s) if(s < size * max_reject_fraction): selection &= ~((new_values >= l) & (new_values <= r)) l = r #print leave, remove = selection.count(True), selection.count(False) #print leave, remove if(remove == 0): break if(size - leave > int(size * max_reject_fraction)): break database_dict = select_dict(database_dict = database_dict, selection = selection) return database_dict
def generate_linear_background_2d(self, size, bmin, bmax): from random import uniform from scitbx.array_family import flex from scitbx import matrix slice_size = (1, size[1], size[2]) data = flex.double(flex.grid(size), 0) mask = flex.bool(flex.grid(size), True) params = [] for k in range(size[0]): a00 = uniform(bmin, bmax) a01 = uniform(bmin, bmax) a10 = uniform(bmin, bmax) p00 = matrix.col((0.5, 0.5, a00)) p01 = matrix.col((8.5, 0.5, a01)) p10 = matrix.col((0.5, 8.5, a10)) n = (p01 - p00).cross(p10 - p00) b = n[0] c = n[1] d = n[2] a = -(b * 0.5 + c * 0.5 + d * a00) a /= -d b /= -d c /= -d for j in range(size[1]): for i in range(size[2]): data[k, j, i] = a + b * (i + 0.5) + c * (j + 0.5) eps = 1e-7 assert abs(data[k, 0, 0] - a00) < eps assert abs(data[k, 8, 0] - a10) < eps assert abs(data[k, 0, 8] - a01) < eps params.append((a, b, c)) return params, data, mask
def get_outliers (self, proxies, unit_cell, sites_cart, pdb_atoms, sigma_cutoff, outliers_only=True, use_segids_in_place_of_chainids=False) : from scitbx.array_family import flex site_labels = flex.bool(sites_cart.size(), True).iselection() sorted_table, not_shown = proxies.get_sorted( by_value="residual", sites_cart=sites_cart, site_labels=site_labels) # this can happen for C-alpha-only models, etc. if (sorted_table is None) : return [] outliers = [] for restraint_info in sorted_table : (i_seqs, ideal, model, slack, delta, sigma, weight, residual, sym_op_j, rt_mx) = restraint_info bond_atoms = get_atoms_info(pdb_atoms, iselection=i_seqs, use_segids_in_place_of_chainids=use_segids_in_place_of_chainids) outlier = bond( atoms_info=bond_atoms, target=ideal, model=model, sigma=sigma, slack=slack, delta=delta, residual=residual, symop=sym_op_j, outlier=True, xyz=get_mean_xyz(bond_atoms)) if (outlier.score > sigma_cutoff) : outliers.append(outlier) elif (not outliers_only) : outlier.outlier=False outliers.append(outlier) return outliers
def sieve_fit (sites_fixed, sites_moving, selection=None, frac_discard=0.5) : """ Reference: Chothia & Lesk??? """ assert (sites_fixed.size() == sites_moving.size() > 0) if (selection is None) : selection = flex.bool(sites_fixed.size(), True) # step 1: superpose using originally selected atoms sites_fixed_aln = sites_fixed.select(selection) sites_moving_aln = sites_moving.select(selection) lsq_fit_obj = least_squares_fit( reference_sites=sites_fixed_aln, other_sites=sites_moving_aln) sites_moving_new = lsq_fit_obj.other_sites_best_fit() # step 2: discard 50% of sites that deviate the most, and superpose again deltas = (sites_fixed_aln - sites_moving_new).norms() deltas_sorted = flex.sorted(deltas) cutoff = deltas_sorted[int((1-frac_discard)*deltas.size())] selection = (deltas > cutoff) if (selection.count(True) == 0) : return lsq_fit_obj sites_fixed_aln = sites_fixed_aln.select(selection) sites_moving_aln = sites_moving_aln.select(selection) lsq_fit_obj = least_squares_fit( reference_sites=sites_fixed_aln, other_sites=sites_moving_aln) return lsq_fit_obj
def selected_positions(selection,positions): """ Returns only the selected indices in the positions specified in "positions" keeping the order Args: selection (flex.size_t): Atoms selection positions (set or list): the allowed positions in the selections Returns: (flex.size_t, flex.size_t): (selected atoms, atoms, not selected) Examples:: >>>a = flex.size_t([1,2,5,6,4]) >>>pos = {0,3,4} >>>s,d = selected_positions(a,pos) >>>list(s) [1,6,4] >>>list(d) [2,5] """ assert isinstance(selection,flex.size_t) if isinstance(positions,set): positions = flex.size_t(list(positions)) if isinstance(positions,list): positions = flex.size_t(positions) include = flex.bool(selection.size(),positions) not_include = ~include return selection.select(include), selection.select(not_include)
def tst_identical(self): from dials.algorithms.integration.fit import fit_profile from scitbx.array_family import flex from tst_profile_helpers import gaussian # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c = p.deep_copy() b = flex.double(flex.grid(9, 9, 9), 0) m = flex.bool(flex.grid(9,9,9), True) # Fit fit = fit_profile(p, m, c, b) I = fit.intensity() V = fit.variance() # Test intensity is the same eps = 1e-7 assert(abs(I - flex.sum(p)) < eps) assert(abs(V - I) < eps) print 'OK'
def __call__(self, predictions, hkllist, pxlsz): if len(self.IT) == 4: # We have only one tile, AnnAdaptor chokes in this case but then there is # only one choice of nearest neighbour anyway! nearest_neighbours = flex.int(len(predictions) * self.NEAR, 0) else: query = flex.double() for pred in predictions: query.append(pred[0] / pxlsz) query.append(pred[1] / pxlsz) self.adapt.query(query) assert len(self.adapt.nn) == len(predictions) * self.NEAR nearest_neighbours = self.adapt.nn selection = flex.bool() self.tile_id = flex.int() for p in xrange(len(predictions)): is_in_active_area = False for n in xrange(self.NEAR): itile = nearest_neighbours[p * self.NEAR + n] if ( self.IT[4 * itile] < predictions[p][0] / pxlsz < self.IT[4 * itile + 2] and self.IT[4 * itile + 1] < predictions[p][1] / pxlsz < self.IT[4 * itile + 3] ): is_in_active_area = True break if is_in_active_area: self.tile_id.append(itile) selection.append(is_in_active_area) assert selection.count(True) == len(self.tile_id) return predictions.select(selection), hkllist.select(selection)
def tst_with_flat_background_partial(self): from dials.algorithms.integration.fit import fit_profile from scitbx.array_family import flex from tst_profile_helpers import gaussian # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c0 = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) b = flex.double(flex.grid(9, 9, 9), 5) m = flex.bool(flex.grid(9,9,9), True) c = c0 + b # Get the partial profiles pp = p[0:5,:,:] mp = m[0:5,:,:] cp = c[0:5,:,:] bp = b[0:5,:,:] c0p = c0[0:5,:,:] # Fit fit = fit_profile(pp, mp, cp, bp) I = fit.intensity() V = fit.variance() # Test intensity is the same eps = 1e-7 assert(abs(I - flex.sum(c0)) < eps) assert(abs(V - (flex.sum(c0p) + flex.sum(bp))) < eps) print 'OK'
def filter_shadowed_reflections(experiments, reflections): from dials.util import mask_untrusted_polygon from dials.util import is_inside_polygon shadowed = flex.bool(reflections.size(), False) for expt_id in range(len(experiments)): expt = experiments[expt_id] imgset = expt.imageset masker = imgset.reader().get_format().get_goniometer_shadow_masker() detector = expt.detector sel = reflections['id'] == expt_id isel = sel.iselection() x,y,z = reflections['xyzcal.px'].select(isel).parts() start, end = expt.scan.get_array_range() for i in range(start, end): shadow = masker.project_extrema( detector, expt.scan.get_angle_from_array_index(i)) img_sel = (z >= i) & (z < (i+1)) img_isel = img_sel.iselection() for p_id in range(len(detector)): panel = reflections['panel'].select(img_isel) if shadow[p_id].size() < 4: continue panel_isel = img_isel.select(panel == p_id) inside = is_inside_polygon( shadow[p_id], flex.vec2_double(x.select(isel.select(panel_isel)), y.select(isel.select(panel_isel)))) shadowed.set_selected(panel_isel, inside) return shadowed
def finalize_model (pdb_hierarchy, xray_structure, set_b_iso=None, convert_to_isotropic=None, selection=None) : """ Prepare a rebuilt model for refinement, optionally including B-factor reset. """ from cctbx import adptbx from scitbx.array_family import flex pdb_atoms = pdb_hierarchy.atoms() if (selection is None) : selection = flex.bool(pdb_atoms.size(), True) elif isinstance(selection, str) : sel_cache = pdb_hierarchy.atom_selection_cache() selection = sel_cache.selection(selection) for i_seq, atom in enumerate(pdb_atoms) : assert (atom.parent() is not None) atom.segid = "" sc = xray_structure.scatterers()[i_seq] sc.label = atom.id_str() if (convert_to_isotropic) : xray_structure.convert_to_isotropic(selection=selection.iselection()) if (set_b_iso is not None) : if (set_b_iso is Auto) : u_iso = xray_structure.extract_u_iso_or_u_equiv() set_b_iso = adptbx.u_as_b(flex.mean(u_iso)) / 2 xray_structure.set_b_iso(value=set_b_iso, selection=selection) pdb_hierarchy.adopt_xray_structure(xray_structure) pdb_atoms.reset_serial() pdb_atoms.reset_i_seq() pdb_atoms.reset_tmp()
def get_outliers (self, proxies, unit_cell, sites_cart, pdb_atoms, sigma_cutoff, outliers_only=True, use_segids_in_place_of_chainids=False) : import cctbx.geometry_restraints from scitbx.array_family import flex site_labels = flex.bool(sites_cart.size(), True).iselection() sorted_table, n_not_shown = proxies.get_sorted( by_value="residual", sites_cart=sites_cart, site_labels=site_labels, unit_cell=unit_cell) if (sorted_table is None) : return [] outliers = [] for restraint_info in sorted_table : (plane_atoms, rms_delta, residual) = restraint_info i_seqs = [ a[0] for a in plane_atoms ] deviation = max([ a[1] / a[2] for a in plane_atoms ]) plane_atoms_ = get_atoms_info(pdb_atoms, iselection=i_seqs) outlier = planarity( atoms_info=plane_atoms_, rms_deltas=rms_delta, residual=residual, delta_max=max([ a[1] for a in plane_atoms ]), score=deviation, outlier=True, xyz=get_mean_xyz(plane_atoms_)) if (outlier.score > sigma_cutoff) : outliers.append(outlier) elif (not outliers_only) : outlier.outlier=False outliers.append(outlier) return outliers
def tst_with_flat_background(self): from dials.algorithms.integration.fit import fit_profile from scitbx.array_family import flex from tst_profile_helpers import gaussian # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c0 = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) b = flex.double(flex.grid(9, 9, 9), 5) m = flex.bool(flex.grid(9,9,9), True) c = c0 + b # Fit fit = fit_profile(p, m, c, b) I = fit.intensity() V = fit.variance() # Test intensity is the same eps = 1e-7 assert(abs(I - flex.sum(c0)) < eps) assert(abs(V - (flex.sum(c0) + flex.sum(b))) < eps) print 'OK'
def __init__ (self, pdb_hierarchy, atom_selection=None, params=None, log=sys.stdout, proxies=None, tables=None, initialize=True): assert pdb_hierarchy is not None assert not pdb_hierarchy.atoms().extract_i_seq().all_eq(0), ""+\ "Probably all atoms have i_seq = 0 which is wrong" adopt_init_args(self, locals()) self.bool_atom_selection = None if self.atom_selection is None: self.bool_atom_selection = flex.bool(pdb_hierarchy.atoms().size(), True) else: cache = pdb_hierarchy.atom_selection_cache() self.bool_atom_selection = cache.selection(atom_selection) if params is None: params = master_phil.fetch().extract() self.params = params if initialize: if(self.params.rama_potential == "oldfield"): self.tables = ramachandran_plot_data( plot_cutoff=self.params.oldfield.plot_cutoff) else : self.tables = load_tables(params) # get proxies self.extract_proxies() else: assert proxies is not None if(self.params.rama_potential == "oldfield"): self.target_phi_psi = self.update_phi_psi_targets( sites_cart = self.pdb_hierarchy.atoms().extract_xyz())
def cross_validate_to_determine_number_of_terms( x_obs, y_obs, w_obs=None, min_terms=10, max_terms=25, n_free=100, n_goes=5 ): if n_goes == None: if min_terms < 2: min_terms = 2 free_residuals = [] free_flags = flex.bool(x_obs.size(), True) free_permut = flex.random_permutation(x_obs.size()) for ii in range(n_free): free_flags[free_permut[ii]] = False for count in range(min_terms, max_terms): fit = chebyshev_lsq_fit(count, x_obs, y_obs, w_obs, free_flags) free_residuals.append(fit.free_f) return flex.double(free_residuals) else: if w_obs is None: w_obs = flex.double(x_obs.size(), 1) free_resid = flex.double(max_terms - min_terms, 0) for jj in range(n_goes): free_resid += cross_validate_to_determine_number_of_terms( x_obs, y_obs, w_obs, min_terms=min_terms, max_terms=max_terms, n_free=n_free, n_goes=None ) return min_terms + flex.min_index(free_resid)
def __init__( self, map_data, xray_structure, ncs_restraints_group_list, real_space_gradients_delta, refine_selection=None, use_ncs_constraints=True, restraints_manager=None, data_weight=None, refine_sites=False, refine_transformations=False): adopt_init_args(self, locals()) self.refine_selection = nu.get_refine_selection( refine_selection=self.refine_selection, number_of_atoms=self.xray_structure.sites_cart().size()) self.extended_ncs_selection = nu.get_extended_ncs_selection( ncs_restraints_group_list=ncs_restraints_group_list, refine_selection=self.refine_selection) self.unit_cell = self.xray_structure.unit_cell() # get selection to refine asu_size = xray_structure.scatterers().size() if refine_sites: # Use all atoms to refine self.selection = flex.bool(asu_size, refine_selection) elif refine_transformations: # use only NCS related atoms (Without the Master NCS copy) self.selection = nu.get_ncs_related_selection( ncs_restraints_group_list=ncs_restraints_group_list, asu_size=asu_size)
def base_pair_selection (self, **kwds) : # assert 0, "Probably shouldn't be used", # used in mmtbx/command_line/find_tls_groups.py whole_selection = flex.bool(self.n_atoms) for sheet in self.base_pair_selections(**kwds) : whole_selection |= sheet return whole_selection
def __init__ (self, object_id, pdb_hierarchy, atomic_bonds, special_position_settings=None, base_color=(0.0,1.0,1.0)) : self.object_id = object_id self.base_color = base_color self.draw_mode = None self.current_bonds = None self.noncovalent_bonds = None self.ribbon = None self.color_mode = None #"rainbow" self.flag_object_visible = True self._color_cache = {} self.flag_show_hydrogens = False self.flag_show_lines = True self.flag_show_labels = True self.flag_show_points = True self.flag_show_spheres = False self.flag_show_ribbon = False self.flag_show_ellipsoids = False self.flag_show_noncovalent_bonds = False self.update_structure(pdb_hierarchy=pdb_hierarchy, atomic_bonds=atomic_bonds, special_position_settings=special_position_settings) from scitbx.array_family import flex self.use_u_aniso = flex.bool(self.atoms.size())
def update_structure (self, pdb_hierarchy, atomic_bonds, special_position_settings=None) : from scitbx.array_family import flex self.pdb_hierarchy = pdb_hierarchy self.atoms = pdb_hierarchy.atoms() self.atom_count = self.atoms.size() assert (self.atom_count == len(atomic_bonds)) if atomic_bonds is None : atomic_bonds = flex.stl_set_unsigned(self.atom_count) self.atomic_bonds = atomic_bonds self.selection_cache = pdb_hierarchy.atom_selection_cache( special_position_settings=special_position_settings) #self.index_atoms() atom_index = [] atom_labels = flex.std_string() for atom in self.pdb_hierarchy.atoms_with_labels() : atom_index.append(atom) atom_labels.append(format_atom_label(atom)) self.atom_index = atom_index self.atom_labels = atom_labels self.trace_bonds = extract_trace(pdb_hierarchy, self.selection_cache) if self.draw_mode is None or self.draw_mode.startswith("trace") : self.current_bonds = self.trace_bonds else : self.current_bonds = self.atomic_bonds atom_radii = flex.double(self.atoms.size(), 1.5) hydrogen_flag = flex.bool(self.atoms.size(), False) for i_seq, atom in enumerate(self.atom_index) : if atom.element.strip() in ["H", "D"] : atom_radii[i_seq] = 0.75 hydrogen_flag[i_seq] = True self.atom_radii = atom_radii self.hydrogen_flag = hydrogen_flag self._color_cache = {} self.is_changed = True
def get_bin_selection (self, bin) : from scitbx.array_family import flex sel = flex.bool(self._bins.size(), False) for k, n in enumerate(self._bins) : if (n == bin) : sel[k] = True return sel
def __init__(self, map_data, xray_structure, ncs_restraints_group_list, real_space_gradients_delta, refine_selection=None, use_ncs_constraints=True, restraints_manager=None, data_weight=None, refine_sites=False): adopt_init_args(self, locals()) self.refine_selection = nu.get_refine_selection( refine_selection=self.refine_selection, number_of_atoms=self.xray_structure.sites_cart().size()) self.extended_ncs_selection = nu.get_extended_ncs_selection( ncs_restraints_group_list=ncs_restraints_group_list, refine_selection=self.refine_selection) self.unit_cell = self.xray_structure.unit_cell() # get selection to refine asu_size = xray_structure.scatterers().size() self.selection = flex.bool(asu_size, refine_selection)
def iselection_ncs_to_asu(iselection_ncs,ncs_chain_id,hierarchy_asu): """ Convert iselection of NCS related atoms in the NCS copy to the ASU iselection Args: ncs_len (int) ncs_chain_id (str) iselection_ncs (flex.size_t) hierarchy_asu hierarchy object) Returns: iselection_asu (flex.size_t) """ asu_len = hierarchy_asu.atoms_size() atom_cache = hierarchy_asu.atom_selection_cache().selection sel = atom_cache('chain ' + ncs_chain_id) sel = sel.iselection(True) offset = min(list(sel)) iselection_ncs += offset selection_bool = flex.bool(asu_len,iselection_ncs) return selection_bool.iselection(True)
def check_flags(self, flags, predictions=None, observations=None, shoeboxes=None, **kwargs): ''' Check the flags are set, if they're not then create a list of Trues equal to the number of items given. :param flags: The input flags :param predictions: The predictions :param observations: The observations :param shoeboxes: The shoeboxes :return: The filtered flags ''' from scitbx.array_family import flex # If flags are not set then create a list of Trues if flags is None: length = 0 if predictions: length = len(predictions) if observations: if length > 0: assert (length == len(observations)) else: length = len(observations) if shoeboxes: if length > 0: assert (length == len(observations)) else: length = len(shoeboxes) # Create an array of flags flags = flex.bool(length, True) # Return the flags return flags
def reduce_raw_data(raw_data, qmax, bandwidth, level=0.05, q_background=None): print print " ==== Data reduction ==== " print print " Preprocessing of data increases efficiency of shape retrieval procedure." print print " - Interpolation stepsize : %4.3e" % bandwidth print " - Uniform density criteria: level is set to : %4.3e" % level print " maximum q to consider : %4.3e" % qmax qmin_indx = flex.max_index(raw_data.i) qmin = raw_data.q[qmin_indx] new_data = get_q_array_uniform_body(raw_data, q_min=qmin, q_max=qmax, level=level) qmax = new_data.q[-1] if qmax > raw_data.q[-1]: qmax = raw_data.q[-1] print " Resulting q range to use in search: q start : %4.3e" % qmin print " q stop : %4.3e" % qmax print raw_q = raw_data.q[qmin_indx:] raw_i = raw_data.i[qmin_indx:] raw_s = raw_data.s[qmin_indx:] ### Take care of the background (set zero at very high q) ### if (q_background is not None): cutoff = flex.bool(raw_q > q_background) q_bk_indx = flex.last_index(cutoff, False) if (q_bk_indx < raw_q.size()): bkgrd = flex.mean(raw_i[q_bk_indx:]) print "Background correction: I=I-background, where background=", bkgrd raw_i = flex.abs(raw_i - bkgrd) q = flex.double(range(int( (qmax - qmin) / bandwidth) + 1)) * bandwidth + qmin raw_data.i = flex.linear_interpolation(raw_q, raw_i, q) raw_data.s = flex.linear_interpolation(raw_q, raw_s, q) raw_data.q = q return raw_data
def test_manhattan(): from scitbx.array_family import flex from dials.algorithms.image.filter import manhattan_distance data = flex.bool(flex.grid(100, 100), True) for j in range(100): for i in range(100): if math.sqrt((j - 50) ** 2 + (i - 50) ** 2) <= 10.5: data[j, i] = False distance = manhattan_distance(data, True) M = distance[1:-1, 1:-1] D = data[1:-1, 1:-1] selection = D.as_1d().select(M.as_1d() == 0) assert selection.all_eq(True) while True: N = data[0:-2, 1:-1] S = data[2:, 1:-1] E = data[1:-1, 0:-2] W = data[1:-1, 2:] selection = M.as_1d() == 1 neighbours = ( N.select(selection) | S.select(selection) | E.select(selection) | W.select(selection) ) assert neighbours.all_eq(True) indices = flex.size_t(range(len(D))).select(selection) for i in indices: D[i] = True data[1:-1, 1:-1] = D M -= 1 if (M > 0).count(True) == 0: break
def __init__(self, fmodel, pdb_hierarchy, processed_pdb_file, params, selection, selection_score=None, nproc=Auto, out=None): adopt_init_args(self, locals()) from scitbx.array_family import flex if (self.out is None) : self.out = sys.stdout self.sites_start = fmodel.xray_structure.sites_cart().deep_copy() if (type(selection).__name__ == 'bool'): assert (self.selection.count(True) > 0) self.iselection = self.selection.iselection() else : # actually an iselection assert (len(self.selection) > 0) self.iselection = self.selection self.selection = flex.bool(self.sites_start.size(), False).set_selected( self.iselection, True) if (self.selection_score is None): self.selection_score = self.selection use_mp = (self.params.n_trials > 1) assert (params.partial_occupancy is not None) if (len(params.partial_occupancy) > 1): use_mp = True if (not use_mp): self._trials = [ self.run_trial(params.partial_occupancy[0]) ] else : self.out = null_out() args = [] for occ in params.partial_occupancy : assert (occ > 0) and (occ <= 1) for k in range(self.params.n_trials): args.append(occ) self._trials = easy_mp.pool_map( fixed_func=self.run_trial, args=args, processes=self.nproc)
def get_ncs_related_selection(ncs_restraints_group_list, asu_size): """ Args: ncs_restraints_group_list (list): list of ncs_restraint_group objects asu_size (int): the total size of the ASU Returns: selection (flex.bool): selection of all ncs related atom in the ASU """ total_master_ncs_selection = set() total_ncs_related_selection = set() for nrg in ncs_restraints_group_list: master_ncs_selection = nrg.master_iselection total_master_ncs_selection.update(set(master_ncs_selection)) for ncs_copy in nrg.copies: asu_selection = ncs_copy.iselection total_ncs_related_selection.update(set(asu_selection)) # total_ncs_related_selection.update(total_master_ncs_selection) ts = flex.size_t(list(total_ncs_related_selection)) selection = flex.bool(asu_size, ts) return selection
def predict_for_reflection_table(self, reflections, skip_derivatives=False): """perform prediction for all reflections in the supplied table""" # set the entering flags if this has not been done from dials.algorithms.refinement.reflection_manager import calculate_entering_flags if "entering" not in reflections: reflections['entering'] = calculate_entering_flags( reflections, self._experiments) # can only predict for experiments that exist and within the scan range # any other reflections will be left unchanged inc = flex.size_t_range(len(reflections)) to_keep = flex.bool(len(inc), False) for iexp, exp in enumerate(self._experiments): sel = reflections['id'] == iexp # keep all reflections if there is no rotation axis if exp.goniometer is None: to_keep.set_selected(sel, True) continue # trim reflections outside the scan range phi = reflections['xyzobs.mm.value'].parts()[2] phi_min, phi_max = exp.scan.get_oscillation_range(deg=False) passed = (phi >= phi_min) & (phi <= phi_max) to_keep.set_selected(sel, passed) # determine indices to include and predict on the subset inc = inc.select(to_keep) sub_refl = reflections.select(inc) preds = self._predict_core(sub_refl, skip_derivatives) # set updated subset back into place reflections.set_selected(inc, preds) return reflections
def test_external_lookup(): from dxtbx.imageset import ExternalLookup mask = flex.bool(flex.grid(10, 10), True) gain = flex.double(flex.grid(10, 10), 1) pedestal = flex.double(flex.grid(10, 10), 2) lookup = ExternalLookup() lookup.mask.data = dxtbx.format.image.ImageBool( dxtbx.format.image.ImageTileBool(mask)) lookup.gain.data = dxtbx.format.image.ImageDouble( dxtbx.format.image.ImageTileDouble(gain)) lookup.pedestal.data = dxtbx.format.image.ImageDouble( dxtbx.format.image.ImageTileDouble(pedestal)) mask2 = lookup.mask.data.tile(0).data() gain2 = lookup.gain.data.tile(0).data() pedestal2 = lookup.pedestal.data.tile(0).data() assert mask2.all_eq(mask) assert gain2.all_eq(gain) assert pedestal2.all_eq(pedestal)
def test_combine_with_user_static_mask(dials_data, tmpdir): master_h5 = (dials_data("image_examples").join( "SACLA-MPCCD-Phase3-21528-5images.h5").strpath) assert FormatHDF5SaclaMPCCD.understand(master_h5) expts_from_filename = ExperimentListFactory.from_filenames([master_h5]) for i, expt in enumerate(expts_from_filename): mask = [] for panel in expt.detector: m = flex.bool(flex.grid(reversed(panel.get_image_size())), True) mask_untrusted_resolution_range(m, expt.beam, panel, 0, 2.23) mask.append(m) mask = tuple(mask) # exact number of masked pixels varies with wavelength between experiments assert [_.count(False) for _ in mask ] == pytest.approx([50643, 0, 0, 48003, 48356, 0, 0, 52191], abs=1e3) mask_file = tmpdir.join("pixel_%i.mask" % i) with mask_file.open("wb") as f: pickle.dump(mask, f) expt.imageset.external_lookup.mask.filename = mask_file.strpath expt.imageset.external_lookup.mask.data = ImageBool(mask) expts_from_dict = ExperimentListFactory.from_dict( expts_from_filename.to_dict()) imageset = expts_from_dict[0].imageset mask = imageset.get_mask(0) assert len(mask) == 8 assert [_.count(False) for _ in mask] == [ 65332, 24296, 24296, 63335, 63660, 24296, 24296, 66691, ] imageset.reader().nullify_format_instance()
def __call__(self, omit_group): from scitbx.array_family import flex fmodel_tmp = self.fmodel.deep_copy() xrs_omit = fmodel_tmp.xray_structure.deep_copy_scatterers() xrs_omit.set_occupancies(self.occupancy, selection=omit_group.selection) if (self.selection_delete is not None): selection_partial = flex.bool(xrs_omit.scatterers().size(), False) selection_partial.set_selected(omit_group.selection, True) selection_delete = selection_partial & self.selection_delete n_del = selection_delete.count(True) if (n_del > 0): #print "setting %d waters to zero occupancy" % n_del xrs_omit.set_occupancies(0., selection=selection_delete) fmodel_tmp.update_xray_structure(xrs_omit, update_f_mask=False, update_f_calc=True) return fmodel_tmp.map_coefficients( map_type=self.map_type, fill_missing=self.fill_missing_f_obs, exclude_free_r_reflections=self.exclude_free_r_reflections, merge_anomalous=True)
def extract_sites_for_alignment(chain_obj): """Extract sequence and sites of c-alphas - adapted from mmtbx.command_line.super""" seq = [] sites = flex.vec3_double() use_sites = flex.bool() for resi in chain_obj.conformers()[0].residues(): if ( iotbx.pdb.common_residue_names_get_class(name=resi.resname) != "common_amino_acid"): continue resn = resi.resname single = iotbx.pdb.amino_acid_codes.one_letter_given_three_letter[resn] seq.append(single) use = False xyz = (0,0,0) for atom in resi.atoms(): if (atom.name == " CA "): xyz = atom.xyz use = True break sites.append(xyz) use_sites.append(use) return "".join(seq), sites, use_sites
def setup_helix_example(pdb_name = "m00_good.pdb", mtz_name = "m00_good.mtz"): # Read good model and compute data from it. xrs_good = iotbx.pdb.input( file_name = os.path.join(qr_unit_tests_data, pdb_name )).xray_structure_simple() f_obs = abs(xrs_good.structure_factors(d_min=1.5).f_calc()) r_free_flags_data = flex.bool() for i in range(f_obs.data().size()): if(i%10 == 0): r_free_flags_data.append(True) else: r_free_flags_data.append(False) r_free_flags = f_obs.array(data = r_free_flags_data) mtz_dataset = f_obs.as_mtz_dataset(column_root_label="F-obs") mtz_dataset.add_miller_array( miller_array = r_free_flags, column_root_label = "R-free-flags") mtz_object = mtz_dataset.mtz_object() mtz_object.write(file_name = mtz_name) xrs_poor = iotbx.pdb.input( file_name = os.path.join(qr_unit_tests_data, "m00_poor.pdb")).xray_structure_simple() return xrs_good,xrs_poor,f_obs,r_free_flags
def resolution_filter(self): """Filter arrays on resolution.""" if not self.params.d_min and self.params.min_completeness: # Update self.params.d_min using resolution estimate params = resolution_analysis.phil_defaults.extract().resolution params.nbins = self.params.resolution_bins r = resolution_analysis.Resolutionizer(self.intensities, params) self.params.d_min = r.resolution( resolution_analysis.metrics.COMPLETENESS, limit=self.params.min_completeness, ).d_min logger.info("Estimated d_min: %.2f", self.params.d_min) if self.params.d_min or self.params.d_max: sel = flex.bool(self.intensities.size(), True) d_spacings = self.intensities.d_spacings().data() if self.params.d_min: sel &= d_spacings >= self.params.d_min if self.params.d_max: sel &= d_spacings <= self.params.d_max self.intensities = self.intensities.select(sel) self.dose = self.dose.select(sel)
def test_with_flat_background_partial(): from dials.algorithms.integration.fit import ProfileFitter from scitbx.array_family import flex from numpy.random import seed seed(0) # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c0 = add_poisson_noise(100 * p) b = flex.double(flex.grid(9, 9, 9), 1) m = flex.bool(flex.grid(9,9,9), True) c = c0 + add_poisson_noise(b) # Get the partial profiles pp = p[0:5,:,:] mp = m[0:5,:,:] cp = c[0:5,:,:] bp = b[0:5,:,:] c0p = c0[0:5,:,:] # Fit fit = ProfileFitter(cp, bp, mp, pp) I = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() Iknown = 99.06932141277105 Vknown = 504.06932141277105 # Test intensity is the same eps = 1e-7 assert(abs(I[0] - Iknown) < eps) assert(abs(V[0] - Vknown) < eps)
def filter_shadowed_reflections(experiments, reflections, experiment_goniometer=False): from dials.util.ext import is_inside_polygon from scitbx.array_family import flex shadowed = flex.bool(reflections.size(), False) for expt_id in range(len(experiments)): expt = experiments[expt_id] imageset = expt.imageset if experiment_goniometer: masker = imageset.masker().format_class( imageset.paths()[0]).get_goniometer_shadow_masker( goniometer=expt.goniometer) else: masker = imageset.masker().format_class( imageset.paths()[0]).get_goniometer_shadow_masker() detector = expt.detector sel = reflections['id'] == expt_id isel = sel.iselection() x, y, z = reflections['xyzcal.px'].select(isel).parts() start, end = expt.scan.get_array_range() for i in range(start, end): shadow = masker.project_extrema( detector, expt.scan.get_angle_from_array_index(i)) img_sel = (z >= i) & (z < (i + 1)) img_isel = img_sel.iselection() for p_id in range(len(detector)): panel = reflections['panel'].select(img_isel) if shadow[p_id].size() < 4: continue panel_isel = img_isel.select(panel == p_id) inside = is_inside_polygon( shadow[p_id], flex.vec2_double(x.select(isel.select(panel_isel)), y.select(isel.select(panel_isel)))) shadowed.set_selected(panel_isel, inside) return shadowed
def apply_default_filter(database_dict, d_min, max_models_for_default_filter, key="high_resolution"): database_dict = order_by_value(database_dict=database_dict, key=key) values = flex.double() for v in database_dict[key]: values.append(float(v)) diff = flex.abs(values - d_min) min_val = flex.min(diff) i_min_sel = (diff == min_val).iselection() assert i_min_sel.size() > 0 i_min = i_min_sel[i_min_sel.size() // 2] i_l = max(0, i_min - max_models_for_default_filter // 2) i_r = min(values.size() - 1, i_min + max_models_for_default_filter // 2) # print "apply_default_filter:" print " found data points dmin->higher =", abs(i_l - i_min) print " found data points dmin->lower =", abs(i_r - i_min) imm = min(abs(i_l - i_min), abs(i_r - i_min)) i_l, i_r = i_min - imm, i_min + imm if (imm == 0): if (i_l == 0): i_r = 100 print " used data points dmin->higher =", 0 print " used data points dmin->lower =", i_r elif (i_l == i_r == len(values) - 1): i_l -= 100 print " used data points dmin->higher =", i_l print " used data points dmin->lower =", 0 else: print " used data points dmin->higher =", imm print " used data points dmin->lower =", imm # selection = flex.bool(values.size(), False) for i in xrange(i_l, i_r): selection[i] = True return select_dict(database_dict=database_dict, selection=selection)
def __call__(self, fragment): from scitbx.array_family import flex frag_selection = flex.bool(self.pdb_atoms.size(), fragment) sites_cart_orig = self.fmodel.xray_structure.sites_cart() sites_start = sites_cart_orig.select(fragment) atom_start = self.pdb_atoms[fragment[0]].fetch_labels() atom_end = self.pdb_atoms[fragment[-1]].fetch_labels() label = "%s%s-%s" % (atom_start.chain_id, atom_start.resid(), atom_end.resid()) # XXX should the partial occupancy be determined automatically? two_fofc_map, fofc_map = alt_confs.get_partial_omit_map( fmodel=self.fmodel.deep_copy(), selection=(frag_selection & self.sele_main_conf), selection_delete=(frag_selection & ~(self.sele_main_conf)), partial_occupancy=0.6) target_map = two_fofc_map if (self.rsr_fofc_map_target): target_map = fofc_map box = self.make_box(selection=frag_selection, target_map=target_map) box.restrain_atoms(selection=alt_confs.SELECTION_OLD, reference_sigma=0.02, reset_first=True) box.restrain_atoms(selection=alt_confs.SELECTION_NEW_REBUILT, reference_sigma=0.05, reset_first=False) # first fix the geometry of adjacent residues box.real_space_refine(alt_confs.SELECTION_NEW_SPLIT) # now the entire B conformer box.real_space_refine(alt_confs.SELECTION_NEW) sites_cart_refined = box.update_original_coordinates() sites_new = sites_cart_refined.select(fragment) self.fmodel.xray_structure.set_sites_cart(sites_cart_orig) self.pdb_atoms.set_xyz(sites_cart_orig) return alt_confs.refined_fragment( label=label, selection=frag_selection, sites_cart=sites_cart_refined, rmsd=sites_new.rms_difference(sites_start))
def convert_to_numeric(values): is_digit_selection = flex.bool() for kv in values: flag = True try: tmp = float(kv) except Exception: flag = False is_digit_selection.append(flag) if (is_digit_selection.count(False) > 0): raise RuntimeError("Non-numerical values.") points = 0 for kv in values: points += kv.count(".") if (points == 0): new_values = flex.int() for kv in values: new_values.append(int(kv)) else: new_values = flex.double() for kv in values: new_values.append(float(kv)) return new_values
def test_deconvolve_zero(): from dials.algorithms.integration.fit import ProfileFitter from scitbx.array_family import flex from numpy.random import seed seed(0) I0 = [1000, 2000, 3000] # Create profile p = generate_3_profiles() # Copy profile c = flex.double(flex.grid(40, 9, 9), 0) b = flex.double(flex.grid(40, 9, 9), 0) m = flex.bool(flex.grid(40,9,9), True) # Fit passed = False with pytest.raises(Exception): fit = ProfileFitter(c, b, m, p) I = fit.intensity() V = fit.variance()
def run(self): from dials.algorithms.image.fill_holes import simple_fill from scitbx.array_family import flex from math import sqrt mask = flex.bool(flex.grid(100, 100), True) data = flex.double(flex.grid(100, 100), True) for j in range(100): for i in range(100): data[j, i] = 10 + j * 0.01 + i * 0.01 if sqrt((j - 50)**2 + (i - 50)**2) <= 10.5: mask[j, i] = False data[j, i] = 0 result = simple_fill(data, mask) known = data.as_1d().select(mask.as_1d()) filled = result.as_1d().select(mask.as_1d() == False) assert flex.max(filled) <= flex.max(known) assert flex.min(filled) >= flex.min(known) # Test passed print 'OK'
def cross_validate_to_determine_number_of_terms(x_obs, y_obs, w_obs=None, min_terms=10, max_terms=25, n_free=100, n_goes=5): if (n_goes == None): if (min_terms < 2): min_terms = 2 free_residuals = [] free_flags = flex.bool(x_obs.size(), True) free_permut = flex.random_permutation(x_obs.size()) for ii in range(n_free): free_flags[free_permut[ii]] = False for count in range(min_terms, max_terms): fit = chebyshev_lsq_fit(count, x_obs, y_obs, w_obs, free_flags) free_residuals.append(fit.free_f) return (flex.double(free_residuals)) else: if w_obs is None: w_obs = flex.double(x_obs.size(), 1) free_resid = flex.double(max_terms - min_terms, 0) for jj in range(n_goes): free_resid += cross_validate_to_determine_number_of_terms( x_obs, y_obs, w_obs, min_terms=min_terms, max_terms=max_terms, n_free=n_free, n_goes=None) return (min_terms + flex.min_index(free_resid))
def update_structure(self, pdb_hierarchy, atomic_bonds, special_position_settings=None): from scitbx.array_family import flex self.pdb_hierarchy = pdb_hierarchy self.atoms = pdb_hierarchy.atoms() self.atom_count = self.atoms.size() assert (self.atom_count == len(atomic_bonds)) if atomic_bonds is None: atomic_bonds = flex.stl_set_unsigned(self.atom_count) self.atomic_bonds = atomic_bonds self.selection_cache = pdb_hierarchy.atom_selection_cache( special_position_settings=special_position_settings) #self.index_atoms() atom_index = [] atom_labels = flex.std_string() for atom in self.pdb_hierarchy.atoms_with_labels(): atom_index.append(atom) atom_labels.append(format_atom_label(atom)) self.atom_index = atom_index self.atom_labels = atom_labels self.trace_bonds = extract_trace(pdb_hierarchy, self.selection_cache) if self.draw_mode is None or self.draw_mode.startswith("trace"): self.current_bonds = self.trace_bonds else: self.current_bonds = self.atomic_bonds atom_radii = flex.double(self.atoms.size(), 1.5) hydrogen_flag = flex.bool(self.atoms.size(), False) for i_seq, atom in enumerate(self.atom_index): if atom.element.strip() in ["H", "D"]: atom_radii[i_seq] = 0.75 hydrogen_flag[i_seq] = True self.atom_radii = atom_radii self.hydrogen_flag = hydrogen_flag self._color_cache = {} self.is_changed = True
def reindex_refl_by_coset(refl,data,symms,uuids,co,anomalous_flag = False, verbose=True): if verbose: cache_miller = refl["miller_index"].deep_copy() cache_asu = refl["miller_index_asymmetric"].deep_copy() for icoset,partition in enumerate(co.partitions): if icoset==0: continue # no change of basis cb_op = change_of_basis_op(partition[0]) mi_new = cb_op.apply(refl["miller_index"]) mi_asu_new = mi_new.deep_copy() miller.map_to_asu(symms[0].space_group().info().type(), anomalous_flag, mi_asu_new) # now select only those expts assigned to that coset good_refls = flex.bool(len(refl), False) all_expt_id = list(data["experiment"]) all_coset = list(data["coset"]) # would like to understand how to use pandas rather than Python list for iexpt in range(len(symms)): iexpt_id = uuids[iexpt] this_coset = all_coset[ all_expt_id.index(iexpt_id) ] if this_coset == icoset: good_refls |= refl["exp_id"] == iexpt_id re_millers = mi_new.select(good_refls) re_asu = mi_asu_new.select(good_refls) refl["miller_index"].set_selected(good_refls, re_millers) refl["miller_index_asymmetric"].set_selected(good_refls, re_asu) if verbose: for x in range(len(refl)): print ("%3d"%x,refl["exp_id"][x], all_coset[ all_expt_id.index(refl["exp_id"][x]) ], "%10s"%(change_of_basis_op(co.partitions[ all_coset[ all_expt_id.index(refl["exp_id"][x]) ] ][0]).as_hkl()), "(%4d%4d%4d)"%(cache_miller[x]), "(%4d%4d%4d)"%(cache_asu[x]), "(%4d%4d%4d)"%(refl["miller_index"][x]), "(%4d%4d%4d)"%(refl["miller_index_asymmetric"][x])) return refl
def chains_and_atoms(pdb_hierarchy, secondary_structure_selection, out=None): if (out is None): out = sys.stdout new_secondary_structure_selection = flex.bool() get_class = iotbx.pdb.common_residue_names_get_class chains_and_residue_selections = [] for model in pdb_hierarchy.models(): for chain in model.chains(): result = [] for rg in chain.residue_groups(): result_ = flex.size_t() is_secondary_structure = False for ag in rg.atom_groups(): #print >> out, ag.resname, get_class(name=ag.resname) if (get_class(name=ag.resname) == "common_amino_acid" or get_class(name=ag.resname) == "common_rna_dna"): for atom in ag.atoms(): result_.append(atom.i_seq) if (not is_secondary_structure): is_secondary_structure = \ secondary_structure_selection[atom.i_seq] new_secondary_structure_selection.append( secondary_structure_selection[atom.i_seq]) if (result_.size() > 0): result.append([ result_, is_secondary_structure, rg.resid(), rg.unique_resnames() ]) if (len(result) > 0): chains_and_residue_selections.append([chain.id, result]) print("Considering these chains:", file=out) for ch in chains_and_residue_selections: print(" chain '%s' (number of residues selected: %d)" % (ch[0], len(ch[1])), file=out) return chains_and_residue_selections, new_secondary_structure_selection
def _expand(self): expansion = expand(pdb_hierarchy=self.pdb_hierarchy, crystal_symmetry=self.crystal_symmetry, select_within_radius=10.0) pdb_hierarchy_super = expansion.ph_super_sphere pdb_hierarchy_super.write_pdb_file(file_name="supersphere.pdb", crystal_symmetry=expansion.cs_box) self.crystal_symmetry_ss = expansion.cs_box if (self.restraints_source.source_of_restraints_qm()): self.pdb_hierarchy_super_completed = model_completion.run( #pdb_hierarchy = pdb_hierarchy_super, crystal_symmetry=expansion.cs_box, model_completion=False, pdb_filename="supersphere.pdb", original_pdb_filename=None) else: self.pdb_hierarchy_super_completed = pdb_hierarchy_super selection = flex.bool( self.pdb_hierarchy_super_completed.atoms().size(), False) self.selection = selection.set_selected( flex.size_t(xrange(self.pdb_hierarchy.atoms().size())), True) self.restraints_manager = self.restraints_source.update( pdb_hierarchy=self.pdb_hierarchy_super_completed, crystal_symmetry=expansion.cs_box)
def tst_with_noisy_flat_background_partial(self): from dials.algorithms.integration.fit import fit_profile from scitbx.array_family import flex from tst_profile_helpers import gaussian # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c0 = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) n = flex.random_double(9 * 9 * 9) b = flex.double(flex.grid(9, 9, 9), 5) + n m = flex.bool(flex.grid(9,9,9), True) c = c0 + b # Get the partial profiles pp = p[0:5,:,:] mp = m[0:5,:,:] cp = c[0:5,:,:] bp = b[0:5,:,:] c0p = c0[0:5,:,:] # Fit fit = fit_profile(pp, mp, cp, bp) I = fit.intensity() V = fit.variance() # Test intensity is the same eps = 1e-7 assert(abs(I - flex.sum(c0)) < eps) assert(abs(V - (flex.sum(c0p) + flex.sum(bp))) < eps) print 'OK'
def test_pickle(): from dials.model.data import PixelList from scitbx.array_family import flex size = (100, 100) sf = 10 image = flex.double(flex.grid(size)) mask = flex.bool(flex.grid(size)) for i in range(len(image)): image[i] = random.randint(0, 100) mask[i] = bool(random.randint(0, 1)) pl = PixelList(sf, image, mask) assert pl.size() == size assert pl.frame() == sf import six.moves.cPickle as pickle obj = pickle.dumps(pl) pl2 = pickle.loads(obj) assert pl2.size() == size assert pl2.frame() == sf assert len(pl2) == len(pl) assert pl2.index().all_eq(pl.index()) assert pl2.value().all_eq(pl.value())
def run (args, image = None): from xfel import radial_average from scitbx.array_family import flex import os, sys import dxtbx # Parse input try: n = len(args) except Exception: params = args else: user_phil = [] for arg in args: if (not "=" in arg): try : user_phil.append(libtbx.phil.parse("""file_path=%s""" % arg)) except ValueError: raise Sorry("Unrecognized argument '%s'" % arg) else: try: user_phil.append(libtbx.phil.parse(arg)) except RuntimeError as e: raise Sorry("Unrecognized argument '%s' (error: %s)" % (arg, str(e))) params = master_phil.fetch(sources=user_phil).extract() if image is None: if params.file_path is None or len(params.file_path) == 0 or not all([os.path.isfile(f) for f in params.file_path]): master_phil.show() raise Usage("file_path must be defined (either file_path=XXX, or the path alone).") assert params.n_bins is not None assert params.verbose is not None assert params.output_bins is not None # Allow writing to a file instead of stdout if params.output_file is None: logger = sys.stdout else: logger = open(params.output_file, 'w') logger.write("%s "%params.output_file) if params.show_plots: from matplotlib import pyplot as plt import numpy as np colormap = plt.cm.gist_ncar plt.gca().set_color_cycle([colormap(i) for i in np.linspace(0, 0.9, len(params.file_path))]) if params.mask is not None: params.mask = easy_pickle.load(params.mask) if image is None: iterable = params.file_path load_func = lambda x: dxtbx.load(x) else: iterable = [image] load_func = lambda x: x # Iterate over each file provided for item in iterable: img = load_func(item) try: n_images = img.get_num_images() subiterable = xrange(n_images) except AttributeError: n_images = None subiterable = [0] for image_number in subiterable: if n_images is None: beam = img.get_beam() detector = img.get_detector() else: beam = img.get_beam(image_number) detector = img.get_detector(image_number) s0 = col(beam.get_s0()) # Search the detector for the panel farthest from the beam. The number of bins in the radial average will be # equal to the farthest point from the beam on the detector, in pixels, unless overridden at the command line panel_res = [p.get_max_resolution_at_corners(s0) for p in detector] farthest_panel = detector[panel_res.index(min(panel_res))] size2, size1 = farthest_panel.get_image_size() corners = [(0,0), (size1-1,0), (0,size2-1), (size1-1,size2-1)] corners_lab = [col(farthest_panel.get_pixel_lab_coord(c)) for c in corners] corner_two_thetas = [farthest_panel.get_two_theta_at_pixel(s0, c) for c in corners] extent_two_theta = max(corner_two_thetas) max_corner = corners_lab[corner_two_thetas.index(extent_two_theta)] extent = int(math.ceil(max_corner.length()*math.sin(extent_two_theta)/max(farthest_panel.get_pixel_size()))) extent_two_theta *= 180/math.pi if params.n_bins < extent: params.n_bins = extent # These arrays will store the radial average info sums = flex.double(params.n_bins) * 0 sums_sq = flex.double(params.n_bins) * 0 counts = flex.int(params.n_bins) * 0 if n_images is None: all_data = img.get_raw_data() else: all_data = img.get_raw_data(image_number) if not isinstance(all_data, tuple): all_data = (all_data,) for tile, (panel, data) in enumerate(zip(detector, all_data)): if params.mask is None: mask = flex.bool(flex.grid(data.focus()), True) else: mask = params.mask[tile] if hasattr(data,"as_double"): data = data.as_double() logger.flush() if params.verbose: logger.write("Average intensity tile %d: %9.3f\n"%(tile, flex.mean(data))) logger.write("N bins: %d\n"%params.n_bins) logger.flush() x1,y1,x2,y2 = 0,0,panel.get_image_size()[1],panel.get_image_size()[0] bc = panel.get_beam_centre_px(beam.get_s0()) bc = int(round(bc[1])), int(round(bc[0])) # compute the average radial_average(data,mask,bc,sums,sums_sq,counts,panel.get_pixel_size()[0],panel.get_distance(), (x1,y1),(x2,y2)) # average the results, avoiding division by zero results = sums.set_selected(counts <= 0, 0) results /= counts.set_selected(counts <= 0, 1).as_double() if params.median_filter_size is not None: logger.write("WARNING, the median filter is not fully propogated to the variances\n") from scipy.ndimage.filters import median_filter results = flex.double(median_filter(results.as_numpy_array(), size = params.median_filter_size)) # calculate standard devations stddev_sel = ((sums_sq-sums*results) >= 0) & (counts > 0) std_devs = flex.double(len(sums), 0) std_devs.set_selected(stddev_sel, (sums_sq.select(stddev_sel)-sums.select(stddev_sel)* \ results.select(stddev_sel))/counts.select(stddev_sel).as_double()) std_devs = flex.sqrt(std_devs) twotheta = flex.double(xrange(len(results)))*extent_two_theta/params.n_bins q_vals = 4*math.pi*flex.sin(math.pi*twotheta/360)/beam.get_wavelength() if params.low_max_two_theta_limit is None: subset = results else: subset = results.select(twotheta >= params.low_max_two_theta_limit) max_result = flex.max(subset) if params.x_axis == 'two_theta': xvals = twotheta max_x = twotheta[flex.first_index(results, max_result)] elif params.x_axis == 'q': xvals = q_vals max_x = q_vals[flex.first_index(results, max_result)] for i in xrange(len(results)): val = xvals[i] if params.output_bins and "%.3f"%results[i] != "nan": #logger.write("%9.3f %9.3f\n"% (val,results[i])) #.xy format for Rex.cell. logger.write("%9.3f %9.3f %9.3f\n"%(val,results[i],std_devs[i])) #.xye format for GSASII #logger.write("%.3f %.3f %.3f\n"%(val,results[i],ds[i])) # include calculated d spacings logger.write("Maximum %s: %f, value: %f\n"%(params.x_axis, max_x, max_result)) if params.show_plots: if params.plot_x_max is not None: results = results.select(xvals <= params.plot_x_max) xvals = xvals.select(xvals <= params.plot_x_max) if params.normalize: plt.plot(xvals.as_numpy_array(),(results/flex.max(results)).as_numpy_array(),'-') else: plt.plot(xvals.as_numpy_array(),results.as_numpy_array(),'-') if params.x_axis == 'two_theta': plt.xlabel("2 theta") elif params.x_axis == 'q': plt.xlabel("q") plt.ylabel("Avg ADUs") if params.plot_y_max is not None: plt.ylim(0, params.plot_y_max) if params.show_plots: #plt.legend([os.path.basename(os.path.splitext(f)[0]) for f in params.file_path], ncol=2) plt.show() return xvals, results
def calculate_scaling(self, miller_array, convergence_crit_perc=0.01, convergence_reject_perc=97.5, max_iter=20): """Calculate the scaling between two arrays""" assert convergence_reject_perc > 90.0 # Convert to intensities and extract d_star_sq new_miller = miller_array.as_intensity_array() new_kernel = self._kernel_normalisation(miller_array=new_miller) # Calculate new range of d_star_sq d_star_sq_min, d_star_sq_max = self._common_d_star_sq_range( d_star_sq=new_kernel.d_star_sq_array) # Create interpolator for the two arrays (new and reference) interpolator = scale_curves.curve_interpolator(d_star_sq_min, d_star_sq_max, self._npoints) # Interpolate the two curves (use full range of the two array) new_itpl_d_star_sq, new_itpl_mean_I, dummy, dummy = interpolator.interpolate( x_array=new_kernel.d_star_sq_array, y_array=new_kernel.mean_I_array) ref_itpl_d_star_sq, ref_itpl_mean_I, dummy, dummy = interpolator.interpolate( x_array=self.ref_kernel.d_star_sq_array, y_array=self.ref_kernel.mean_I_array) # Initalise convergence loop - begin by scaling over all points selection = flex.bool(self._npoints, True) # Set initial scale factor to small value curr_b = 1e-6 # Percent change between iterations - convergence when delta <convergence_criterion n_iter = 0 # Report in case of error report = Report('Scaling log:', verbose=False) while n_iter < max_iter: report('---') report('ITER: ' + str(n_iter)) if selection.all_eq(False): print("Selection now empty, breaking") break # Run optimisation on the linear scaling lsc = ExponentialScaling(x_values=interpolator.target_x, ref_values=ref_itpl_mean_I, scl_values=new_itpl_mean_I, weights=selection.as_double()) # Calculate scaling B-factor lsc.scaling_b_factor = -0.5 * list(lsc.optimised_values)[0] # Break if fitted to 0 if approx_equal_relatively(0.0, lsc.scaling_b_factor, 1e-6): report('Scaling is approximately 0.0 - stopping') break # Calculate percentage change report('Curr/New: ' + str(curr_b) + '\t' + str(lsc.scaling_b_factor)) delta = abs((curr_b - lsc.scaling_b_factor) / curr_b) report('Delta: ' + str(delta)) if delta < convergence_crit_perc: report('Scaling has converged to within tolerance - stopping') break # Update selection report('Curr Selection Size: ' + str(sum(selection))) ref_diffs = flex.log(lsc.ref_values) - flex.log(lsc.out_values) #abs_diffs = flex.abs(ref_diffs) sel_diffs = ref_diffs.select(selection) rej_val_high = numpy.percentile(sel_diffs, convergence_reject_perc) rej_val_low = numpy.percentile(sel_diffs, 100.0 - convergence_reject_perc) report('Percentile: ' + str(convergence_reject_perc) + '\t<' + str(rej_val_low) + '\t>' + str(rej_val_high)) selection.set_selected(ref_diffs > rej_val_high, False) selection.set_selected(ref_diffs < rej_val_low, False) report('New Selection Size: ' + str(sum(selection))) # Update loop params curr_b = lsc.scaling_b_factor n_iter += 1 lsc.unscaled_ln_rmsd = (flex.log(lsc.ref_values) - flex.log( lsc.scl_values)).norm() / (lsc.ref_values.size()**0.5) lsc.scaled_ln_rmsd = (flex.log(lsc.ref_values) - flex.log( lsc.out_values)).norm() / (lsc.ref_values.size()**0.5) lsc.unscaled_ln_dev = flex.sum( flex.abs(flex.log(lsc.ref_values) - flex.log(lsc.scl_values))) lsc.scaled_ln_dev = flex.sum( flex.abs(flex.log(lsc.ref_values) - flex.log(lsc.out_values))) return lsc