def exercise () :
g = graphics_utils.make_rainbow_gradient(nbins=21)
assert approx_equal(g[0], (0.0, 0.0, 1.0))
assert approx_equal(g[10], (0.0, 1.0, 0.0))
assert approx_equal(g[-1], (1.0, 0.0, 0.0))
sel = flex.bool([True] * 10)
sel[5] = False
r = graphics_utils.color_rainbow(
selection=sel,
color_all=False)
assert approx_equal(r[0], (0.0,0.0,1.0))
assert approx_equal(r[-1], (1.0,0.0,0.0))
assert approx_equal(r[4], (0.0,1.0,0.0))
r2 = graphics_utils.color_rainbow(
selection=sel,
color_all=True)
assert approx_equal(r2[0], (0.0,0.0,1.0))
assert approx_equal(r2[-1], (1.0,0.0,0.0))
assert approx_equal(r2[6], (2/3.,1.0,0.0))
b = flex.double([4.0,5.2,1.7,6.9,9.5,24.3])
c = graphics_utils.color_by_property(
properties=b,
selection=flex.bool(b.size(), True))
assert approx_equal(c[2], (0.0,0.0,1.0))
c2 = graphics_utils.scale_selected_colors(
input_colors=c,
selection=flex.bool(c.size(), True),
scale=0.9)
assert approx_equal(c2[2], (0.0,0.0,0.9))
c3 = graphics_utils.grayscale_by_property(
properties=b,
selection=flex.bool(b.size(), True))
assert approx_equal(c3[2], (0.95,0.95,0.95))
def selection_around_to_negate(
xray_structure,
selection_within_radius,
iselection,
selection_good=None,
iselection_backbone=None,
iselection_n_external=None,
iselection_c_external=None):
# takes ~0.002 seconds
if([selection_good,iselection_backbone].count(None)==0):
selection_backbone = flex.bool(selection_good.size(), iselection_backbone)
selection_good = selection_good.set_selected(selection_backbone, True)
sel_around = xray_structure.selection_within(
radius = selection_within_radius,
selection = flex.bool(xray_structure.scatterers().size(), iselection))
if(selection_good is not None):
ssb = flex.bool(selection_good.size(), iselection)
sel_around_minus_self = sel_around.set_selected(ssb, False)
else:
sel_around_minus_self = flex.size_t(tuple(
set(sel_around.iselection()).difference(set(iselection))))
if(selection_good is not None):
negate_selection = sel_around_minus_self & selection_good
else:
negate_selection = sel_around_minus_self
if(iselection_n_external is not None and iselection_n_external.size()>0):
negate_selection[iselection_n_external[0]]=False
if(iselection_c_external is not None and iselection_c_external.size()>0):
negate_selection[iselection_c_external[0]]=False
return negate_selection
def __init__(self,maxima):
self.verbose=False
#for the next spot, define the universe of possible pixels (working targets)
# and the map of which pixels have been visited already (pixel_visited)
working_targets = list(xrange(len(maxima)))
pixel_visited = flex.bool(len(working_targets),False)
self.spots = []
while len(working_targets) > 0:
# for this spot, indices of pixels known to be in the spot (pixel_members)
# and a stack of indices of pixels still in process of testing for connections
# (connection_stack). Pop/push operates on the end of stack
pixel_members = [working_targets[0]]
connection_stack = [0]
assert len(pixel_visited)==len(working_targets)
pixel_visited[0]=True
while len(connection_stack) > 0:
idx_current = connection_stack[-1]
for idx_target in xrange(len(working_targets)):
if not pixel_visited[idx_target]:
distance = math.hypot( maxima[working_targets[idx_current]][0]-
maxima[working_targets[idx_target]][0],
maxima[working_targets[idx_current]][1]-
maxima[working_targets[idx_target]][1])
if distance >= 2.0: continue
pixel_visited[idx_target]=True
pixel_members.append(working_targets[idx_target])
connection_stack.append(idx_target)
if connection_stack[-1] == idx_current: connection_stack.pop()
if self.verbose: print "new spot with %d pixels"%len(pixel_members),pixel_members
for idx in pixel_members:#[working_targets[i] for i in pixel_members]:
working_targets.remove(idx)
pixel_visited = flex.bool(len(working_targets),False)
self.spots.append( [maxima[i] for i in pixel_members] )
def get_mask(self, goniometer=None):
'''Creates a mask merging untrusted pixels with active areas.'''
from scitbx.array_family import flex
detector_base = self.detectorbase
# get effective active area coordinates
tile_manager = detector_base.get_tile_manager(detector_base.horizons_phil_cache)
tiling = tile_manager.effective_tiling_as_flex_int(reapply_peripheral_margin = True)
# get the raw data to get the size of the mask
data = self.get_raw_data()
if tiling is None or len(tiling) == 0:
return None
# set the mask to the same dimensions as the data
mask = flex.bool(flex.grid(data.focus()))
# set active areas to True so they are not masked
for i in xrange(len(tiling)//4):
x1,y1,x2,y2=tiling[4*i:(4*i)+4]
sub_array = flex.bool(flex.grid(x2-x1,y2-y1),True)
mask.matrix_paste_block_in_place(sub_array,x1,y1)
# create untrusted pixel mask
detector = self.get_detector()
assert len(detector) == 1
trusted_mask = detector[0].get_trusted_range_mask(data)
# returns merged untrusted pixels and active areas using bitwise AND (pixels are accepted
# if they are inside of the active areas AND inside of the trusted range)
return (mask & trusted_mask,)
def random_selection(n_candidates, n_keep):
assert n_keep >= 0 and n_keep <= n_candidates
n_discard = n_candidates - n_keep
if (n_keep > n_discard):
selection = flex.bool(n_candidates, True)
if (n_discard > 0):
_random_selection_core(selection, n_keep, False)
else:
selection = flex.bool(n_candidates, False)
if (n_keep > 0):
_random_selection_core(selection, n_discard, True)
return selection
def psana_mask_to_dials_mask(self, psana_mask):
if psana_mask.dtype == np.bool:
psana_mask = flex.bool(psana_mask)
else:
psana_mask = flex.bool(psana_mask == 1)
assert psana_mask.focus() == (32, 185, 388)
dials_mask = []
for i in xrange(32):
dials_mask.append(psana_mask[i:i+1,:,:194])
dials_mask[-1].reshape(flex.grid(185,194))
dials_mask.append(psana_mask[i:i+1,:,194:])
dials_mask[-1].reshape(flex.grid(185,194))
return dials_mask
def get_bool_selection_to_keep(big_selection, small_selection):
"""
given 2 iselections (they are sorted), returns bool selection of size
big selection showing what are the matches with small selection.
Rather fast algorithm but may be beneficial to transfer to C++
O(n+m), where n,m - sizes of selections
"""
assert big_selection.size >= small_selection.size()
result = flex.bool(big_selection.size(), False)
i_in_big = 0
i_in_small = 0
size_small = small_selection.size()
size_big = big_selection.size()
n_matches = 0
nw = 0
while (i_in_big < size_big) and (i_in_small < size_small):
if big_selection[i_in_big] == small_selection[i_in_small]:
result[i_in_big] = True
i_in_big += 1
i_in_small += 1
n_matches += 1
elif big_selection[i_in_big] > small_selection[i_in_small]:
i_in_small += 1
nw += 1
else:
i_in_big += 1
# this assert is optional, in general case it is not guaranteed that
# all numbers from small selection are present in big selection.
assert n_matches == size_small, "%d %d" % (n_matches, size_small)
return result
def combine_maps (
map_arrays,
omit_groups,
background_map_coeffs,
resolution_factor,
flatten_background=False,
sigma_scaling=False,
control_map=False) :
"""
For each box, FFT the corresponding omit map coefficients, extract the
omit regions, and copy them to the combined map, using Marat's asymmetric
map class to handle symmetry expansion internally.
"""
assert len(map_arrays) == len(omit_groups)
space_group = background_map_coeffs.space_group()
fft_map = background_map_coeffs.fft_map(
symmetry_flags=maptbx.use_space_group_symmetry,
resolution_factor=resolution_factor).apply_volume_scaling()
if (sigma_scaling) :
fft_map.apply_sigma_scaling()
background_map = fft_map.real_map_unpadded()
#print "full map:", background_map.focus()
if (flatten_background) :
sel_all = flex.bool(background_map.as_1d().size(), True)
background_map.as_1d().set_selected(sel_all, 0)
asym_map = asu_map_ext.asymmetric_map(space_group.type(), background_map)
origin = asym_map.data().origin()
n_real_all = background_map.focus()
n_real_asu = asym_map.data().focus()
m_real_asu = asym_map.data().all()
#print "ORIGIN:", origin
#print "N_REAL:", n_real_asu
#print "M_REAL:", m_real_asu
if (not control_map) :
f2g = maptbx.frac2grid(n_real_all)
n = 0
for group, map_coeffs in zip(omit_groups, map_arrays) :
space_group = map_coeffs.space_group()
omit_fft_map = map_coeffs.fft_map(
resolution_factor=resolution_factor,
symmetry_flags=maptbx.use_space_group_symmetry).apply_volume_scaling()
if (sigma_scaling) :
omit_fft_map.apply_sigma_scaling()
omit_map = omit_fft_map.real_map_unpadded()
omit_asym_map = asu_map_ext.asymmetric_map(space_group.type(), omit_map)
assert omit_asym_map.data().focus() == n_real_asu
for box in group.boxes :
if (control_map) : continue
grid_start = tuple(f2g(box.frac_min))
grid_end = grid_max(f2g(box.frac_max), n_real_asu)
#print grid_start, grid_end
for u in range(grid_start[0], grid_end[0]+1) :
for v in range(grid_start[1], grid_end[1]+1) :
for w in range(grid_start[2], grid_end[2]+1) :
try :
asym_map.data()[(u,v,w)] = omit_asym_map.data()[(u,v,w)]
except IndexError :
raise IndexError("n_real: %s origin: %s index: %s" %
(str(n_real_asu), str(origin), str((u,v,w))))
return asym_map.map_for_fft()
def join_selections (sel1, sel2) : intersections = sel1.intersection_i_seqs(sel2) unique_sel = flex.bool(len(sel1), True) unique_sel.set_selected(intersections[0], False) sel1 = sel1.select(unique_sel) sel1.extend(sel2) return flex.sorted(sel1)
def chains_and_atoms(pdb_hierarchy, secondary_structure_selection,
out=None) :
if (out is None) :
out = sys.stdout
new_secondary_structure_selection = flex.bool()
get_class = iotbx.pdb.common_residue_names_get_class
chains_and_residue_selections = []
for model in pdb_hierarchy.models():
for chain in model.chains():
result = []
for rg in chain.residue_groups():
result_ = flex.size_t()
is_secondary_structure = False
for ag in rg.atom_groups():
#print >> out, ag.resname, get_class(name=ag.resname)
if(get_class(name=ag.resname) == "common_amino_acid" or
get_class(name=ag.resname) == "common_rna_dna"):
for atom in ag.atoms():
result_.append(atom.i_seq)
if(not is_secondary_structure):
is_secondary_structure = \
secondary_structure_selection[atom.i_seq]
new_secondary_structure_selection.append(
secondary_structure_selection[atom.i_seq])
if(result_.size()>0):
result.append(
[result_, is_secondary_structure, rg.resid(), rg.unique_resnames()])
if(len(result)>0):
chains_and_residue_selections.append([chain.id, result])
print >> out, "Considering these chains:"
for ch in chains_and_residue_selections:
print >> out, " chain '%s' (number of residues selected: %d)" % (ch[0], len(ch[1]))
return chains_and_residue_selections, new_secondary_structure_selection
def apply_default_filter(database_dict, d_min, max_models_for_default_filter,
key = "high_resolution"):
database_dict = order_by_value(database_dict = database_dict, key = key)
values = flex.double()
for v in database_dict[key]: values.append(float(v))
diff = flex.abs(values-d_min)
min_val = flex.min(diff)
i_min_sel = (diff == min_val).iselection()
assert i_min_sel.size() > 0
i_min = i_min_sel[i_min_sel.size()//2]
i_l = max(0, i_min-max_models_for_default_filter//2)
i_r = min(values.size()-1, i_min+max_models_for_default_filter//2)
#
print "apply_default_filter:"
print " found data points dmin->higher =", abs(i_l-i_min)
print " found data points dmin->lower =", abs(i_r-i_min)
imm = min(abs(i_l-i_min), abs(i_r-i_min))
i_l, i_r = i_min-imm, i_min+imm
if (imm == 0) :
if (i_l == 0) :
i_r = 100
print " used data points dmin->higher =", 0
print " used data points dmin->lower =", i_r
elif (i_l == i_r == len(values) - 1) :
i_l -= 100
print " used data points dmin->higher =", i_l
print " used data points dmin->lower =", 0
else :
print " used data points dmin->higher =", imm
print " used data points dmin->lower =", imm
#
selection = flex.bool(values.size(), False)
for i in xrange(i_l,i_r): selection[i] = True
return select_dict(database_dict = database_dict, selection = selection)
def filter_histogram_of_key_value(database_dict, key, max_reject_fraction,
edge_tolerance_small = 1.e-4, n_slots = 3):
values = database_dict[key]
new_values = convert_to_numeric(values = values)
#print flex.min(new_values), flex.max(new_values)
size = new_values.size()
#print size
if(size == 0): return
while True:
values = database_dict[key]
new_values = convert_to_numeric(values = values)
selection = flex.bool(new_values.size(), True)
histogram = flex.histogram(data = new_values, n_slots = n_slots)
l = histogram.data_min()
for i, s in enumerate(histogram.slots()):
r = histogram.data_min() + histogram.slot_width() * (i+1)
r = r+edge_tolerance_small
l = max(0, l-edge_tolerance_small)
#print "%8.4f %8.4f %d" % (l, r, s)
if(s < size * max_reject_fraction):
selection &= ~((new_values >= l) & (new_values <= r))
l = r
#print
leave, remove = selection.count(True), selection.count(False)
#print leave, remove
if(remove == 0): break
if(size - leave > int(size * max_reject_fraction)): break
database_dict = select_dict(database_dict = database_dict,
selection = selection)
return database_dict
def generate_linear_background_2d(self, size, bmin, bmax):
from random import uniform
from scitbx.array_family import flex
from scitbx import matrix
slice_size = (1, size[1], size[2])
data = flex.double(flex.grid(size), 0)
mask = flex.bool(flex.grid(size), True)
params = []
for k in range(size[0]):
a00 = uniform(bmin, bmax)
a01 = uniform(bmin, bmax)
a10 = uniform(bmin, bmax)
p00 = matrix.col((0.5, 0.5, a00))
p01 = matrix.col((8.5, 0.5, a01))
p10 = matrix.col((0.5, 8.5, a10))
n = (p01 - p00).cross(p10 - p00)
b = n[0]
c = n[1]
d = n[2]
a = -(b * 0.5 + c * 0.5 + d * a00)
a /= -d
b /= -d
c /= -d
for j in range(size[1]):
for i in range(size[2]):
data[k, j, i] = a + b * (i + 0.5) + c * (j + 0.5)
eps = 1e-7
assert abs(data[k, 0, 0] - a00) < eps
assert abs(data[k, 8, 0] - a10) < eps
assert abs(data[k, 0, 8] - a01) < eps
params.append((a, b, c))
return params, data, mask
def get_outliers (self, proxies, unit_cell, sites_cart, pdb_atoms,
sigma_cutoff, outliers_only=True,
use_segids_in_place_of_chainids=False) :
from scitbx.array_family import flex
site_labels = flex.bool(sites_cart.size(), True).iselection()
sorted_table, not_shown = proxies.get_sorted(
by_value="residual",
sites_cart=sites_cart,
site_labels=site_labels)
# this can happen for C-alpha-only models, etc.
if (sorted_table is None) :
return []
outliers = []
for restraint_info in sorted_table :
(i_seqs, ideal, model, slack, delta, sigma, weight, residual, sym_op_j,
rt_mx) = restraint_info
bond_atoms = get_atoms_info(pdb_atoms, iselection=i_seqs,
use_segids_in_place_of_chainids=use_segids_in_place_of_chainids)
outlier = bond(
atoms_info=bond_atoms,
target=ideal,
model=model,
sigma=sigma,
slack=slack,
delta=delta,
residual=residual,
symop=sym_op_j,
outlier=True,
xyz=get_mean_xyz(bond_atoms))
if (outlier.score > sigma_cutoff) :
outliers.append(outlier)
elif (not outliers_only) :
outlier.outlier=False
outliers.append(outlier)
return outliers
def sieve_fit (sites_fixed,
sites_moving,
selection=None,
frac_discard=0.5) :
"""
Reference: Chothia & Lesk???
"""
assert (sites_fixed.size() == sites_moving.size() > 0)
if (selection is None) :
selection = flex.bool(sites_fixed.size(), True)
# step 1: superpose using originally selected atoms
sites_fixed_aln = sites_fixed.select(selection)
sites_moving_aln = sites_moving.select(selection)
lsq_fit_obj = least_squares_fit(
reference_sites=sites_fixed_aln,
other_sites=sites_moving_aln)
sites_moving_new = lsq_fit_obj.other_sites_best_fit()
# step 2: discard 50% of sites that deviate the most, and superpose again
deltas = (sites_fixed_aln - sites_moving_new).norms()
deltas_sorted = flex.sorted(deltas)
cutoff = deltas_sorted[int((1-frac_discard)*deltas.size())]
selection = (deltas > cutoff)
if (selection.count(True) == 0) :
return lsq_fit_obj
sites_fixed_aln = sites_fixed_aln.select(selection)
sites_moving_aln = sites_moving_aln.select(selection)
lsq_fit_obj = least_squares_fit(
reference_sites=sites_fixed_aln,
other_sites=sites_moving_aln)
return lsq_fit_obj
def selected_positions(selection,positions):
"""
Returns only the selected indices in the positions specified in "positions"
keeping the order
Args:
selection (flex.size_t): Atoms selection
positions (set or list): the allowed positions in the selections
Returns:
(flex.size_t, flex.size_t): (selected atoms, atoms, not selected)
Examples::
>>>a = flex.size_t([1,2,5,6,4])
>>>pos = {0,3,4}
>>>s,d = selected_positions(a,pos)
>>>list(s)
[1,6,4]
>>>list(d)
[2,5]
"""
assert isinstance(selection,flex.size_t)
if isinstance(positions,set): positions = flex.size_t(list(positions))
if isinstance(positions,list): positions = flex.size_t(positions)
include = flex.bool(selection.size(),positions)
not_include = ~include
return selection.select(include), selection.select(not_include)
def tst_identical(self):
from dials.algorithms.integration.fit import fit_profile
from scitbx.array_family import flex
from tst_profile_helpers import gaussian
# Create profile
p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2))
s = flex.sum(p)
p = p / s
# Copy profile
c = p.deep_copy()
b = flex.double(flex.grid(9, 9, 9), 0)
m = flex.bool(flex.grid(9,9,9), True)
# Fit
fit = fit_profile(p, m, c, b)
I = fit.intensity()
V = fit.variance()
# Test intensity is the same
eps = 1e-7
assert(abs(I - flex.sum(p)) < eps)
assert(abs(V - I) < eps)
print 'OK'
def __call__(self, predictions, hkllist, pxlsz):
if len(self.IT) == 4:
# We have only one tile, AnnAdaptor chokes in this case but then there is
# only one choice of nearest neighbour anyway!
nearest_neighbours = flex.int(len(predictions) * self.NEAR, 0)
else:
query = flex.double()
for pred in predictions:
query.append(pred[0] / pxlsz)
query.append(pred[1] / pxlsz)
self.adapt.query(query)
assert len(self.adapt.nn) == len(predictions) * self.NEAR
nearest_neighbours = self.adapt.nn
selection = flex.bool()
self.tile_id = flex.int()
for p in xrange(len(predictions)):
is_in_active_area = False
for n in xrange(self.NEAR):
itile = nearest_neighbours[p * self.NEAR + n]
if (
self.IT[4 * itile] < predictions[p][0] / pxlsz < self.IT[4 * itile + 2]
and self.IT[4 * itile + 1] < predictions[p][1] / pxlsz < self.IT[4 * itile + 3]
):
is_in_active_area = True
break
if is_in_active_area:
self.tile_id.append(itile)
selection.append(is_in_active_area)
assert selection.count(True) == len(self.tile_id)
return predictions.select(selection), hkllist.select(selection)
def tst_with_flat_background_partial(self):
from dials.algorithms.integration.fit import fit_profile
from scitbx.array_family import flex
from tst_profile_helpers import gaussian
# Create profile
p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2))
s = flex.sum(p)
p = p / s
# Copy profile
c0 = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2))
b = flex.double(flex.grid(9, 9, 9), 5)
m = flex.bool(flex.grid(9,9,9), True)
c = c0 + b
# Get the partial profiles
pp = p[0:5,:,:]
mp = m[0:5,:,:]
cp = c[0:5,:,:]
bp = b[0:5,:,:]
c0p = c0[0:5,:,:]
# Fit
fit = fit_profile(pp, mp, cp, bp)
I = fit.intensity()
V = fit.variance()
# Test intensity is the same
eps = 1e-7
assert(abs(I - flex.sum(c0)) < eps)
assert(abs(V - (flex.sum(c0p) + flex.sum(bp))) < eps)
print 'OK'
def filter_shadowed_reflections(experiments, reflections):
from dials.util import mask_untrusted_polygon
from dials.util import is_inside_polygon
shadowed = flex.bool(reflections.size(), False)
for expt_id in range(len(experiments)):
expt = experiments[expt_id]
imgset = expt.imageset
masker = imgset.reader().get_format().get_goniometer_shadow_masker()
detector = expt.detector
sel = reflections['id'] == expt_id
isel = sel.iselection()
x,y,z = reflections['xyzcal.px'].select(isel).parts()
start, end = expt.scan.get_array_range()
for i in range(start, end):
shadow = masker.project_extrema(
detector, expt.scan.get_angle_from_array_index(i))
img_sel = (z >= i) & (z < (i+1))
img_isel = img_sel.iselection()
for p_id in range(len(detector)):
panel = reflections['panel'].select(img_isel)
if shadow[p_id].size() < 4:
continue
panel_isel = img_isel.select(panel == p_id)
inside = is_inside_polygon(
shadow[p_id],
flex.vec2_double(x.select(isel.select(panel_isel)),
y.select(isel.select(panel_isel))))
shadowed.set_selected(panel_isel, inside)
return shadowed
def finalize_model (pdb_hierarchy,
xray_structure,
set_b_iso=None,
convert_to_isotropic=None,
selection=None) :
"""
Prepare a rebuilt model for refinement, optionally including B-factor reset.
"""
from cctbx import adptbx
from scitbx.array_family import flex
pdb_atoms = pdb_hierarchy.atoms()
if (selection is None) :
selection = flex.bool(pdb_atoms.size(), True)
elif isinstance(selection, str) :
sel_cache = pdb_hierarchy.atom_selection_cache()
selection = sel_cache.selection(selection)
for i_seq, atom in enumerate(pdb_atoms) :
assert (atom.parent() is not None)
atom.segid = ""
sc = xray_structure.scatterers()[i_seq]
sc.label = atom.id_str()
if (convert_to_isotropic) :
xray_structure.convert_to_isotropic(selection=selection.iselection())
if (set_b_iso is not None) :
if (set_b_iso is Auto) :
u_iso = xray_structure.extract_u_iso_or_u_equiv()
set_b_iso = adptbx.u_as_b(flex.mean(u_iso)) / 2
xray_structure.set_b_iso(value=set_b_iso, selection=selection)
pdb_hierarchy.adopt_xray_structure(xray_structure)
pdb_atoms.reset_serial()
pdb_atoms.reset_i_seq()
pdb_atoms.reset_tmp()
def get_outliers (self, proxies, unit_cell, sites_cart, pdb_atoms,
sigma_cutoff, outliers_only=True,
use_segids_in_place_of_chainids=False) :
import cctbx.geometry_restraints
from scitbx.array_family import flex
site_labels = flex.bool(sites_cart.size(), True).iselection()
sorted_table, n_not_shown = proxies.get_sorted(
by_value="residual",
sites_cart=sites_cart,
site_labels=site_labels,
unit_cell=unit_cell)
if (sorted_table is None) : return []
outliers = []
for restraint_info in sorted_table :
(plane_atoms, rms_delta, residual) = restraint_info
i_seqs = [ a[0] for a in plane_atoms ]
deviation = max([ a[1] / a[2] for a in plane_atoms ])
plane_atoms_ = get_atoms_info(pdb_atoms, iselection=i_seqs)
outlier = planarity(
atoms_info=plane_atoms_,
rms_deltas=rms_delta,
residual=residual,
delta_max=max([ a[1] for a in plane_atoms ]),
score=deviation,
outlier=True,
xyz=get_mean_xyz(plane_atoms_))
if (outlier.score > sigma_cutoff) :
outliers.append(outlier)
elif (not outliers_only) :
outlier.outlier=False
outliers.append(outlier)
return outliers
def tst_with_flat_background(self):
from dials.algorithms.integration.fit import fit_profile
from scitbx.array_family import flex
from tst_profile_helpers import gaussian
# Create profile
p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2))
s = flex.sum(p)
p = p / s
# Copy profile
c0 = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2))
b = flex.double(flex.grid(9, 9, 9), 5)
m = flex.bool(flex.grid(9,9,9), True)
c = c0 + b
# Fit
fit = fit_profile(p, m, c, b)
I = fit.intensity()
V = fit.variance()
# Test intensity is the same
eps = 1e-7
assert(abs(I - flex.sum(c0)) < eps)
assert(abs(V - (flex.sum(c0) + flex.sum(b))) < eps)
print 'OK'
def __init__ (self, pdb_hierarchy, atom_selection=None, params=None,
log=sys.stdout, proxies=None, tables=None, initialize=True):
assert pdb_hierarchy is not None
assert not pdb_hierarchy.atoms().extract_i_seq().all_eq(0), ""+\
"Probably all atoms have i_seq = 0 which is wrong"
adopt_init_args(self, locals())
self.bool_atom_selection = None
if self.atom_selection is None:
self.bool_atom_selection = flex.bool(pdb_hierarchy.atoms().size(), True)
else:
cache = pdb_hierarchy.atom_selection_cache()
self.bool_atom_selection = cache.selection(atom_selection)
if params is None:
params = master_phil.fetch().extract()
self.params = params
if initialize:
if(self.params.rama_potential == "oldfield"):
self.tables = ramachandran_plot_data(
plot_cutoff=self.params.oldfield.plot_cutoff)
else :
self.tables = load_tables(params)
# get proxies
self.extract_proxies()
else:
assert proxies is not None
if(self.params.rama_potential == "oldfield"):
self.target_phi_psi = self.update_phi_psi_targets(
sites_cart = self.pdb_hierarchy.atoms().extract_xyz())
def cross_validate_to_determine_number_of_terms(
x_obs, y_obs, w_obs=None, min_terms=10, max_terms=25, n_free=100, n_goes=5
):
if n_goes == None:
if min_terms < 2:
min_terms = 2
free_residuals = []
free_flags = flex.bool(x_obs.size(), True)
free_permut = flex.random_permutation(x_obs.size())
for ii in range(n_free):
free_flags[free_permut[ii]] = False
for count in range(min_terms, max_terms):
fit = chebyshev_lsq_fit(count, x_obs, y_obs, w_obs, free_flags)
free_residuals.append(fit.free_f)
return flex.double(free_residuals)
else:
if w_obs is None:
w_obs = flex.double(x_obs.size(), 1)
free_resid = flex.double(max_terms - min_terms, 0)
for jj in range(n_goes):
free_resid += cross_validate_to_determine_number_of_terms(
x_obs, y_obs, w_obs, min_terms=min_terms, max_terms=max_terms, n_free=n_free, n_goes=None
)
return min_terms + flex.min_index(free_resid)
def __init__(
self,
map_data,
xray_structure,
ncs_restraints_group_list,
real_space_gradients_delta,
refine_selection=None,
use_ncs_constraints=True,
restraints_manager=None,
data_weight=None,
refine_sites=False,
refine_transformations=False):
adopt_init_args(self, locals())
self.refine_selection = nu.get_refine_selection(
refine_selection=self.refine_selection,
number_of_atoms=self.xray_structure.sites_cart().size())
self.extended_ncs_selection = nu.get_extended_ncs_selection(
ncs_restraints_group_list=ncs_restraints_group_list,
refine_selection=self.refine_selection)
self.unit_cell = self.xray_structure.unit_cell()
# get selection to refine
asu_size = xray_structure.scatterers().size()
if refine_sites:
# Use all atoms to refine
self.selection = flex.bool(asu_size, refine_selection)
elif refine_transformations:
# use only NCS related atoms (Without the Master NCS copy)
self.selection = nu.get_ncs_related_selection(
ncs_restraints_group_list=ncs_restraints_group_list,
asu_size=asu_size)
def base_pair_selection (self, **kwds) :
# assert 0, "Probably shouldn't be used",
# used in mmtbx/command_line/find_tls_groups.py
whole_selection = flex.bool(self.n_atoms)
for sheet in self.base_pair_selections(**kwds) :
whole_selection |= sheet
return whole_selection
def __init__ (self, object_id, pdb_hierarchy, atomic_bonds,
special_position_settings=None,
base_color=(0.0,1.0,1.0)) :
self.object_id = object_id
self.base_color = base_color
self.draw_mode = None
self.current_bonds = None
self.noncovalent_bonds = None
self.ribbon = None
self.color_mode = None #"rainbow"
self.flag_object_visible = True
self._color_cache = {}
self.flag_show_hydrogens = False
self.flag_show_lines = True
self.flag_show_labels = True
self.flag_show_points = True
self.flag_show_spheres = False
self.flag_show_ribbon = False
self.flag_show_ellipsoids = False
self.flag_show_noncovalent_bonds = False
self.update_structure(pdb_hierarchy=pdb_hierarchy,
atomic_bonds=atomic_bonds,
special_position_settings=special_position_settings)
from scitbx.array_family import flex
self.use_u_aniso = flex.bool(self.atoms.size())
def update_structure (self, pdb_hierarchy, atomic_bonds,
special_position_settings=None) :
from scitbx.array_family import flex
self.pdb_hierarchy = pdb_hierarchy
self.atoms = pdb_hierarchy.atoms()
self.atom_count = self.atoms.size()
assert (self.atom_count == len(atomic_bonds))
if atomic_bonds is None :
atomic_bonds = flex.stl_set_unsigned(self.atom_count)
self.atomic_bonds = atomic_bonds
self.selection_cache = pdb_hierarchy.atom_selection_cache(
special_position_settings=special_position_settings)
#self.index_atoms()
atom_index = []
atom_labels = flex.std_string()
for atom in self.pdb_hierarchy.atoms_with_labels() :
atom_index.append(atom)
atom_labels.append(format_atom_label(atom))
self.atom_index = atom_index
self.atom_labels = atom_labels
self.trace_bonds = extract_trace(pdb_hierarchy, self.selection_cache)
if self.draw_mode is None or self.draw_mode.startswith("trace") :
self.current_bonds = self.trace_bonds
else :
self.current_bonds = self.atomic_bonds
atom_radii = flex.double(self.atoms.size(), 1.5)
hydrogen_flag = flex.bool(self.atoms.size(), False)
for i_seq, atom in enumerate(self.atom_index) :
if atom.element.strip() in ["H", "D"] :
atom_radii[i_seq] = 0.75
hydrogen_flag[i_seq] = True
self.atom_radii = atom_radii
self.hydrogen_flag = hydrogen_flag
self._color_cache = {}
self.is_changed = True
def get_bin_selection (self, bin) :
from scitbx.array_family import flex
sel = flex.bool(self._bins.size(), False)
for k, n in enumerate(self._bins) :
if (n == bin) :
sel[k] = True
return sel
def __init__(self,
map_data,
xray_structure,
ncs_restraints_group_list,
real_space_gradients_delta,
refine_selection=None,
use_ncs_constraints=True,
restraints_manager=None,
data_weight=None,
refine_sites=False):
adopt_init_args(self, locals())
self.refine_selection = nu.get_refine_selection(
refine_selection=self.refine_selection,
number_of_atoms=self.xray_structure.sites_cart().size())
self.extended_ncs_selection = nu.get_extended_ncs_selection(
ncs_restraints_group_list=ncs_restraints_group_list,
refine_selection=self.refine_selection)
self.unit_cell = self.xray_structure.unit_cell()
# get selection to refine
asu_size = xray_structure.scatterers().size()
self.selection = flex.bool(asu_size, refine_selection)
def iselection_ncs_to_asu(iselection_ncs,ncs_chain_id,hierarchy_asu):
"""
Convert iselection of NCS related atoms in the NCS copy to the ASU iselection
Args:
ncs_len (int)
ncs_chain_id (str)
iselection_ncs (flex.size_t)
hierarchy_asu hierarchy object)
Returns:
iselection_asu (flex.size_t)
"""
asu_len = hierarchy_asu.atoms_size()
atom_cache = hierarchy_asu.atom_selection_cache().selection
sel = atom_cache('chain ' + ncs_chain_id)
sel = sel.iselection(True)
offset = min(list(sel))
iselection_ncs += offset
selection_bool = flex.bool(asu_len,iselection_ncs)
return selection_bool.iselection(True)
def check_flags(self,
flags,
predictions=None,
observations=None,
shoeboxes=None,
**kwargs):
'''
Check the flags are set, if they're not then create a list
of Trues equal to the number of items given.
:param flags: The input flags
:param predictions: The predictions
:param observations: The observations
:param shoeboxes: The shoeboxes
:return: The filtered flags
'''
from scitbx.array_family import flex
# If flags are not set then create a list of Trues
if flags is None:
length = 0
if predictions:
length = len(predictions)
if observations:
if length > 0:
assert (length == len(observations))
else:
length = len(observations)
if shoeboxes:
if length > 0:
assert (length == len(observations))
else:
length = len(shoeboxes)
# Create an array of flags
flags = flex.bool(length, True)
# Return the flags
return flags
def reduce_raw_data(raw_data, qmax, bandwidth, level=0.05, q_background=None):
print
print " ==== Data reduction ==== "
print
print " Preprocessing of data increases efficiency of shape retrieval procedure."
print
print " - Interpolation stepsize : %4.3e" % bandwidth
print " - Uniform density criteria: level is set to : %4.3e" % level
print " maximum q to consider : %4.3e" % qmax
qmin_indx = flex.max_index(raw_data.i)
qmin = raw_data.q[qmin_indx]
new_data = get_q_array_uniform_body(raw_data,
q_min=qmin,
q_max=qmax,
level=level)
qmax = new_data.q[-1]
if qmax > raw_data.q[-1]:
qmax = raw_data.q[-1]
print " Resulting q range to use in search: q start : %4.3e" % qmin
print " q stop : %4.3e" % qmax
print
raw_q = raw_data.q[qmin_indx:]
raw_i = raw_data.i[qmin_indx:]
raw_s = raw_data.s[qmin_indx:]
### Take care of the background (set zero at very high q) ###
if (q_background is not None):
cutoff = flex.bool(raw_q > q_background)
q_bk_indx = flex.last_index(cutoff, False)
if (q_bk_indx < raw_q.size()):
bkgrd = flex.mean(raw_i[q_bk_indx:])
print "Background correction: I=I-background, where background=", bkgrd
raw_i = flex.abs(raw_i - bkgrd)
q = flex.double(range(int(
(qmax - qmin) / bandwidth) + 1)) * bandwidth + qmin
raw_data.i = flex.linear_interpolation(raw_q, raw_i, q)
raw_data.s = flex.linear_interpolation(raw_q, raw_s, q)
raw_data.q = q
return raw_data
def test_manhattan():
from scitbx.array_family import flex
from dials.algorithms.image.filter import manhattan_distance
data = flex.bool(flex.grid(100, 100), True)
for j in range(100):
for i in range(100):
if math.sqrt((j - 50) ** 2 + (i - 50) ** 2) <= 10.5:
data[j, i] = False
distance = manhattan_distance(data, True)
M = distance[1:-1, 1:-1]
D = data[1:-1, 1:-1]
selection = D.as_1d().select(M.as_1d() == 0)
assert selection.all_eq(True)
while True:
N = data[0:-2, 1:-1]
S = data[2:, 1:-1]
E = data[1:-1, 0:-2]
W = data[1:-1, 2:]
selection = M.as_1d() == 1
neighbours = (
N.select(selection)
| S.select(selection)
| E.select(selection)
| W.select(selection)
)
assert neighbours.all_eq(True)
indices = flex.size_t(range(len(D))).select(selection)
for i in indices:
D[i] = True
data[1:-1, 1:-1] = D
M -= 1
if (M > 0).count(True) == 0:
break
def __init__(self,
fmodel,
pdb_hierarchy,
processed_pdb_file,
params,
selection,
selection_score=None,
nproc=Auto,
out=None):
adopt_init_args(self, locals())
from scitbx.array_family import flex
if (self.out is None) : self.out = sys.stdout
self.sites_start = fmodel.xray_structure.sites_cart().deep_copy()
if (type(selection).__name__ == 'bool'):
assert (self.selection.count(True) > 0)
self.iselection = self.selection.iselection()
else : # actually an iselection
assert (len(self.selection) > 0)
self.iselection = self.selection
self.selection = flex.bool(self.sites_start.size(), False).set_selected(
self.iselection, True)
if (self.selection_score is None):
self.selection_score = self.selection
use_mp = (self.params.n_trials > 1)
assert (params.partial_occupancy is not None)
if (len(params.partial_occupancy) > 1):
use_mp = True
if (not use_mp):
self._trials = [ self.run_trial(params.partial_occupancy[0]) ]
else :
self.out = null_out()
args = []
for occ in params.partial_occupancy :
assert (occ > 0) and (occ <= 1)
for k in range(self.params.n_trials):
args.append(occ)
self._trials = easy_mp.pool_map(
fixed_func=self.run_trial,
args=args,
processes=self.nproc)
def get_ncs_related_selection(ncs_restraints_group_list, asu_size):
"""
Args:
ncs_restraints_group_list (list): list of ncs_restraint_group objects
asu_size (int): the total size of the ASU
Returns:
selection (flex.bool): selection of all ncs related atom in the ASU
"""
total_master_ncs_selection = set()
total_ncs_related_selection = set()
for nrg in ncs_restraints_group_list:
master_ncs_selection = nrg.master_iselection
total_master_ncs_selection.update(set(master_ncs_selection))
for ncs_copy in nrg.copies:
asu_selection = ncs_copy.iselection
total_ncs_related_selection.update(set(asu_selection))
#
total_ncs_related_selection.update(total_master_ncs_selection)
ts = flex.size_t(list(total_ncs_related_selection))
selection = flex.bool(asu_size, ts)
return selection
def predict_for_reflection_table(self,
reflections,
skip_derivatives=False):
"""perform prediction for all reflections in the supplied table"""
# set the entering flags if this has not been done
from dials.algorithms.refinement.reflection_manager import calculate_entering_flags
if "entering" not in reflections:
reflections['entering'] = calculate_entering_flags(
reflections, self._experiments)
# can only predict for experiments that exist and within the scan range
# any other reflections will be left unchanged
inc = flex.size_t_range(len(reflections))
to_keep = flex.bool(len(inc), False)
for iexp, exp in enumerate(self._experiments):
sel = reflections['id'] == iexp
# keep all reflections if there is no rotation axis
if exp.goniometer is None:
to_keep.set_selected(sel, True)
continue
# trim reflections outside the scan range
phi = reflections['xyzobs.mm.value'].parts()[2]
phi_min, phi_max = exp.scan.get_oscillation_range(deg=False)
passed = (phi >= phi_min) & (phi <= phi_max)
to_keep.set_selected(sel, passed)
# determine indices to include and predict on the subset
inc = inc.select(to_keep)
sub_refl = reflections.select(inc)
preds = self._predict_core(sub_refl, skip_derivatives)
# set updated subset back into place
reflections.set_selected(inc, preds)
return reflections
def test_external_lookup():
from dxtbx.imageset import ExternalLookup
mask = flex.bool(flex.grid(10, 10), True)
gain = flex.double(flex.grid(10, 10), 1)
pedestal = flex.double(flex.grid(10, 10), 2)
lookup = ExternalLookup()
lookup.mask.data = dxtbx.format.image.ImageBool(
dxtbx.format.image.ImageTileBool(mask))
lookup.gain.data = dxtbx.format.image.ImageDouble(
dxtbx.format.image.ImageTileDouble(gain))
lookup.pedestal.data = dxtbx.format.image.ImageDouble(
dxtbx.format.image.ImageTileDouble(pedestal))
mask2 = lookup.mask.data.tile(0).data()
gain2 = lookup.gain.data.tile(0).data()
pedestal2 = lookup.pedestal.data.tile(0).data()
assert mask2.all_eq(mask)
assert gain2.all_eq(gain)
assert pedestal2.all_eq(pedestal)
def test_combine_with_user_static_mask(dials_data, tmpdir):
master_h5 = (dials_data("image_examples").join(
"SACLA-MPCCD-Phase3-21528-5images.h5").strpath)
assert FormatHDF5SaclaMPCCD.understand(master_h5)
expts_from_filename = ExperimentListFactory.from_filenames([master_h5])
for i, expt in enumerate(expts_from_filename):
mask = []
for panel in expt.detector:
m = flex.bool(flex.grid(reversed(panel.get_image_size())), True)
mask_untrusted_resolution_range(m, expt.beam, panel, 0, 2.23)
mask.append(m)
mask = tuple(mask)
# exact number of masked pixels varies with wavelength between experiments
assert [_.count(False) for _ in mask
] == pytest.approx([50643, 0, 0, 48003, 48356, 0, 0, 52191],
abs=1e3)
mask_file = tmpdir.join("pixel_%i.mask" % i)
with mask_file.open("wb") as f:
pickle.dump(mask, f)
expt.imageset.external_lookup.mask.filename = mask_file.strpath
expt.imageset.external_lookup.mask.data = ImageBool(mask)
expts_from_dict = ExperimentListFactory.from_dict(
expts_from_filename.to_dict())
imageset = expts_from_dict[0].imageset
mask = imageset.get_mask(0)
assert len(mask) == 8
assert [_.count(False) for _ in mask] == [
65332,
24296,
24296,
63335,
63660,
24296,
24296,
66691,
]
imageset.reader().nullify_format_instance()
def __call__(self, omit_group):
from scitbx.array_family import flex
fmodel_tmp = self.fmodel.deep_copy()
xrs_omit = fmodel_tmp.xray_structure.deep_copy_scatterers()
xrs_omit.set_occupancies(self.occupancy,
selection=omit_group.selection)
if (self.selection_delete is not None):
selection_partial = flex.bool(xrs_omit.scatterers().size(), False)
selection_partial.set_selected(omit_group.selection, True)
selection_delete = selection_partial & self.selection_delete
n_del = selection_delete.count(True)
if (n_del > 0):
#print "setting %d waters to zero occupancy" % n_del
xrs_omit.set_occupancies(0., selection=selection_delete)
fmodel_tmp.update_xray_structure(xrs_omit,
update_f_mask=False,
update_f_calc=True)
return fmodel_tmp.map_coefficients(
map_type=self.map_type,
fill_missing=self.fill_missing_f_obs,
exclude_free_r_reflections=self.exclude_free_r_reflections,
merge_anomalous=True)
def extract_sites_for_alignment(chain_obj):
"""Extract sequence and sites of c-alphas - adapted from mmtbx.command_line.super"""
seq = []
sites = flex.vec3_double()
use_sites = flex.bool()
for resi in chain_obj.conformers()[0].residues():
if ( iotbx.pdb.common_residue_names_get_class(name=resi.resname) != "common_amino_acid"):
continue
resn = resi.resname
single = iotbx.pdb.amino_acid_codes.one_letter_given_three_letter[resn]
seq.append(single)
use = False
xyz = (0,0,0)
for atom in resi.atoms():
if (atom.name == " CA "):
xyz = atom.xyz
use = True
break
sites.append(xyz)
use_sites.append(use)
return "".join(seq), sites, use_sites
def setup_helix_example(pdb_name = "m00_good.pdb",
mtz_name = "m00_good.mtz"):
# Read good model and compute data from it.
xrs_good = iotbx.pdb.input(
file_name = os.path.join(qr_unit_tests_data,
pdb_name )).xray_structure_simple()
f_obs = abs(xrs_good.structure_factors(d_min=1.5).f_calc())
r_free_flags_data = flex.bool()
for i in range(f_obs.data().size()):
if(i%10 == 0): r_free_flags_data.append(True)
else: r_free_flags_data.append(False)
r_free_flags = f_obs.array(data = r_free_flags_data)
mtz_dataset = f_obs.as_mtz_dataset(column_root_label="F-obs")
mtz_dataset.add_miller_array(
miller_array = r_free_flags,
column_root_label = "R-free-flags")
mtz_object = mtz_dataset.mtz_object()
mtz_object.write(file_name = mtz_name)
xrs_poor = iotbx.pdb.input(
file_name = os.path.join(qr_unit_tests_data,
"m00_poor.pdb")).xray_structure_simple()
return xrs_good,xrs_poor,f_obs,r_free_flags
def resolution_filter(self):
"""Filter arrays on resolution."""
if not self.params.d_min and self.params.min_completeness:
# Update self.params.d_min using resolution estimate
params = resolution_analysis.phil_defaults.extract().resolution
params.nbins = self.params.resolution_bins
r = resolution_analysis.Resolutionizer(self.intensities, params)
self.params.d_min = r.resolution(
resolution_analysis.metrics.COMPLETENESS,
limit=self.params.min_completeness,
).d_min
logger.info("Estimated d_min: %.2f", self.params.d_min)
if self.params.d_min or self.params.d_max:
sel = flex.bool(self.intensities.size(), True)
d_spacings = self.intensities.d_spacings().data()
if self.params.d_min:
sel &= d_spacings >= self.params.d_min
if self.params.d_max:
sel &= d_spacings <= self.params.d_max
self.intensities = self.intensities.select(sel)
self.dose = self.dose.select(sel)
def test_with_flat_background_partial(): from dials.algorithms.integration.fit import ProfileFitter from scitbx.array_family import flex from numpy.random import seed seed(0) # Create profile p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2)) s = flex.sum(p) p = p / s # Copy profile c0 = add_poisson_noise(100 * p) b = flex.double(flex.grid(9, 9, 9), 1) m = flex.bool(flex.grid(9,9,9), True) c = c0 + add_poisson_noise(b) # Get the partial profiles pp = p[0:5,:,:] mp = m[0:5,:,:] cp = c[0:5,:,:] bp = b[0:5,:,:] c0p = c0[0:5,:,:] # Fit fit = ProfileFitter(cp, bp, mp, pp) I = fit.intensity() V = fit.variance() assert fit.niter() < fit.maxiter() Iknown = 99.06932141277105 Vknown = 504.06932141277105 # Test intensity is the same eps = 1e-7 assert(abs(I[0] - Iknown) < eps) assert(abs(V[0] - Vknown) < eps)
def filter_shadowed_reflections(experiments,
reflections,
experiment_goniometer=False):
from dials.util.ext import is_inside_polygon
from scitbx.array_family import flex
shadowed = flex.bool(reflections.size(), False)
for expt_id in range(len(experiments)):
expt = experiments[expt_id]
imageset = expt.imageset
if experiment_goniometer:
masker = imageset.masker().format_class(
imageset.paths()[0]).get_goniometer_shadow_masker(
goniometer=expt.goniometer)
else:
masker = imageset.masker().format_class(
imageset.paths()[0]).get_goniometer_shadow_masker()
detector = expt.detector
sel = reflections['id'] == expt_id
isel = sel.iselection()
x, y, z = reflections['xyzcal.px'].select(isel).parts()
start, end = expt.scan.get_array_range()
for i in range(start, end):
shadow = masker.project_extrema(
detector, expt.scan.get_angle_from_array_index(i))
img_sel = (z >= i) & (z < (i + 1))
img_isel = img_sel.iselection()
for p_id in range(len(detector)):
panel = reflections['panel'].select(img_isel)
if shadow[p_id].size() < 4:
continue
panel_isel = img_isel.select(panel == p_id)
inside = is_inside_polygon(
shadow[p_id],
flex.vec2_double(x.select(isel.select(panel_isel)),
y.select(isel.select(panel_isel))))
shadowed.set_selected(panel_isel, inside)
return shadowed
def apply_default_filter(database_dict,
d_min,
max_models_for_default_filter,
key="high_resolution"):
database_dict = order_by_value(database_dict=database_dict, key=key)
values = flex.double()
for v in database_dict[key]:
values.append(float(v))
diff = flex.abs(values - d_min)
min_val = flex.min(diff)
i_min_sel = (diff == min_val).iselection()
assert i_min_sel.size() > 0
i_min = i_min_sel[i_min_sel.size() // 2]
i_l = max(0, i_min - max_models_for_default_filter // 2)
i_r = min(values.size() - 1, i_min + max_models_for_default_filter // 2)
#
print "apply_default_filter:"
print " found data points dmin->higher =", abs(i_l - i_min)
print " found data points dmin->lower =", abs(i_r - i_min)
imm = min(abs(i_l - i_min), abs(i_r - i_min))
i_l, i_r = i_min - imm, i_min + imm
if (imm == 0):
if (i_l == 0):
i_r = 100
print " used data points dmin->higher =", 0
print " used data points dmin->lower =", i_r
elif (i_l == i_r == len(values) - 1):
i_l -= 100
print " used data points dmin->higher =", i_l
print " used data points dmin->lower =", 0
else:
print " used data points dmin->higher =", imm
print " used data points dmin->lower =", imm
#
selection = flex.bool(values.size(), False)
for i in xrange(i_l, i_r):
selection[i] = True
return select_dict(database_dict=database_dict, selection=selection)
def __call__(self, fragment):
from scitbx.array_family import flex
frag_selection = flex.bool(self.pdb_atoms.size(), fragment)
sites_cart_orig = self.fmodel.xray_structure.sites_cart()
sites_start = sites_cart_orig.select(fragment)
atom_start = self.pdb_atoms[fragment[0]].fetch_labels()
atom_end = self.pdb_atoms[fragment[-1]].fetch_labels()
label = "%s%s-%s" % (atom_start.chain_id, atom_start.resid(),
atom_end.resid())
# XXX should the partial occupancy be determined automatically?
two_fofc_map, fofc_map = alt_confs.get_partial_omit_map(
fmodel=self.fmodel.deep_copy(),
selection=(frag_selection & self.sele_main_conf),
selection_delete=(frag_selection & ~(self.sele_main_conf)),
partial_occupancy=0.6)
target_map = two_fofc_map
if (self.rsr_fofc_map_target):
target_map = fofc_map
box = self.make_box(selection=frag_selection, target_map=target_map)
box.restrain_atoms(selection=alt_confs.SELECTION_OLD,
reference_sigma=0.02,
reset_first=True)
box.restrain_atoms(selection=alt_confs.SELECTION_NEW_REBUILT,
reference_sigma=0.05,
reset_first=False)
# first fix the geometry of adjacent residues
box.real_space_refine(alt_confs.SELECTION_NEW_SPLIT)
# now the entire B conformer
box.real_space_refine(alt_confs.SELECTION_NEW)
sites_cart_refined = box.update_original_coordinates()
sites_new = sites_cart_refined.select(fragment)
self.fmodel.xray_structure.set_sites_cart(sites_cart_orig)
self.pdb_atoms.set_xyz(sites_cart_orig)
return alt_confs.refined_fragment(
label=label,
selection=frag_selection,
sites_cart=sites_cart_refined,
rmsd=sites_new.rms_difference(sites_start))
def convert_to_numeric(values):
is_digit_selection = flex.bool()
for kv in values:
flag = True
try:
tmp = float(kv)
except Exception:
flag = False
is_digit_selection.append(flag)
if (is_digit_selection.count(False) > 0):
raise RuntimeError("Non-numerical values.")
points = 0
for kv in values:
points += kv.count(".")
if (points == 0):
new_values = flex.int()
for kv in values:
new_values.append(int(kv))
else:
new_values = flex.double()
for kv in values:
new_values.append(float(kv))
return new_values
def test_deconvolve_zero():
from dials.algorithms.integration.fit import ProfileFitter
from scitbx.array_family import flex
from numpy.random import seed
seed(0)
I0 = [1000, 2000, 3000]
# Create profile
p = generate_3_profiles()
# Copy profile
c = flex.double(flex.grid(40, 9, 9), 0)
b = flex.double(flex.grid(40, 9, 9), 0)
m = flex.bool(flex.grid(40,9,9), True)
# Fit
passed = False
with pytest.raises(Exception):
fit = ProfileFitter(c, b, m, p)
I = fit.intensity()
V = fit.variance()
def run(self):
from dials.algorithms.image.fill_holes import simple_fill
from scitbx.array_family import flex
from math import sqrt
mask = flex.bool(flex.grid(100, 100), True)
data = flex.double(flex.grid(100, 100), True)
for j in range(100):
for i in range(100):
data[j, i] = 10 + j * 0.01 + i * 0.01
if sqrt((j - 50)**2 + (i - 50)**2) <= 10.5:
mask[j, i] = False
data[j, i] = 0
result = simple_fill(data, mask)
known = data.as_1d().select(mask.as_1d())
filled = result.as_1d().select(mask.as_1d() == False)
assert flex.max(filled) <= flex.max(known)
assert flex.min(filled) >= flex.min(known)
# Test passed
print 'OK'
def cross_validate_to_determine_number_of_terms(x_obs,
y_obs,
w_obs=None,
min_terms=10,
max_terms=25,
n_free=100,
n_goes=5):
if (n_goes == None):
if (min_terms < 2):
min_terms = 2
free_residuals = []
free_flags = flex.bool(x_obs.size(), True)
free_permut = flex.random_permutation(x_obs.size())
for ii in range(n_free):
free_flags[free_permut[ii]] = False
for count in range(min_terms, max_terms):
fit = chebyshev_lsq_fit(count, x_obs, y_obs, w_obs, free_flags)
free_residuals.append(fit.free_f)
return (flex.double(free_residuals))
else:
if w_obs is None:
w_obs = flex.double(x_obs.size(), 1)
free_resid = flex.double(max_terms - min_terms, 0)
for jj in range(n_goes):
free_resid += cross_validate_to_determine_number_of_terms(
x_obs,
y_obs,
w_obs,
min_terms=min_terms,
max_terms=max_terms,
n_free=n_free,
n_goes=None)
return (min_terms + flex.min_index(free_resid))
def update_structure(self,
pdb_hierarchy,
atomic_bonds,
special_position_settings=None):
from scitbx.array_family import flex
self.pdb_hierarchy = pdb_hierarchy
self.atoms = pdb_hierarchy.atoms()
self.atom_count = self.atoms.size()
assert (self.atom_count == len(atomic_bonds))
if atomic_bonds is None:
atomic_bonds = flex.stl_set_unsigned(self.atom_count)
self.atomic_bonds = atomic_bonds
self.selection_cache = pdb_hierarchy.atom_selection_cache(
special_position_settings=special_position_settings)
#self.index_atoms()
atom_index = []
atom_labels = flex.std_string()
for atom in self.pdb_hierarchy.atoms_with_labels():
atom_index.append(atom)
atom_labels.append(format_atom_label(atom))
self.atom_index = atom_index
self.atom_labels = atom_labels
self.trace_bonds = extract_trace(pdb_hierarchy, self.selection_cache)
if self.draw_mode is None or self.draw_mode.startswith("trace"):
self.current_bonds = self.trace_bonds
else:
self.current_bonds = self.atomic_bonds
atom_radii = flex.double(self.atoms.size(), 1.5)
hydrogen_flag = flex.bool(self.atoms.size(), False)
for i_seq, atom in enumerate(self.atom_index):
if atom.element.strip() in ["H", "D"]:
atom_radii[i_seq] = 0.75
hydrogen_flag[i_seq] = True
self.atom_radii = atom_radii
self.hydrogen_flag = hydrogen_flag
self._color_cache = {}
self.is_changed = True
def reindex_refl_by_coset(refl,data,symms,uuids,co,anomalous_flag = False, verbose=True):
if verbose:
cache_miller = refl["miller_index"].deep_copy()
cache_asu = refl["miller_index_asymmetric"].deep_copy()
for icoset,partition in enumerate(co.partitions):
if icoset==0: continue # no change of basis
cb_op = change_of_basis_op(partition[0])
mi_new = cb_op.apply(refl["miller_index"])
mi_asu_new = mi_new.deep_copy()
miller.map_to_asu(symms[0].space_group().info().type(), anomalous_flag, mi_asu_new)
# now select only those expts assigned to that coset
good_refls = flex.bool(len(refl), False)
all_expt_id = list(data["experiment"])
all_coset = list(data["coset"]) # would like to understand how to use pandas rather than Python list
for iexpt in range(len(symms)):
iexpt_id = uuids[iexpt]
this_coset = all_coset[ all_expt_id.index(iexpt_id) ]
if this_coset == icoset:
good_refls |= refl["exp_id"] == iexpt_id
re_millers = mi_new.select(good_refls)
re_asu = mi_asu_new.select(good_refls)
refl["miller_index"].set_selected(good_refls, re_millers)
refl["miller_index_asymmetric"].set_selected(good_refls, re_asu)
if verbose:
for x in range(len(refl)):
print ("%3d"%x,refl["exp_id"][x],
all_coset[ all_expt_id.index(refl["exp_id"][x]) ],
"%10s"%(change_of_basis_op(co.partitions[ all_coset[ all_expt_id.index(refl["exp_id"][x]) ] ][0]).as_hkl()),
"(%4d%4d%4d)"%(cache_miller[x]), "(%4d%4d%4d)"%(cache_asu[x]),
"(%4d%4d%4d)"%(refl["miller_index"][x]), "(%4d%4d%4d)"%(refl["miller_index_asymmetric"][x]))
return refl
def chains_and_atoms(pdb_hierarchy, secondary_structure_selection, out=None):
if (out is None):
out = sys.stdout
new_secondary_structure_selection = flex.bool()
get_class = iotbx.pdb.common_residue_names_get_class
chains_and_residue_selections = []
for model in pdb_hierarchy.models():
for chain in model.chains():
result = []
for rg in chain.residue_groups():
result_ = flex.size_t()
is_secondary_structure = False
for ag in rg.atom_groups():
#print >> out, ag.resname, get_class(name=ag.resname)
if (get_class(name=ag.resname) == "common_amino_acid"
or get_class(name=ag.resname) == "common_rna_dna"):
for atom in ag.atoms():
result_.append(atom.i_seq)
if (not is_secondary_structure):
is_secondary_structure = \
secondary_structure_selection[atom.i_seq]
new_secondary_structure_selection.append(
secondary_structure_selection[atom.i_seq])
if (result_.size() > 0):
result.append([
result_, is_secondary_structure,
rg.resid(),
rg.unique_resnames()
])
if (len(result) > 0):
chains_and_residue_selections.append([chain.id, result])
print("Considering these chains:", file=out)
for ch in chains_and_residue_selections:
print(" chain '%s' (number of residues selected: %d)" %
(ch[0], len(ch[1])),
file=out)
return chains_and_residue_selections, new_secondary_structure_selection
def _expand(self):
expansion = expand(pdb_hierarchy=self.pdb_hierarchy,
crystal_symmetry=self.crystal_symmetry,
select_within_radius=10.0)
pdb_hierarchy_super = expansion.ph_super_sphere
pdb_hierarchy_super.write_pdb_file(file_name="supersphere.pdb",
crystal_symmetry=expansion.cs_box)
self.crystal_symmetry_ss = expansion.cs_box
if (self.restraints_source.source_of_restraints_qm()):
self.pdb_hierarchy_super_completed = model_completion.run(
#pdb_hierarchy = pdb_hierarchy_super,
crystal_symmetry=expansion.cs_box,
model_completion=False,
pdb_filename="supersphere.pdb",
original_pdb_filename=None)
else:
self.pdb_hierarchy_super_completed = pdb_hierarchy_super
selection = flex.bool(
self.pdb_hierarchy_super_completed.atoms().size(), False)
self.selection = selection.set_selected(
flex.size_t(xrange(self.pdb_hierarchy.atoms().size())), True)
self.restraints_manager = self.restraints_source.update(
pdb_hierarchy=self.pdb_hierarchy_super_completed,
crystal_symmetry=expansion.cs_box)
def tst_with_noisy_flat_background_partial(self):
from dials.algorithms.integration.fit import fit_profile
from scitbx.array_family import flex
from tst_profile_helpers import gaussian
# Create profile
p = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2))
s = flex.sum(p)
p = p / s
# Copy profile
c0 = gaussian((9, 9, 9), 1, (4, 4, 4), (2, 2, 2))
n = flex.random_double(9 * 9 * 9)
b = flex.double(flex.grid(9, 9, 9), 5) + n
m = flex.bool(flex.grid(9,9,9), True)
c = c0 + b
# Get the partial profiles
pp = p[0:5,:,:]
mp = m[0:5,:,:]
cp = c[0:5,:,:]
bp = b[0:5,:,:]
c0p = c0[0:5,:,:]
# Fit
fit = fit_profile(pp, mp, cp, bp)
I = fit.intensity()
V = fit.variance()
# Test intensity is the same
eps = 1e-7
assert(abs(I - flex.sum(c0)) < eps)
assert(abs(V - (flex.sum(c0p) + flex.sum(bp))) < eps)
print 'OK'
def test_pickle():
from dials.model.data import PixelList
from scitbx.array_family import flex
size = (100, 100)
sf = 10
image = flex.double(flex.grid(size))
mask = flex.bool(flex.grid(size))
for i in range(len(image)):
image[i] = random.randint(0, 100)
mask[i] = bool(random.randint(0, 1))
pl = PixelList(sf, image, mask)
assert pl.size() == size
assert pl.frame() == sf
import six.moves.cPickle as pickle
obj = pickle.dumps(pl)
pl2 = pickle.loads(obj)
assert pl2.size() == size
assert pl2.frame() == sf
assert len(pl2) == len(pl)
assert pl2.index().all_eq(pl.index())
assert pl2.value().all_eq(pl.value())
def run (args, image = None):
from xfel import radial_average
from scitbx.array_family import flex
import os, sys
import dxtbx
# Parse input
try:
n = len(args)
except Exception:
params = args
else:
user_phil = []
for arg in args:
if (not "=" in arg):
try :
user_phil.append(libtbx.phil.parse("""file_path=%s""" % arg))
except ValueError:
raise Sorry("Unrecognized argument '%s'" % arg)
else:
try:
user_phil.append(libtbx.phil.parse(arg))
except RuntimeError as e:
raise Sorry("Unrecognized argument '%s' (error: %s)" % (arg, str(e)))
params = master_phil.fetch(sources=user_phil).extract()
if image is None:
if params.file_path is None or len(params.file_path) == 0 or not all([os.path.isfile(f) for f in params.file_path]):
master_phil.show()
raise Usage("file_path must be defined (either file_path=XXX, or the path alone).")
assert params.n_bins is not None
assert params.verbose is not None
assert params.output_bins is not None
# Allow writing to a file instead of stdout
if params.output_file is None:
logger = sys.stdout
else:
logger = open(params.output_file, 'w')
logger.write("%s "%params.output_file)
if params.show_plots:
from matplotlib import pyplot as plt
import numpy as np
colormap = plt.cm.gist_ncar
plt.gca().set_color_cycle([colormap(i) for i in np.linspace(0, 0.9, len(params.file_path))])
if params.mask is not None:
params.mask = easy_pickle.load(params.mask)
if image is None:
iterable = params.file_path
load_func = lambda x: dxtbx.load(x)
else:
iterable = [image]
load_func = lambda x: x
# Iterate over each file provided
for item in iterable:
img = load_func(item)
try:
n_images = img.get_num_images()
subiterable = xrange(n_images)
except AttributeError:
n_images = None
subiterable = [0]
for image_number in subiterable:
if n_images is None:
beam = img.get_beam()
detector = img.get_detector()
else:
beam = img.get_beam(image_number)
detector = img.get_detector(image_number)
s0 = col(beam.get_s0())
# Search the detector for the panel farthest from the beam. The number of bins in the radial average will be
# equal to the farthest point from the beam on the detector, in pixels, unless overridden at the command line
panel_res = [p.get_max_resolution_at_corners(s0) for p in detector]
farthest_panel = detector[panel_res.index(min(panel_res))]
size2, size1 = farthest_panel.get_image_size()
corners = [(0,0), (size1-1,0), (0,size2-1), (size1-1,size2-1)]
corners_lab = [col(farthest_panel.get_pixel_lab_coord(c)) for c in corners]
corner_two_thetas = [farthest_panel.get_two_theta_at_pixel(s0, c) for c in corners]
extent_two_theta = max(corner_two_thetas)
max_corner = corners_lab[corner_two_thetas.index(extent_two_theta)]
extent = int(math.ceil(max_corner.length()*math.sin(extent_two_theta)/max(farthest_panel.get_pixel_size())))
extent_two_theta *= 180/math.pi
if params.n_bins < extent:
params.n_bins = extent
# These arrays will store the radial average info
sums = flex.double(params.n_bins) * 0
sums_sq = flex.double(params.n_bins) * 0
counts = flex.int(params.n_bins) * 0
if n_images is None:
all_data = img.get_raw_data()
else:
all_data = img.get_raw_data(image_number)
if not isinstance(all_data, tuple):
all_data = (all_data,)
for tile, (panel, data) in enumerate(zip(detector, all_data)):
if params.mask is None:
mask = flex.bool(flex.grid(data.focus()), True)
else:
mask = params.mask[tile]
if hasattr(data,"as_double"):
data = data.as_double()
logger.flush()
if params.verbose:
logger.write("Average intensity tile %d: %9.3f\n"%(tile, flex.mean(data)))
logger.write("N bins: %d\n"%params.n_bins)
logger.flush()
x1,y1,x2,y2 = 0,0,panel.get_image_size()[1],panel.get_image_size()[0]
bc = panel.get_beam_centre_px(beam.get_s0())
bc = int(round(bc[1])), int(round(bc[0]))
# compute the average
radial_average(data,mask,bc,sums,sums_sq,counts,panel.get_pixel_size()[0],panel.get_distance(),
(x1,y1),(x2,y2))
# average the results, avoiding division by zero
results = sums.set_selected(counts <= 0, 0)
results /= counts.set_selected(counts <= 0, 1).as_double()
if params.median_filter_size is not None:
logger.write("WARNING, the median filter is not fully propogated to the variances\n")
from scipy.ndimage.filters import median_filter
results = flex.double(median_filter(results.as_numpy_array(), size = params.median_filter_size))
# calculate standard devations
stddev_sel = ((sums_sq-sums*results) >= 0) & (counts > 0)
std_devs = flex.double(len(sums), 0)
std_devs.set_selected(stddev_sel,
(sums_sq.select(stddev_sel)-sums.select(stddev_sel)* \
results.select(stddev_sel))/counts.select(stddev_sel).as_double())
std_devs = flex.sqrt(std_devs)
twotheta = flex.double(xrange(len(results)))*extent_two_theta/params.n_bins
q_vals = 4*math.pi*flex.sin(math.pi*twotheta/360)/beam.get_wavelength()
if params.low_max_two_theta_limit is None:
subset = results
else:
subset = results.select(twotheta >= params.low_max_two_theta_limit)
max_result = flex.max(subset)
if params.x_axis == 'two_theta':
xvals = twotheta
max_x = twotheta[flex.first_index(results, max_result)]
elif params.x_axis == 'q':
xvals = q_vals
max_x = q_vals[flex.first_index(results, max_result)]
for i in xrange(len(results)):
val = xvals[i]
if params.output_bins and "%.3f"%results[i] != "nan":
#logger.write("%9.3f %9.3f\n"% (val,results[i])) #.xy format for Rex.cell.
logger.write("%9.3f %9.3f %9.3f\n"%(val,results[i],std_devs[i])) #.xye format for GSASII
#logger.write("%.3f %.3f %.3f\n"%(val,results[i],ds[i])) # include calculated d spacings
logger.write("Maximum %s: %f, value: %f\n"%(params.x_axis, max_x, max_result))
if params.show_plots:
if params.plot_x_max is not None:
results = results.select(xvals <= params.plot_x_max)
xvals = xvals.select(xvals <= params.plot_x_max)
if params.normalize:
plt.plot(xvals.as_numpy_array(),(results/flex.max(results)).as_numpy_array(),'-')
else:
plt.plot(xvals.as_numpy_array(),results.as_numpy_array(),'-')
if params.x_axis == 'two_theta':
plt.xlabel("2 theta")
elif params.x_axis == 'q':
plt.xlabel("q")
plt.ylabel("Avg ADUs")
if params.plot_y_max is not None:
plt.ylim(0, params.plot_y_max)
if params.show_plots:
#plt.legend([os.path.basename(os.path.splitext(f)[0]) for f in params.file_path], ncol=2)
plt.show()
return xvals, results
def calculate_scaling(self,
miller_array,
convergence_crit_perc=0.01,
convergence_reject_perc=97.5,
max_iter=20):
"""Calculate the scaling between two arrays"""
assert convergence_reject_perc > 90.0
# Convert to intensities and extract d_star_sq
new_miller = miller_array.as_intensity_array()
new_kernel = self._kernel_normalisation(miller_array=new_miller)
# Calculate new range of d_star_sq
d_star_sq_min, d_star_sq_max = self._common_d_star_sq_range(
d_star_sq=new_kernel.d_star_sq_array)
# Create interpolator for the two arrays (new and reference)
interpolator = scale_curves.curve_interpolator(d_star_sq_min,
d_star_sq_max,
self._npoints)
# Interpolate the two curves (use full range of the two array)
new_itpl_d_star_sq, new_itpl_mean_I, dummy, dummy = interpolator.interpolate(
x_array=new_kernel.d_star_sq_array,
y_array=new_kernel.mean_I_array)
ref_itpl_d_star_sq, ref_itpl_mean_I, dummy, dummy = interpolator.interpolate(
x_array=self.ref_kernel.d_star_sq_array,
y_array=self.ref_kernel.mean_I_array)
# Initalise convergence loop - begin by scaling over all points
selection = flex.bool(self._npoints, True)
# Set initial scale factor to small value
curr_b = 1e-6
# Percent change between iterations - convergence when delta <convergence_criterion
n_iter = 0
# Report in case of error
report = Report('Scaling log:', verbose=False)
while n_iter < max_iter:
report('---')
report('ITER: ' + str(n_iter))
if selection.all_eq(False):
print("Selection now empty, breaking")
break
# Run optimisation on the linear scaling
lsc = ExponentialScaling(x_values=interpolator.target_x,
ref_values=ref_itpl_mean_I,
scl_values=new_itpl_mean_I,
weights=selection.as_double())
# Calculate scaling B-factor
lsc.scaling_b_factor = -0.5 * list(lsc.optimised_values)[0]
# Break if fitted to 0
if approx_equal_relatively(0.0, lsc.scaling_b_factor, 1e-6):
report('Scaling is approximately 0.0 - stopping')
break
# Calculate percentage change
report('Curr/New: ' + str(curr_b) + '\t' +
str(lsc.scaling_b_factor))
delta = abs((curr_b - lsc.scaling_b_factor) / curr_b)
report('Delta: ' + str(delta))
if delta < convergence_crit_perc:
report('Scaling has converged to within tolerance - stopping')
break
# Update selection
report('Curr Selection Size: ' + str(sum(selection)))
ref_diffs = flex.log(lsc.ref_values) - flex.log(lsc.out_values)
#abs_diffs = flex.abs(ref_diffs)
sel_diffs = ref_diffs.select(selection)
rej_val_high = numpy.percentile(sel_diffs, convergence_reject_perc)
rej_val_low = numpy.percentile(sel_diffs,
100.0 - convergence_reject_perc)
report('Percentile: ' + str(convergence_reject_perc) + '\t<' +
str(rej_val_low) + '\t>' + str(rej_val_high))
selection.set_selected(ref_diffs > rej_val_high, False)
selection.set_selected(ref_diffs < rej_val_low, False)
report('New Selection Size: ' + str(sum(selection)))
# Update loop params
curr_b = lsc.scaling_b_factor
n_iter += 1
lsc.unscaled_ln_rmsd = (flex.log(lsc.ref_values) - flex.log(
lsc.scl_values)).norm() / (lsc.ref_values.size()**0.5)
lsc.scaled_ln_rmsd = (flex.log(lsc.ref_values) - flex.log(
lsc.out_values)).norm() / (lsc.ref_values.size()**0.5)
lsc.unscaled_ln_dev = flex.sum(
flex.abs(flex.log(lsc.ref_values) - flex.log(lsc.scl_values)))
lsc.scaled_ln_dev = flex.sum(
flex.abs(flex.log(lsc.ref_values) - flex.log(lsc.out_values)))
return lsc
