def run(self):
from dials.algorithms.image.fill_holes import simple_fill
from scitbx.array_family import flex
from random import randint
from math import sqrt
import sys
mask = flex.bool(flex.grid(100, 100), True)
data = flex.double(flex.grid(100, 100), True)
for j in range(100):
for i in range(100):
data[j,i] = 10 + j * 0.01 + i * 0.01
if sqrt((j - 50)**2 + (i - 50)**2) <= 10.5:
mask[j,i] = False
data[j,i] = 0
result = simple_fill(data, mask)
known = data.as_1d().select(mask.as_1d())
filled = result.as_1d().select(mask.as_1d() == False)
assert flex.max(filled) <= flex.max(known)
assert flex.min(filled) >= flex.min(known)
# Test passed
print 'OK'
def __init__(self, xray_structure, k_anisotropic, k_masks, ss):
self.xray_structure = xray_structure
self.k_anisotropic = k_anisotropic
self.k_masks = k_masks
self.ss = ss
#
k_total = self.k_anisotropic
r = scitbx.math.gaussian_fit_1d_analytical(x=flex.sqrt(self.ss), y=k_total)
k,b = r.a, r.b
#
k,b,r = mmtbx.bulk_solvent.fit_k_exp_b_to_k_total(k_total, self.ss, k, b)
k_exp_overall, b_exp_overall = None,None
if(r<0.7): k_exp_overall, b_exp_overall = k,b
if(self.xray_structure is None): return None
b_adj = 0
if([k_exp_overall, b_exp_overall].count(None)==0 and k != 0):
bs1 = self.xray_structure.extract_u_iso_or_u_equiv()*adptbx.u_as_b(1.)
def split(b_trace, xray_structure):
b_min = xray_structure.min_u_cart_eigenvalue()*adptbx.u_as_b(1.)
b_res = min(0, b_min + b_trace+1.e-6)
b_adj = b_trace-b_res
xray_structure.shift_us(b_shift = b_adj)
return b_adj, b_res
b_adj,b_res=split(b_trace=b_exp_overall,xray_structure=self.xray_structure)
k_new = k_exp_overall*flex.exp(-self.ss*b_adj)
bs2 = self.xray_structure.extract_u_iso_or_u_equiv()*adptbx.u_as_b(1.)
diff = bs2-bs1
assert approx_equal(flex.min(diff), flex.max(diff))
assert approx_equal(flex.max(diff), b_adj)
self.k_anisotropic = self.k_anisotropic/k_new
self.k_masks = [m*flex.exp(-self.ss*b_adj) for m in self.k_masks]
def tst_curve_interpolator():
x = flex.double( range(25) )/24.0
y = x*x
ip = curve_interpolator(0,2.0,200)
x_target = ip.target_x
y_ref = x_target*x_target
nx,ny,a,b = ip.interpolate(x,y)
count = 0
for xx in x_target:
if flex.max(x) >= xx:
count += 1
assert count==len(nx)
for yy,yyy in zip(ny,y_ref):
assert approx_equal(yy,yyy,eps=1e-3)
assert a[0]==0
assert a[1]==24
assert b[0]==0
assert b[1] in (99,100)
x = flex.double( range(5,23) )/24.0
y = x*x
ip = curve_interpolator(0,2.0,200)
nx,ny,a,b = ip.interpolate(x,y)
assert nx[0] >= flex.min(x)
assert nx[-1] <= flex.max(x)
y_ref= nx*nx
for yy,yyy in zip(ny,y_ref):
assert approx_equal(yy,yyy,eps=1e-3)
def apply_back_trace_of_overall_exp_scale_matrix(self, xray_structure=None):
k,b=self.overall_isotropic_kb_estimate()
k_total = self.core.k_isotropic * self.core.k_anisotropic * \
self.core.k_isotropic_exp
k,b,r = mmtbx.bulk_solvent.fit_k_exp_b_to_k_total(k_total, self.ss, k, b)
if(r<0.7): self.k_exp_overall,self.b_exp_overall = k,b
if(xray_structure is None): return None
b_adj = 0
if([self.k_exp_overall,self.b_exp_overall].count(None)==0 and k != 0):
bs1 = xray_structure.extract_u_iso_or_u_equiv()*adptbx.u_as_b(1.)
def split(b_trace, xray_structure):
b_min = xray_structure.min_u_cart_eigenvalue()*adptbx.u_as_b(1.)
b_res = min(0, b_min + b_trace+1.e-6)
b_adj = b_trace-b_res
xray_structure.shift_us(b_shift = b_adj)
return b_adj, b_res
b_adj,b_res=split(b_trace=self.b_exp_overall,xray_structure=xray_structure)
k_new = self.k_exp_overall*flex.exp(-self.ss*b_adj)
bs2 = xray_structure.extract_u_iso_or_u_equiv()*adptbx.u_as_b(1.)
diff = bs2-bs1
assert approx_equal(flex.min(diff), flex.max(diff))
assert approx_equal(flex.max(diff), b_adj)
self.core = self.core.update(
k_isotropic = self.core.k_isotropic,
k_isotropic_exp = self.core.k_isotropic_exp/k_new,
k_masks = [m*flex.exp(-self.ss*b_adj) for m in self.core.k_masks])
return group_args(
xray_structure = xray_structure,
k_isotropic = self.k_isotropic(),
k_anisotropic = self.k_anisotropic(),
k_mask = self.k_masks(),
b_adj = b_adj)
def blank_integrated_analysis(reflections, scan, phi_step, fractional_loss):
prf_sel = reflections.get_flags(reflections.flags.integrated_prf)
if prf_sel.count(True) > 0:
reflections = reflections.select(prf_sel)
intensities = reflections["intensity.prf.value"]
variances = reflections["intensity.prf.variance"]
else:
sum_sel = reflections.get_flags(reflections.flags.integrated_sum)
reflections = reflections.select(sum_sel)
intensities = reflections["intensity.sum.value"]
variances = reflections["intensity.sum.variance"]
i_sigi = intensities / flex.sqrt(variances)
xyz_px = reflections["xyzobs.px.value"]
x_px, y_px, z_px = xyz_px.parts()
phi = scan.get_angle_from_array_index(z_px)
osc = scan.get_oscillation()[1]
n_images_per_step = iceil(phi_step / osc)
phi_step = n_images_per_step * osc
phi_min = flex.min(phi)
phi_max = flex.max(phi)
n_steps = iceil((phi_max - phi_min) / phi_step)
hist = flex.histogram(z_px, n_slots=n_steps)
mean_i_sigi = flex.double()
for i, slot_info in enumerate(hist.slot_infos()):
sel = (z_px >= slot_info.low_cutoff) & (z_px < slot_info.high_cutoff)
if sel.count(True) == 0:
mean_i_sigi.append(0)
else:
mean_i_sigi.append(flex.mean(i_sigi.select(sel)))
fractional_mean_i_sigi = mean_i_sigi / flex.max(mean_i_sigi)
potential_blank_sel = mean_i_sigi <= (fractional_loss * flex.max(mean_i_sigi))
xmin, xmax = zip(*[(slot_info.low_cutoff, slot_info.high_cutoff) for slot_info in hist.slot_infos()])
d = {
"data": [
{
"x": list(hist.slot_centers()),
"y": list(mean_i_sigi),
"xlow": xmin,
"xhigh": xmax,
"blank": list(potential_blank_sel),
"type": "bar",
"name": "blank_counts_analysis",
}
],
"layout": {"xaxis": {"title": "z observed (images)"}, "yaxis": {"title": "Number of reflections"}, "bargap": 0},
}
blank_regions = blank_regions_from_sel(d["data"][0])
d["blank_regions"] = blank_regions
return d
def exercise_reference_impl_long(n_dynamics_steps, out):
sim = fmri.simulation()
e_tots = flex.double([sim.e_tot])
print >> out, "i_step, [e_pot, e_kin_ang, e_kin_lin, e_kin, e_tot]"
def show(i_step):
print >> out, i_step, [sim.e_pot, sim.e_kin_ang, sim.e_kin_lin, sim.e_kin, sim.e_tot]
out.flush()
n_show = max(1, n_dynamics_steps // 10)
for i_step in xrange(n_dynamics_steps):
sim.dynamics_step(delta_t=0.001)
e_tots.append(sim.e_tot)
if i_step % n_show == 0:
show(i_step)
show(n_dynamics_steps)
print >> out
print >> out, "number of dynamics steps:", n_dynamics_steps
print >> out, "e_tot start:", e_tots[0]
print >> out, " final:", e_tots[-1]
print >> out, " min:", flex.min(e_tots)
print >> out, " max:", flex.max(e_tots)
print >> out, " max-min:", flex.max(e_tots) - flex.min(e_tots)
print >> out
out.flush()
def exercise():
"""
Exercise refine "easy" with DNA/RNA.
"""
pi_good = get_pdb_inputs(pdb_str=pdb_str_answer, restraints=False)
map_data = get_map(xrs=pi_good.xrs)
xrs_good = pi_good.xrs.deep_copy_scatterers()
pi_good.ph.write_pdb_file(file_name="answer.pdb",
crystal_symmetry=xrs_good.crystal_symmetry())
#
pi_poor = get_pdb_inputs(pdb_str=pdb_str_poor, restraints=True)
pi_poor.ph.write_pdb_file(file_name="poor.pdb")
xrs_poor = pi_poor.xrs.deep_copy_scatterers()
#
d = xrs_good.distances(other=xrs_poor)
print d.min_max_mean().as_tuple()
assert flex.max(d)>2
assert flex.mean(d)>0.7
#
xrs_refined = xrs_poor
for i in xrange(3):
ero = individual_sites.easy(
map_data = map_data,
xray_structure = xrs_refined,
pdb_hierarchy = pi_poor.ph,
geometry_restraints_manager = pi_poor.grm)
xrs_refined = ero.xray_structure
# comapre
d = xrs_good.distances(other=xrs_refined)
print d.min_max_mean().as_tuple()
assert flex.max(d)<0.15
assert flex.mean(d)<0.03
ero.pdb_hierarchy.write_pdb_file(file_name="refined.pdb",
crystal_symmetry=xrs_good.crystal_symmetry())
def exercise_simulation(
out, n_trials, n_dynamics_steps, delta_t=0.0001, random_seed=0):
mersenne_twister = flex.mersenne_twister(seed=random_seed)
sim_labels = None
relative_ranges_accu = None
rms_max_list_accu = None
for i_trial in xrange(n_trials):
sim_labels_new, e_tots_list, \
relative_ranges, rms_max_list = run_simulations(
out=out,
mersenne_twister=mersenne_twister,
n_dynamics_steps=n_dynamics_steps,
delta_t=delta_t)
if (sim_labels is None):
sim_labels = sim_labels_new
else:
assert sim_labels == sim_labels_new
if (relative_ranges_accu is None):
relative_ranges_accu=[flex.double() for i in xrange(len(relative_ranges))]
else:
assert len(relative_ranges) == len(relative_ranges_accu)
for r,a in zip(relative_ranges, relative_ranges_accu):
a.append(r)
if (rms_max_list_accu is None):
rms_max_list_accu = [flex.double() for i in xrange(len(rms_max_list))]
else:
assert len(rms_max_list) == len(rms_max_list_accu)
for r,a in zip(rms_max_list, rms_max_list_accu):
a.append(r)
if (out is sys.stdout):
f = open("tmp_e_tots_%02d_%02d.xy" % (plot_prefix, i_trial), "w")
print >> f, "@with g0"
for i,l in enumerate(sim_labels):
l = l[l.find('"')+1:].replace('"','')[:-1]
print >> f, '@ s%d legend "%s"' % (i, l)
for es in e_tots_list:
for e in es:
print >> f, e
print >> f, "&"
f.close()
print >> out, "Accumulated results:"
print >> out
for sim_label,accu in zip(sim_labels, relative_ranges_accu):
print >> out, "relative ranges %s:" % sim_label
accu.min_max_mean().show(out=out, prefix=" ")
print >> out
for sim_label,accu in zip(sim_labels[1:], rms_max_list_accu):
print >> out, "rms max %s" % sim_labels[0]
print >> out, " vs. %s:" % sim_label
accu.min_max_mean().show(out=out, prefix=" ")
print >> out
if (out is not sys.stdout):
for accu in relative_ranges_accu:
assert flex.max(accu) < 1.e-4
for i,accu in enumerate(rms_max_list_accu):
assert flex.max(accu) < 1.e-4
def __init__(self, rs_vectors, percentile=0.05):
from scitbx.array_family import flex
NEAR = 10
self.NNBIN = 5 # target number of neighbors per histogram bin
# nearest neighbor analysis
from annlib_ext import AnnAdaptor
query = flex.double()
for spot in rs_vectors: # spots, in reciprocal space xyz
query.append(spot[0])
query.append(spot[1])
query.append(spot[2])
assert len(rs_vectors)>NEAR # Can't do nearest neighbor with too few spots
IS_adapt = AnnAdaptor(data=query,dim=3,k=1)
IS_adapt.query(query)
direct = flex.double()
for i in xrange(len(rs_vectors)):
direct.append(1.0/math.sqrt(IS_adapt.distances[i]))
# determine the most probable nearest neighbor distance (direct space)
hst = flex.histogram(direct, n_slots=int(len(rs_vectors)/self.NNBIN))
centers = hst.slot_centers()
islot = hst.slots()
highest_bin_height = flex.max(islot)
most_probable_neighbor = centers[list(islot).index(highest_bin_height)]
if False: # to print out the histogramming analysis
smin, smax = flex.min(direct), flex.max(direct)
stats = flex.mean_and_variance(direct)
import sys
out = sys.stdout
print >> out, " range: %6.2f - %.2f" % (smin, smax)
print >> out, " mean: %6.2f +/- %6.2f on N = %d" % (
stats.mean(), stats.unweighted_sample_standard_deviation(), direct.size())
hst.show(f=out, prefix=" ", format_cutoffs="%6.2f")
print >> out, ""
# determine the 5th-percentile direct-space distance
perm = flex.sort_permutation(direct, reverse=True)
percentile = direct[perm[int(percentile * len(rs_vectors))]]
MAXTOL = 1.5 # Margin of error for max unit cell estimate
self.max_cell = max( MAXTOL * most_probable_neighbor,
MAXTOL * percentile)
if False:
self.plot(direct)
def run(args):
import libtbx.load_env
from dials.array_family import flex
from dials.util import log
from dials.util.version import dials_version
usage = "%s [options] experiment.json indexed.pickle" % \
libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
# Configure the logging
log.config(info=params.output.log, debug=params.output.debug_log)
logger.info(dials_version())
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reflections) == 0 or len(experiments) == 0:
parser.print_help()
return
assert(len(reflections) == 1)
assert(len(experiments) == 1)
experiment = experiments[0]
reflections = reflections[0]
# remove reflections with 0, 0, 0 index
zero = (reflections['miller_index'] == (0, 0, 0))
logger.info('Removing %d unindexed reflections' % zero.count(True))
reflections = reflections.select(~zero)
h, k, l = reflections['miller_index'].as_vec3_double().parts()
h = h.iround()
k = k.iround()
l = l.iround()
logger.info('Range on h: %d to %d' % (flex.min(h), flex.max(h)))
logger.info('Range on k: %d to %d' % (flex.min(k), flex.max(k)))
logger.info('Range on l: %d to %d' % (flex.min(l), flex.max(l)))
test_P1_crystal_indexing(reflections, experiment, params)
test_crystal_pointgroup_symmetry(reflections, experiment, params)
def from_cctbx_altlocs(ph, cs, method="subtract"):
assert method in ["subtract", "average"]
g_result = flex.vec3_double(ph.atoms().size(), [0,0,0])
conf_ind = ph.get_conformer_indices()
n_altlocs = flex.max(conf_ind)
sel_W = conf_ind == 0
g_blanks = flex.vec3_double(sel_W.count(True))
for ci in range(1, n_altlocs+1):
sel_ci = conf_ind == ci
sel_ci_blank = (conf_ind == ci) | sel_W
ph_ci_blank = ph.select(sel_ci_blank)
g_ci_blank_ = get_cctbx_gradients(ph=ph_ci_blank, cs=cs).gradients
g_ci = g_ci_blank_.select(ph_ci_blank.get_conformer_indices() == 1)
g_result = g_result.set_selected(sel_ci, g_ci)
g_blanks += g_ci_blank_.select( ph_ci_blank.get_conformer_indices()==0 )
if(method=="subtract"):
# Option 1
# g_blank = get_cctbx_gradients(ph=ph.select(sel_W), cs=cs).gradients
# g_result_1 = g_result.set_selected(sel_W, g_blanks-((n_altlocs-1)*g_blank))
# Option 2
g_blank = get_cctbx_gradients(ph=ph, cs=cs).gradients.select(sel_W)
g_result_2 = g_result.set_selected(sel_W, g_blank)
#
# Options 1 and 2 are identical. Disabled for performance and because it
# expectedly crashes when altloc is ' '.
# assert approx_equal(g_result_1, g_result_2)
elif(method=="average"):
g_result_2 = g_result.set_selected(sel_W, g_blanks*(1/n_altlocs))
else: assert 0
return g_result_2
def another_example(np=41,nt=5):
x = flex.double( range(np) )/(np-1)
y = 0.99*flex.exp(-x*x*0.5)
y = -flex.log(1.0/y-1)
w = y*y/1.0
d = (flex.random_double(np)-0.5)*w
y_obs = y+d
y = 1.0/( 1.0 + flex.exp(-y) )
fit_w = chebyshev_lsq_fit.chebyshev_lsq_fit(nt,
x,
y_obs,
w )
fit_w_f = chebyshev_polynome(
nt, fit_w.low_limit, fit_w.high_limit, fit_w.coefs)
fit_nw = chebyshev_lsq_fit.chebyshev_lsq_fit(nt,
x,
y_obs)
fit_nw_f = chebyshev_polynome(
nt, fit_nw.low_limit, fit_nw.high_limit, fit_nw.coefs)
print
print "Coefficients from weighted lsq"
print list( fit_w.coefs )
print "Coefficients from non-weighted lsq"
print list( fit_nw.coefs )
assert flex.max( flex.abs(fit_nw.coefs-fit_w.coefs) ) > 0
def single_peak_fit(self, hist, lower_threshold, upper_threshold, mean,
zero_peak_gaussian=None):
lower_slot = 0
for slot in hist.slot_centers():
lower_slot += 1
if slot > lower_threshold: break
upper_slot = 0
for slot in hist.slot_centers():
upper_slot += 1
if slot > upper_threshold: break
x = hist.slot_centers()
y = hist.slots().as_double()
starting_gaussians = [curve_fitting.gaussian(
a=flex.max(y[lower_slot:upper_slot]), b=mean, c=3)]
# print starting_gaussians
#mamin: fit gaussian will take the maximum between starting point (lower_slot) and ending (upper_slot) as a
if zero_peak_gaussian is not None:
y -= zero_peak_gaussian(x)
if 1:
fit = curve_fitting.lbfgs_minimiser(
starting_gaussians, x[lower_slot:upper_slot], y[lower_slot:upper_slot])
sigma = abs(fit.functions[0].params[2])
if sigma < 1 or sigma > 10:
if flex.sum(y[lower_slot:upper_slot]) < 15: #mamin I changed 15 to 5
# No point wasting time attempting to fit a gaussian if there aren't any counts
#raise PixelFitError("Not enough counts to fit gaussian")
return fit
print "using cma_es:", sigma
fit = curve_fitting.cma_es_minimiser(
starting_gaussians, x[lower_slot:upper_slot], y[lower_slot:upper_slot])
else:
fit = curve_fitting.cma_es_minimiser(
starting_gaussians, x[lower_slot:upper_slot], y[lower_slot:upper_slot])
return fit
def outer_loop(self):
missing_reflections_manager = mmtbx.map_tools.model_missing_reflections(
coeffs=self.mc, fmodel=self.fmodel)
missing = missing_reflections_manager.get_missing(deterministic=True)
wam = kick.weighted_average(fmodel=self.fmodel, map_coefficients=self.mc)
progress_counter = counter(
n1=self.n_inner_loop, n2=self.n_outer_loop, log=self.log)
map_accumulator = maptbx.map_accumulator(
n_real = self.crystal_gridding.n_real(), smearing_b=1, max_peak_scale=100,
smearing_span=5, use_max_map=self.use_max_map)
for i in xrange(self.n_outer_loop):
m = inner_loop(
fmodel = self.fmodel,
wam = wam,
missing = missing,
crystal_gridding = self.crystal_gridding,
n = self.n_inner_loop,
progress_counter = progress_counter,
use_max_map = self.use_max_map)
m = low_volume_density_elimination(m=m, fmodel=self.fmodel,
selection=self.selection)
if(self.sharp and self.use_unsharp_masking):
maptbx.sharpen(map_data=m, index_span=1, n_averages=2,
allow_negatives=False)
maptbx.gamma_compression(map_data=m, gamma=0.1)
self.zero_below_threshold(m = m)
m = m/flex.max(m)
map_accumulator.add(map_data=m)
m = map_accumulator.as_median_map()
sd = m.sample_standard_deviation()
print >> self.log
return m/sd
def do(self):
self.mu_history = flex.double()
self.n_iterations = 0
nu = 2
self.non_linear_ls.build_up()
if self.has_gradient_converged_to_zero(): return
a = self.non_linear_ls.normal_matrix_packed_u()
self.mu = self.tau*flex.max(a.matrix_packed_u_diagonal())
while self.n_iterations < self.n_max_iterations:
a.matrix_packed_u_diagonal_add_in_place(self.mu)
objective = self.non_linear_ls.objective()
g = -self.non_linear_ls.opposite_of_gradient()
self.non_linear_ls.solve()
if self.had_too_small_a_step(): break
self.n_iterations += 1
h = self.non_linear_ls.step()
expected_decrease = 0.5*h.dot(self.mu*h - g)
self.non_linear_ls.step_forward()
self.non_linear_ls.build_up(objective_only=True)
objective_new = self.non_linear_ls.objective()
actual_decrease = objective - objective_new
rho = actual_decrease/expected_decrease
if rho > 0:
if self.has_gradient_converged_to_zero(): break
self.mu *= max(1/3, 1 - (2*rho - 1)**3)
nu = 2
else:
self.non_linear_ls.step_backward()
self.mu *= nu
nu *= 2
self.non_linear_ls.build_up()
def get_summary(self):
"""
Returns a simple object for harvesting statistics elsewhere.
"""
n_anom_peaks = None
if (self.anom_peaks is not None):
n_anom_peaks = len(self.anom_peaks.heights)
n_water_peaks = n_water_anom_peaks = None
if (self.water_peaks is not None):
n_water_peaks = len(self.water_peaks)
if (self.water_anom_peaks is not None):
n_water_anom_peaks = len(self.water_anom_peaks)
hole_max = peak_max = None
if (len(self.peaks.heights) > 0):
peak_max = flex.max(self.peaks.heights)
if (len(self.holes.heights) > 0):
hole_max = flex.min(self.holes.heights)
n_non_water_anom_peaks = None
if (getattr(self, "non_water_anom_peaks", None) is not None):
n_non_water_anom_peaks = len(self.non_water_anom_peaks)
return summary(
n_peaks_1=(self.peaks.heights > self.map_cutoff).count(True),
n_peaks_2=(self.peaks.heights > self.map_cutoff + 3).count(True),
n_peaks_3=(self.peaks.heights > self.map_cutoff + 6).count(True),
n_holes_1=(self.holes.heights < -self.map_cutoff).count(True),
n_holes_2=(self.holes.heights < -self.map_cutoff - 3).count(True),
n_holes_3=(self.holes.heights < -self.map_cutoff - 6).count(True),
peak_max=peak_max,
hole_max=hole_max,
n_anom_peaks=n_anom_peaks,
n_water_peaks=n_water_peaks,
n_water_anom_peaks=n_water_anom_peaks,
map_cutoff=self.map_cutoff,
anom_map_cutoff=self.anom_map_cutoff,
n_non_water_anom_peaks=n_non_water_anom_peaks)
def __init__(self,rawdata,projection_vector,spotfinder_spot,verbose=False):
# projection vector is either the radial or azimuthal unit vector
# at a specific Bragg spot position
model_center = col((spotfinder_spot.ctr_mass_x(),spotfinder_spot.ctr_mass_y()))
px_x,px_y = project_2d_response_onto_line(projection_vector)
point_projections = flex.double()
pixel_values = flex.double()
for point in spotfinder_spot.bodypixels:
point_projection = (col((point.x,point.y)) - model_center).dot( projection_vector )
point_projections.append(point_projection)
pxval = rawdata[(point.x,point.y)]
if verbose:
print "point_projection",point_projection,
print "signal",pxval
pixel_values.append( pxval )
Lmin = flex.min(point_projections)
Lmax = flex.max(point_projections)
#print "Range %6.2f"%(Lmax-Lmin)
Rmin = round(Lmin-2.0,1)
Rmax = round(Lmax+2.0,1)
#print "Range %6.2f"%(Rmax-Rmin)
def histogram_bin (j) : return int(10.*(j-Rmin)) # bin units of 1/10 pixel
histo_x = flex.double((int(10*(Rmax-Rmin))))
histo_y = flex.double(len(histo_x))
for ihis in xrange(len(histo_x)): histo_x[ihis] = Rmin + 0.1*ihis
for ipp, point_projection in enumerate(point_projections):
value = pixel_values[ipp]
for isample in xrange(len(px_x)):
histo_y[int(10*(point_projection + px_x[isample] - Rmin))] += value * px_y[isample]
self.histo_x = histo_x
self.histo_y = histo_y
def output_image(flex_img, filename, invert=False, scale=False):
try:
import PIL.Image as Image
except ImportError:
import Image
flex_img = flex_img.deep_copy()
flex_img -= flex.min(flex_img)
if scale:
img_max_value = 2**16
scale = img_max_value / flex.max(flex_img)
flex_img = flex_img.as_double() * scale
flex_img = flex_img
if invert:
img_max_value = 2**16
flex_img = img_max_value - flex_img # invert image for display
dim = flex_img.all()
#easy_pickle.dump("%s/avg_img.pickle" %output_dirname, flex_img)
byte_str = flex_img.slice_to_byte_str(0, flex_img.size())
try:
im = Image.fromstring(mode="I", size=(dim[1], dim[0]), data=byte_str)
except NotImplementedError:
im = Image.frombytes(mode="I", size=(dim[1], dim[0]), data=byte_str)
im = im.crop((0, 185, 391, 370))
#im.save("avg.tiff", "TIFF") # XXX This does not work (phenix.python -Qnew option)
im.save(filename, "PNG")
def test_rs_mapper(dials_regression, tmpdir):
tmpdir.chdir()
result = procrunner.run_process([
'dials.rs_mapper',
os.path.join(dials_regression, "centroid_test_data", "datablock.json"),
'map_file="junk.ccp4"',
])
assert result['exitcode'] == 0
assert result['stderr'] == ''
assert os.path.exists('junk.ccp4')
# load results
from iotbx import ccp4_map
from scitbx.array_family import flex
m = ccp4_map.map_reader(file_name="junk.ccp4")
assert len(m.data) == 7189057
assert m.header_min == -1.0
assert flex.min(m.data) == -1.0
assert m.header_max == 2052.75
assert flex.max(m.data) == 2052.75
assert m.header_mean == pytest.approx(0.018606403842568398, abs=1e-6)
assert flex.mean(m.data) == pytest.approx(0.018606403842568398, abs=1e-6)
def get_q_array_uniform_body(data,
level=1.0e-2,
n_last=5,
q_min=0.0,
q_max=0.25):
# first we need to define a 'background level'
n_points = data.q.size()
#bkg = flex.mean(data.i[n_points-n_last:])
bkg = 0
i_tmp = data.i - bkg
location = n_points
maxi = flex.max(data.i) - bkg
for ii in range(n_points):
this_i = data.i[n_points - 1 - ii] - bkg
# print data.q[n_points-1 - ii],this_i,this_i/maxi,level
if this_i >= maxi * level:
location = ii
break
# now cut the data please
location = n_points - 1 - location
if q_max > data.q[location]:
q_max = data.q[location]
new_data = data.cut_data(q_min, q_max)
return new_data
def exercise_tardy_model(out, n_dynamics_steps, delta_t, tardy_model):
tardy_model.check_d_e_pot_d_q()
e_pots = flex.double([tardy_model.e_pot()])
e_kins = flex.double([tardy_model.e_kin()])
for i_step in xrange(n_dynamics_steps):
tardy_model.dynamics_step(delta_t=delta_t)
e_pots.append(tardy_model.e_pot())
e_kins.append(tardy_model.e_kin())
e_tots = e_pots + e_kins
tardy_model.check_d_e_pot_d_q()
print >> out, "degrees of freedom:", tardy_model.degrees_of_freedom
print >> out, "energy samples:", e_tots.size()
print >> out, "e_pot min, max:", min(e_pots), max(e_pots)
print >> out, "e_kin min, max:", min(e_kins), max(e_kins)
print >> out, "e_tot min, max:", min(e_tots), max(e_tots)
print >> out, "start e_tot:", e_tots[0]
print >> out, "final e_tot:", e_tots[-1]
ave = flex.sum(e_tots) / e_tots.size()
range = flex.max(e_tots) - flex.min(e_tots)
if (ave == 0): relative_range = 0
else: relative_range = range / ave
print >> out, "ave:", ave
print >> out, "range:", range
print >> out, "relative range:", relative_range
print >> out
out.flush()
return relative_range
def test1():
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression",
test=os.path.isdir)
data_dir = os.path.join(dials_regression, "centroid_test_data")
datablock_path = os.path.join(data_dir, "datablock.json")
# work in a temporary directory
cwd = os.path.abspath(os.curdir)
tmp_dir = open_tmp_directory(suffix="tst_rs_mapper")
os.chdir(tmp_dir)
cmd = 'dials.rs_mapper ' + datablock_path + ' map_file="junk.ccp4"'
result = easy_run.fully_buffered(command=cmd).raise_if_errors()
# load results
from iotbx import ccp4_map
from scitbx.array_family import flex
m = ccp4_map.map_reader(file_name="junk.ccp4")
assert len(m.data) == 7189057
assert approx_equal(m.header_min, -1.0)
assert approx_equal(flex.min(m.data), -1.0)
assert approx_equal(m.header_max, 2052.75)
assert approx_equal(flex.max(m.data), 2052.75)
assert approx_equal(m.header_mean, 0.018606403842568398)
assert approx_equal(flex.mean(m.data), 0.018606403842568398)
print "OK"
return
def do(self):
self.mu_history = flex.double()
self.n_iterations = 0
nu = 2
self.non_linear_ls.build_up()
if self.has_gradient_converged_to_zero(): return
a_diag = self.non_linear_ls.get_normal_matrix_diagonal()
self.mu = self.tau*flex.max(a_diag)
while self.n_iterations < self.n_max_iterations:
self.non_linear_ls.add_constant_to_diagonal(self.mu)
objective = self.non_linear_ls.objective()
print "%5d %18.4f"%(self.n_iterations,objective), "%12.7f"%(self.mu),
g = -self.non_linear_ls.opposite_of_gradient()
self.non_linear_ls.solve()
if self.had_too_small_a_step(): break
self.n_iterations += 1
h = self.non_linear_ls.step()
expected_decrease = 0.5*h.dot(self.mu*h - g)
self.non_linear_ls.step_forward()
self.non_linear_ls.build_up(objective_only=True)
objective_new = self.non_linear_ls.objective()
actual_decrease = objective - objective_new
rho = actual_decrease/expected_decrease
if self.objective_decrease_threshold is not None:
if actual_decrease/objective < self.objective_decrease_threshold: break
if rho > 0:
if self.has_gradient_converged_to_zero(): break
self.mu *= max(1/3, 1 - (2*rho - 1)**3)
nu = 2
else:
self.non_linear_ls.step_backward()
self.mu *= nu
nu *= 2
self.non_linear_ls.build_up()
def jacobian(self, x):
analytical = self.jacobian_analytical(x=x)
if (self.check_with_finite_differences):
finite = self.jacobian_finite(x=x)
scale = max(1, flex.max(flex.abs(analytical)))
assert approx_equal(analytical/scale, finite/scale, 1.e-5)
return analytical
def quasi_normalisation(intensities):
"""Quasi-normalisation of the input intensities.
Args:
intensities (cctbx.miller.array): The intensities to be normalised.
Returns:
cctbx.miller.array: The normalised intensities.
"""
# handle negative reflections to minimise effect on mean I values.
intensities.data().set_selected(intensities.data() < 0.0, 0.0)
# set up binning objects
if intensities.size() > 20000:
n_refl_shells = 20
elif intensities.size() > 15000:
n_refl_shells = 15
else:
n_refl_shells = 10
d_star_sq = intensities.d_star_sq().data()
step = (flex.max(d_star_sq) - flex.min(d_star_sq) +
1e-8) / n_refl_shells
intensities.setup_binner_d_star_sq_step(d_star_sq_step=step)
normalisations = intensities.intensity_quasi_normalisations()
return intensities.customized_copy(
data=(intensities.data() / normalisations.data()),
sigmas=(intensities.sigmas() / normalisations.data()),
)
def get_mean_statistic_for_resolution (d_min, stat_type, range=0.2, out=None) :
if (out is None) :
out = sys.stdout
from scitbx.array_family import flex
pkl_file = libtbx.env.find_in_repositories(
relative_path = "chem_data/polygon_data/all_mvd.pickle",
test = os.path.isfile)
db = easy_pickle.load(pkl_file)
all_d_min = db['high_resolution']
stat_values = db[stat_type]
values_for_range = flex.double()
for (d_, v_) in zip(all_d_min, stat_values) :
try :
d = float(d_)
v = float(v_)
except ValueError : continue
else :
if (d > (d_min - range)) and (d < (d_min + range)) :
values_for_range.append(v)
h = flex.histogram(values_for_range, n_slots=10)
print >> out, " %s for d_min = %.3f - %.3f A" % (stat_names[stat_type], d_min-range,
d_min+range)
min = flex.min(values_for_range)
max = flex.max(values_for_range)
mean = flex.mean(values_for_range)
print >> out, " count: %d" % values_for_range.size()
print >> out, " min: %.2f" % min
print >> out, " max: %.2f" % max
print >> out, " mean: %.2f" % mean
print >> out, " histogram of values:"
h.show(prefix=" ")
return mean
def compare_two_raw_images(
reference,
test,
tol=1.E-7): # TODO: run more tests to decide on the default tolerance
from six.moves import cPickle as pickle
from scitbx.array_family import flex
with open(reference, 'rb') as F:
if six.PY3:
ref_array = pickle.load(F, encoding="bytes")
else:
ref_array = pickle.load(F)
with open(test, 'rb') as F:
test_array = pickle.load(F)
print("\nComparing raw image: '%s' with the reference: '%s'" %
(test, reference))
diff_array = test_array - ref_array
if diff_array.all_eq(0.0):
print("There are 0 differences\n")
else:
stats = flex.mean_and_variance(diff_array.as_1d(
)) # flex.mean_and_variance works only on 1d arrays
diff_mean = stats.mean()
diff_std = stats.unweighted_sample_standard_deviation()
diff_min = flex.min(diff_array)
diff_max = flex.max(diff_array)
print("Differences: range (%.2E to %.2E); mean %.2E; std %.2E" %
(diff_min, diff_max, diff_mean, diff_std))
# assert acceptable differences
assert abs(
diff_mean) < tol, "The raw image is different from the reference."
def run_simulations(out, mersenne_twister, n_dynamics_steps, delta_t):
mt_state = mersenne_twister.getstate()
sim_labels = []
sites_moved_accu = []
e_tots_list = []
relative_ranges = []
for r_is_qr in [True, False]:
for six_dof_type in ["euler_params", "euler_angles_xyz"]:
mersenne_twister.setstate(mt_state)
sim, sim_label, sites_moved, e_tots, relative_range = run_simulation(
out=out,
six_dof_type=six_dof_type,
r_is_qr=r_is_qr,
mersenne_twister=mersenne_twister,
n_dynamics_steps=n_dynamics_steps,
delta_t=delta_t)
sim_labels.append(sim_label)
e_tots_list.append(e_tots)
sites_moved_accu.append(sites_moved)
relative_ranges.append(relative_range)
rms_max_list = flex.double()
for sim_label, other in zip(sim_labels[1:], sites_moved_accu[1:]):
print >> out, "rms joints %s" % sim_labels[0]
print >> out, " vs. %s:" % sim_label
rms = flex.double()
for sites_ref, sites_other in zip(sites_moved_accu[0], other):
sites_ref = flex.vec3_double(sites_ref)
rms.append(sites_ref.rms_difference(flex.vec3_double(sites_other)))
rms.min_max_mean().show(out=out, prefix=" ")
rms_max_list.append(flex.max(rms))
print >> out
out.flush()
return sim_labels, e_tots_list, relative_ranges, rms_max_list
def extract_proxies(self):
self.proxies = ext.shared_phi_psi_proxy()
from mmtbx.conformation_dependent_library import generate_protein_threes
selected_h = self.pdb_hierarchy.select(self.bool_atom_selection)
n_seq = flex.max(selected_h.atoms().extract_i_seq())
for three in generate_protein_threes(
hierarchy=selected_h,
geometry=None):
rc = three.get_phi_psi_atoms()
if rc is None: continue
phi_atoms, psi_atoms = rc
rama_key = three.get_ramalyze_key()
i_seqs = [atom.i_seq for atom in phi_atoms] + [psi_atoms[-1].i_seq]
resnames = three.get_resnames()
r_name = resnames[1]
assert rama_key in range(6)
text_rama_key = ramalyze.res_types[rama_key]
assert text_rama_key in ["general", "glycine", "cis-proline", "trans-proline",
"pre-proline", "isoleucine or valine"]
proxy = ext.phi_psi_proxy(
residue_name=r_name,
residue_type=text_rama_key,
i_seqs=i_seqs)
if not is_proxy_present(self.proxies, n_seq, proxy):
self.proxies.append(proxy)
print >> self.log, ""
print >> self.log, " %d Ramachandran restraints generated." % (
self.get_n_proxies())
def show_summary(self, out=None, nn=50):
if out is None:
out = sys.stdout
qq = self.q.size()
if qq <= nn:
nn = int(qq) / 2
nn = int(nn)
print >> out, "Data id : ", self.data_id
print >> out, "Data type : ", self.data_type
print >> out, "Concentratation : %s" % (str(self.concentration))
print >> out, "q-range : %5.3e %5.3e" % (flex.min(
self.q), flex.max(self.q))
print >> out, "N obs : %5.3e" % (self.q.size())
print >> out, "<I/sigI> low : %5.3e" % (flex.mean(
self.i[0:nn] / (self.s[0:nn] + 1e-13)))
print >> out, "<I/sigI> high : %5.3e" % (flex.mean(
self.i[qq - nn:] / (self.s[qq - nn:] + 1e-13)))
print >> out, " low q range : %5.3e %5.3e" % (self.q[0],
self.q[nn])
print >> out, " high q range : %5.3e %5.3e" % (self.q[qq - nn],
self.q[qq - 1])
print >> out
for cmm in self.comments:
print >> out, cmm
print >> out
def __init__(self, model, diffs, params):
self.params = params
self.model = model
if params.group_sulfurs:
self.n = 1 + flex.max(self.model.scatterer_model_idx)
else:
self.n = self.model.N_anom_scatterers
self.x = flex.double(self.n, 0.)
from cctbx import miller
matches = miller.match_indices(self.model.f_model_real.indices(),
diffs.indices())
self.sel0 = flex.size_t([p[0] for p in matches.pairs()])
self.sel1 = flex.size_t([p[1] for p in matches.pairs()])
self.diffs = diffs.select(self.sel1)
print("SELECTED %d diffs out of %d" %
(len(self.diffs.data()), len(diffs.data())))
self.minimizer = scitbx.lbfgs.run(
target_evaluator=self,
termination_params=scitbx.lbfgs.termination_parameters(
traditional_convergence_test=True,
traditional_convergence_test_eps=1.e-4,
max_iterations=20))
def extract_proxies(self, log):
self.proxies = ext.shared_phi_psi_proxy()
from mmtbx.conformation_dependent_library import generate_protein_threes
selected_h = self.pdb_hierarchy.select(self.bool_atom_selection)
n_seq = flex.max(selected_h.atoms().extract_i_seq())
for three in generate_protein_threes(
hierarchy=selected_h,
geometry=None):
rc = three.get_phi_psi_atoms()
if rc is None: continue
phi_atoms, psi_atoms = rc
rama_key = three.get_ramalyze_key()
i_seqs = [atom.i_seq for atom in phi_atoms] + [psi_atoms[-1].i_seq]
resnames = three.get_resnames()
r_name = resnames[1]
assert rama_key in range(6)
text_rama_key = ramalyze.res_types[rama_key]
assert text_rama_key in ["general", "glycine", "cis-proline", "trans-proline",
"pre-proline", "isoleucine or valine"]
proxy = ext.phi_psi_proxy(
residue_name=r_name,
residue_type=text_rama_key,
i_seqs=i_seqs)
if not is_proxy_present(self.proxies, n_seq, proxy):
self.proxies.append(proxy)
print >> log, ""
print >> log, " %d Ramachandran restraints generated." % (
self.get_n_proxies())
def show(self):
b = self.bss_result
print >> self.log, " Statistics in resolution bins:"
#assert k_mask.size() == len(self.bin_selections)
fmt=" %7.5f %6.2f -%6.2f %5.1f %5d %-6s %-6s %-6s %6.3f %6.3f %8.2f %6.4f"
f_model = self.core.f_model.data()
print >> self.log, " s^2 Resolution Compl Nrefl k_mask k_iso k_ani <Fobs> R"
print >> self.log, " (A) (%) orig smooth average"
k_mask_bin_orig_ = str(None)
k_mask_bin_smooth_ = str(None)
k_mask_bin_approx_ = str(None)
for i_sel, cas in enumerate(self.cores_and_selections):
selection, core, selection_use, sel_work = cas
sel = sel_work
ss_ = self.ss_bin_values[i_sel][2]
if(b is not None and self.bss_result.k_mask_bin_orig is not None):
k_mask_bin_orig_ = "%6.4f"%self.bss_result.k_mask_bin_orig[i_sel]
if(b is not None and self.bss_result.k_mask_bin_smooth is not None):
k_mask_bin_smooth_ = "%6.4f"%self.bss_result.k_mask_bin_smooth[i_sel]
k_mask_bin_averaged_ = "%6.4f"%flex.mean(self.core.k_mask().select(sel))
d_ = self.d_spacings.data().select(sel)
d_min_ = flex.min(d_)
d_max_ = flex.max(d_)
n_ref_ = d_.size()
f_obs_ = self.f_obs.select(sel)
f_obs_mean_ = flex.mean(f_obs_.data())
k_isotropic_ = flex.mean(self.core.k_isotropic.select(sel))
k_anisotropic_ = flex.mean(self.core.k_anisotropic.select(sel))
cmpl_ = f_obs_.completeness(d_max=d_max_)*100.
r_ = bulk_solvent.r_factor(f_obs_.data(),f_model.select(sel),1)
print >> self.log, fmt%(ss_, d_max_, d_min_, cmpl_, n_ref_,
k_mask_bin_orig_, k_mask_bin_smooth_,k_mask_bin_averaged_,
k_isotropic_, k_anisotropic_, f_obs_mean_, r_)
def tst_nsd():
moving1 = flex.vec3_double()
moving2 = flex.vec3_double()
fixed = flex.vec3_double()
max_noise = 0
for ii in range(10):
noise = flex.random_double(3)*2-1.0
if noise.norm() > max_noise:
max_noise = noise.norm()
xyz = flex.random_double(3)*5
fixed.append( list(xyz) )
moving1.append( list(xyz + noise/10) )
moving2.append( list(xyz + noise/2) )
ne = nsd_engine(fixed)
a = ne.nsd(fixed)
b = ne.nsd(moving1)
c = ne.nsd(moving2)
assert abs(a)<1e-6
assert(b<=c)
matrix = euler.zyz_matrix(0.7,1.3,2.1)
fixed_r = matrix*moving1+(8,18,28)
fitter = nsd_rigid_body_fitter( fixed,fixed_r)
nxyz = fitter.best_shifted()
dd = nxyz[0:fixed.size()]-fixed
dd = dd.norms()
dd = flex.max(dd)
assert (dd<2.00*max_noise/10)
def exercise_tardy_model(out, n_dynamics_steps, delta_t, tardy_model):
tardy_model.check_d_e_pot_d_q()
e_pots = flex.double([tardy_model.e_pot()])
e_kins = flex.double([tardy_model.e_kin()])
for i_step in range(n_dynamics_steps):
tardy_model.dynamics_step(delta_t=delta_t)
e_pots.append(tardy_model.e_pot())
e_kins.append(tardy_model.e_kin())
e_tots = e_pots + e_kins
tardy_model.check_d_e_pot_d_q()
print("degrees of freedom:", tardy_model.degrees_of_freedom, file=out)
print("energy samples:", e_tots.size(), file=out)
print("e_pot min, max:", min(e_pots), max(e_pots), file=out)
print("e_kin min, max:", min(e_kins), max(e_kins), file=out)
print("e_tot min, max:", min(e_tots), max(e_tots), file=out)
print("start e_tot:", e_tots[0], file=out)
print("final e_tot:", e_tots[-1], file=out)
ave = flex.sum(e_tots) / e_tots.size()
range_ = flex.max(e_tots) - flex.min(e_tots)
if (ave == 0): relative_range = 0
else: relative_range = range_ / ave
print("ave:", ave, file=out)
print("range:", range_, file=out)
print("relative range:", relative_range, file=out)
print(file=out)
out.flush()
return relative_range
def get_summary (self) :
"""
Returns a simple object for harvesting statistics elsewhere.
"""
n_anom_peaks = None
if (self.anom_peaks is not None) :
n_anom_peaks = len(self.anom_peaks.heights)
n_water_peaks = n_water_anom_peaks = None
if (self.water_peaks is not None) :
n_water_peaks = len(self.water_peaks)
if (self.water_anom_peaks is not None) :
n_water_anom_peaks = len(self.water_anom_peaks)
hole_max = peak_max = None
if (len(self.peaks.heights) > 0) :
peak_max = flex.max(self.peaks.heights)
if (len(self.holes.heights) > 0) :
hole_max = flex.min(self.holes.heights)
n_non_water_anom_peaks = None
if (getattr(self, "non_water_anom_peaks", None) is not None) :
n_non_water_anom_peaks = len(self.non_water_anom_peaks)
return summary(
n_peaks_1=(self.peaks.heights > self.map_cutoff).count(True),
n_peaks_2=(self.peaks.heights > self.map_cutoff + 3).count(True),
n_peaks_3=(self.peaks.heights > self.map_cutoff + 6).count(True),
n_holes_1=(self.holes.heights < -self.map_cutoff).count(True),
n_holes_2=(self.holes.heights < -self.map_cutoff - 3).count(True),
n_holes_3=(self.holes.heights < -self.map_cutoff - 6).count(True),
peak_max=peak_max,
hole_max=hole_max,
n_anom_peaks=n_anom_peaks,
n_water_peaks=n_water_peaks,
n_water_anom_peaks=n_water_anom_peaks,
map_cutoff=self.map_cutoff,
anom_map_cutoff=self.anom_map_cutoff,
n_non_water_anom_peaks=n_non_water_anom_peaks)
def exercise_complex_to_complex_3d():
print "complex_to_complex_3d"
for n_complex,n_repeats in [((100,80,90),2), ((200,160,180),1)]:
print " dimensions:", n_complex
print " repeats:", n_repeats
np = n_complex[0]*n_complex[1]*n_complex[2]
d0 = (flex.random_double(size=np)*2-1) * flex.polar(
1, flex.random_double(size=np)*2-1)
d0.reshape(flex.grid(n_complex))
#
t0 = time.time()
for i_trial in xrange(n_repeats):
d = d0.deep_copy()
overhead = time.time()-t0
print " overhead: %.2f seconds" % overhead
#
t0 = time.time()
for i_trial in xrange(n_repeats):
d = d0.deep_copy()
fftw3tbx.complex_to_complex_3d_in_place(data=d, exp_sign=-1)
fftw3tbx.complex_to_complex_3d_in_place(data=d, exp_sign=+1)
print " fftw: %.2f seconds" % (time.time()-t0-overhead)
rw = d / np
#
t0 = time.time()
for i_trial in xrange(n_repeats):
d = d0.deep_copy()
fftpack.complex_to_complex_3d(n_complex).forward(d)
fftpack.complex_to_complex_3d(n_complex).backward(d)
print " fftpack: %.2f seconds" % (time.time()-t0-overhead)
sys.stdout.flush()
rp = d / np
#
assert flex.max(flex.abs(rw-rp)) < 1.e-6
def test_rs_mapper(dials_data, tmpdir):
result = procrunner.run(
[
"dials.rs_mapper",
dials_data("centroid_test_data").join("datablock.json").strpath,
'map_file="junk.ccp4"',
],
working_directory=tmpdir.strpath,
)
assert not result.returncode and not result.stderr
assert tmpdir.join("junk.ccp4").check()
# load results
from iotbx import ccp4_map
from scitbx.array_family import flex
m = ccp4_map.map_reader(file_name=tmpdir.join("junk.ccp4").strpath)
assert len(m.data) == 7189057
assert m.header_min == 0.0
assert flex.min(m.data) == 0.0
assert m.header_max == 2052.75
assert flex.max(m.data) == 2052.75
assert m.header_mean == pytest.approx(0.018905939534306526, abs=1e-6)
assert flex.mean(m.data) == pytest.approx(0.018905939534306526, abs=1e-6)
def prepare_maps(fofc, two_fofc, fem, fofc_cutoff=2, two_fofc_cutoff=0.5,
fem_cutoff=0.5, connectivity_cutoff=0.5, local_average=True):
"""
- This takes 3 maps: mFo-DFc, 2mFo-DFc and FEM and combines them into one map
that is most suitable for real-space refinement.
- Maps are the boxes extracted around region of interest from the whole unit
cell map.
- All maps are expected to be normalized by standard deviation (sigma-scaled)
BEFORE extracting the box. There is no way to assert it at this point.
- Map gridding equivalence is asserted.
"""
m1,m2,m3 = fofc, two_fofc, fem
# assert identical gridding
for m_ in [m1,m2,m3]:
for m__ in [m1,m2,m3]:
assert m_.all() == m__.all()
assert m_.focus() == m__.focus()
assert m_.origin() == m__.origin()
# binarize residual map
sel = m1 <= fofc_cutoff
mask = m1 .set_selected( sel, 0)
mask = mask.set_selected(~sel, 1)
del sel, m1
assert approx_equal([flex.max(mask), flex.min(mask)], [1,0])
def truncate_and_filter(m, cutoff, mask):
return m.set_selected(m<=cutoff, 0)*mask
# truncate and filter 2mFo-DFc map
m2 = truncate_and_filter(m2, two_fofc_cutoff, mask)
# truncate and filter FEM
m3 = truncate_and_filter(m3, fem_cutoff, mask)
del mask
# combined maps
def scale(m):
sd = m.sample_standard_deviation()
if(sd != 0): return m/sd
else: return m
m2 = scale(m2)
m3 = scale(m3)
m = (m2+m3)/2.
del m2, m3
m = scale(m)
# connectivity analysis
co = maptbx.connectivity(map_data=m, threshold=connectivity_cutoff)
v_max=-1.e+9
i_max=None
for i, v in enumerate(co.regions()):
if(i>0):
if(v>v_max):
v_max=v
i_max=i
mask2 = co.result()
selection = mask2==i_max
mask2 = mask2.set_selected(selection, 1)
mask2 = mask2.set_selected(~selection, 0)
assert mask2.count(1) == v_max
# final filter
m = m * mask2.as_double()
if(local_average):
maptbx.map_box_average(map_data=m, cutoff=0.5, index_span=1)
return m
def inner_loop(fmodel, wam, missing, crystal_gridding, n, progress_counter,
use_max_map):
mac = maptbx.map_accumulator(n_real=crystal_gridding.n_real(),
smearing_b=1,
max_peak_scale=100,
smearing_span=5,
use_max_map=use_max_map)
for j in xrange(n):
mc_w = wam.random_weight_averaged_map_coefficients(
missing=missing,
random_scale=random.choice([0, 1, 2, 3, 4, 5]),
random_seed=random.choice(range(1, 9754365, 10000)),
n_cycles=100,
fraction_keep=random.choice([0.9, 0.95, 1.0]))
if (random.choice([True, False])):
mc_w = kick.randomize_struture_factors(map_coeffs=mc_w,
number_of_kicks=100)
m = get_map(mc=mc_w, cg=crystal_gridding)
m = m / flex.max(m)
sel = m < 0
m = m.set_selected(sel, 0)
mac.add(map_data=m)
progress_counter.show()
mm = mac.as_median_map()
sd = mm.sample_standard_deviation()
r = mm / sd
return r
def max_change_so_far(x):
result = flex.double()
if(self.cc_to_answer.size()):
for i in xrange(self.cc_to_answer.size()):
if(i>0):
result.append(self.cc_to_answer[i]-self.cc_to_answer[i-1])
return flex.max(result)
def add_imagesets_buttons(self):
if 'imageset_id' not in self.parent.reflections_input:
self.imgset_btn = None
return
n = flex.max(self.parent.reflections_input['imageset_id'])
if n <= 0:
self.imgset_btn = None
return
box = wx.BoxSizer(wx.VERTICAL)
self.panel_sizer.Add(box)
label = wx.StaticText(self,-1,"Imageset ids:")
box.Add(label, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL, 5)
from wxtbx.segmentedctrl import SegmentedToggleControl, SEGBTN_HORIZONTAL
self.imgset_btn = SegmentedToggleControl(self, style=SEGBTN_HORIZONTAL)
for i in range(n+1):
self.imgset_btn.AddSegment(str(i))
if (self.settings.imageset_ids is not None and
i in self.settings.imageset_ids):
self.imgset_btn.SetValue(i+1, True)
self.imgset_btn.Realize()
self.Bind(wx.EVT_TOGGLEBUTTON, self.OnChangeSettings, self.imgset_btn)
box.Add(self.imgset_btn, 0, wx.ALL, 5)
def box_iterator(self):
b = maptbx.boxes(
n_real = self.atom_map_asu.focus(),
fraction = self.box_size_as_fraction,
max_boxes= self.max_boxes,
log = self.log)
def get_wide_box(s,e): # define wide box: neutral + phased volumes
if(self.neutral_volume_box_cushion_width>0):
sh = self.neutral_volume_box_cushion_width
ss = [max(s[i]-sh,0) for i in [0,1,2]]
ee = [min(e[i]+sh,n_real_asu[i]) for i in [0,1,2]]
else: ss,ee = s,e
return ss,ee
n_real_asu = b.n_real
n_boxes = len(b.starts)
i_box = 0
for s,e in zip(b.starts, b.ends):
i_box+=1
sw,ew = get_wide_box(s=s,e=e)
fmodel_omit = self.omit_box(start=sw, end=ew)
r = fmodel_omit.r_work()
self.r.append(r) # for tests only
if(self.log):
print >> self.log, "r(curr,min,max,mean)=%6.4f %6.4f %6.4f %6.4f"%(r,
flex.min(self.r), flex.max(self.r), flex.mean(self.r)), i_box, n_boxes
omit_map_data = self.asu_map_from_fmodel(
fmodel=fmodel_omit, map_type=self.map_type)
maptbx.copy_box(
map_data_from = omit_map_data,
map_data_to = self.map_result_asu,
start = s,
end = e)
self.map_result_asu.reshape(self.acc_asu)
def test_flatten():
from dials.algorithms.shoebox import MaskCode
from dials.array_family import flex
for shoebox, (XC, I) in random_shoeboxes(10, mask=True):
assert not shoebox.flat
zs = shoebox.zsize()
ys = shoebox.ysize()
xs = shoebox.xsize()
expected_data = flex.real(flex.grid(1, ys, xs), 0)
expected_mask = flex.int(flex.grid(1, ys, xs), 0)
for k in range(zs):
for j in range(ys):
for i in range(xs):
expected_data[0, j, i] += shoebox.data[k, j, i]
expected_mask[0, j, i] |= shoebox.mask[k, j, i]
if not (expected_mask[0, j, i] & MaskCode.Valid) or not (
shoebox.mask[k, j, i] & MaskCode.Valid):
expected_mask[0, j, i] &= ~MaskCode.Valid
shoebox.flatten()
diff = expected_data.as_double() - shoebox.data.as_double()
max_diff = flex.max(flex.abs(diff))
assert max_diff < 1e-7
assert expected_mask.all_eq(shoebox.mask)
assert shoebox.flat
assert shoebox.is_consistent()
def exercise () :
if (os.path.isfile("tst_fmodel_anomalous.mtz")) :
os.remove("tst_fmodel_anomalous.mtz")
pdb_file = make_fake_anomalous_data.write_pdb_input_cd_cl(
file_base="tst_fmodel_anomalous")
# phenix.fmodel (with wavelength)
args = [
pdb_file,
"high_resolution=1.0",
"wavelength=1.116",
"label=F",
"type=real",
"output.file_name=tst_fmodel_anomalous.mtz",
"r_free_flags_fraction=0.1",
]
fmodel.run(args=args, log=null_out())
assert os.path.isfile("tst_fmodel_anomalous.mtz")
mtz_in = file_reader.any_file("tst_fmodel_anomalous.mtz")
array = mtz_in.file_server.miller_arrays[0]
assert (array.anomalous_flag())
anom_diffs = array.anomalous_differences()
assert approx_equal(flex.max(anom_diffs.data()), 5.72, eps=0.01)
# mmtbx.fmodel_simple
result = easy_run.call(
"mmtbx.fmodel_simple \"%s\" tst_fmodel_anomalous.mtz high_resolution=2.0"
% pdb_file)
print "OK"
def box_iterator(self):
b = maptbx.boxes(n_real=self.atom_map_asu.focus(),
fraction=self.box_size_as_fraction,
max_boxes=self.max_boxes,
log=self.log)
def get_wide_box(s, e): # define wide box: neutral + phased volumes
if (self.neutral_volume_box_cushion_width > 0):
sh = self.neutral_volume_box_cushion_width
ss = [max(s[i] - sh, 0) for i in [0, 1, 2]]
ee = [min(e[i] + sh, n_real_asu[i]) for i in [0, 1, 2]]
else:
ss, ee = s, e
return ss, ee
n_real_asu = b.n_real
n_boxes = len(b.starts)
i_box = 0
for s, e in zip(b.starts, b.ends):
i_box += 1
sw, ew = get_wide_box(s=s, e=e)
fmodel_omit = self.omit_box(start=sw, end=ew)
r = fmodel_omit.r_work()
self.r.append(r) # for tests only
if (self.log):
print >> self.log, "r(curr,min,max,mean)=%6.4f %6.4f %6.4f %6.4f" % (
r, flex.min(self.r), flex.max(self.r), flex.mean(
self.r)), i_box, n_boxes
omit_map_data = self.asu_map_from_fmodel(fmodel=fmodel_omit,
map_type=self.map_type)
maptbx.copy_box(map_data_from=omit_map_data,
map_data_to=self.map_result_asu,
start=s,
end=e)
self.map_result_asu.reshape(self.acc_asu)
