def randomize_struture_factors(map_coeffs, number_of_kicks, phases_only=False):
map_coeff_data = None
for kick in xrange(number_of_kicks):
rc, ar, pr = random.choice([(0.1, 0.10, 40),
(0.2, 0.09, 40),
(0.3, 0.08, 30),
(0.4, 0.07, 30),
(0.5, 0.06, 20),
(0.6, 0.05, 20),
(0.7, 0.04, 20),
(0.8, 0.03, 10),
(0.9, 0.02, 10),
(1.0, 0.01, 10)
])
if(phases_only): ar = 0
sel = flex.random_bool(map_coeffs.size(), rc)
mc = map_coeffs.randomize_amplitude_and_phase(
amplitude_error=ar, phase_error_deg=pr, selection=sel)
if(map_coeff_data is None): map_coeff_data = mc.data()
else: map_coeff_data = map_coeff_data + mc.data()
map_coeff_data = map_coeff_data/number_of_kicks
return miller.set(
crystal_symmetry = map_coeffs.crystal_symmetry(),
indices = map_coeffs.indices(),
anomalous_flag = False).array(data = map_coeff_data)
def setup_class(self):
from dials.algorithms.image.connected_components import LabelImageStack3d
self.size = (500, 500)
self.label_images = LabelImageStack3d(self.size)
from scitbx.array_family import flex
self.data_list = []
self.mask_list = []
for i in range(10):
data = flex.random_int_gaussian_distribution(
self.size[0] * self.size[1], 100, 10
)
data.reshape(flex.grid(self.size))
mask = flex.random_bool(self.size[0] * self.size[1], 0.1)
mask.reshape(flex.grid(self.size))
self.data_list.append(data)
self.mask_list.append(mask)
for i in range(10):
self.label_images.add_image(self.data_list[i], self.mask_list[i])
self.labels = self.label_images.labels()
self.coords = self.label_images.coords()
self.values = list(self.label_images.values())
assert len(self.labels) > 0
assert len(self.labels) == len(self.coords)
assert len(self.labels) == len(self.values)
def compute_f_calc(fmodel, params):
from cctbx import miller
coeffs_partial_set = fmodel.f_obs().structure_factors_from_scatterers(
xray_structure = fmodel.xray_structure).f_calc()
if(hasattr(params,"dev") and params.dev.complete_set_up_to_d_min):
coeffs = fmodel.xray_structure.structure_factors(
d_min = fmodel.f_obs().d_min()).f_calc()
frac_inc = 1.*coeffs_partial_set.data().size()/coeffs.data().size()
n_miss = coeffs.data().size() - coeffs_partial_set.data().size()
if(params.dev.aply_same_incompleteness_to_complete_set_at == "randomly"):
sel = flex.random_bool(coeffs.data().size(), frac_inc)
coeffs = coeffs.select(sel)
elif(params.dev.aply_same_incompleteness_to_complete_set_at == "low"):
coeffs = coeffs.sort()
coeffs = miller.set(
crystal_symmetry = coeffs,
indices = coeffs.indices()[n_miss+1:],
anomalous_flag = coeffs.anomalous_flag()).array(
data = coeffs.data()[n_miss+1:])
elif(params.dev.aply_same_incompleteness_to_complete_set_at == "high"):
coeffs = coeffs.sort(reverse=True)
coeffs = miller.set(
crystal_symmetry = coeffs,
indices = coeffs.indices()[n_miss+1:],
anomalous_flag = coeffs.anomalous_flag()).array(
data = coeffs.data()[n_miss+1:])
else:
coeffs = coeffs_partial_set
return coeffs
def compute_f_calc(fmodel, params):
from cctbx import miller
coeffs_partial_set = fmodel.f_obs().structure_factors_from_scatterers(
xray_structure=fmodel.xray_structure).f_calc()
if (hasattr(params, "dev") and params.dev.complete_set_up_to_d_min):
coeffs = fmodel.xray_structure.structure_factors(
d_min=fmodel.f_obs().d_min()).f_calc()
frac_inc = 1. * coeffs_partial_set.data().size() / coeffs.data().size()
n_miss = coeffs.data().size() - coeffs_partial_set.data().size()
if (params.dev.aply_same_incompleteness_to_complete_set_at ==
"randomly"):
sel = flex.random_bool(coeffs.data().size(), frac_inc)
coeffs = coeffs.select(sel)
elif (params.dev.aply_same_incompleteness_to_complete_set_at == "low"):
coeffs = coeffs.sort()
coeffs = miller.set(crystal_symmetry=coeffs,
indices=coeffs.indices()[n_miss + 1:],
anomalous_flag=coeffs.anomalous_flag()).array(
data=coeffs.data()[n_miss + 1:])
elif (params.dev.aply_same_incompleteness_to_complete_set_at == "high"
):
coeffs = coeffs.sort(reverse=True)
coeffs = miller.set(crystal_symmetry=coeffs,
indices=coeffs.indices()[n_miss + 1:],
anomalous_flag=coeffs.anomalous_flag()).array(
data=coeffs.data()[n_miss + 1:])
else:
coeffs = coeffs_partial_set
return coeffs
def run(self):
from scitbx.array_family import flex
data_list = []
mask_list = []
for i in range(10):
data = flex.random_int_gaussian_distribution(
self.size[0] * self.size[1], 100, 10)
data.reshape(flex.grid(self.size))
mask = flex.random_bool(self.size[0] * self.size[1], 0.1)
mask.reshape(flex.grid(self.size))
data_list.append(data)
mask_list.append(mask)
for i in range(10):
self.label_images.add_image(data_list[i], mask_list[i])
labels = self.label_images.labels()
coords = self.label_images.coords()
values = self.label_images.values()
assert(len(labels) > 0)
assert(len(labels) == len(coords))
assert(len(labels) == len(values))
self.tst_coords_are_valid(mask_list, coords)
self.tst_values_are_valid(data_list, mask_list, values)
self.tst_labels_are_valid(data_list, mask_list, coords, labels)
def kick_map_coeffs(
map_coeffs,
crystal_gridding,
number_of_kicks,
macro_cycles,
missing = None,
kick_completeness = 0.95,
phases_only = False):
map_data = None
if(macro_cycles==0):
map_data = compute_map_and_combine(
map_coeffs = map_coeffs,
crystal_gridding = crystal_gridding,
map_data = map_data)
for it in xrange(macro_cycles):
print " %d"%it
if(number_of_kicks>0):
mc = randomize_struture_factors(map_coeffs=map_coeffs,
number_of_kicks=number_of_kicks, phases_only=phases_only)
else:
mc = map_coeffs.deep_copy()
if(missing is not None):
mc = mc.complete_with(other=missing, scale=True)
if(kick_completeness):
mc = mc.select(flex.random_bool(mc.size(), kick_completeness))
map_data = compute_map_and_combine(
map_coeffs = mc,
crystal_gridding = crystal_gridding,
map_data = map_data)
return map_data
def setup_class(self):
from scitbx.array_family import flex
spot = flex.double(flex.grid(11, 11))
for j in range(11):
for i in range(11):
spot[j, i] = exp(-((j - 5) ** 2 + (i - 5) ** 2) / 2 ** 2)
self.image = flex.double(flex.grid(2000, 2000))
for n in range(200):
x = randint(0, 2000)
y = randint(0, 2000)
for j in range(0, 11):
for i in range(0, 11):
xx = x + i - 5
yy = y + j - 5
if xx >= 0 and yy >= 0 and xx < 2000 and yy < 2000:
self.image[yy, xx] = poisson(100 * spot[j, i])
background = flex.double(list(poisson(5, 2000 * 2000)))
background.reshape(flex.grid(2000, 2000))
self.image += background
# Create an image
self.mask = flex.random_bool(2000 * 2000, 0.99)
self.mask.reshape(flex.grid(2000, 2000))
self.gain = flex.random_double(2000 * 2000) + 1.0
self.gain.reshape(flex.grid(2000, 2000))
self.size = (3, 3)
self.min_count = 2
def randomize_completeness2(map_coeffs, crystal_gridding, n_cycles=10):
map_filter = None
from mmtbx.maps import fem
for cycle in xrange(n_cycles):
if (cycle > 0):
mc = map_coeffs.select(flex.random_bool(map_coeffs.size(), 0.99))
else:
mc = map_coeffs.deep_copy()
m = get_map(map_coeffs=mc, crystal_gridding=crystal_gridding)
maptbx.reset(data=m,
substitute_value=0.0,
less_than_threshold=0.5,
greater_than_threshold=-9999,
use_and=True)
m = maptbx.volume_scale(map=m, n_bins=10000).map_data()
if (map_filter is not None):
map_filter = maptbx.volume_scale(map=map_filter,
n_bins=10000).map_data()
map_filter = fem.intersection(m1=map_filter,
m2=m,
thresholds=flex.double(
[i / 100. for i in range(0, 55, 5)]),
average=True)
#map_filter = map_filter.set_selected(map_filter< 0.5, 0)
#map_filter = map_filter.set_selected(map_filter>=0.5, 1)
return map_filter
def test_masked_mean_filter():
from scitbx.array_family import flex
from dials.algorithms.image.filter import mean_filter
# Create an image
image = flex.random_double(2000 * 2000)
image.reshape(flex.grid(2000, 2000))
mask = flex.random_bool(2000 * 2000, 0.99).as_int()
mask.reshape(flex.grid(2000, 2000))
# Calculate the summed area table
mask2 = mask.deep_copy()
mean = mean_filter(image, mask2, (3, 3), 1)
# For a selection of random points, ensure that the value is the
# sum of the area under the kernel
eps = 1e-7
for i in range(10000):
i = random.randint(10, 1990)
j = random.randint(10, 1990)
m1 = mean[j, i]
p = image[j - 3:j + 4, i - 3:i + 4]
m = mask[j - 3:j + 4, i - 3:i + 4]
if mask[j, i] == 0:
m2 = 0.0
else:
p = flex.select(p, flags=m)
mv = flex.mean_and_variance(flex.double(p))
m2 = mv.mean()
assert m1 == pytest.approx(m2, abs=eps)
def kick_map_coeffs(map_coeffs,
crystal_gridding,
number_of_kicks,
macro_cycles,
missing=None,
kick_completeness=0.95,
phases_only=False):
map_data = None
if (macro_cycles == 0):
map_data = compute_map_and_combine(map_coeffs=map_coeffs,
crystal_gridding=crystal_gridding,
map_data=map_data)
for it in xrange(macro_cycles):
print " %d" % it
if (number_of_kicks > 0):
mc = randomize_struture_factors(map_coeffs=map_coeffs,
number_of_kicks=number_of_kicks,
phases_only=phases_only)
else:
mc = map_coeffs.deep_copy()
if (missing is not None):
mc = mc.complete_with(other=missing, scale=True)
if (kick_completeness):
mc = mc.select(flex.random_bool(mc.size(), kick_completeness))
map_data = compute_map_and_combine(map_coeffs=mc,
crystal_gridding=crystal_gridding,
map_data=map_data)
return map_data
def test_setup(config): import cctbx.miller from iotbx import mtz, pdb from scitbx.array_family import flex mtz_name = config['mtz_filename'] mtz_file = mtz.object(mtz_filename) pdb_name = config['pdb_name'] pdb_inp = pdb.input(file_name=pdb_name) structure = pdb_inp.xray_structure_simple() miller = structure.structure_factors(d_min=2.85).f_calc() miller_sub = miller[20000:20002] flex.random_generator.seed(82364) size = miller.size() rand_sel_1 = flex.random_bool(size, 0.5) rand_sel_2 = flex.random_bool(size, 0.5) miller_1 = miller.select(rand_sel_1).randomize_phases() miller_2 = miller.select(rand_sel_2).randomize_phases() rand_doub_1 = flex.random_double(miller_1.size(), 0.1) + 0.015 rand_doub_2 = flex.random_double(miller_2.size(), 0.1) + 0.015 sigmas_1 = rand_doub_1 * miller_1.amplitudes().data() sigmas_2 = rand_doub_2 * miller_2.amplitudes().data() miller_1.set_sigmas(sigmas_1) miller_2.set_sigmas(sigmas_2) miller_1.set_observation_type_xray_amplitude() miller_2.set_observation_type_xray_amplitude() miller_1.as_intensity_array().i_over_sig_i() miller_2.as_intensity_array().i_over_sig_i() binner = miller.setup_binner(n_bins=20) indices = miller.indices() mtch_indcs = miller_1.match_indices(miller_2) mset = miller.set() # doc = wikify_all_methods(cctbx.miller.binning, config) # doc = wikify_all_methods(type(mset), config) # doc = wikify_all_methods(type(binner), config) # doc = wikify_all_methods(type(miller), config) # doc = wikify_all_methods(type(miller.data()), config) # doc = wikify_all_methods(type(indices), config) # doc = wikify_all_methods(type(mtch_indcs), config, # module=["cctbx", "miller"]) return (mtch_indcs, ["cctbx", "miller"])
def __init__(self):
from scitbx.array_family import flex
# Create an image
self.image = flex.random_double(2000 * 2000)
self.image.reshape(flex.grid(2000, 2000))
self.mask = flex.random_bool(2000 * 2000, 0.99)
self.mask.reshape(flex.grid(2000, 2000))
self.gain = flex.random_double(2000 * 2000) + 0.5
self.gain.reshape(flex.grid(2000, 2000))
self.size = (3, 3)
self.min_count = 2
def __init__(self):
from scitbx.array_family import flex
# Create an image
self.image = flex.random_double(2000 * 2000)
self.image.reshape(flex.grid(2000, 2000))
self.mask = flex.random_bool(2000 * 2000, 0.99)
self.mask.reshape(flex.grid(2000, 2000))
self.gain = flex.random_double(2000 * 2000) + 0.5
self.gain.reshape(flex.grid(2000, 2000))
self.size = (3, 3)
self.min_count = 2
def kick_fmodel(
fmodel,
map_type,
crystal_gridding,
number_of_kicks,
macro_cycles,
missing = None,
kick_completeness = 0.95):
f_model = fmodel.f_model_no_scales()
zero = fmodel.f_calc().customized_copy(data =
flex.complex_double(fmodel.f_calc().data().size(), 0))
fmodel_dc = mmtbx.f_model.manager(
f_obs = fmodel.f_obs(),
r_free_flags = fmodel.r_free_flags(),
k_isotropic = fmodel.k_isotropic(),
k_anisotropic = fmodel.k_anisotropic(),
f_calc = fmodel.f_model_no_scales(),
f_part1 = fmodel.f_part1(),
f_part2 = fmodel.f_part2(),
f_mask = zero)
r1 = fmodel.r_work()
r2 = fmodel_dc.r_work()
assert approx_equal(r1, r2, 1.e-4), [r1, r2]
def get_mc(fm):
return fm.electron_density_map().map_coefficients(
map_type = map_type,
isotropize = True,
fill_missing = False)
def recreate_r_free_flags(fmodel):
rc = random.choice([0.05, 0.9])
r_free_flags = flex.random_bool(fmodel.f_obs().indices().size(), rc)
fmodel._r_free_flags._data = r_free_flags
return fmodel
map_data = None
for it in xrange(macro_cycles):
print " %d"%it
f_model_kick = randomize_struture_factors(map_coeffs=f_model,
number_of_kicks=number_of_kicks)
fmodel_dc = recreate_r_free_flags(fmodel = fmodel_dc)
fmodel_dc.update(f_calc = f_model_kick)
mc = get_mc(fm=fmodel_dc)
if(missing is not None):
mc = mc.complete_with(missing, scale=True)
if(kick_completeness):
mc = mc.select(flex.random_bool(mc.size(), kick_completeness))
map_data = compute_map_and_combine(
map_coeffs = mc,
crystal_gridding = crystal_gridding,
map_data = map_data)
return map_data
def randomize_completeness(map_coeffs, crystal_gridding, n_cycles=10,
thresholds=[0,0.1,0.2,0.3,0.4,0.5]):
map_data = None
for cycle in xrange(n_cycles):
if(cycle>0):
mc = map_coeffs.select(flex.random_bool(map_coeffs.size(), 0.95))
else:
mc = map_coeffs.deep_copy()
map_data = compute_map_and_combine(
map_coeffs = mc,
crystal_gridding = crystal_gridding,
map_data = map_data,
thresholds = thresholds)
return map_data
def tst_labels_3d(self):
from dials.model.data import PixelList, PixelListLabeller
from scitbx.array_family import flex
size = (500, 500)
sf = 0
labeller = PixelListLabeller()
count = 0
mask_list = []
for i in range(3):
image = flex.random_int_gaussian_distribution(
size[0] * size[1], 100, 5)
mask = flex.random_bool(size[0] * size[1], 0.5)
image.reshape(flex.grid(size))
mask.reshape(flex.grid(size))
pl = PixelList(sf + i, image, mask)
count += len(mask.as_1d().select(mask.as_1d()))
labeller.add(pl)
mask_list.append(mask)
coords = labeller.coords()
labels = labeller.labels_3d()
# Create a map of labels
label_map = flex.int(flex.grid(3, size[0], size[1]))
for c, l in zip(coords, labels):
label_map[c] = l
# Ensure all labels are correct
vi = 0
for k in range(3):
for j in range(size[0]):
for i in range(size[1]):
if mask_list[k][j, i]:
l1 = labels[vi]
if k > 0 and mask_list[k - 1][j, i]:
l2 = label_map[k - 1, j, i]
assert (l2 == l1)
if j > 0 and mask_list[k][j - 1, i]:
l2 = label_map[k, j - 1, i]
assert (l2 == l1)
if i > 0 and mask_list[k][j, i - 1]:
l2 = label_map[k, j, i - 1]
assert (l2 == l1)
vi += 1
# Test passed
print 'OK'
def kick_fmodel(fmodel,
map_type,
crystal_gridding,
number_of_kicks,
macro_cycles,
missing=None,
kick_completeness=0.95):
f_model = fmodel.f_model_no_scales()
zero = fmodel.f_calc().customized_copy(
data=flex.complex_double(fmodel.f_calc().data().size(), 0))
fmodel_dc = mmtbx.f_model.manager(f_obs=fmodel.f_obs(),
r_free_flags=fmodel.r_free_flags(),
k_isotropic=fmodel.k_isotropic(),
k_anisotropic=fmodel.k_anisotropic(),
f_calc=fmodel.f_model_no_scales(),
f_part1=fmodel.f_part1(),
f_part2=fmodel.f_part2(),
f_mask=zero)
r1 = fmodel.r_work()
r2 = fmodel_dc.r_work()
assert approx_equal(r1, r2, 1.e-4), [r1, r2]
def get_mc(fm):
return fm.electron_density_map().map_coefficients(map_type=map_type,
isotropize=True,
fill_missing=False)
def recreate_r_free_flags(fmodel):
rc = random.choice([0.05, 0.9])
r_free_flags = flex.random_bool(fmodel.f_obs().indices().size(), rc)
fmodel._r_free_flags._data = r_free_flags
return fmodel
map_data = None
for it in xrange(macro_cycles):
print " %d" % it
f_model_kick = randomize_struture_factors(
map_coeffs=f_model, number_of_kicks=number_of_kicks)
fmodel_dc = recreate_r_free_flags(fmodel=fmodel_dc)
fmodel_dc.update(f_calc=f_model_kick)
mc = get_mc(fm=fmodel_dc)
if (missing is not None):
mc = mc.complete_with(missing, scale=True)
if (kick_completeness):
mc = mc.select(flex.random_bool(mc.size(), kick_completeness))
map_data = compute_map_and_combine(map_coeffs=mc,
crystal_gridding=crystal_gridding,
map_data=map_data)
return map_data
def randomize_completeness(map_coeffs,
crystal_gridding,
n_cycles=10,
thresholds=[0, 0.1, 0.2, 0.3, 0.4, 0.5]):
map_data = None
for cycle in xrange(n_cycles):
if (cycle > 0):
mc = map_coeffs.select(flex.random_bool(map_coeffs.size(), 0.95))
else:
mc = map_coeffs.deep_copy()
map_data = compute_map_and_combine(map_coeffs=mc,
crystal_gridding=crystal_gridding,
map_data=map_data,
thresholds=thresholds)
return map_data
def random_weight_averaged_map_coefficients(self,
random_scale, random_seed, n_cycles, fraction_keep, missing=None):
assert self.map_coefficients_data.size() == \
self.r.size() == \
self.so.size()
mc_data = maptbx.fem_averaging_loop(
map_coefficients = self.map_coefficients_data,
r_factors = self.r,
sigma_over_f_obs = self.so,
random_scale = random_scale,
random_seed = random_seed,
n_cycles = n_cycles)
mc_wa = self.map_coefficients.customized_copy(data = mc_data)
if(missing is not None):
mc_wa = mc_wa.complete_with(other=missing, scale=True)
return mc_wa.select(flex.random_bool(mc_wa.size(), fraction_keep))
def run(self):
from dials.algorithms.image.filter import fano_filter
from scitbx.array_family import flex
from random import randint
# Create an image
image = flex.random_double(2000 * 2000)
image.reshape(flex.grid(2000, 2000))
mask = flex.random_bool(2000 * 2000, 0.99).as_int()
mask.reshape(flex.grid(2000, 2000))
# Calculate the summed area table
mask2 = mask.deep_copy()
fano_filter = fano_filter(image, mask2, (3, 3), 2)
mean = fano_filter.mean()
var = fano_filter.sample_variance()
fano = fano_filter.fano()
# For a selection of random points, ensure that the value is the
# sum of the area under the kernel
eps = 1e-7
for i in range(10000):
i = randint(10, 1990)
j = randint(10, 1990)
m1 = mean[j,i]
v1 = var[j,i]
f1 = fano[j,i]
p = image[j-3:j+4,i-3:i+4]
m = mask[j-3:j+4,i-3:i+4]
if mask[j,i] == 0:
m2 = 0.0
v2 = 0.0
f2 = 1.0
else:
p = flex.select(p, flags=m)
mv = flex.mean_and_variance(flex.double(p))
m2 = mv.mean()
v2 = mv.unweighted_sample_variance()
f2 = v2 / m2
assert(abs(m1 - m2) <= eps)
assert(abs(v1 - v2) <= eps)
assert(abs(f1 - f2) <= eps)
# Test passed
print 'OK'
def run(self):
from dials.algorithms.image.filter import fano_filter
from scitbx.array_family import flex
from random import randint
# Create an image
image = flex.random_double(2000 * 2000)
image.reshape(flex.grid(2000, 2000))
mask = flex.random_bool(2000 * 2000, 0.99).as_int()
mask.reshape(flex.grid(2000, 2000))
# Calculate the summed area table
mask2 = mask.deep_copy()
fano_filter = fano_filter(image, mask2, (3, 3), 2)
mean = fano_filter.mean()
var = fano_filter.sample_variance()
fano = fano_filter.fano()
# For a selection of random points, ensure that the value is the
# sum of the area under the kernel
eps = 1e-7
for i in range(10000):
i = randint(10, 1990)
j = randint(10, 1990)
m1 = mean[j, i]
v1 = var[j, i]
f1 = fano[j, i]
p = image[j - 3:j + 4, i - 3:i + 4]
m = mask[j - 3:j + 4, i - 3:i + 4]
if mask[j, i] == 0:
m2 = 0.0
v2 = 0.0
f2 = 1.0
else:
p = flex.select(p, flags=m)
mv = flex.mean_and_variance(flex.double(p))
m2 = mv.mean()
v2 = mv.unweighted_sample_variance()
f2 = v2 / m2
assert (abs(m1 - m2) <= eps)
assert (abs(v1 - v2) <= eps)
assert (abs(f1 - f2) <= eps)
# Test passed
print 'OK'
def test_add_image():
from dials.model.data import PixelList, PixelListLabeller
from scitbx.array_family import flex
size = (2000, 2000)
sf = 10
labeller = PixelListLabeller()
count = 0
for i in range(3):
image = flex.random_int_gaussian_distribution(size[0] * size[1], 100,
5)
mask = flex.random_bool(size[0] * size[1], 0.5)
image.reshape(flex.grid(size))
mask.reshape(flex.grid(size))
pl = PixelList(sf + i, image, mask)
count += len(mask.as_1d().select(mask.as_1d()))
labeller.add(pl)
assert len(labeller.values()) == count
def randomize_struture_factors(map_coeffs, number_of_kicks, phases_only=False):
map_coeff_data = None
for kick in xrange(number_of_kicks):
rc, ar, pr = random.choice([(0.1, 0.10, 40), (0.2, 0.09, 40),
(0.3, 0.08, 30), (0.4, 0.07, 30),
(0.5, 0.06, 20), (0.6, 0.05, 20),
(0.7, 0.04, 20), (0.8, 0.03, 10),
(0.9, 0.02, 10), (1.0, 0.01, 10)])
if (phases_only): ar = 0
sel = flex.random_bool(map_coeffs.size(), rc)
mc = map_coeffs.randomize_amplitude_and_phase(amplitude_error=ar,
phase_error_deg=pr,
selection=sel)
if (map_coeff_data is None): map_coeff_data = mc.data()
else: map_coeff_data = map_coeff_data + mc.data()
map_coeff_data = map_coeff_data / number_of_kicks
return miller.set(crystal_symmetry=map_coeffs.crystal_symmetry(),
indices=map_coeffs.indices(),
anomalous_flag=False).array(data=map_coeff_data)
def test():
from scitbx.array_family import flex
from dials.algorithms.image.filter import index_of_dispersion_filter
# Create an image
image = flex.random_double(2000 * 2000)
image.reshape(flex.grid(2000, 2000))
mask = flex.random_bool(2000 * 2000, 0.99).as_int()
mask.reshape(flex.grid(2000, 2000))
# Calculate the summed area table
mask2 = mask.deep_copy()
index_of_dispersion_filter = index_of_dispersion_filter(
image, mask2, (3, 3), 2)
mean = index_of_dispersion_filter.mean()
var = index_of_dispersion_filter.sample_variance()
index_of_dispersion = index_of_dispersion_filter.index_of_dispersion()
# For a selection of random points, ensure that the value is the
# sum of the area under the kernel
eps = 1e-7
for i in range(10000):
i = random.randint(10, 1990)
j = random.randint(10, 1990)
m1 = mean[j, i]
v1 = var[j, i]
f1 = index_of_dispersion[j, i]
p = image[j - 3:j + 4, i - 3:i + 4]
m = mask[j - 3:j + 4, i - 3:i + 4]
if mask[j, i] == 0:
m2 = 0.0
v2 = 0.0
f2 = 1.0
else:
p = flex.select(p, flags=m)
mv = flex.mean_and_variance(flex.double(p))
m2 = mv.mean()
v2 = mv.unweighted_sample_variance()
f2 = v2 / m2
assert m1 == pytest.approx(m2, abs=eps)
assert v1 == pytest.approx(v2, abs=eps)
assert f1 == pytest.approx(f2, abs=eps)
def random_weight_averaged_map_coefficients(self,
random_scale,
random_seed,
n_cycles,
fraction_keep,
missing=None):
assert self.map_coefficients_data.size() == \
self.r.size() == \
self.so.size()
mc_data = maptbx.fem_averaging_loop(
map_coefficients=self.map_coefficients_data,
r_factors=self.r,
sigma_over_f_obs=self.so,
random_scale=random_scale,
random_seed=random_seed,
n_cycles=n_cycles)
mc_wa = self.map_coefficients.customized_copy(data=mc_data)
if (missing is not None):
mc_wa = mc_wa.complete_with(other=missing, scale=True)
return mc_wa.select(flex.random_bool(mc_wa.size(), fraction_keep))
def randomize_completeness2(map_coeffs, crystal_gridding, n_cycles=10):
map_filter = None
from mmtbx.maps import fem
for cycle in xrange(n_cycles):
if(cycle>0):
mc = map_coeffs.select(flex.random_bool(map_coeffs.size(), 0.99))
else:
mc = map_coeffs.deep_copy()
m = get_map(map_coeffs=mc, crystal_gridding=crystal_gridding)
maptbx.reset(
data=m,
substitute_value=0.0,
less_than_threshold=0.5,
greater_than_threshold=-9999,
use_and=True)
m = maptbx.volume_scale(map = m, n_bins = 10000).map_data()
if(map_filter is not None):
map_filter = maptbx.volume_scale(map = map_filter, n_bins = 10000).map_data()
map_filter = fem.intersection(m1=map_filter, m2=m,
thresholds=flex.double([i/100. for i in range(0,55,5)]),
average=True)
#map_filter = map_filter.set_selected(map_filter< 0.5, 0)
#map_filter = map_filter.set_selected(map_filter>=0.5, 1)
return map_filter
def tst_masked_mean_filter(self):
from dials.algorithms.image.filter import mean_filter
from scitbx.array_family import flex
from random import randint
# Create an image
image = flex.random_double(2000 * 2000)
image.reshape(flex.grid(2000, 2000))
mask = flex.random_bool(2000 * 2000, 0.99).as_int()
mask.reshape(flex.grid(2000, 2000))
# Calculate the summed area table
mask2 = mask.deep_copy()
mean = mean_filter(image, mask2, (3, 3), 1)
# For a selection of random points, ensure that the value is the
# sum of the area under the kernel
eps = 1e-7
for i in range(10000):
i = randint(10, 1990)
j = randint(10, 1990)
m1 = mean[j,i]
p = image[j-3:j+4,i-3:i+4]
m = mask[j-3:j+4,i-3:i+4]
if mask[j,i] == 0:
m2 = 0.0
else:
p = flex.select(p, flags=m)
mv = flex.mean_and_variance(flex.double(p))
m2 = mv.mean()
s1 = flex.sum(flex.double(p))
s2 = flex.sum(m.as_1d())
assert(abs(m1 - m2) <= eps)
# Test passed
print 'OK'
def recreate_r_free_flags(fmodel): rc = random.choice([0.05, 0.9]) r_free_flags = flex.random_bool(fmodel.f_obs().indices().size(), rc) fmodel._r_free_flags._data = r_free_flags return fmodel
def recreate_r_free_flags(fmodel):
rc = random.choice([0.05, 0.9])
r_free_flags = flex.random_bool(fmodel.f_obs().indices().size(), rc)
fmodel._r_free_flags._data = r_free_flags
return fmodel
def example():
# Read in PDB file and get xray_structure object
xray_structure = iotbx.pdb.input(
file_name = "model.pdb").xray_structure_simple()
# compute Fcalc from atoms
f_calc_1 = xray_structure.structure_factors(d_min=1.5).f_calc()
print f_calc_1.indices().size()
# create mtz file
mtz_dataset = f_calc_1.as_mtz_dataset(column_root_label="FC1")
#
sel = flex.random_bool(f_calc_1.data().size(), 0.5)
f_calc_2 = f_calc_1.select(sel)
print f_calc_2.data().size()
# add data to mtz
mtz_dataset.add_miller_array(
miller_array=f_calc_2,
column_root_label="FC2")
# write out mtz
mtz_object = mtz_dataset.mtz_object()
mtz_object.write(file_name = "data.mtz")
#
f1, f2 = f_calc_1.common_sets(f_calc_2)
print f1.data().size(), f2.data().size()
#
n = flex.sum( flex.abs( flex.abs(f1.data())-flex.abs(f2.data()) ) )
d = flex.sum(flex.abs(f2.data()))
print n/d
#
miller_arrays = reflection_file_reader.any_reflection_file(
file_name = "data.mtz").as_miller_arrays()
ma_2 = None
for ma in miller_arrays:
if(ma.info().labels == ['FC2', 'PHIFC2']):
ma_2 = ma
break
print ma_2
#
f_obs1 = abs(f_calc_2)
f_obs2 = f_calc_2.customized_copy(data = flex.abs(f_calc_2.data()))
assert approx_equal(f_obs1.data(), f_obs2.data())
fmodel = mmtbx.f_model.manager(
f_obs = f_obs1,
r_free_flags = f_obs1.generate_r_free_flags(),
xray_structure = xray_structure)
print dir(fmodel)
print fmodel.r_work()
print fmodel.r_free()
#
x = flex.double([1,2,3,4])
print x
print list(x)
#
fft_map = f_calc_1.fft_map(resolution_factor = 0.3)
fft_map.apply_sigma_scaling()
m1 = fft_map.real_map_unpadded()
print "m1:", m1
print fft_map.n_real()
#
m2 = m1.deep_copy()
#
m3 = m1+m2
#
map_coefficients = f_calc_1.structure_factors_from_map(map = m3)
#
sf = miller_set.structure_factors_from_map(
map = map_data,
use_scale = True,
anomalous_flag = False,
use_sg = False)
def run():
import iotbx.pdb
xrs_str = """
CRYST1 8.000 8.000 8.000 90.00 90.00 90.00 P 1
HETATM 115 O HOH A 18 4.000 4.000 4.000 1.00 10.00 O
TER
END
"""
pdb_inp = iotbx.pdb.input(source_info=None, lines=xrs_str)
xrs = pdb_inp.xray_structure_simple()
xrs.scattering_type_registry(table = "wk1995")
cntr = 0
for remove_fraction in [0.2, 0.5]:
print
print "remove_fraction:", remove_fraction
for option in ["random", "smallest", "highest"]:
print
print "data incompleteness:",option,"-"*30
#
f_exact = xrs.structure_factors(d_min = 1.0).f_calc()
#
if(option=="highest"):
s = flex.sort_permutation(abs(f_exact).data(), reverse=True)
f_exact = f_exact.select(s)
n_remove = int(s.size()*remove_fraction)
f_poor = f_exact.customized_copy(
data = f_exact.data()[n_remove:],
indices = f_exact.indices()[n_remove:])
elif(option == "smallest"):
s = flex.sort_permutation(abs(f_exact).data(), reverse=True)
f_exact = f_exact.select(s)
n_remove = int(s.size()*remove_fraction)
sz = f_exact.data().size()
f_poor = f_exact.customized_copy(
data = f_exact.data()[:sz-n_remove],
indices = f_exact.indices()[:sz-n_remove])
elif(option == "random"):
s = flex.random_bool(f_exact.data().size(), 1.-remove_fraction)
f_poor = f_exact.select(s)
else: assert 0
#
print "number of all data:", f_exact.data().size()
print "number of incomplete data:", f_poor.data().size()
cc1 = map_cc(map_coeffs_1=f_exact, map_coeffs_2=f_poor)
print "start CC(exact_map, poor_map): ", cc1
#
f_dsf = f_poor.double_step_filtration(
vol_cutoff_plus_percent =0.1,
vol_cutoff_minus_percent=0.1,
complete_set=f_exact)
f_new = f_poor.complete_with(other = f_dsf)
cc2 = map_cc(map_coeffs_1=f_exact, map_coeffs_2=f_new)
print "start CC(exact_map, filled_map)1: ", cc2
#
f_dsf = f_poor.double_step_filtration(
vol_cutoff_plus_percent =0.1,
vol_cutoff_minus_percent=0.1,
complete_set=f_exact,
scale_to=f_exact)
f_new = f_poor.complete_with(other = f_dsf)
cc3 = map_cc(map_coeffs_1=f_exact, map_coeffs_2=f_new)
print "start CC(exact_map, filled_map)2: ", cc3
#
if(option=="highest"):
if(remove_fraction==0.2):
assert cc1<0.92
assert cc2>0.99 and cc3>0.99
cntr += 1
elif(remove_fraction==0.5):
assert cc1<0.16
assert cc2>0.76
assert cc3>0.99
cntr += 1
#
assert cntr == 2 # make sure it's gone through all if statements
def run():
import iotbx.pdb
xrs_str = """
CRYST1 8.000 8.000 8.000 90.00 90.00 90.00 P 1
HETATM 115 O HOH A 18 4.000 4.000 4.000 1.00 10.00 O
TER
END
"""
pdb_inp = iotbx.pdb.input(source_info=None, lines=xrs_str)
xrs = pdb_inp.xray_structure_simple()
xrs.scattering_type_registry(table="wk1995")
cntr = 0
for remove_fraction in [0.2, 0.5]:
print
print "remove_fraction:", remove_fraction
for option in ["random", "smallest", "highest"]:
print
print "data incompleteness:", option, "-" * 30
#
f_exact = xrs.structure_factors(d_min=1.0).f_calc()
#
if (option == "highest"):
s = flex.sort_permutation(abs(f_exact).data(), reverse=True)
f_exact = f_exact.select(s)
n_remove = int(s.size() * remove_fraction)
f_poor = f_exact.customized_copy(
data=f_exact.data()[n_remove:],
indices=f_exact.indices()[n_remove:])
elif (option == "smallest"):
s = flex.sort_permutation(abs(f_exact).data(), reverse=True)
f_exact = f_exact.select(s)
n_remove = int(s.size() * remove_fraction)
sz = f_exact.data().size()
f_poor = f_exact.customized_copy(
data=f_exact.data()[:sz - n_remove],
indices=f_exact.indices()[:sz - n_remove])
elif (option == "random"):
s = flex.random_bool(f_exact.data().size(),
1. - remove_fraction)
f_poor = f_exact.select(s)
else:
assert 0
#
print "number of all data:", f_exact.data().size()
print "number of incomplete data:", f_poor.data().size()
cc1 = map_cc(map_coeffs_1=f_exact, map_coeffs_2=f_poor)
print "start CC(exact_map, poor_map): ", cc1
#
f_dsf = f_poor.double_step_filtration(vol_cutoff_plus_percent=0.1,
vol_cutoff_minus_percent=0.1,
complete_set=f_exact)
f_new = f_poor.complete_with(other=f_dsf)
cc2 = map_cc(map_coeffs_1=f_exact, map_coeffs_2=f_new)
print "start CC(exact_map, filled_map)1: ", cc2
#
f_dsf = f_poor.double_step_filtration(vol_cutoff_plus_percent=0.1,
vol_cutoff_minus_percent=0.1,
complete_set=f_exact,
scale_to=f_exact)
f_new = f_poor.complete_with(other=f_dsf)
cc3 = map_cc(map_coeffs_1=f_exact, map_coeffs_2=f_new)
print "start CC(exact_map, filled_map)2: ", cc3
#
if (option == "highest"):
if (remove_fraction == 0.2):
assert cc1 < 0.92
assert cc2 > 0.99 and cc3 > 0.99
cntr += 1
elif (remove_fraction == 0.5):
assert cc1 < 0.16
assert cc2 > 0.76
assert cc3 > 0.99
cntr += 1
#
assert cntr == 2 # make sure it's gone through all if statements
def generate_mask(xsize, ysize):
from scitbx.array_family import flex
mask = flex.random_bool(xsize * ysize, 0.9)
mask.reshape(flex.grid(ysize, xsize))
return mask
def example():
# Read in PDB file and get xray_structure object
xray_structure = iotbx.pdb.input(
file_name="model.pdb").xray_structure_simple()
# compute Fcalc from atoms
f_calc_1 = xray_structure.structure_factors(d_min=1.5).f_calc()
print f_calc_1.indices().size()
# create mtz file
mtz_dataset = f_calc_1.as_mtz_dataset(column_root_label="FC1")
#
sel = flex.random_bool(f_calc_1.data().size(), 0.5)
f_calc_2 = f_calc_1.select(sel)
print f_calc_2.data().size()
# add data to mtz
mtz_dataset.add_miller_array(miller_array=f_calc_2,
column_root_label="FC2")
# write out mtz
mtz_object = mtz_dataset.mtz_object()
mtz_object.write(file_name="data.mtz")
#
f1, f2 = f_calc_1.common_sets(f_calc_2)
print f1.data().size(), f2.data().size()
#
n = flex.sum(flex.abs(flex.abs(f1.data()) - flex.abs(f2.data())))
d = flex.sum(flex.abs(f2.data()))
print n / d
#
miller_arrays = reflection_file_reader.any_reflection_file(
file_name="data.mtz").as_miller_arrays()
ma_2 = None
for ma in miller_arrays:
if (ma.info().labels == ['FC2', 'PHIFC2']):
ma_2 = ma
break
print ma_2
#
f_obs1 = abs(f_calc_2)
f_obs2 = f_calc_2.customized_copy(data=flex.abs(f_calc_2.data()))
assert approx_equal(f_obs1.data(), f_obs2.data())
fmodel = mmtbx.f_model.manager(f_obs=f_obs1,
r_free_flags=f_obs1.generate_r_free_flags(),
xray_structure=xray_structure)
print dir(fmodel)
print fmodel.r_work()
print fmodel.r_free()
#
x = flex.double([1, 2, 3, 4])
print x
print list(x)
#
fft_map = f_calc_1.fft_map(resolution_factor=0.3)
fft_map.apply_sigma_scaling()
m1 = fft_map.real_map_unpadded()
print "m1:", m1
print fft_map.n_real()
#
m2 = m1.deep_copy()
#
m3 = m1 + m2
#
map_coefficients = f_calc_1.structure_factors_from_map(map=m3)
#
sf = miller_set.structure_factors_from_map(map=map_data,
use_scale=True,
anomalous_flag=False,
use_sg=False)
def generate_mask(self, xsize, ysize): from scitbx.array_family import flex mask = flex.random_bool(xsize * ysize, 0.9) mask.reshape(flex.grid(ysize, xsize)) return mask
