def get_raw_data(self, index=0):
import numpy
self.tag = self._images[index]
if self._raw_data is None:
from scitbx.array_family import flex
if self.RECONST_MODE:
return flex.int(self.reconst_image())
else:
print "get_raw_data(%d) for %s" % (index, self.tag)
data = self._h5_handle[self.tag]["data"][()].astype(numpy.int32)
# [()] forces conversion to ndarray
# this is 8192x512 (slow/fast) tiled image
self._raw_data = []
for i in range(8):
xmin, ymin, xmax, ymax = 0, i * 1024, 512, (i + 1) * 1024
# To avoid "numpy.ndarray instance is not contiguous"
# TODO: Is this the right way?
source = numpy.ascontiguousarray(data[ymin:ymax,xmin:xmax])
self._raw_data.append(flex.int(source))
if index is not None:
return self._raw_data[index]
return self._raw_data[0]
def run(args):
assert args in [[], ["--forever"]]
verbose = True
while True:
if (flex.int().as_numpy_array(optional=True) is None):
try:
flex.int().as_numpy_array()
except RuntimeError, e:
assert not show_diff(str(e), "numpy API not available")
else:
raise Exception_expected
else:
for flex_type in [
flex.bool,
flex.int,
flex.long,
flex.float,
flex.double,
flex.complex_double,
flex.size_t]:
exercise_basic(flex_type, verbose)
exercise_int()
if (len(args) == 0):
break
verbose = False
def __call__(self, predictions, hkllist, pxlsz):
if len(self.IT) == 4:
# We have only one tile, AnnAdaptor chokes in this case but then there is
# only one choice of nearest neighbour anyway!
nearest_neighbours = flex.int(len(predictions) * self.NEAR, 0)
else:
query = flex.double()
for pred in predictions:
query.append(pred[0] / pxlsz)
query.append(pred[1] / pxlsz)
self.adapt.query(query)
assert len(self.adapt.nn) == len(predictions) * self.NEAR
nearest_neighbours = self.adapt.nn
selection = flex.bool()
self.tile_id = flex.int()
for p in xrange(len(predictions)):
is_in_active_area = False
for n in xrange(self.NEAR):
itile = nearest_neighbours[p * self.NEAR + n]
if (
self.IT[4 * itile] < predictions[p][0] / pxlsz < self.IT[4 * itile + 2]
and self.IT[4 * itile + 1] < predictions[p][1] / pxlsz < self.IT[4 * itile + 3]
):
is_in_active_area = True
break
if is_in_active_area:
self.tile_id.append(itile)
selection.append(is_in_active_area)
assert selection.count(True) == len(self.tile_id)
return predictions.select(selection), hkllist.select(selection)
def assign_random_r_free_flags (n_refl, fraction_free, format="cns") :
assert (fraction_free > 0) and (fraction_free < 0.5)
from scitbx.array_family import flex
from libtbx.math_utils import iround
group_size = 1/(fraction_free)
assert group_size >= 2
if (format == "cns") or (format == "shelx") :
result = flex.bool(n_refl, False)
i_start = 0
for i_group in count(1):
i_end = min(n_refl, iround(i_group*group_size) )
if (i_start == i_end):
break
if (i_end + 1 == n_refl):
i_end += 1
assert i_end - i_start >= 2
result[random.randrange(i_start, i_end)] = True
i_start = i_end
if (format == "shelx") :
result_ = flex.int(n_refl, 1)
result_.set_selected(result, -1)
result = result_
elif (format == "ccp4") :
result = flex.int()
flag_max = iround(group_size) - 1
for i in range(n_refl) :
result.append(random.randint(0, flag_max))
return result
def exercise_int():
fa = flex.int_range(1,7)
na = fa.as_numpy_array()
assert na.tolist() == list(fa)
fna = flex.int(na)
assert fna.all() == (6,)
assert fna.origin() == (0,)
assert fna.focus() == (6,)
assert fna.all_eq(fa)
fa[0] = 99
assert na[0] == 1
fa[0] = 1
#
fa.reshape(flex.grid(2,3))
na = fa.as_numpy_array()
assert na.tolist() == [[1, 2, 3], [4, 5, 6]]
fna = flex.int(na)
assert fna.all() == (2,3)
assert fna.origin() == (0,0)
assert fna.focus() == (2,3)
assert fna.all_eq(fa)
#
fa = flex.int_range(4*2*3) + 1
fa.reshape(flex.grid(4,2,3))
na = fa.as_numpy_array()
assert na.tolist() == [
[[1, 2, 3], [4, 5, 6]],
[[7, 8, 9], [10, 11, 12]],
[[13, 14, 15], [16, 17, 18]],
[[19, 20, 21], [22, 23, 24]]]
fna = flex.int(na)
assert fna.all() == (4,2,3)
assert fna.origin() == (0,0,0)
assert fna.focus() == (4,2,3)
assert fna.all_eq(fa)
def convert_detector(raw_data):
# https://confluence.slac.stanford.edu/display/PCDS/CSPad+metrology+and+calibration+files%2C+links
data3d = []
if raw_data.shape == (5920, 388):
raise NotImplementedError("Please contact the authors. This is an older gain map format.")
asic_start = 0
# input dictionary for writing the CBF
tiles = {}
# iterate through and extract the data, converting it to asic-sized chunks
for i_quad in range(4):
asic_size = 185 * 194
section_size = asic_size * 4
quad_start = i_quad * section_size * 4
quad_asics = []
# the data is laid out by quadrants, and then by 4 sections per quadrant, each with 2 sensors,
# each with two asics
for i_2x2 in range(4):
for i_sensor in range(2):
asic_end = asic_start + 185
a = raw_data[asic_start:asic_end, :]
asic_start = asic_end
tiles[(0, i_quad, (i_2x2 * 2) + i_sensor, 0)] = flex.int(a.astype(numpy.int32))
asic_end = asic_start + 185
b = raw_data[asic_start:asic_end, :]
asic_start = asic_end
tiles[(0, i_quad, (i_2x2 * 2) + i_sensor, 1)] = flex.int(b.astype(numpy.int32))
return tiles
def selections_as_ints (self) : assert 0, "Anybody using this?" sec_str = flex.int(self.n_atoms, 0) all_alpha = flex.int(self.n_atoms, 1) all_beta = flex.int(self.n_atoms, 2) helices = self.alpha_selection() sheets = self.beta_selection() sec_str.set_selected(helices, all_alpha.select(helices)) sec_str.set_selected(sheets, all_beta.select(sheets)) return sec_str
def distl_filter(self,
address,
cspad_img,
distance,
timestamp,
wavelength):
self.hitfinder_d["DATA"] = cspad_img
self.hitfinder_d["DISTANCE"] = distance
self.hitfinder_d["TIMESTAMP"] = timestamp
self.hitfinder_d["WAVELENGTH"] = wavelength
self.hitfinder_d["DETECTOR_ADDRESS"] = address
args = ["indexing.data=dummy",
"distl.bins.verbose=False",
self.asic_filter,
]
detector_format_version = detector_format_function(
address, reverse_timestamp(timestamp)[0])
args += ["distl.detector_format_version=%s" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = self.hitfinder_d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
from rstbx.new_horizons.index import pre_indexing_validation,pack_names
pre_indexing_validation(horizons_phil)
imagefile_arguments = pack_names(horizons_phil)
from spotfinder.applications import signal_strength
info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments)
imgdata = info.Files.images[0].linearintdata
active_data = self.get_active_data(info.Files.images[0],horizons_phil)
peak_heights = flex.int( [
imgdata[ spot.max_pxl_x(), spot.max_pxl_y() ]
for spot in info.S.images[info.frames[0]]["spots_total"]
])
outscale = 256
corrected = peak_heights.as_double() * self.correction
outvalue = outscale *(1.0-corrected)
outvalue.set_selected(outvalue<0.0,0.)
outvalue.set_selected(outvalue>=outscale,int(outscale)-1)
outvalue = flex.int(outvalue.as_numpy_array().astype(numpy.int32))
# essentially, select a peak if the peak's ADU value is > 2.5 * the 90-percentile pixel value
#work = display_spots.wrapper_of_callback(info)
#work.display_with_callback(horizons_phil.indexing.data)
return peak_heights,outvalue
def set_up_hitfinder(self):
# See r17537 of mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
img_dim = (1765, 1765)
pixel_size = cspad_tbx.pixel_size
elif device == 'marccd':
img_dim = (4500, 4500)
pixel_size = 0.079346
elif device == 'Rayonix':
img_dim = rayonix_tbx.get_rayonix_detector_dimensions(self.bin_size)
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
else:
raise RuntimeError("Unsupported device %s" % self.address)
if self.beam_center is None:
self.beam_center = [0,0]
self.hitfinder_d = cspad_tbx.dpack(
active_areas=self.active_areas,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=flex.int(flex.grid(img_dim[0], img_dim[1]), 0),
xtal_target=self.m_xtal_target)
if device == 'Cspad':
# Figure out which ASIC:s are on the central four sensors. This
# only applies to the CSPAD.
assert len(self.active_areas) % 4 == 0
distances = flex.double()
for i in range(0, len(self.active_areas), 4):
cenasic = ((self.active_areas[i + 0] + self.active_areas[i + 2]) / 2,
(self.active_areas[i + 1] + self.active_areas[i + 3]) / 2)
distances.append(math.hypot(cenasic[0] - self.beam_center[0],
cenasic[1] - self.beam_center[1]))
orders = flex.sort_permutation(distances)
# Use the central 8 ASIC:s (central 4 sensors).
flags = flex.int(len(self.active_areas) // 4, 0)
for i in range(8):
flags[orders[i]] = 1
self.asic_filter = "distl.tile_flags=" + ",".join(
["%1d" % b for b in flags])
elif device == 'marccd':
# There is only one active area for the MAR CCD, so use it.
self.asic_filter = "distl.tile_flags=1"
elif device == 'Rayonix':
# There is only one active area for the Rayonix, so use it.
self.asic_filter = "distl.tile_flags=1"
def tst_labels_are_valid(self, data_list, mask_list, coords, labels):
from scitbx.array_family import flex
# Create a map of labels
label_map = flex.int(flex.grid(10, self.size[0], self.size[1]))
for c, l in zip(coords, labels):
assert(c[0] >= 0 and c[0] < 10)
assert(c[1] >= 0 and c[1] < self.size[0])
assert(c[2] >= 0 and c[2] < self.size[1])
label_map[c] = l
# Ensure all labels are correct
vi = 0
for k in range(10):
for j in range(self.size[0]):
for i in range(self.size[1]):
if mask_list[k][j,i]:
l1 = labels[vi]
if k > 0 and mask_list[k-1][j,i]:
l2 = label_map[k-1,j,i]
assert(l2 != l1)
if j > 0 and mask_list[k][j-1,i]:
l2 = label_map[k,j-1,i]
assert(l2 == l1)
if i > 0 and mask_list[k][j,i-1]:
l2 = label_map[k,j,i-1]
assert(l2 == l1)
vi += 1
# Test passed
print 'OK'
def ref_gen_static(experiments):
"""Generate some reflections using the static predictor"""
beam = experiments[0].beam
crystal = experiments[0].crystal
goniometer = experiments[0].goniometer
detector = experiments[0].detector
scan = experiments[0].scan
# All indices to the detector max resolution
dmin = detector.get_max_resolution(beam.get_s0())
index_generator = IndexGenerator(crystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(), dmin)
indices = index_generator.to_array()
# Predict rays within the sweep range
sweep_range = scan.get_oscillation_range(deg=False)
ray_predictor = ScansRayPredictor(experiments, sweep_range)
refs = ray_predictor(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(detector, refs)
refs = refs.select(intersects)
# Make a reflection predictor and re-predict for these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
ref_predictor = ExperimentsPredictor(experiments)
refs['id'] = flex.int(len(refs), 0)
refs = ref_predictor(refs)
return refs
def get_data(self):
"""Get the gain array from the file"""
import numpy
from scitbx.array_family import flex
# Select the first datablock and rewind all the categories
self.cbf_handle.select_datablock(0)
self.cbf_handle.select_category(0)
self.cbf_handle.select_column(2)
self.cbf_handle.select_row(0)
# Check the type of the element to ensure it's a binary
# otherwise raise an exception
type = self.cbf_handle.get_typeofvalue()
if type.find('bnry') > -1:
# Read the image data into an array
image_string = self.cbf_handle.get_integerarray_as_string()
image = flex.int(numpy.fromstring(image_string, numpy.int32))
# Get the array parameters
parameters = self.cbf_handle.get_integerarrayparameters_wdims()
image_size = (parameters[10], parameters[9])
# Resize the image
image.reshape(flex.grid(*image_size))
else:
raise TypeError('Can\'t find image')
# Return the image
return image
def get_symop_correlation_coefficients(miller_array, use_binning=False):
from copy import deepcopy
from scitbx.array_family import flex
from cctbx import miller
corr_coeffs = flex.double()
n_refs = flex.int()
space_group = miller_array.space_group()
for smx in space_group.smx():
reindexed_array = miller_array.change_basis(sgtbx.change_of_basis_op(smx))
intensity, intensity_rdx = reindexed_array.common_sets(miller_array)
if use_binning:
intensity.use_binning_of(miller_array)
intensity_rdx.use_binning_of(miller_array)
cc = intensity.correlation(intensity_rdx, use_binning=use_binning)
corr_coeffs.append(
flex.mean_weighted(
flex.double(i for i in cc.data if i is not None),
flex.double(j for i, j in zip(cc.data, cc.binner.counts()) if i is not None),
)
)
else:
corr_coeffs.append(intensity.correlation(intensity_rdx, use_binning=use_binning).coefficient())
n_refs.append(intensity.size())
return corr_coeffs, n_refs
def scale_down_array_py(image, scale_factor):
'''Scale the data in image in a manner which retains the statistical structure
of the input data. Input data type must be integers; negative values assumed
to be flags of some kind (i.e. similar to Pilatus data) and hence preserved
as input.'''
from scitbx.random import variate, uniform_distribution
from scitbx.array_family import flex
assert (scale_factor <= 1)
assert (scale_factor >= 0)
# construct a random number generator in range [0, 1]
dist = variate(uniform_distribution(0.0, 1.0))
scaled_image = flex.int(len(image), 0)
for j, pixel in enumerate(image):
if pixel < 0:
scaled_image[j] = pixel
else:
for c in range(pixel):
if dist.next() < scale_factor:
scaled_image[j] += 1
return scaled_image
def _filter_maximum_centroid(self, coords, values, spots, cpos):
'''Filter the reflections by the distance between the maximum pixel
value and the centroid position. If the centroid is a greater than the
maximum separation from maximum pixel (in pixel coords) then discard.
Params:
coords The list of coordinates
values The list of values
cpos The list of centroids
Returns:
An index list of valid spots
'''
from scitbx.array_family import flex
from scitbx import matrix
index = []
for si, (s, c) in enumerate(zip(spots, cpos)):
im = flex.max_index(flex.int([values[i] for i in s]))
xc = matrix.col(c)
xm = matrix.col(coords[s[im]])
if (xc - xm).length() <= self._max_separation:
index.append(si)
# Return the list of indices
return index
def compare_fftpack_with_hermft_1d():
mt = flex.mersenne_twister(seed=0)
for n_cmpl in xrange(1, 101):
primes = prime_factors_of(n_cmpl)
if (n_cmpl != 1 and max(primes) > 19): continue
n_real = n_cmpl * 2
m_real = n_real + 2
z = (mt.random_double(size=m_real)*2-1).as_float()
# The imaginary parts of the first and last complex values should
# be zero, but z has random values in these places, to prove that
# they don't change the result.
hermft_xy = z.deep_copy()
hermft_xy[1] = hermft_xy[-2]
# real part of last complex value stored in imaginary part of first
# (see ccp4/doc/libfft.doc)
hermft_xy[-2] = 253
# random value, for consistency check below
d = flex.int((m_real,2,m_real,m_real,m_real))
ccp4io_dev_ext.fftlib_hermft(xy=hermft_xy, n=n_cmpl, d=d)
# consistency check first
assert hermft_xy[-2] == 253
assert hermft_xy[-1] == z[-1]
# reset to zero for comparision with fftpack result further down
hermft_xy[-2] = 0
hermft_xy[-1] = 0
fft = scitbx.fftpack.real_to_complex(n_real)
assert fft.m_real() == m_real
fftpack_xy = z.as_double()
for i in xrange(n_cmpl): fftpack_xy[i*2+1] *= -1 # conjugate
fft.backward(fftpack_xy)
fftpack_xy = fftpack_xy.as_float()
if (flex.max_absolute(hermft_xy-fftpack_xy) > 1e-5):
assert approx_equal(hermft_xy, fftpack_xy, eps=1e-5)
def __init__(self,imgobj,phil,inputpd,verbose=False):
adopt_init_args(self,locals())
self.active_areas = imgobj.get_tile_manager(phil).effective_tiling_as_flex_int()
B = self.active_areas
#figure out which asics are on the central four sensors
assert len(self.active_areas)%4 == 0
# apply an additional margin of 1 pixel, since we don't seem to be
# registering the global margin.
asics = [(B[i]+1,B[i+1]+1,B[i+2]-1,B[i+3]-1) for i in xrange(0,len(B),4)]
from scitbx.matrix import col
centre_mm = col((float(inputpd["xbeam"]),float(inputpd["ybeam"])))
centre = centre_mm / float(inputpd["pixel_size"])
distances = flex.double()
cenasics = flex.vec2_double()
self.corners = []
for iasic in xrange(len(asics)):
cenasic = ((asics[iasic][2] + asics[iasic][0])/2. ,
(asics[iasic][3] + asics[iasic][1])/2. )
cenasics.append(cenasic)
distances.append(math.hypot(cenasic[0]-centre[0], cenasic[1]-centre[1]))
orders = flex.sort_permutation(distances)
self.flags = flex.int(len(asics),0)
#Use the central 8 asics (central 4 sensors)
self.green = []
for i in xrange(32):
#self.green.append( cenasics[orders[i]] )
self.corners.append (asics[orders[i]])
#self.green.append((self.corners[-1][0],self.corners[-1][1]))
self.flags[orders[i]]=1
self.asic_filter = "distl.tile_flags="+",".join(["%1d"%b for b in self.flags])
def __init__(self):
self.ilimits = flex.int()
for line in self.limits.split("\n"):
for ituple in line.split(") ("):
for dint in ituple.split(" "):
self.ilimits.append(
int(dint.replace(")","").replace("(","").replace(",","")))
def _to_array_all(self, panel=0):
""" Get the array from all the sweep elements. """
from scitbx.array_family import flex
# Get the image dimensions
size_z = len(self)
size_y = self.reader().get_image_size(panel)[1]
size_x = self.reader().get_image_size(panel)[0]
# Check sizes are valid
if size_z <= 0 or size_y <= 0 or size_x <= 0:
raise RuntimeError("Invalid dimensions")
# Allocate the array
array = flex.int(flex.grid(size_z, size_y, size_x))
# Loop through all the images and set the image data
for k, image in enumerate(self):
if not isinstance(image, tuple):
image = (image,)
im = image[panel]
im.reshape(flex.grid(1, *im.all()))
array[k : k + 1, :, :] = im
# Return the array
return array
def _to_array_w_range(self, item, panel=0):
""" Get the array from the user specified range. """
from scitbx.array_family import flex
# Get the range from the given index item
z0, z1, y0, y1, x0, x1 = self._get_data_range(item, panel)
# Get the image dimensions
size_z = z1 - z0
size_y = y1 - y0
size_x = x1 - x0
# Check sizes are valid
if size_z <= 0 or size_y <= 0 or size_x <= 0:
raise RuntimeError("Invalid dimensions")
# Allocate the array
array = flex.int(flex.grid(size_z, size_y, size_x))
# Loop through all the images and set the image data
for k, index in enumerate(self.indices()[z0:z1]):
image = self.reader().read(index)
if not isinstance(image, tuple):
image = (image,)
im = image[panel]
im.reshape(flex.grid(1, *im.all()))
array[k : k + 1, :, :] = im[0:1, y0:y1, x0:x1]
# Return the array
return array
def exercise_writer():
from iotbx import csv_utils
x = (1,2,3,4,5)
y = (6,7,8,9,10)
f = open_tmp_file()
field_names = ('x','y')
csv_utils.writer(f, (x,y), field_names=field_names)
f.close()
f = open(f.name, 'r')
content = f.readlines()
text = ['x,y\r\n']
text += ['%s,%s\r\n' %(row[0],row[1]) for row in zip(x,y)]
assert content == text
f.close()
x = (1,2,3,4,5)
y = (6,7,8,9,10)
f = open_tmp_file()
csv_utils.writer(f, (x,y), delimiter=';')
f.close()
f = open(f.name, 'r')
content = f.readlines()
text = ['%s;%s\r\n' %(row[0],row[1]) for row in zip(x,y)]
assert content == text
x = flex.int(x)
y = flex.int(y)
f = open_tmp_file()
csv_utils.writer(f, (x,y), field_names=field_names)
f.close()
f = open(f.name, 'r')
content = f.readlines()
text = ['x,y\r\n']
text += ['%s,%s\r\n' %(row[0],row[1]) for row in zip(x,y)]
assert content == text
f.close()
y.append(11)
f = open_tmp_file()
try:
csv_utils.writer(f, (x,y), field_names=field_names)
except AssertionError:
pass
else:
raise Exception_expected
f.close()
def cspad_unbound_pixel_mask():
# Every 10th pixel along the diagonal from the top left hand corner are not
# bonded, hence we ignore them in the summed spectrum
mask = flex.int(flex.grid(370, 391), 0)
for section_offset in ((0,0), (0, 197), (185, 197), (185, 0)):
for i in range(19):
mask[section_offset[0] + i * 10, section_offset[1] + i * 10] = 1
return mask
def merge_counts(images):
from scitbx.array_family import flex
image = flex.int(flex.grid(images[0].focus()), 0)
negative = images[0].as_1d() < 0
for i in images:
image += i
image.as_1d().set_selected(negative, images[0].as_1d())
return image
def slice_callback_with_object_data(self): self.object.read() start_index = 2463*((195+17)*self.sliceindex) stop_index = start_index + 2463*195 linearintdata = flex.int(self.object.linearintdata[start_index:stop_index]) linearintdata.reshape(flex.grid((195,2463))) del self.object #once the data are copied, no need to keep the original return linearintdata
def cspad2x2_bad_pixel_mask_cxi_run7():
bad_pixels = set((
(279,289), (277,254), (273,295), (268,247), (269,260), (281,383), (268,57),
(262,184), (291,206), (275,106), (261,28), (286,189), (265,304), (284,277)))
mask = flex.int(flex.grid(370, 391), 0)
for bad_pixel in bad_pixels:
mask[bad_pixel] = 1
return mask
def compare_fftpack_with_cmplft_3d():
mt = flex.mersenne_twister(seed=0)
for nx,ny,nz in [(30,20,40), (7,19,13), (5,11,4)]:
z = (mt.random_double(size=2*nx*ny*nz)*2-1).as_float()
cmplft_xy = z.deep_copy()
d = flex.int([2*nx*ny*nz, 2*nx*ny, 2*nx*ny*nz, 2*nx*ny, 2])
ccp4io_dev_ext.fftlib_cmplft(xy=cmplft_xy, n=nz, d=d)
d = flex.int([2*nx*ny*nz, 2*nx, 2*nx*ny, 2*nx, 2])
ccp4io_dev_ext.fftlib_cmplft(xy=cmplft_xy, n=ny, d=d)
d = flex.int([2*nx*ny*nz, 2, 2*nx*ny*nz, 2*nx*ny*nz, 2*nx])
ccp4io_dev_ext.fftlib_cmplft(xy=cmplft_xy, n=nx, d=d)
fft = scitbx.fftpack.complex_to_complex_3d((nz,ny,nx))
fftpack_xy = z.as_double()
fftpack_xy.reshape(flex.grid(nz,ny,2*nx))
fft.forward(fftpack_xy)
fftpack_xy = fftpack_xy.as_float().as_1d()
if (flex.max_absolute(cmplft_xy-fftpack_xy) > 1e-4):
assert approx_equal(cmplft_xy, fftpack_xy, eps=1e-4)
def run_fast(args):
islow = args[0]
result_val = flex.int(388)
for jfast in xrange(388):
histo1 = histograms[islow*388+jfast,:]
nphotons = fit_3_gaussian.test_fit(histo1,plot=False)
print islow*388+jfast, "of %d, # photons= %d"%(len(histograms),nphotons)
result_val[jfast]=nphotons
return {islow:result_val}
def save_hit(self):
#self.set_ssx()
self.result_folder = self.options['output_directory']
self.num = self.options['num']
if self.options['roi'].lower() is not 'none':
if 'eiger' in self.options['detector'].lower() and 'h5' in self.options['file_extension']:
self.data = self.h5[self.group][self.index,::]
self.data[self.data >= self.ovl] = 0
else: self.data = self.img.data
# Conversion to edf
if 'edf' in self.options['output_formats']:
OutputFileName = os.path.join(self.result_folder, 'EDF_%s' % self.num.zfill(3), "%s.edf" % self.root)
edfout = fabio.edfimage.edfimage(data=self.data.astype(np.float32))
edfout.write(OutputFileName)
if 'cbf' in self.options['output_formats']:
OutputFileName = os.path.join(self.result_folder, 'CBF_%s' % self.num.zfill(3), "%s.cbf" % self.root)
cbfout = fabio.cbfimage.cbfimage(data=self.data.astype(np.float32))
cbfout.write(OutputFileName)
# Conversion to H5
if 'hdf5' in self.options['output_formats']:
OutputFileName = os.path.join(self.result_folder,
'HDF5_%s_%s' % (self.options['filename_root'], self.num.zfill(3)),
"%s.h5" % self.root)
OutputFile = h5py.File(OutputFileName, 'w')
OutputFile.create_dataset("data", data=self.data, compression="gzip", dtype=self.type)
if self.options['bragg_search']:
OutputFile.create_dataset("processing/hitfinder/peakinfo", data=self.peaks.astype(np.int))
OutputFile.close()
# Conversion to Pickle
if cctbx and 'pickles' in self.options['output_formats']:
# def get_ovl(det):
if 'pilatus' in self.detector.name.lower(): ovl = 1048500
if 'eiger' in self.detector.name.lower(): ovl = self.ovl
pixels = flex.int(self.data.astype(np.int32))
pixel_size = self.detector.pixel1
data = dpack(data=pixels,
distance=self.options['distance'],
pixel_size=pixel_size,
wavelength=self.options['wavelength'],
beam_center_x=self.options['beam_y'] * pixel_size,
beam_center_y=self.options['beam_x'] * pixel_size,
ccd_image_saturation= ovl,
saturated_value= ovl)
#data = crop_image_pickle(data)
OutputFileName = os.path.join(self.result_folder,
'PICKLES_%s_%s' %(self.options['filename_root'],
self.num.zfill(3)),
"%s.pickle" % self.root)
easy_pickle.dump(OutputFileName, data)
def exercise_debug_write():
#---------Write out a test file
D = DetectorImageBase('no_file')
D.parameters = {'SIZE1':filesize,
'SIZE2':filesize,
'PIXEL_SIZE':0.1,
'DISTANCE':100.0,
'TWOTHETA':0.0,
'OSC_START':0.0,
'OSC_RANGE':1.0,
'WAVELENGTH':1.0,
'BEAM_CENTER_X':12.5,
'BEAM_CENTER_Y':12.5,
}
def getEndian():
return 0
D.getEndian = getEndian
sindata = flex.int()
for x in xrange(filesize):
sindata.append(min(255,abs(int(256*sinfunc(x)))))
moddata = flex.int()
accum = 0
for x in xrange(filesize):
for y in xrange(filesize):
value = sindata[x]*sindata[y]
moddata.append(value)
accum+=value
D.debug_write(filename,mod_data=moddata)
#---------Read back the test file
a = adsc.ADSCImage(filename)
a.read()
assert a.size1 == filesize
assert a.size2 == filesize
assert a.npixels == filesize*filesize
assert approx_equal(a.pixel_size, 0.1)
assert approx_equal(a.osc_start, 0)
checkaccum = 0
for x in xrange(a.npixels):
if moddata[x]!=a.linearintdata[x]:
print x,moddata[x],a.linearintdata[x]
checkaccum+=a.linearintdata[x]
assert accum == checkaccum
os.remove(filename)
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_datablocks
from dials.util.options import flatten_experiments
from dials.util.options import flatten_reflections
import libtbx.load_env
usage = "%s [options] datablock.json reflections.pickle" %(
libtbx.env.dispatcher_name)
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_datablocks=True,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if (len(datablocks) == 0 and len(experiments) == 0) or len(reflections) == 0:
parser.print_help()
exit(0)
if len(datablocks) == 0 and len(experiments) > 0:
imagesets = experiments.imagesets()
else:
imagesets = []
for datablock in datablocks:
imagesets.extend(datablock.extract_imagesets())
if len(reflections) > 1:
assert len(reflections) == len(imagesets)
from scitbx.array_family import flex
for i in range(len(reflections)):
reflections[i]['imageset_id'] = flex.int(len(reflections[i]), i)
if i > 0:
reflections[0].extend(reflections[i])
reflections = reflections[0]
import wxtbx.app
a = wxtbx.app.CCTBXApp(0)
a.settings = params
f = ReciprocalLatticeViewer(
None, -1, "Reflection data viewer", size=(1024,768))
f.load_models(imagesets, reflections)
f.Show()
a.SetTopWindow(f)
#a.Bind(wx.EVT_WINDOW_DESTROY, lambda evt: tb_icon.Destroy(), f)
a.MainLoop()
def test_compress_decompress():
x, y = 10, 10
data = flex.int(x * y, 1)
data[10] = 44369
data[11] = 214
compressed = compress(data)
uncompressed = uncompress(compressed, x, y)
assert list(data) == list(uncompressed)
def tst_no_mask(self):
from scitbx.array_family import flex
from dials.algorithms.background import NormalDiscriminator
discriminate = NormalDiscriminator(min_data=10)
shoebox_d = flex.random_double(5 * 5 * 5) * 100
shoebox = flex.int([int(s) for s in shoebox_d])
shoebox.reshape(flex.grid((5, 5, 5)))
mask = discriminate(shoebox)
self.is_correct(shoebox, mask, 3.0, 10)
print 'OK'
def _minimize_divide_clustering(self):
assert self.params.cluster.n_clusters in (2, Auto)
x = self.coords_reduced[:, :1].as_1d()
y = self.coords_reduced[:, 1:2].as_1d()
from cctbx.merging.brehm_diederichs import minimize_divide
selection = minimize_divide(x, y).plus_minus()
cluster_labels = flex.int(x.size(), 0)
cluster_labels.set_selected(selection, 1)
return cluster_labels
def test_image_buffer():
data = flex.int(flex.grid(10, 10))
name = "TileName0"
tile0 = dxtbx.format.image.ImageTileInt(data, name)
image = dxtbx.format.image.ImageInt(tile0)
b = dxtbx.format.image.ImageBuffer(image)
assert b.is_int() is True
assert b.is_double() is False
assert b.is_empty() is False
def __init__(self):
self.ilimits = flex.int()
for line in self.limits.split("\n"):
for ituple in line.split(") ("):
for dint in ituple.split(" "):
self.ilimits.append(
int(
dint.replace(")", "").replace("(",
"").replace(",",
"")))
def fake_it(self, size):
selection_set = self.non_para_bootstrap.draw(size)
isel = flex.int()
for element in selection_set:
isel.append(int(element))
new_x = flex.double(flex.select(self.x_data, isel))
new_y = flex.double(flex.select(self.y_data, isel))
return new_x, new_y
def get_channcalc(params):
log_dir = os.path.basename(os.path.abspath("."))
log_file = os.path.join("..","%s%s.out"%(params.prefix,log_dir))
channcalc=flex.double()
channrank=flex.int()
sbcalc = flex.double()
sbrank = flex.int()
with open(log_file,"r") as F:
lines = F.read().strip().split("\n")
for line in lines:
if "finished with the calculation" in line:
tokens = line.split()
channcalc.append(float(tokens[1]))
channrank.append(int(tokens[0]))
if "finished with single" in line:
tokens = line.split()
sbcalc.append(float(tokens[1]))
sbrank.append(int(tokens[0]))
return channcalc, channrank, sbcalc, sbrank
def run_fast(args):
islow = args[0]
result_val = flex.int(388)
for jfast in range(388):
histo1 = histograms[islow * 388 + jfast, :]
nphotons = fit_3_gaussian.test_fit(histo1, plot=False)
print islow * 388 + jfast, "of %d, # photons= %d" % (
len(histograms), nphotons)
result_val[jfast] = nphotons
return {islow: result_val}
def crop_image_pickle(
data, preserve_active_areas_even_though_cropping_would_invalidate_them=False
):
"""
Given an image pickle dictionary, crop the pixels such that the beam center is as close
as possile to the image center. Then adjust SIZE1/SIZE2, ACTIVE_AREAS and the beam
center accordingly.
@param data The image dictionary of interest
"""
# only one active area is allowed, and it should be the size of the image.
from scitbx.array_family import flex
if preserve_active_areas_even_though_cropping_would_invalidate_them is False:
test = flex.int([0, 0, data["SIZE1"], data["SIZE2"]]) == data["ACTIVE_AREAS"]
assert test.all_eq(True)
# retrieve parameters from the dictionary
pixel_size = data["PIXEL_SIZE"]
beam_x = int(round(data["BEAM_CENTER_X"] / pixel_size))
beam_y = int(round(data["BEAM_CENTER_Y"] / pixel_size))
width = data["SIZE1"]
height = data["SIZE2"]
pixels = data["DATA"]
# the new image will be twice the size as the smallest distance between the
# beam center and one of the edges of the images
new_half_size = min([beam_x, width - beam_x, beam_y, height - beam_y])
new_size = new_half_size * 2
min_x = beam_x - new_half_size
min_y = beam_y - new_half_size
pixels = pixels[min_y : min_y + new_size, min_x : min_x + new_size]
assert pixels.focus()[0] == pixels.focus()[1]
# save the results
data["DATA"] = pixels
data["SIZE1"] = new_size
data["SIZE2"] = new_size
data["BEAM_CENTER_X"] -= min_x * pixel_size
data["BEAM_CENTER_Y"] -= min_y * pixel_size
if preserve_active_areas_even_though_cropping_would_invalidate_them is False:
data["ACTIVE_AREAS"] = flex.int([0, 0, new_size, new_size])
return data
def test_basics():
with pytest.raises(ValueError):
s = flumpy.Scuffer(1)
memoryview(s)
i = flex.int(10)
d = flex.double(10)
flumpy.Scuffer(d)
flumpy.Scuffer(i)
def run(self, experiment, table):
# first set ids to zero so can integrate (this is how integration
# finds the image in the imageset)
ids_map = dict(table.experiment_identifiers())
table["id"] = flex.int(table.size(), 0)
del table.experiment_identifiers()[list(ids_map.keys())[0]]
table.experiment_identifiers()[0] = list(ids_map.values())[0]
fix_list = []
if self.params.profile.ellipsoid.unit_cell.fixed:
fix_list.append("unit_cell")
if self.params.profile.ellipsoid.orientation.fixed:
fix_list.append("orientation")
self.collector.initial_collect(experiment, table)
for _ in range(
self.params.profile.ellipsoid.refinement.n_macro_cycles):
try:
table, sigma_d = self.preprocess(experiment, table,
self.params)
self.collector.collect_after_preprocess(experiment, table)
except ToFewReflections as e:
raise RuntimeError(e)
else:
experiment, table, refiner_output = self.refine(
experiment,
table,
sigma_d,
profile_model=self.params.profile.ellipsoid.rlp_mosaicity.
model,
fix_list=fix_list,
n_cycles=self.params.profile.ellipsoid.refinement.n_cycles,
capture_progress=isinstance(self.collector,
EllipsoidOutputCollector),
)
self.collector.collect_after_refinement(
experiment, table,
refiner_output["refiner_output"]["history"])
predicted = self.predict(
experiment,
table,
d_min=self.params.prediction.d_min,
prediction_probability=self.params.profile.ellipsoid.prediction.
probability,
)
# do we want to add unmatched i.e. strong spots which weren't predicted?
self.collector.collect_after_prediction(predicted, table)
predicted = self.integrate(experiment, predicted, sigma_d)
self.collector.collect_after_integration(experiment, predicted)
return experiment, predicted, self.collector
def __init__(self,
current_x=None,
parameterization=None,
refinery=None,
ISIGI=None,
indices=None,
bins=None,
out=None,
min_iterations=0,
max_calls=1000,
max_drop_eps=1.e-10,
show_finite_differences=False):
adopt_init_args(self, locals())
self.n = current_x.size()
self.x = current_x
if False:
self.diag_mode = "always"
from scitbx import lbfgs
self.minimizer = lbfgs.run(
target_evaluator=self,
termination_params=lbfgs.termination_parameters(
traditional_convergence_test=False,
drop_convergence_test_max_drop_eps=max_drop_eps,
min_iterations=min_iterations,
max_iterations=None,
max_calls=max_calls),
exception_handling_params=lbfgs.exception_handling_parameters(
ignore_line_search_failed_rounding_errors=True,
ignore_line_search_failed_step_at_lower_bound=
True, #the only change from default
ignore_line_search_failed_step_at_upper_bound=False,
ignore_line_search_failed_maxfev=False,
ignore_line_search_failed_xtol=False,
ignore_search_direction_not_descent=False))
else:
from scitbx import lbfgsb
l = flex.double(self.n, 1e-8)
if len(l) > 3:
for p in xrange(7, len(l)):
l[p] = 1e-15 # g*
self.minimizer = lbfgsb.minimizer(
n=self.n,
l=l,
u=flex.double(self.n, 0),
nbd=flex.int(self.n, 1),
)
while True:
self.compute_functional_and_gradients()
if self.minimizer.process(self.x, self.f, self.g):
pass
elif self.minimizer.is_terminated():
break
def plot_coords(coords, labels=None, key="cosym_coordinates"):
coord_x = coords[:, 0:1].as_1d()
coord_y = coords[:, 1:2].as_1d()
assert coord_x.size() == coord_y.size(), (coord_x.size(), coord_y.size())
if labels is None:
labels = flex.int(len(coord_x), -1)
unique_labels = set(labels)
unique_labels = sorted(unique_labels)
n_clusters = max(len(unique_labels) - (1 if -1 in unique_labels else 0), 1)
# XXX should avoid relying on matplotlib here to determine colours
from matplotlib import pyplot as plt
import numpy
colours = plt.cm.Spectral(numpy.linspace(0, 1, n_clusters)).tolist()
data = []
for k, col in zip(unique_labels, colours):
isel = (labels == k).iselection()
coord_x_sel = coord_x.select(isel)
coord_y_sel = coord_y.select(isel)
data.append({
"x": list(coord_x_sel),
"y": list(coord_y_sel),
"mode": "markers",
"type": "scatter",
"marker": {
"size": 2,
"alpha": 0.5,
"color": "rgb(%f,%f,%f)" % tuple(col[:3]),
},
"name": "Cluster %i" % k,
})
d = {
key: {
"data": data,
"layout": {
"title": "Cosym coordinates",
"xaxis": {
"range": [-1, 1],
"constrain": "domain"
},
"yaxis": {
"range": [-1, 1],
"scaleanchor": "x",
"constrain": "domain"
},
},
}
}
return d
def test_plot_coords():
coords = flex.double([0, 1, 0, 1, 1, 0, 1, 0])
coords.reshape(flex.grid((4, 2)))
labels = flex.int([0, 0, 1, 1])
d = plots.plot_coords(coords, labels=labels)
assert "cosym_coordinates" in d
assert set(d["cosym_coordinates"]) == {"layout", "data"}
assert d["cosym_coordinates"]["data"][0]["x"] == [0.0, 0.0]
assert d["cosym_coordinates"]["data"][0]["y"] == [1.0, 1.0]
assert d["cosym_coordinates"]["data"][1]["x"] == [1.0, 1.0]
assert d["cosym_coordinates"]["data"][1]["y"] == [0.0, 0.0]
def create_mask(size, x0, value):
from scitbx.array_family import flex
mask = flex.int(flex.grid(size), 0)
rad = min(s - c for s, c in zip(size, x0))
for k in range(size[0]):
for j in range(size[1]):
for i in range(size[2]):
d = math.sqrt((j - x0[1])**2 + (i - x0[2])**2)
if d < rad:
mask[k, j, i] = value
return mask
def build_sphere_list(self):
self.indx_list=flex.int()
indx_range = range(self.np_on_grid*2+1)
np = self.np_on_grid
np2 = np**2
for ix in indx_range:
for iy in indx_range:
for iz in indx_range:
if( (ix-np)**2 + (iy-np)**2 + (iz-np)**2 < np2 ):
self.indx_list.append( self.convert_indx_3_1( (ix,iy,iz) ) )
return
def __init__(self,file_name,scale):
self.CA_indx=flex.int()
self.label=[]
self.xyz=flex.vec3_double()
self.natm=0
self.readPDB(file_name)
self.crd=flex.double()
self.eigens=[]
for xyz in self.xyz:
for value in xyz:
self.crd.append(value)
def get_raw_data(self, index=None):
if index is not None and self.index != index:
self.set_index(index)
if self._raw_data is None:
h5_handle = h5py.File(self.image_filename, "r")
data = h5_handle[self.tag]["data"][()].astype(numpy.int32)
h5_handle.close()
self._raw_data = flex.int(data)
return self._raw_data
def compare_fftpack_with_cmplft_3d():
mt = flex.mersenne_twister(seed=0)
for nx, ny, nz in [(30, 20, 40), (7, 19, 13), (5, 11, 4)]:
z = (mt.random_double(size=2 * nx * ny * nz) * 2 - 1).as_float()
cmplft_xy = z.deep_copy()
d = flex.int(
[2 * nx * ny * nz, 2 * nx * ny, 2 * nx * ny * nz, 2 * nx * ny, 2])
ccp4io_dev_ext.fftlib_cmplft(xy=cmplft_xy, n=nz, d=d)
d = flex.int([2 * nx * ny * nz, 2 * nx, 2 * nx * ny, 2 * nx, 2])
ccp4io_dev_ext.fftlib_cmplft(xy=cmplft_xy, n=ny, d=d)
d = flex.int(
[2 * nx * ny * nz, 2, 2 * nx * ny * nz, 2 * nx * ny * nz, 2 * nx])
ccp4io_dev_ext.fftlib_cmplft(xy=cmplft_xy, n=nx, d=d)
fft = scitbx.fftpack.complex_to_complex_3d((nz, ny, nx))
fftpack_xy = z.as_double()
fftpack_xy.reshape(flex.grid(nz, ny, 2 * nx))
fft.forward(fftpack_xy)
fftpack_xy = fftpack_xy.as_float().as_1d()
if (flex.max_absolute(cmplft_xy - fftpack_xy) > 1e-4):
assert approx_equal(cmplft_xy, fftpack_xy, eps=1e-4)
def dbscan_clustering(self):
from sklearn.preprocessing import StandardScaler
X = self.coords_reduced.as_numpy_array()
X = StandardScaler().fit_transform(X)
# Perform cluster analysis
from sklearn.cluster import DBSCAN
db = DBSCAN(eps=self.params.cluster.dbscan.eps,
min_samples=self.params.cluster.dbscan.min_samples).fit(X)
import numpy as np
self.cluster_labels = flex.int(db.labels_.astype(np.int32))
def get_raw_data(self):
"""Get the pixel intensities (i.e. read the image and return as a
flex array."""
if self._raw_data is not None:
return self._raw_data
# Override parent's get_raw_data, which relies on iotbx, which in turn
# relies on cbflib_adaptbx, which in turn expects a gonio
cbf = self._get_cbf_handle()
cbf.find_category(b"array_structure")
cbf.find_column(b"encoding_type")
cbf.select_row(0)
types = []
for i in range(cbf.count_rows()):
types.append(cbf.get_value())
cbf.next_row()
assert (len(types) == cbf.count_rows() == 1
) # multi-tile data read by different class
dtype = types[0]
# find the data
cbf.select_category(0)
while cbf.category_name().lower() != b"array_data":
try:
cbf.next_category()
except Exception:
return None
cbf.select_column(0)
cbf.select_row(0)
cbf.find_column(b"data")
assert cbf.get_typeofvalue().find(b"bnry") > -1
# handle floats vs ints
if dtype == b"signed 32-bit integer":
array_string = cbf.get_integerarray_as_string()
self._raw_data = flex.int(np.fromstring(array_string, np.int32))
parameters = cbf.get_integerarrayparameters_wdims_fs()
slow, mid, fast = (parameters[11], parameters[10], parameters[9])
assert slow == 1 # sections not supported
array_size = mid, fast
elif dtype == b"signed 64-bit real IEEE":
array_string = cbf.get_realarray_as_string()
self._raw_data = flex.double(np.fromstring(array_string, np.float))
parameters = cbf.get_realarrayparameters_wdims_fs()
slow, mid, fast = (parameters[7], parameters[6], parameters[5])
assert slow == 1 # sections not supported
array_size = mid, fast
else:
return None # type not supported
self._raw_data.reshape(flex.grid(*array_size))
return self._raw_data
def _filter_by_distance(self, nn, dist):
"""
Filter the matches by distance.
:param nn: The nearest neighbour list
:param dist: The distances
:returns: A reduced list of nearest neighbours
"""
index = range(len(nn))
return flex.int(i for i in index if dist[i] <= self._max_separation)
def echelon_constraints(group,reciprocal_space = 1):
# direct space : XT G X = G
#reciprocal space : X G* XT = G*
#as tuple: g = g00,g01,g02,g11,g12,g22 = A F E B D C
''' G = g00 g01 g02 = A F E
g01 g11 g12 F B D
g02 g12 g22 E D C'''
n0 = 6*len(group)
n1 = 6
m = flex.int(flex.grid(n0,n1))
i = 0
for x in group:
if reciprocal_space==1:
Rm = matrix.sqr(x.as_double_array()[0:9])
else:
Rm = matrix.sqr(x.as_double_array()[0:9]).transpose()
R = Rm.elems
Rt = Rm.transpose()
e = (R[0]*R[0]-1, 2*R[0]*R[1], 2*R[0]*R[2], R[1]*R[1],2*R[1]*R[2], R[2]*R[2])
for x in xrange(6): m[i] = int(e[x]); i+=1
e = (R[0]*R[3], R[1]*R[3]+R[0]*R[4]-1, R[2]*R[3]+R[0]*R[5], R[1]*R[4], R[2]*R[4]+R[1]*R[5], R[2]*R[5])
for x in xrange(6): m[i] = int(e[x]); i+=1
e = (R[0]*R[6], R[1]*R[6]+R[0]*R[7], R[2]*R[6]+R[0]*R[8]-1,R[1]*R[7], R[2]*R[7]+R[1]*R[8], R[2]*R[8])
for x in xrange(6): m[i] = int(e[x]); i+=1
e = (R[3]*R[3], 2*R[3]*R[4], 2*R[3]*R[5], R[4]*R[4]-1,2*R[4]*R[5], R[5]*R[5])
for x in xrange(6): m[i] = int(e[x]); i+=1
e = (R[3]*R[6], R[4]*R[6]+R[3]*R[7], R[5]*R[6]+R[3]*R[8], R[4]*R[7], R[5]*R[7]+R[4]*R[8]-1,R[5]*R[8])
for x in xrange(6): m[i] = int(e[x]); i+=1
e = (R[6]*R[6], 2*R[6]*R[7], 2*R[6]*R[8], R[7]*R[7],2*R[7]*R[8], R[8]*R[8]-1)
for x in xrange(6): m[i] = int(e[x]); i+=1
mnew = scitbx.math.row_echelon_form(m)
i = 0
#Rearrange row echelon changing coefficient order AFEBDC to ABCDEF
n0 = mnew
i=0
C = flex.int(flex.grid(n0,n1))
for x in xrange(mnew):
C[i]=m[i]; C[i+1]=m[i+3]; C[i+2]=m[i+5]; C[i+3]=m[i+4]; C[i+4]=m[i+2]; C[i+5]=m[i+1]
i+=6
i=0
return C
def effective_tiling_as_flex_int(self, reapply_peripheral_margin=False,
reference_image=None,
encode_inactive_as_zeroes=False, **kwargs):
"""Some documentation goes here"""
if reference_image is not None:
self.reference_image = reference_image
IT = self.effective_tiling_as_flex_int_impl(**kwargs)
# Inactive margin around the edge of the sensor
if reapply_peripheral_margin:
try: peripheral_margin = self.working_params.distl.peripheral_margin
except Exception: peripheral_margin = 2
for i in xrange(len(IT) // 4):
IT[4 * i + 0] += peripheral_margin
IT[4 * i + 1] += peripheral_margin
IT[4 * i + 2] -= peripheral_margin
IT[4 * i + 3] -= peripheral_margin
if self.working_params.distl.tile_flags is not None and encode_inactive_as_zeroes is False:
#sensors whose flags are set to zero are not analyzed by spotfinder
#this returns an active list with fewer tiles
expand_flags=[]
for flag in self.working_params.distl.tile_flags :
expand_flags=expand_flags + [flag]*4
bool_flags = flex.bool( flex.int(expand_flags)==1 )
return IT.select(bool_flags)
if self.working_params.distl.tile_flags is not None and encode_inactive_as_zeroes is True:
#sensors whose flags are set to zero are identified
#this returns a same-length active list with some tiles set to 0,0,0,0
expand_flags=[]
for flag in self.working_params.distl.tile_flags :
expand_flags=expand_flags + [flag]*4
Zero_IT = flex.int()
for idx in xrange(len(IT)):
Zero_IT.append(expand_flags[idx]*IT[idx])
return Zero_IT
return IT
def get_binned_intensities(self, n_bins=100):
"""
Using self.ISIGI, bin the intensities using the following procedure:
1) Find the minimum and maximum intensity values.
2) Divide max-min by n_bins. This is the bin step size
The effect is
@param n_bins number of bins to use.
@return a tuple with an array of selections for each bin and an array of median
intensity values for each bin.
"""
print("Computing intensity bins.", end=' ', file=self.log)
ISIGI = self.scaler.ISIGI
meanI = ISIGI['mean_scaled_intensity']
sels = []
binned_intensities = []
if True:
# intensity range per bin is the same
min_meanI = flex.min(meanI)
step = (flex.max(meanI)-min_meanI)/n_bins
print("Bin size:", step, file=self.log)
self.bin_indices = flex.int(len(ISIGI), -1)
for i in range(n_bins):
if i+1 == n_bins:
sel = (meanI >= (min_meanI + step * i))
else:
sel = (meanI >= (min_meanI + step * i)) & (meanI < (min_meanI + step * (i+1)))
if sel.all_eq(False): continue
sels.append(sel)
self.bin_indices.set_selected(sel, len(sels)-1)
binned_intensities.append((step/2 + step*i)+min_meanI)
assert(self.bin_indices == -1).count(True) == False
else:
# n obs per bin is the same
sorted_meanI = meanI.select(flex.sort_permutation(meanI))
bin_size = len(meanI)/n_bins
for i in range(n_bins):
bin_min = sorted_meanI[int(i*bin_size)]
sel = meanI >= bin_min
if i+1 == n_bins:
bin_max = sorted_meanI[-1]
else:
bin_max = sorted_meanI[int((i+1)*bin_size)]
sel &= meanI < bin_max
sels.append(sel)
binned_intensities.append(bin_min + ((bin_max-bin_min)/2))
for i, (sel, intensity) in enumerate(zip(sels, binned_intensities)):
print("Bin %02d, number of observations: % 10d, midpoint intensity: %f"%(i, sel.count(True), intensity), file=self.log)
return sels, binned_intensities
def agglomerative_clustering(self):
X = self.coords.as_numpy_array()
# Perform cluster analysis
from sklearn.cluster import AgglomerativeClustering
import numpy as np
model = AgglomerativeClustering(
n_clusters=self.params.cluster.agglomerative.n_clusters,
linkage='average',
affinity='cosine')
model.fit(X)
self.cluster_labels = flex.int(model.labels_.astype(np.int32))
def test_image_tile():
from dxtbx.format.image import ImageTileInt
from scitbx.array_family import flex
data = flex.int(flex.grid(10, 10))
name = "TileName"
tile = ImageTileInt(data, name)
assert tile.data().all_eq(data)
assert tile.name() == name
assert tile.empty() is False
def __init__(self, Ih_table, zmax):
"""Set up and run the outlier rejection algorithm."""
assert (Ih_table.n_work_blocks == 1), """
Outlier rejection algorithms require an Ih_table with nblocks = 1"""
# Note: could be possible to code for nblocks > 1
self._Ih_table_block = Ih_table.blocked_data_list[0]
self._n_datasets = Ih_table.n_datasets
self._block_selections = Ih_table.blocked_selection_list[0]
self._datasets = flex.int([])
self._zmax = zmax
self._outlier_indices = flex.size_t([])
self.final_outlier_arrays = None
def sort_cosets(self):
new_partitions = []
for pp in self.partitions:
orders = []
for op in pp:
orders.append(op.r().info().type())
orders = flex.sort_permutation(flex.int(orders))
tmp = partition_t()
for ii in orders:
tmp.append(pp[ii])
new_partitions.append(tmp)
self.partitions = new_partitions
def run(self):
from dxtbx.format.image import ImageTileInt
from dxtbx.format.image import ImageInt
from scitbx.array_family import flex
data = flex.int(flex.grid(10, 10))
name = "TileName0"
tile0 = ImageTileInt(data, name)
image = ImageInt(tile0)
for i in range(1, 4):
data = flex.int(flex.grid(10, 10))
name = "TileName%d" % i
tile = ImageTileInt(data, name)
image.append(tile)
assert image.n_tiles() == 4
for i in range(image.n_tiles()):
tile = image.tile(i)
assert tile.name() == "TileName%d" % i
print "OK"
