def extract_shoeboxes(self,
imageset,
mask=None,
nthreads=1,
verbose=False):
'''
Helper function to read a load of shoebox data.
:param imageset: The imageset
:param mask: The mask to apply
:param nthreads: The number of threads to use
:param verbose: The verbosity
:return: A tuple containing read time and extract time
'''
from dials.model.data import make_image
from time import time
assert ("shoebox" in self)
detector = imageset.get_detector()
try:
frame0, frame1 = imageset.get_array_range()
except Exception:
frame0, frame1 = (0, len(imageset))
extractor = ShoeboxExtractor(self, len(detector), frame0, frame1)
logger.info(" Beginning to read images")
read_time = 0
extract_time = 0
for i in range(len(imageset)):
if verbose:
logger.info(' reading image %d' % i)
st = time()
image = imageset.get_corrected_data(i)
mask2 = imageset.get_mask(i)
if mask is not None:
assert (len(mask) == len(mask2))
mask2 = tuple(m1 & m2 for m1, m2 in zip(mask, mask2))
read_time += time() - st
st = time()
extractor.next(make_image(image, mask2))
extract_time += time() - st
del image
assert (extractor.finished())
logger.info(' successfully read %d images' % (frame1 - frame0))
logger.info(' read time: %g seconds' % read_time)
logger.info(' extract time: %g seconds' % extract_time)
return read_time, extract_time
def extract_shoeboxes(self, imageset, mask=None, nthreads=1, verbose=False):
'''
Helper function to read a load of shoebox data.
:param imageset: The imageset
:param mask: The mask to apply
:param nthreads: The number of threads to use
:param verbose: The verbosity
:return: A tuple containing read time and extract time
'''
from dials.model.data import make_image
from logging import info
from time import time
assert("shoebox" in self)
detector = imageset.get_detector()
try:
frame0, frame1 = imageset.get_array_range()
except Exception:
frame0, frame1 = (0, len(imageset))
extractor = ShoeboxExtractor(self, len(detector), frame0, frame1)
info(" Beginning to read images")
read_time = 0
extract_time = 0
for i in range(len(imageset)):
if verbose:
info(' reading image %d' % i)
st = time()
image = imageset.get_corrected_data(i)
mask2 = imageset.get_mask(i)
if mask is not None:
assert(len(mask) == len(mask2))
mask2 = tuple(m1 & m2 for m1, m2 in zip(mask, mask2))
read_time += time() - st
st = time()
extractor.next(make_image(image, mask2))
extract_time += time() - st
del image
assert(extractor.finished())
info(' successfully read %d images' % (frame1 - frame0))
info(' read time: %g seconds' % read_time)
info(' extract time: %g seconds' % extract_time)
return read_time, extract_time
def extract_shoeboxes(self, imageset, mask=None, nthreads=1):
"""
Helper function to read a load of shoebox data.
:param imageset: The imageset
:param mask: The mask to apply
:param nthreads: The number of threads to use
:return: A tuple containing read time and extract time
"""
from time import time
from dials.model.data import make_image
assert "shoebox" in self
detector = imageset.get_detector()
try:
frame0, frame1 = imageset.get_array_range()
except Exception:
frame0, frame1 = (0, len(imageset))
extractor = dials_array_family_flex_ext.ShoeboxExtractor(
self, len(detector), frame0, frame1
)
logger.info(" Beginning to read images")
read_time = 0
extract_time = 0
for i in range(len(imageset)):
logger.debug(" reading image %d", i)
st = time()
image = imageset.get_corrected_data(i)
mask2 = imageset.get_mask(i)
if mask is not None:
assert len(mask) == len(mask2)
mask2 = tuple(m1 & m2 for m1, m2 in zip(mask, mask2))
read_time += time() - st
st = time()
extractor.next(make_image(image, mask2))
extract_time += time() - st
del image
assert extractor.finished()
logger.info(" successfully read %d images", frame1 - frame0)
logger.info(" read time: %.1f seconds", read_time)
logger.info(" extract time: %.1f seconds", extract_time)
return read_time, extract_time
def __call__(self):
"""
Do the processing.
:return: The processed data
"""
# Set the job index
job.index = self.index
# Get the start time
start_time = time()
# Check all reflections have same imageset and get it
exp_id = list(set(self.reflections["id"]))
imageset = self.experiments[exp_id[0]].imageset
for i in exp_id[1:]:
assert (self.experiments[i].imageset == imageset
), "Task can only handle 1 imageset"
# Get the sub imageset
frame00, frame01 = self.frames
try:
frame10, frame11 = imageset.get_array_range()
except Exception:
frame10, frame11 = (0, len(imageset))
try:
assert frame00 < frame01
assert frame10 < frame11
assert frame00 >= frame10
assert frame01 <= frame11
index0 = frame00 - frame10
index1 = index0 + (frame01 - frame00)
assert index0 < index1
assert index0 >= 0
assert index1 <= len(imageset)
imageset = imageset[index0:index1]
except Exception:
raise RuntimeError("Programmer Error: bad array range")
try:
frame0, frame1 = imageset.get_array_range()
except Exception:
frame0, frame1 = (0, len(imageset))
# Initialise the dataset
image_volume = MultiPanelImageVolume()
for panel in self.experiments[0].detector:
image_volume.add(
ImageVolume(frame0, frame1,
panel.get_image_size()[1],
panel.get_image_size()[0]))
# Read all the images into a block of data
read_time = 0.0
for i in range(len(imageset)):
st = time()
image = imageset.get_corrected_data(i)
mask = imageset.get_mask(i)
if self.params.integration.lookup.mask is not None:
assert len(mask) == len(
self.params.lookup.mask
), "Mask/Image are incorrect size %d %d" % (
len(mask),
len(self.params.integration.lookup.mask),
)
mask = tuple(m1 & m2 for m1, m2 in zip(
self.params.integration.lookup.mask, mask))
image_volume.set_image(frame0 + i, make_image(image, mask))
read_time += time() - st
del image
del mask
# Process the data
st = time()
data = self.executor.process(image_volume, self.experiments,
self.reflections)
process_time = time() - st
# Set the result values
return dials.algorithms.integration.Result(
index=self.index,
reflections=self.reflections,
read_time=read_time,
process_time=process_time,
total_time=time() - start_time,
extract_time=0,
data=data,
)
def __call__(self):
"""
Do the processing.
:return: The processed data
"""
# Get the start time
start_time = time()
# Set the global process ID
job.index = self.index
# Check all reflections have same imageset and get it
exp_id = list(set(self.reflections["id"]))
imageset = self.experiments[exp_id[0]].imageset
for i in exp_id[1:]:
assert (self.experiments[i].imageset == imageset
), "Task can only handle 1 imageset"
# Get the sub imageset
frame0, frame1 = self.job
try:
allowed_range = imageset.get_array_range()
except Exception:
allowed_range = 0, len(imageset)
try:
# range increasing
assert frame0 < frame1
# within an increasing range
assert allowed_range[1] > allowed_range[0]
# we are processing data which is within range
assert frame0 >= allowed_range[0]
assert frame1 <= allowed_range[1]
# I am 99% sure this is implied by all the code above
assert (frame1 - frame0) <= len(imageset)
if len(imageset) > 1:
imageset = imageset[frame0:frame1]
except Exception as e:
raise RuntimeError(f"Programmer Error: bad array range: {e}")
try:
frame0, frame1 = imageset.get_array_range()
except Exception:
frame0, frame1 = (0, len(imageset))
self.executor.initialize(frame0, frame1, self.reflections)
# Set the shoeboxes (don't allocate)
self.reflections["shoebox"] = flex.shoebox(
self.reflections["panel"],
self.reflections["bbox"],
allocate=False,
flatten=self.params.shoebox.flatten,
)
# Create the processor
processor = ShoeboxProcessor(
self.reflections,
len(imageset.get_detector()),
frame0,
frame1,
self.params.debug.output,
)
# Loop through the imageset, extract pixels and process reflections
read_time = 0.0
for i in range(len(imageset)):
st = time()
image = imageset.get_corrected_data(i)
if imageset.is_marked_for_rejection(i):
mask = tuple(flex.bool(im.accessor(), False) for im in image)
else:
mask = imageset.get_mask(i)
if self.params.lookup.mask is not None:
assert len(mask) == len(
self.params.lookup.mask
), "Mask/Image are incorrect size %d %d" % (
len(mask),
len(self.params.lookup.mask),
)
mask = tuple(
m1 & m2
for m1, m2 in zip(self.params.lookup.mask, mask))
read_time += time() - st
processor.next(make_image(image, mask), self.executor)
del image
del mask
assert processor.finished(), "Data processor is not finished"
# Optionally save the shoeboxes
if self.params.debug.output and self.params.debug.separate_files:
output = self.reflections
if self.params.debug.select is not None:
output = output.select(self.params.debug.select(output))
if self.params.debug.split_experiments:
output = output.split_by_experiment_id()
for table in output:
i = table["id"][0]
table.as_file("shoeboxes_%d_%d.refl" % (self.index, i))
else:
output.as_file("shoeboxes_%d.refl" % self.index)
# Delete the shoeboxes
if self.params.debug.separate_files or not self.params.debug.output:
del self.reflections["shoebox"]
# Finalize the executor
self.executor.finalize()
# Return the result
return dials.algorithms.integration.Result(
index=self.index,
reflections=self.reflections,
data=self.executor.data(),
read_time=read_time,
extract_time=processor.extract_time(),
process_time=processor.process_time(),
total_time=time() - start_time,
)
def __call__(self):
'''
Do the processing.
:return: The processed data
'''
from dials.array_family import flex
from time import time
from dials.model.data import make_image
from libtbx.introspection import machine_memory_info
# Get the start time
start_time = time()
# Set the global process ID
job.index = self.index
# Check all reflections have same imageset and get it
exp_id = list(set(self.reflections['id']))
imageset = self.experiments[exp_id[0]].imageset
for i in exp_id[1:]:
assert self.experiments[
i].imageset == imageset, "Task can only handle 1 imageset"
# Get the sub imageset
frame00, frame01 = self.job
try:
frame10, frame11 = imageset.get_array_range()
except Exception:
frame10, frame11 = (0, len(imageset))
try:
assert frame00 < frame01
assert frame10 < frame11
assert frame00 >= frame10
assert frame01 <= frame11
index0 = frame00 - frame10
index1 = index0 + (frame01 - frame00)
assert index0 < index1
assert index0 >= 0
assert index1 <= len(imageset)
imageset = imageset[index0:index1]
except Exception:
raise RuntimeError('Programmer Error: bad array range')
try:
frame0, frame1 = imageset.get_array_range()
except Exception:
frame0, frame1 = (0, len(imageset))
# Initlize the executor
self.executor.initialize(frame0, frame1, self.reflections)
# Set the shoeboxes (dont't allocate)
self.reflections['shoebox'] = flex.shoebox(
self.reflections['panel'],
self.reflections['bbox'],
allocate=False,
flatten=self.params.shoebox.flatten)
# Create the processor
processor = ShoeboxProcessor(self.reflections,
len(imageset.get_detector()), frame0,
frame1, self.params.debug.output)
# Compute percentage of max available. The function is not portable to
# windows so need to add a check if the function fails. On windows no
# warning will be printed
memory_info = machine_memory_info()
total_memory = memory_info.memory_total()
sbox_memory = processor.compute_max_memory_usage()
if total_memory is not None:
assert total_memory > 0, "Your system appears to have no memory!"
assert self.params.block.max_memory_usage > 0.0, "maximum memory usage must be > 0"
assert self.params.block.max_memory_usage <= 1.0, "maximum memory usage must be <= 1"
limit_memory = total_memory * self.params.block.max_memory_usage
if sbox_memory > limit_memory:
raise RuntimeError('''
There was a problem allocating memory for shoeboxes. Possible solutions
include increasing the percentage of memory allowed for shoeboxes or
decreasing the block size. This could also be caused by a highly mosaic
crystal model - is your crystal really this mosaic?
Total system memory: %g GB
Limit shoebox memory: %g GB
Required shoebox memory: %g GB
''' % (total_memory / 1e9, limit_memory / 1e9, sbox_memory / 1e9))
else:
logger.info(' Memory usage:')
logger.info(' Total system memory: %g GB' %
(total_memory / 1e9))
logger.info(' Limit shoebox memory: %g GB' %
(limit_memory / 1e9))
logger.info(' Required shoebox memory: %g GB' %
(sbox_memory / 1e9))
logger.info('')
# Loop through the imageset, extract pixels and process reflections
read_time = 0.0
for i in range(len(imageset)):
st = time()
image = imageset.get_corrected_data(i)
mask = imageset.get_mask(i)
if self.params.lookup.mask is not None:
assert len(mask) == len(self.params.lookup.mask), \
"Mask/Image are incorrect size %d %d" % (
len(mask),
len(self.params.lookup.mask))
mask = tuple(m1 & m2
for m1, m2 in zip(self.params.lookup.mask, mask))
read_time += time() - st
processor.next(make_image(image, mask), self.executor)
del image
del mask
assert processor.finished(), "Data processor is not finished"
# Optionally save the shoeboxes
if self.params.debug.output and self.params.debug.separate_files:
output = self.reflections
if self.params.debug.select is not None:
output = output.select(self.params.debug.select(output))
if self.params.debug.split_experiments:
output = output.split_by_experiment_id()
for table in output:
i = table['id'][0]
table.as_pickle('shoeboxes_%d_%d.pickle' % (self.index, i))
else:
output.as_pickle('shoeboxes_%d.pickle' % self.index)
# Delete the shoeboxes
if self.params.debug.separate_files or not self.params.debug.output:
del self.reflections['shoebox']
# Finalize the executor
self.executor.finalize()
# Return the result
result = Result(self.index, self.reflections, self.executor.data())
result.read_time = read_time
result.extract_time = processor.extract_time()
result.process_time = processor.process_time()
result.total_time = time() - start_time
return result
def __call__(self):
'''
Do the processing.
:return: The processed data
'''
from dials.array_family import flex
from time import time
from dials.model.data import make_image
from libtbx.introspection import machine_memory_info
# Get the start time
start_time = time()
# Set the global process ID
job.index = self.index
# Check all reflections have same imageset and get it
exp_id = list(set(self.reflections['id']))
imageset = self.experiments[exp_id[0]].imageset
for i in exp_id[1:]:
assert self.experiments[i].imageset == imageset, "Task can only handle 1 imageset"
# Get the sub imageset
frame00, frame01 = self.job
try:
frame10, frame11 = imageset.get_array_range()
except Exception:
frame10, frame11 = (0, len(imageset))
try:
assert frame00 < frame01
assert frame10 < frame11
assert frame00 >= frame10
assert frame01 <= frame11
index0 = frame00 - frame10
index1 = index0 + (frame01 - frame00)
assert index0 < index1
assert index0 >= 0
assert index1 <= len(imageset)
imageset = imageset[index0:index1]
except Exception:
raise RuntimeError('Programmer Error: bad array range')
try:
frame0, frame1 = imageset.get_array_range()
except Exception:
frame0, frame1 = (0, len(imageset))
# Initlize the executor
self.executor.initialize(frame0, frame1, self.reflections)
# Set the shoeboxes (dont't allocate)
self.reflections['shoebox'] = flex.shoebox(
self.reflections['panel'],
self.reflections['bbox'],
allocate=False,
flatten=self.params.shoebox.flatten)
# Create the processor
processor = ShoeboxProcessor(
self.reflections,
len(imageset.get_detector()),
frame0,
frame1,
self.params.debug.output)
# Compute percentage of max available. The function is not portable to
# windows so need to add a check if the function fails. On windows no
# warning will be printed
memory_info = machine_memory_info()
total_memory = memory_info.memory_total()
sbox_memory = processor.compute_max_memory_usage()
if total_memory is not None:
assert total_memory > 0, "Your system appears to have no memory!"
assert self.params.block.max_memory_usage > 0.0, "maximum memory usage must be > 0"
assert self.params.block.max_memory_usage <= 1.0, "maximum memory usage must be <= 1"
limit_memory = total_memory * self.params.block.max_memory_usage
if sbox_memory > limit_memory:
raise RuntimeError('''
There was a problem allocating memory for shoeboxes. Possible solutions
include increasing the percentage of memory allowed for shoeboxes or
decreasing the block size. This could also be caused by a highly mosaic
crystal model - is your crystal really this mosaic?
Total system memory: %g GB
Limit shoebox memory: %g GB
Required shoebox memory: %g GB
''' % (total_memory/1e9, limit_memory/1e9, sbox_memory/1e9))
else:
logger.info(' Memory usage:')
logger.info(' Total system memory: %g GB' % (total_memory/1e9))
logger.info(' Limit shoebox memory: %g GB' % (limit_memory/1e9))
logger.info(' Required shoebox memory: %g GB' % (sbox_memory/1e9))
logger.info('')
# Loop through the imageset, extract pixels and process reflections
read_time = 0.0
for i in range(len(imageset)):
st = time()
image = imageset.get_corrected_data(i)
mask = imageset.get_mask(i)
if self.params.lookup.mask is not None:
assert len(mask) == len(self.params.lookup.mask), \
"Mask/Image are incorrect size %d %d" % (
len(mask),
len(self.params.lookup.mask))
mask = tuple(m1 & m2 for m1, m2 in zip(self.params.lookup.mask, mask))
read_time += time() - st
processor.next(make_image(image, mask), self.executor)
del image
del mask
assert processor.finished(), "Data processor is not finished"
# Optionally save the shoeboxes
if self.params.debug.output and self.params.debug.separate_files:
output = self.reflections
if self.params.debug.select is not None:
output = output.select(self.params.debug.select(output))
if self.params.debug.split_experiments:
output = output.split_by_experiment_id()
for table in output:
i = table['id'][0]
table.as_pickle('shoeboxes_%d_%d.pickle' % (self.index, i))
else:
output.as_pickle('shoeboxes_%d.pickle' % self.index)
# Delete the shoeboxes
if self.params.debug.separate_files or not self.params.debug.output:
del self.reflections['shoebox']
# Finalize the executor
self.executor.finalize()
# Return the result
result = Result(self.index, self.reflections, self.executor.data())
result.read_time = read_time
result.extract_time = processor.extract_time()
result.process_time = processor.process_time()
result.total_time = time() - start_time
return result
def __call__(self):
"""
Do the processing.
:return: The processed data
"""
from dials.array_family import flex
from dials.model.data import make_image
# Get the start time
start_time = time()
# Set the global process ID
job.index = self.index
# Check all reflections have same imageset and get it
exp_id = list(set(self.reflections["id"]))
imageset = self.experiments[exp_id[0]].imageset
for i in exp_id[1:]:
assert (self.experiments[i].imageset == imageset
), "Task can only handle 1 imageset"
# Get the sub imageset
frame00, frame01 = self.job
try:
frame10, frame11 = imageset.get_array_range()
except Exception:
frame10, frame11 = (0, len(imageset))
try:
assert frame00 < frame01
assert frame10 < frame11
assert frame00 >= frame10
assert frame01 <= frame11
index0 = frame00 - frame10
index1 = index0 + (frame01 - frame00)
assert index0 < index1
assert index0 >= 0
assert index1 <= len(imageset)
imageset = imageset[index0:index1]
except Exception:
raise RuntimeError("Programmer Error: bad array range")
try:
frame0, frame1 = imageset.get_array_range()
except Exception:
frame0, frame1 = (0, len(imageset))
# Initlize the executor
self.executor.initialize(frame0, frame1, self.reflections)
# Set the shoeboxes (dont't allocate)
self.reflections["shoebox"] = flex.shoebox(
self.reflections["panel"],
self.reflections["bbox"],
allocate=False,
flatten=self.params.shoebox.flatten,
)
# Create the processor
processor = ShoeboxProcessor(
self.reflections,
len(imageset.get_detector()),
frame0,
frame1,
self.params.debug.output,
)
# Compute expected memory usage and warn if not enough
total_memory = psutil.virtual_memory().total
sbox_memory = processor.compute_max_memory_usage()
assert (self.params.block.max_memory_usage >
0.0), "maximum memory usage must be > 0"
assert (self.params.block.max_memory_usage <=
1.0), "maximum memory usage must be <= 1"
limit_memory = total_memory * self.params.block.max_memory_usage
if sbox_memory > limit_memory:
raise RuntimeError("""
There was a problem allocating memory for shoeboxes. Possible solutions
include increasing the percentage of memory allowed for shoeboxes or
decreasing the block size. This could also be caused by a highly mosaic
crystal model - is your crystal really this mosaic?
Total system memory: %g GB
Limit shoebox memory: %g GB
Required shoebox memory: %g GB
""" % (total_memory / 1e9, limit_memory / 1e9, sbox_memory / 1e9))
else:
logger.info(" Memory usage:")
logger.info(" Total system memory: %g GB" % (total_memory / 1e9))
logger.info(" Limit shoebox memory: %g GB" % (limit_memory / 1e9))
logger.info(" Required shoebox memory: %g GB" %
(sbox_memory / 1e9))
logger.info("")
# Loop through the imageset, extract pixels and process reflections
read_time = 0.0
for i in range(len(imageset)):
st = time()
image = imageset.get_corrected_data(i)
if imageset.is_marked_for_rejection(i):
mask = tuple(flex.bool(im.accessor(), False) for im in image)
else:
mask = imageset.get_mask(i)
if self.params.lookup.mask is not None:
assert len(mask) == len(self.params.lookup.mask), (
"Mask/Image are incorrect size %d %d" %
(len(mask), len(self.params.lookup.mask)))
mask = tuple(
m1 & m2
for m1, m2 in zip(self.params.lookup.mask, mask))
read_time += time() - st
processor.next(make_image(image, mask), self.executor)
del image
del mask
assert processor.finished(), "Data processor is not finished"
# Optionally save the shoeboxes
if self.params.debug.output and self.params.debug.separate_files:
output = self.reflections
if self.params.debug.select is not None:
output = output.select(self.params.debug.select(output))
if self.params.debug.split_experiments:
output = output.split_by_experiment_id()
for table in output:
i = table["id"][0]
table.as_file("shoeboxes_%d_%d.refl" % (self.index, i))
else:
output.as_file("shoeboxes_%d.refl" % self.index)
# Delete the shoeboxes
if self.params.debug.separate_files or not self.params.debug.output:
del self.reflections["shoebox"]
# Finalize the executor
self.executor.finalize()
# Return the result
result = Result(self.index, self.reflections, self.executor.data())
result.read_time = read_time
result.extract_time = processor.extract_time()
result.process_time = processor.process_time()
result.total_time = time() - start_time
return result
def integrate_job(block,
experiments,
reflections,
reference,
grid_size=5,
detector_space=False):
from dials.algorithms.profile_model.modeller import CircleSampler
from dials.array_family import flex
from dials.algorithms.profile_model.gaussian_rs.transform import TransformReverse
from dials.algorithms.profile_model.gaussian_rs.transform import TransformForward
from dials.algorithms.profile_model.gaussian_rs.transform import (
TransformReverseNoModel, )
from dials.algorithms.profile_model.gaussian_rs.transform import TransformSpec
from dials.algorithms.profile_model.gaussian_rs import CoordinateSystem
from dials.algorithms.integration.fit import ProfileFitter
from dials.array_family import flex
from dials.model.data import make_image
reflections["shoebox"] = flex.shoebox(reflections["panel"],
reflections["bbox"],
allocate=True)
frame0, frame1 = experiments[0].scan.get_array_range()
frame0 = frame0 + block[0]
frame1 = frame0 + block[1]
reflections["shoebox"] = flex.shoebox(reflections["panel"],
reflections["bbox"],
allocate=True)
extractor = flex.ShoeboxExtractor(reflections, 1, frame0, frame1)
iset = experiments[0].imageset[block[0]:block[1]]
for i in range(len(iset)):
print("Reading image %d" % i)
data = iset.get_raw_data(i)
mask = iset.get_mask(i)
extractor.next(make_image(data, mask))
print("Computing mask")
reflections.compute_mask(experiments)
print("Computing background")
reflections.compute_background(experiments)
print("Computing centroid")
reflections.compute_centroid(experiments)
print("Computing summed intensity")
reflections.compute_summed_intensity()
assert len(reference) % 9 == 0
num_scan_points = len(reference) // 9
sampler = CircleSampler(
experiments[0].detector[0].get_image_size(),
experiments[0].scan.get_array_range(),
num_scan_points,
)
spec = TransformSpec(
experiments[0].beam,
experiments[0].detector,
experiments[0].goniometer,
experiments[0].scan,
experiments[0].profile.sigma_b(deg=False),
experiments[0].profile.sigma_m(deg=False),
experiments[0].profile.n_sigma() * 1.5,
grid_size,
)
m2 = experiments[0].goniometer.get_rotation_axis()
s0 = experiments[0].beam.get_s0()
Iprf = flex.double(len(reflections))
Vprf = flex.double(len(reflections))
Cprf = flex.double(len(reflections))
Fprf = flex.bool(len(reflections))
Part = reflections["partiality"]
reflections["intensity.prf_old.value"] = reflections["intensity.prf.value"]
reflections["intensity.prf_old.variance"] = reflections[
"intensity.prf.variance"]
selection = reflections.get_flags(reflections.flags.integrated_prf)
reflections.unset_flags(~Fprf, reflections.flags.integrated_prf)
for i, r in enumerate(reflections):
if selection[i] == False:
continue
s1 = r["s1"]
phi = r["xyzcal.mm"][2]
xyz = r["xyzcal.px"]
bbox = r["bbox"]
panel = r["panel"]
image = r["shoebox"].data.as_double()
background = r["shoebox"].background.as_double()
mask = r["shoebox"].mask.as_1d(
) == 5 # | (r['shoebox'].mask.as_1d() == 3)
mask.reshape(image.accessor())
cs = CoordinateSystem(m2, s0, s1, phi)
index = sampler.nearest(0, xyz)
profile, profile_mask = reference[index]
# print flex.sum(profile)
# print r['partiality']
if detector_space:
transform = TransformReverseNoModel(spec, cs, bbox, panel, profile)
p = transform.profile()
d = image
m = mask
b = background
# print flex.sum(p)
Part[i] = flex.sum(p)
# ysize, xsize = p.all()[1:3]
# p1 = flex.double(flex.grid(1, ysize , xsize))
# d1 = flex.double(flex.grid(1, ysize , xsize))
# b1 = flex.double(flex.grid(1, ysize , xsize))
# m1 = flex.double(flex.grid(1, ysize , xsize))
# for k in range(p.all()[0]):
# p1 += p[k:k+1,:,:]
# d1 += d[k:k+1,:,:]
# b1 += b[k:k+1,:,:]
# m1 = m[k:k+1,:,:]
try:
fit = ProfileFitter(d, b, m, p, 1e-3, 100)
assert fit.niter() < 100
Iprf[i] = fit.intensity()
Vprf[i] = fit.variance()
Cprf[i] = fit.correlation()
Fprf[i] = True
# if r['intensity.sum.value'] > 10 and abs(fit.intensity()) < 1e-3:
print(
r["miller_index"],
i,
fit.intensity(),
r["intensity.sum.value"],
r["intensity.prf_old.value"],
Part[i],
fit.niter(),
)
# from matplotlib import pylab
# pylab.imshow(p1.as_numpy_array()[0,:,:], interpolation='none')
# pylab.show()
except Exception as e:
print(e)
pass
else:
try:
transform = TransformForward(spec, cs, bbox, panel, image,
background, mask)
p = profile
d = transform.profile()
b = transform.background()
m = transform.mask() & profile_mask
# if r['miller_index'] == (9, -25, 74):
# print list(p)
# print list(m)
# print list(b)
# print list(d)
# exit(0)
fit = ProfileFitter(d, b, m, p, 1e-3, 100)
assert fit.niter() < 100
Iprf[i] = fit.intensity()[0]
Vprf[i] = fit.variance()[0]
Cprf[i] = fit.correlation()
Fprf[i] = True
print(r["miller_index"], i, fit.intensity(),
r["intensity.prf_old.value"])
# from matplotlib import pylab
# pylab.imshow(p1.as_numpy_array()[0,:,:], interpolation='none')
# pylab.show()
except Exception as e:
pass
reflections["intensity.prf.value"] = Iprf
reflections["intensity.prf.variance"] = Vprf
reflections["intensity.prf.correlation"] = Cprf
reflections.set_flags(Fprf, reflections.flags.integrated_prf)
del reflections["shoebox"]
return reflections
def __call__(self):
'''
Do the processing.
:return: The processed data
'''
from dials.model.data import make_image
from dials.model.data import MultiPanelImageVolume
from dials.model.data import ImageVolume
from dials.algorithms.integration.processor import job
from time import time
# Set the job index
job.index = self.index
# Get the start time
start_time = time()
# Check all reflections have same imageset and get it
exp_id = list(set(self.reflections['id']))
imageset = self.experiments[exp_id[0]].imageset
for i in exp_id[1:]:
assert self.experiments[i].imageset == imageset, "Task can only handle 1 imageset"
# Get the sub imageset
frame00, frame01 = self.frames
try:
frame10, frame11 = imageset.get_array_range()
except Exception:
frame10, frame11 = (0, len(imageset))
try:
assert frame00 < frame01
assert frame10 < frame11
assert frame00 >= frame10
assert frame01 <= frame11
index0 = frame00 - frame10
index1 = index0 + (frame01 - frame00)
assert index0 < index1
assert index0 >= 0
assert index1 <= len(imageset)
imageset = imageset[index0:index1]
except Exception:
raise RuntimeError('Programmer Error: bad array range')
try:
frame0, frame1 = imageset.get_array_range()
except Exception:
frame0, frame1 = (0, len(imageset))
# Initialise the dataset
image_volume = MultiPanelImageVolume()
for panel in self.experiments[0].detector:
image_volume.add(ImageVolume(
frame0,
frame1,
panel.get_image_size()[1],
panel.get_image_size()[0]))
# Read all the images into a block of data
read_time = 0.0
for i in range(len(imageset)):
st = time()
image = imageset.get_corrected_data(i)
mask = imageset.get_mask(i)
if self.params.integration.lookup.mask is not None:
assert len(mask) == len(self.params.lookup.mask), \
"Mask/Image are incorrect size %d %d" % (
len(mask),
len(self.params.integration.lookup.mask))
mask = tuple(m1 & m2 for m1, m2 in zip(self.params.integration.lookup.mask, mask))
image_volume.set_image(frame0 + i, make_image(image, mask))
read_time += time() - st
del image
del mask
# Process the data
st = time()
data = self.executor.process(
image_volume,
self.experiments,
self.reflections)
process_time = time() - st
# Set the result values
result = Result(self.index, self.reflections)
result.read_time = read_time
result.process_time = process_time
result.total_time = time() - start_time
result.data = data
return result
