def run (out=None, omit_unicode_experiment=False):
if (out is None): out = sys.stdout
out.write(boost.python.platform_info)
tag = libtbx.env.under_dist("boost", "TAG")
if (op.isfile(tag)):
tag = open(tag).read().strip()
else:
tag = None
print >> out, "boost/TAG:", tag
print >> out, "os.name:", os.name
print >> out, "sys.platform:", sys.platform
print >> out, "sys.byteorder:", sys.byteorder
print >> out, "platform.platform():", platform.platform()
print >> out, "platform.architecture():", platform.architecture()
for attr in ["division_by_zero", "invalid", "overflow"]:
attr = "floating_point_exceptions.%s_trapped" % attr
print >> out, "%s:" % attr, eval("boost.python.%s" % attr)
print >> out, "number of processors:", introspection.number_of_processors(
return_value_if_unknown="unknown")
introspection.machine_memory_info().show(out=out)
try: import thread
except ImportError: print >> out, "import thread: NO"
else: print >> out, "import thread: OK"
print "Division operator semantics: %s division" % (div_probe() / 0)
c = getattr(boost.python.ext, "str_or_unicode_as_char_list", None)
if (c is not None and not omit_unicode_experiment):
print >> out, '"hello" =', c("hello")
print >> out, 'u"hello" =', c(u"hello")
e = u"\u00C5".encode("utf-8", "strict")
print >> out, 'u"\u00C5" =', c(u"\u00C5"), 'as utf-8 =', c(e)
print >> out, "LATIN CAPITAL LETTER A WITH RING ABOVE =", e
from libtbx.utils import format_cpu_times
print format_cpu_times()
def __cubicles_max_memory_allocation_set():
from libtbx.introspection import machine_memory_info
m = machine_memory_info().memory_total()
if (m is None): m = 1000000000
l = 2**(8 * bp.sizeof_void_ptr - 1) - 1
if (m > l): m = l
cubicles_max_memory_allocation_set(number_of_bytes=m // 2)
def run(self):
''' Run the script. '''
from dials.util.command_line import Command
from dials.util.options import flatten_experiments
from libtbx.utils import Sorry
from libtbx.introspection import machine_memory_info
from math import floor
memory_info = machine_memory_info()
total_memory = memory_info.memory_total()
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0:
self.parser.print_help()
return
elif len(experiments) > 1:
raise RuntimeError("Only 1 experiment can be specified")
imageset = experiments[0].imageset
from dials.algorithms.integration.parallel_integrator import MultiThreadedIntegrator
max_block_size = MultiThreadedIntegrator.compute_max_block_size(
imageset, params.use_dynamic_mask, total_memory)
print("Total memory: %.2f GB" % (total_memory / 1e9))
print("Max Block Size: %d" % max_block_size)
for i in range(1, min(max_block_size, len(imageset) + 1)):
memory = MultiThreadedIntegrator.compute_required_memory(
imageset[0:i], params.use_dynamic_mask)
print("Block Size = %d, Memory = %.2f GB" % (i, memory / 1e9))
def __cubicles_max_memory_allocation_set(): from libtbx.introspection import machine_memory_info m = machine_memory_info().memory_total() if (m is None): m = 1000000000 l = 2**(8*boost.python.sizeof_void_ptr-1)-1 if (m > l): m = l cubicles_max_memory_allocation_set(number_of_bytes=m//2)
def run(out=None, omit_unicode_experiment=False):
if (out is None): out = sys.stdout
out.write(bp.platform_info)
tag = libtbx.env.under_dist("boost", "TAG")
if (op.isfile(tag)):
tag = open(tag).read().strip()
else:
tag = None
print("boost/TAG:", tag, file=out)
print("os.name:", os.name, file=out)
print("sys.platform:", sys.platform, file=out)
print("sys.byteorder:", sys.byteorder, file=out)
print("platform.platform():", platform.platform(), file=out)
print("platform.architecture():", platform.architecture(), file=out)
for attr in ["division_by_zero", "invalid", "overflow"]:
attr = "floating_point_exceptions.%s_trapped" % attr
print("%s:" % attr, eval("bp.%s" % attr), file=out)
print("number of processors:", introspection.number_of_processors(
return_value_if_unknown="unknown"), file=out)
introspection.machine_memory_info().show(out=out)
try: import thread
except ImportError: print("import thread: NO", file=out)
else: print("import thread: OK", file=out)
print("Division operator semantics: %s division" % (div_probe() / 0))
c = getattr(bp.ext, "str_or_unicode_as_char_list", None)
if (c is not None and not omit_unicode_experiment):
print('"hello" =', c("hello"), file=out)
print('u"hello" =', c(u"hello"), file=out)
e = u"\u00C5".encode("utf-8", "strict")
# XXX temp fix for python3 failure: unicode failure in show_platform_info.py
# Reason: c(u"\u00C5") fails and c(e) fails
try:
cc = c(u"\u00C5")
ce = c(e)
except Exception as ignore_exception:
cc = u"\u00C5"
ce = e
print('u"\u00C5" =', cc, 'as utf-8 =', ce, file=out)
print("LATIN CAPITAL LETTER A WITH RING ABOVE =", e, file=out)
from libtbx.utils import format_cpu_times
print(format_cpu_times())
def run(out=None, omit_unicode_experiment=False):
if (out is None): out = sys.stdout
out.write(boost.python.platform_info)
tag = libtbx.env.under_dist("boost", "TAG")
if (op.isfile(tag)):
tag = open(tag).read().strip()
else:
tag = None
print >> out, "boost/TAG:", tag
print >> out, "os.name:", os.name
print >> out, "sys.platform:", sys.platform
print >> out, "sys.byteorder:", sys.byteorder
print >> out, "platform.platform():", platform.platform()
print >> out, "platform.architecture():", platform.architecture()
for attr in ["division_by_zero", "invalid", "overflow"]:
attr = "floating_point_exceptions.%s_trapped" % attr
print >> out, "%s:" % attr, eval("boost.python.%s" % attr)
print >> out, "number of processors:", introspection.number_of_processors(
return_value_if_unknown="unknown")
introspection.machine_memory_info().show(out=out)
try:
import thread
except ImportError:
print >> out, "import thread: NO"
else:
print >> out, "import thread: OK"
print "Division operator semantics: %s division" % (div_probe() / 0)
c = getattr(boost.python.ext, "str_or_unicode_as_char_list", None)
if (c is not None and not omit_unicode_experiment):
print >> out, '"hello" =', c("hello")
print >> out, 'u"hello" =', c(u"hello")
e = u"\u00C5".encode("utf-8", "strict")
print >> out, 'u"\u00C5" =', c(u"\u00C5"), 'as utf-8 =', c(e)
print >> out, "LATIN CAPITAL LETTER A WITH RING ABOVE =", e
from libtbx.utils import format_cpu_times
print format_cpu_times()
def compute_max_block_size(self):
'''
Compute the required memory
'''
from libtbx.introspection import machine_memory_info
from math import floor
memory_info = machine_memory_info()
total_memory = memory_info.memory_total()
max_memory_usage = self.params.integration.block.max_memory_usage
if total_memory is None:
raise RuntimeError("Inspection of system memory failed")
assert total_memory > 0, "Your system appears to have no memory!"
assert max_memory_usage > 0.0, "maximum memory usage must be > 0"
assert max_memory_usage <= 1.0, "maximum memory usage must be <= 1"
limit_memory = int(floor(total_memory * max_memory_usage))
return MultiThreadedIntegrator.compute_max_block_size(
self.experiments[0].imageset,
max_memory_usage = limit_memory)
def assert_enough_memory(required_memory, max_memory_usage):
"""
Check there is enough memory available or fail
:param required_memory: The required number of bytes
:param max_memory_usage: The maximum memory usage allowed
"""
from libtbx.introspection import machine_memory_info
# Compute percentage of max available. The function is not portable to
# windows so need to add a check if the function fails. On windows no
# warning will be printed
memory_info = machine_memory_info()
total_memory = memory_info.memory_total()
if total_memory is None:
raise RuntimeError("Inspection of system memory failed")
assert total_memory > 0, "Your system appears to have no memory!"
assert max_memory_usage > 0.0, "maximum memory usage must be > 0"
assert max_memory_usage <= 1.0, "maximum memory usage must be <= 1"
limit_memory = total_memory * max_memory_usage
if required_memory > limit_memory:
raise RuntimeError(
"""
There was a problem allocating memory for image data. Possible solutions
include increasing the percentage of memory allowed for shoeboxes or
decreasing the block size. This could also be caused by a highly mosaic
crystal model - is your crystal really this mosaic?
Total system memory: %g GB
Limit image memory: %g GB
Required image memory: %g GB
"""
% (total_memory / 1e9, limit_memory / 1e9, required_memory / 1e9)
)
else:
logger.info("")
logger.info(" Memory usage:")
logger.info(" Total system memory: %g GB" % (total_memory / 1e9))
logger.info(" Limit image memory: %g GB" % (limit_memory / 1e9))
logger.info(" Required image memory: %g GB" % (required_memory / 1e9))
logger.info("")
def compute_processors(self):
'''
Compute the number of processors
'''
from libtbx.introspection import machine_memory_info
from math import floor
from dials.array_family import flex
# Set the memory usage per processor
if self.params.mp.method == 'multiprocessing' and self.params.mp.nproc > 1:
# Get the maximum shoebox memory
max_memory = flex.max(
self.jobs.shoebox_memory(self.reflections,
self.params.shoebox.flatten))
# Compute percentage of max available. The function is not portable to
# windows so need to add a check if the function fails. On windows no
# warning will be printed
memory_info = machine_memory_info()
total_memory = memory_info.memory_total()
if total_memory is not None:
assert total_memory > 0, "Your system appears to have no memory!"
limit_memory = total_memory * self.params.block.max_memory_usage
njobs = int(floor(limit_memory / max_memory))
if njobs < 1:
raise RuntimeError('''
No enough memory to run integration jobs. Possible solutions
include increasing the percentage of memory allowed for shoeboxes or
decreasing the block size.
Total system memory: %g GB
Limit shoebox memory: %g GB
Max shoebox memory: %g GB
''' % (total_memory / 1e9, limit_memory / 1e9, max_memory / 1e9))
else:
self.params.mp.nproc = min(self.params.mp.nproc, njobs)
self.params.block.max_memory_usage /= self.params.mp.nproc
def compute_processors(self):
'''
Compute the number of processors
'''
from libtbx.introspection import machine_memory_info
from math import floor
from dials.array_family import flex
# Set the memory usage per processor
if self.params.mp.method == 'multiprocessing' and self.params.mp.nproc > 1:
# Get the maximum shoebox memory
max_memory = flex.max(self.jobs.shoebox_memory(
self.reflections, self.params.shoebox.flatten))
# Compute percentage of max available. The function is not portable to
# windows so need to add a check if the function fails. On windows no
# warning will be printed
memory_info = machine_memory_info()
total_memory = memory_info.memory_total()
if total_memory is not None:
assert total_memory > 0, "Your system appears to have no memory!"
limit_memory = total_memory * self.params.block.max_memory_usage
njobs = int(floor(limit_memory / max_memory))
if njobs < 1:
raise RuntimeError('''
No enough memory to run integration jobs. Possible solutions
include increasing the percentage of memory allowed for shoeboxes or
decreasing the block size.
Total system memory: %g GB
Limit shoebox memory: %g GB
Max shoebox memory: %g GB
''' % (total_memory/1e9, limit_memory/1e9, max_memory/1e9))
else:
self.params.mp.nproc = min(self.params.mp.nproc, njobs)
self.params.block.max_memory_usage /= self.params.mp.nproc
def _build_components(cls, params, reflections, experiments):
"""low level build"""
# Currently a refinement job can only have one parameterisation of the
# prediction equation. This can either be of the XYDelPsi (stills) type, the
# XYPhi (scans) type or the scan-varying XYPhi type with a varying crystal
# model
single_as_still = params.refinement.parameterisation.treat_single_image_as_still
exps_are_stills = []
for exp in experiments:
if exp.scan is None:
exps_are_stills.append(True)
elif exp.scan.get_num_images() == 1:
if single_as_still:
exps_are_stills.append(True)
elif exp.scan.get_oscillation()[1] == 0.0:
exps_are_stills.append(True)
else:
exps_are_stills.append(False)
else:
if exp.scan.get_oscillation()[1] <= 0.0:
raise DialsRefineConfigError(
"Cannot refine a zero-width scan")
exps_are_stills.append(False)
# check experiment types are consistent
if not all(exps_are_stills[0] == e for e in exps_are_stills):
raise DialsRefineConfigError(
"Cannot refine a mixture of stills and scans")
do_stills = exps_are_stills[0]
# If experiments are stills, ensure scan-varying refinement won't be attempted
if do_stills:
params.refinement.parameterisation.scan_varying = False
# Refiner does not accept scan_varying=Auto. This is a special case for
# doing macrocycles of refinement in dials.refine.
if params.refinement.parameterisation.scan_varying is libtbx.Auto:
params.refinement.parameterisation.scan_varying = False
# calculate reflection block_width if required for scan-varying refinement
if params.refinement.parameterisation.scan_varying:
from dials.algorithms.refinement.reflection_manager import BlockCalculator
block_calculator = BlockCalculator(experiments, reflections)
if params.refinement.parameterisation.compose_model_per == "block":
reflections = block_calculator.per_width(
params.refinement.parameterisation.block_width, deg=True)
elif params.refinement.parameterisation.compose_model_per == "image":
reflections = block_calculator.per_image()
logger.debug("\nBuilding reflection manager")
logger.debug("Input reflection list size = %d observations",
len(reflections))
# create reflection manager
from dials.algorithms.refinement.reflection_manager import (
ReflectionManagerFactory, )
refman = ReflectionManagerFactory.from_parameters_reflections_experiments(
params.refinement.reflections, reflections, experiments, do_stills)
logger.debug(
"Number of observations that pass initial inclusion criteria = %d",
refman.get_accepted_refs_size(),
)
sample_size = refman.get_sample_size()
if sample_size > 0:
logger.debug("Working set size = %d observations", sample_size)
logger.debug("Reflection manager built\n")
# configure use of sparse data types
params = cls.config_sparse(params, experiments)
do_sparse = params.refinement.parameterisation.sparse
# create managed reflection predictor
from dials.algorithms.refinement.prediction.managed_predictors import (
ExperimentsPredictorFactory, )
ref_predictor = ExperimentsPredictorFactory.from_experiments(
experiments,
force_stills=do_stills,
spherical_relp=params.refinement.parameterisation.
spherical_relp_model,
)
# Predict for the managed observations, set columns for residuals and set
# the used_in_refinement flag to the predictions
obs = refman.get_obs()
ref_predictor(obs)
x_obs, y_obs, phi_obs = obs["xyzobs.mm.value"].parts()
x_calc, y_calc, phi_calc = obs["xyzcal.mm"].parts()
obs["x_resid"] = x_calc - x_obs
obs["y_resid"] = y_calc - y_obs
obs["phi_resid"] = phi_calc - phi_obs
# determine whether to do basic centroid analysis to automatically
# determine outlier rejection block
if params.refinement.reflections.outlier.block_width is libtbx.Auto:
ca = refman.get_centroid_analyser()
analysis = ca(calc_average_residuals=False,
calc_periodograms=False)
else:
analysis = None
# Now predictions and centroid analysis are available, so we can finalise
# the reflection manager
refman.finalise(analysis)
# Create model parameterisations
logger.debug("Building prediction equation parameterisation")
pred_param, param_reporter = cls.config_parameterisation(
params.refinement.parameterisation, experiments, refman, do_stills)
logger.debug("Prediction equation parameterisation built")
logger.debug("Parameter order : name mapping")
for i, e in enumerate(pred_param.get_param_names()):
logger.debug("Parameter %03d : %s", i + 1, e)
# Build a restraints parameterisation (if requested).
# Only unit cell restraints are supported at the moment.
restraints_parameterisation = cls.config_restraints(
params.refinement.parameterisation, pred_param)
# Build a constraints manager, if requested
from dials.algorithms.refinement.constraints import ConstraintManagerFactory
cmf = ConstraintManagerFactory(params, pred_param)
constraints_manager = cmf()
# Create target function
logger.debug("Building target function")
target = cls.config_target(
params.refinement.target,
experiments,
refman,
ref_predictor,
pred_param,
restraints_parameterisation,
do_stills,
do_sparse,
)
logger.debug("Target function built")
# create refinery
logger.debug("Building refinement engine")
refinery = cls.config_refinery(params, target, pred_param,
constraints_manager)
logger.debug("Refinement engine built")
nparam = len(pred_param)
ndim = target.dim
nref = len(refman.get_matches())
logger.info(
"There are {0} parameters to refine against {1} reflections in {2} dimensions"
.format(nparam, nref, ndim))
from dials.algorithms.refinement.engine import AdaptLstbx
if not params.refinement.parameterisation.sparse and isinstance(
refinery, AdaptLstbx):
dense_jacobian_gigabytes = (nparam * nref * ndim *
flex.double.element_size()) / 1e9
tot_memory_gigabytes = machine_memory_info().memory_total() / 1e9
# Report if the Jacobian requires a large amount of storage
if (dense_jacobian_gigabytes > 0.2 * tot_memory_gigabytes
or dense_jacobian_gigabytes > 0.5):
logger.info(
"Storage of the Jacobian matrix requires {:.1f} GB".format(
dense_jacobian_gigabytes))
# build refiner interface and return
if params.refinement.parameterisation.scan_varying:
refiner = ScanVaryingRefiner
else:
refiner = Refiner
return refiner(experiments, pred_param, param_reporter, refman, target,
refinery)
def __call__(self):
'''
Do the processing.
:return: The processed data
'''
from dials.array_family import flex
from time import time
from dials.model.data import make_image
from libtbx.introspection import machine_memory_info
# Get the start time
start_time = time()
# Set the global process ID
job.index = self.index
# Check all reflections have same imageset and get it
exp_id = list(set(self.reflections['id']))
imageset = self.experiments[exp_id[0]].imageset
for i in exp_id[1:]:
assert self.experiments[
i].imageset == imageset, "Task can only handle 1 imageset"
# Get the sub imageset
frame00, frame01 = self.job
try:
frame10, frame11 = imageset.get_array_range()
except Exception:
frame10, frame11 = (0, len(imageset))
try:
assert frame00 < frame01
assert frame10 < frame11
assert frame00 >= frame10
assert frame01 <= frame11
index0 = frame00 - frame10
index1 = index0 + (frame01 - frame00)
assert index0 < index1
assert index0 >= 0
assert index1 <= len(imageset)
imageset = imageset[index0:index1]
except Exception:
raise RuntimeError('Programmer Error: bad array range')
try:
frame0, frame1 = imageset.get_array_range()
except Exception:
frame0, frame1 = (0, len(imageset))
# Initlize the executor
self.executor.initialize(frame0, frame1, self.reflections)
# Set the shoeboxes (dont't allocate)
self.reflections['shoebox'] = flex.shoebox(
self.reflections['panel'],
self.reflections['bbox'],
allocate=False,
flatten=self.params.shoebox.flatten)
# Create the processor
processor = ShoeboxProcessor(self.reflections,
len(imageset.get_detector()), frame0,
frame1, self.params.debug.output)
# Compute percentage of max available. The function is not portable to
# windows so need to add a check if the function fails. On windows no
# warning will be printed
memory_info = machine_memory_info()
total_memory = memory_info.memory_total()
sbox_memory = processor.compute_max_memory_usage()
if total_memory is not None:
assert total_memory > 0, "Your system appears to have no memory!"
assert self.params.block.max_memory_usage > 0.0, "maximum memory usage must be > 0"
assert self.params.block.max_memory_usage <= 1.0, "maximum memory usage must be <= 1"
limit_memory = total_memory * self.params.block.max_memory_usage
if sbox_memory > limit_memory:
raise RuntimeError('''
There was a problem allocating memory for shoeboxes. Possible solutions
include increasing the percentage of memory allowed for shoeboxes or
decreasing the block size. This could also be caused by a highly mosaic
crystal model - is your crystal really this mosaic?
Total system memory: %g GB
Limit shoebox memory: %g GB
Required shoebox memory: %g GB
''' % (total_memory / 1e9, limit_memory / 1e9, sbox_memory / 1e9))
else:
logger.info(' Memory usage:')
logger.info(' Total system memory: %g GB' %
(total_memory / 1e9))
logger.info(' Limit shoebox memory: %g GB' %
(limit_memory / 1e9))
logger.info(' Required shoebox memory: %g GB' %
(sbox_memory / 1e9))
logger.info('')
# Loop through the imageset, extract pixels and process reflections
read_time = 0.0
for i in range(len(imageset)):
st = time()
image = imageset.get_corrected_data(i)
mask = imageset.get_mask(i)
if self.params.lookup.mask is not None:
assert len(mask) == len(self.params.lookup.mask), \
"Mask/Image are incorrect size %d %d" % (
len(mask),
len(self.params.lookup.mask))
mask = tuple(m1 & m2
for m1, m2 in zip(self.params.lookup.mask, mask))
read_time += time() - st
processor.next(make_image(image, mask), self.executor)
del image
del mask
assert processor.finished(), "Data processor is not finished"
# Optionally save the shoeboxes
if self.params.debug.output and self.params.debug.separate_files:
output = self.reflections
if self.params.debug.select is not None:
output = output.select(self.params.debug.select(output))
if self.params.debug.split_experiments:
output = output.split_by_experiment_id()
for table in output:
i = table['id'][0]
table.as_pickle('shoeboxes_%d_%d.pickle' % (self.index, i))
else:
output.as_pickle('shoeboxes_%d.pickle' % self.index)
# Delete the shoeboxes
if self.params.debug.separate_files or not self.params.debug.output:
del self.reflections['shoebox']
# Finalize the executor
self.executor.finalize()
# Return the result
result = Result(self.index, self.reflections, self.executor.data())
result.read_time = read_time
result.extract_time = processor.extract_time()
result.process_time = processor.process_time()
result.total_time = time() - start_time
return result
def __call__(self):
'''
Do the processing.
:return: The processed data
'''
from dials.array_family import flex
from time import time
from dials.model.data import make_image
from libtbx.introspection import machine_memory_info
# Get the start time
start_time = time()
# Set the global process ID
job.index = self.index
# Check all reflections have same imageset and get it
exp_id = list(set(self.reflections['id']))
imageset = self.experiments[exp_id[0]].imageset
for i in exp_id[1:]:
assert self.experiments[i].imageset == imageset, "Task can only handle 1 imageset"
# Get the sub imageset
frame00, frame01 = self.job
try:
frame10, frame11 = imageset.get_array_range()
except Exception:
frame10, frame11 = (0, len(imageset))
try:
assert frame00 < frame01
assert frame10 < frame11
assert frame00 >= frame10
assert frame01 <= frame11
index0 = frame00 - frame10
index1 = index0 + (frame01 - frame00)
assert index0 < index1
assert index0 >= 0
assert index1 <= len(imageset)
imageset = imageset[index0:index1]
except Exception:
raise RuntimeError('Programmer Error: bad array range')
try:
frame0, frame1 = imageset.get_array_range()
except Exception:
frame0, frame1 = (0, len(imageset))
# Initlize the executor
self.executor.initialize(frame0, frame1, self.reflections)
# Set the shoeboxes (dont't allocate)
self.reflections['shoebox'] = flex.shoebox(
self.reflections['panel'],
self.reflections['bbox'],
allocate=False,
flatten=self.params.shoebox.flatten)
# Create the processor
processor = ShoeboxProcessor(
self.reflections,
len(imageset.get_detector()),
frame0,
frame1,
self.params.debug.output)
# Compute percentage of max available. The function is not portable to
# windows so need to add a check if the function fails. On windows no
# warning will be printed
memory_info = machine_memory_info()
total_memory = memory_info.memory_total()
sbox_memory = processor.compute_max_memory_usage()
if total_memory is not None:
assert total_memory > 0, "Your system appears to have no memory!"
assert self.params.block.max_memory_usage > 0.0, "maximum memory usage must be > 0"
assert self.params.block.max_memory_usage <= 1.0, "maximum memory usage must be <= 1"
limit_memory = total_memory * self.params.block.max_memory_usage
if sbox_memory > limit_memory:
raise RuntimeError('''
There was a problem allocating memory for shoeboxes. Possible solutions
include increasing the percentage of memory allowed for shoeboxes or
decreasing the block size. This could also be caused by a highly mosaic
crystal model - is your crystal really this mosaic?
Total system memory: %g GB
Limit shoebox memory: %g GB
Required shoebox memory: %g GB
''' % (total_memory/1e9, limit_memory/1e9, sbox_memory/1e9))
else:
logger.info(' Memory usage:')
logger.info(' Total system memory: %g GB' % (total_memory/1e9))
logger.info(' Limit shoebox memory: %g GB' % (limit_memory/1e9))
logger.info(' Required shoebox memory: %g GB' % (sbox_memory/1e9))
logger.info('')
# Loop through the imageset, extract pixels and process reflections
read_time = 0.0
for i in range(len(imageset)):
st = time()
image = imageset.get_corrected_data(i)
mask = imageset.get_mask(i)
if self.params.lookup.mask is not None:
assert len(mask) == len(self.params.lookup.mask), \
"Mask/Image are incorrect size %d %d" % (
len(mask),
len(self.params.lookup.mask))
mask = tuple(m1 & m2 for m1, m2 in zip(self.params.lookup.mask, mask))
read_time += time() - st
processor.next(make_image(image, mask), self.executor)
del image
del mask
assert processor.finished(), "Data processor is not finished"
# Optionally save the shoeboxes
if self.params.debug.output and self.params.debug.separate_files:
output = self.reflections
if self.params.debug.select is not None:
output = output.select(self.params.debug.select(output))
if self.params.debug.split_experiments:
output = output.split_by_experiment_id()
for table in output:
i = table['id'][0]
table.as_pickle('shoeboxes_%d_%d.pickle' % (self.index, i))
else:
output.as_pickle('shoeboxes_%d.pickle' % self.index)
# Delete the shoeboxes
if self.params.debug.separate_files or not self.params.debug.output:
del self.reflections['shoebox']
# Finalize the executor
self.executor.finalize()
# Return the result
result = Result(self.index, self.reflections, self.executor.data())
result.read_time = read_time
result.extract_time = processor.extract_time()
result.process_time = processor.process_time()
result.total_time = time() - start_time
return result
