def test_cached_log_records(caplog):
# Generate some cached log messages in easy_mp child processes.
results = multi_node_parallel_map(
log_something,
iterable=range(5),
njobs=2,
nproc=2,
cluster_method="multiprocessing",
)
# Get all the resulting log records in a single flattened list.
results = [record for records in results for record in records]
results_batch = batch_multi_node_parallel_map(
log_something,
range(5),
njobs=2,
nproc=2,
cluster_method="multiprocessing",
callback=lambda _: None,
)
# Get all the resulting log records in a single flattened list.
results_batch = [
record for batch in results_batch for records in batch
for record in records
]
# Check that re-handling the messages in a logger in this process with a
# threshold severity of WARNING results in no log records being emitted.
with dials.util.log.LoggingContext("dials", logging.WARNING):
dials.util.log.rehandle_cached_records(results)
assert not caplog.records
dials.util.log.rehandle_cached_records(results_batch)
assert not caplog.records
# Check that re-handling the messages in a logger in this process with a
# threshold severity of INFO results in all the log records being emitted.
with dials.util.log.LoggingContext("dials", logging.INFO):
dials.util.log.rehandle_cached_records(results)
assert caplog.records == results
caplog.clear()
dials.util.log.rehandle_cached_records(results_batch)
assert caplog.records == results_batch
caplog.clear()
def _find_spots_2d_no_shoeboxes(self, imageset):
"""
Find the spots in the imageset
:param imageset: The imageset to process
:return: The list of spot shoeboxes
"""
from dials.util.mp import batch_multi_node_parallel_map
# Change the number of processors if necessary
mp_nproc = self.mp_nproc
mp_njobs = self.mp_njobs
if os.name == "nt" and (mp_nproc > 1 or mp_njobs > 1):
logger.warning(_no_multiprocessing_on_windows)
mp_nproc = 1
mp_njobs = 1
if mp_nproc * mp_njobs > len(imageset):
mp_nproc = min(mp_nproc, len(imageset))
mp_njobs = int(math.ceil(len(imageset) / mp_nproc))
mp_method = self.mp_method
mp_chunksize = self.mp_chunksize
if mp_chunksize == libtbx.Auto:
mp_chunksize = self._compute_chunksize(len(imageset),
mp_njobs * mp_nproc,
self.min_chunksize)
logger.info("Setting chunksize=%i" % mp_chunksize)
len_by_nproc = int(math.floor(len(imageset) / (mp_njobs * mp_nproc)))
if mp_chunksize > len_by_nproc:
mp_chunksize = len_by_nproc
assert mp_nproc > 0, "Invalid number of processors"
assert mp_njobs > 0, "Invalid number of jobs"
assert mp_njobs == 1 or mp_method is not None, "Invalid cluster method"
assert mp_chunksize > 0, "Invalid chunk size"
# The extract pixels function
function = ExtractPixelsFromImage2DNoShoeboxes(
imageset=imageset,
threshold_function=self.threshold_function,
mask=self.mask,
max_strong_pixel_fraction=self.max_strong_pixel_fraction,
compute_mean_background=self.compute_mean_background,
region_of_interest=self.region_of_interest,
min_spot_size=self.min_spot_size,
max_spot_size=self.max_spot_size,
filter_spots=self.filter_spots,
)
# The indices to iterate over
indices = list(range(len(imageset)))
# The resulting reflections
reflections = flex.reflection_table()
# Do the processing
logger.info("Extracting strong spots from images")
if mp_njobs > 1:
logger.info(
" Using %s with %d parallel job(s) and %d processes per node\n"
% (mp_method, mp_njobs, mp_nproc))
else:
logger.info(" Using multiprocessing with %d parallel job(s)\n" %
(mp_nproc))
if mp_nproc > 1 or mp_njobs > 1:
def process_output(result):
for message in result[1]:
logger.log(message.levelno, message.msg)
reflections.extend(result[0][0])
result[0][0] = None
batch_multi_node_parallel_map(
func=ExtractSpotsParallelTask(function),
iterable=indices,
nproc=mp_nproc,
njobs=mp_njobs,
cluster_method=mp_method,
chunksize=mp_chunksize,
callback=process_output,
)
else:
for task in indices:
reflections.extend(function(task)[0])
# Return the reflections
return reflections, None
def _find_spots(self, imageset):
"""
Find the spots in the imageset
:param imageset: The imageset to process
:return: The list of spot shoeboxes
"""
from dials.model.data import PixelListLabeller
from dials.util.mp import batch_multi_node_parallel_map
# Change the number of processors if necessary
mp_nproc = self.mp_nproc
mp_njobs = self.mp_njobs
if os.name == "nt" and (mp_nproc > 1 or mp_njobs > 1):
logger.warning(_no_multiprocessing_on_windows)
mp_nproc = 1
mp_njobs = 1
if mp_nproc * mp_njobs > len(imageset):
mp_nproc = min(mp_nproc, len(imageset))
mp_njobs = int(math.ceil(len(imageset) / mp_nproc))
mp_method = self.mp_method
mp_chunksize = self.mp_chunksize
if mp_chunksize == libtbx.Auto:
mp_chunksize = self._compute_chunksize(len(imageset),
mp_njobs * mp_nproc,
self.min_chunksize)
logger.info("Setting chunksize=%i" % mp_chunksize)
len_by_nproc = int(math.floor(len(imageset) / (mp_njobs * mp_nproc)))
if mp_chunksize > len_by_nproc:
mp_chunksize = len_by_nproc
if mp_chunksize == 0:
mp_chunksize = 1
assert mp_nproc > 0, "Invalid number of processors"
assert mp_njobs > 0, "Invalid number of jobs"
assert mp_njobs == 1 or mp_method is not None, "Invalid cluster method"
assert mp_chunksize > 0, "Invalid chunk size"
# The extract pixels function
function = ExtractPixelsFromImage(
imageset=imageset,
threshold_function=self.threshold_function,
mask=self.mask,
max_strong_pixel_fraction=self.max_strong_pixel_fraction,
compute_mean_background=self.compute_mean_background,
region_of_interest=self.region_of_interest,
)
# The indices to iterate over
indices = list(range(len(imageset)))
# Initialise the pixel labeller
num_panels = len(imageset.get_detector())
pixel_labeller = [PixelListLabeller() for p in range(num_panels)]
# Do the processing
logger.info("Extracting strong pixels from images")
if mp_njobs > 1:
logger.info(
" Using %s with %d parallel job(s) and %d processes per node\n"
% (mp_method, mp_njobs, mp_nproc))
else:
logger.info(" Using multiprocessing with %d parallel job(s)\n" %
(mp_nproc))
if mp_nproc > 1 or mp_njobs > 1:
def process_output(result):
for message in result[1]:
logger.log(message.levelno, message.msg)
assert len(pixel_labeller) == len(
result[0].pixel_list), "Inconsistent size"
for plabeller, plist in zip(pixel_labeller,
result[0].pixel_list):
plabeller.add(plist)
result[0].pixel_list = None
batch_multi_node_parallel_map(
func=ExtractSpotsParallelTask(function),
iterable=indices,
nproc=mp_nproc,
njobs=mp_njobs,
cluster_method=mp_method,
chunksize=mp_chunksize,
callback=process_output,
)
else:
for task in indices:
result = function(task)
assert len(pixel_labeller) == len(
result.pixel_list), "Inconsistent size"
for plabeller, plist in zip(pixel_labeller, result.pixel_list):
plabeller.add(plist)
result.pixel_list = None
# Create shoeboxes from pixel list
converter = PixelListToReflectionTable(
self.min_spot_size,
self.max_spot_size,
self.filter_spots,
self.write_hot_pixel_mask,
)
return converter(imageset, pixel_labeller)
def _find_spots_2d_no_shoeboxes(self, imageset):
'''
Find the spots in the imageset
:param imageset: The imageset to process
:return: The list of spot shoeboxes
'''
from dials.util.command_line import Command
from dials.array_family import flex
from dxtbx.imageset import ImageSweep
from dials.model.data import PixelListLabeller
from dials.util.mp import batch_multi_node_parallel_map
from math import floor, ceil
import platform
# Change the number of processors if necessary
mp_nproc = self.mp_nproc
mp_njobs = self.mp_njobs
if (mp_nproc > 1 or mp_njobs > 1) and platform.system() == "Windows": # platform.system() forks which is bad for MPI, so don't use it unless nproc > 1
logger.warn("")
logger.warn("*" * 80)
logger.warn("Multiprocessing is not available on windows. Setting nproc = 1, njobs = 1")
logger.warn("*" * 80)
logger.warn("")
mp_nproc = 1
mp_njobs = 1
if mp_nproc * mp_njobs > len(imageset):
mp_nproc = min(mp_nproc, len(imageset))
mp_njobs = int(ceil(len(imageset) / mp_nproc))
mp_method = self.mp_method
mp_chunksize = self.mp_chunksize
import libtbx
if mp_chunksize == libtbx.Auto:
mp_chunksize = self._compute_chunksize(
len(imageset),
mp_njobs * mp_nproc,
self.min_chunksize)
logger.info("Setting chunksize=%i" %mp_chunksize)
len_by_nproc = int(floor(len(imageset) / (mp_njobs * mp_nproc)))
if mp_chunksize > len_by_nproc:
mp_chunksize = len_by_nproc
assert mp_nproc > 0, "Invalid number of processors"
assert mp_njobs > 0, "Invalid number of jobs"
assert mp_njobs == 1 or mp_method is not None, "Invalid cluster method"
assert mp_chunksize > 0, "Invalid chunk size"
# The extract pixels function
function = ExtractPixelsFromImage2DNoShoeboxes(
imageset = imageset,
threshold_function = self.threshold_function,
mask = self.mask,
max_strong_pixel_fraction = self.max_strong_pixel_fraction,
compute_mean_background = self.compute_mean_background,
region_of_interest = self.region_of_interest,
min_spot_size = self.min_spot_size,
max_spot_size = self.max_spot_size,
filter_spots = self.filter_spots)
# The indices to iterate over
indices = list(range(len(imageset)))
# The resulting reflections
reflections = flex.reflection_table()
# Do the processing
logger.info('Extracting strong spots from images')
if mp_njobs > 1:
logger.info(' Using %s with %d parallel job(s) and %d processes per node\n' % (mp_method, mp_njobs, mp_nproc))
else:
logger.info(' Using multiprocessing with %d parallel job(s)\n' % (mp_nproc))
if mp_nproc > 1 or mp_njobs > 1:
def process_output(result):
for message in result[1]:
logger.log(message.levelno, message.msg)
reflections.extend(result[0][0])
result[0][0] = None
batch_multi_node_parallel_map(
func = ExtractSpotsParallelTask(function),
iterable = indices,
nproc = mp_nproc,
njobs = mp_njobs,
cluster_method = mp_method,
chunksize = mp_chunksize,
callback = process_output)
else:
for task in indices:
reflections.extend(function(task)[0])
# Return the reflections
return reflections, None
def _find_spots(self, imageset):
"""
Find the spots in the imageset
:param imageset: The imageset to process
:return: The list of spot shoeboxes
"""
# Change the number of processors if necessary
mp_nproc = self.mp_nproc
mp_njobs = self.mp_njobs
if mp_nproc is libtbx.Auto:
mp_nproc = available_cores()
logger.info(f"Setting nproc={mp_nproc}")
if mp_nproc * mp_njobs > len(imageset):
mp_nproc = min(mp_nproc, len(imageset))
mp_njobs = int(math.ceil(len(imageset) / mp_nproc))
mp_method = self.mp_method
mp_chunksize = self.mp_chunksize
if mp_chunksize is libtbx.Auto:
mp_chunksize = self._compute_chunksize(len(imageset),
mp_njobs * mp_nproc,
self.min_chunksize)
logger.info("Setting chunksize=%i", mp_chunksize)
len_by_nproc = int(math.floor(len(imageset) / (mp_njobs * mp_nproc)))
if mp_chunksize > len_by_nproc:
mp_chunksize = len_by_nproc
if mp_chunksize == 0:
mp_chunksize = 1
assert mp_nproc > 0, "Invalid number of processors"
assert mp_njobs > 0, "Invalid number of jobs"
assert mp_njobs == 1 or mp_method is not None, "Invalid cluster method"
assert mp_chunksize > 0, "Invalid chunk size"
# The extract pixels function
function = ExtractPixelsFromImage(
imageset=imageset,
threshold_function=self.threshold_function,
mask=self.mask,
max_strong_pixel_fraction=self.max_strong_pixel_fraction,
compute_mean_background=self.compute_mean_background,
region_of_interest=self.region_of_interest,
)
# The indices to iterate over
indices = list(range(len(imageset)))
# Initialise the pixel labeller
num_panels = len(imageset.get_detector())
pixel_labeller = [PixelListLabeller() for p in range(num_panels)]
# Do the processing
logger.info("Extracting strong pixels from images")
if mp_njobs > 1:
logger.info(
" Using %s with %d parallel job(s) and %d processes per node\n",
mp_method,
mp_njobs,
mp_nproc,
)
else:
logger.info(" Using multiprocessing with %d parallel job(s)\n",
mp_nproc)
if mp_nproc > 1 or mp_njobs > 1:
def process_output(result):
rehandle_cached_records(result[1])
assert len(pixel_labeller) == len(
result[0]), "Inconsistent size"
for plabeller, plist in zip(pixel_labeller, result[0]):
plabeller.add(plist)
batch_multi_node_parallel_map(
func=ExtractSpotsParallelTask(function),
iterable=indices,
nproc=mp_nproc,
njobs=mp_njobs,
cluster_method=mp_method,
chunksize=mp_chunksize,
callback=process_output,
)
else:
for task in indices:
result = function(task)
assert len(pixel_labeller) == len(result), "Inconsistent size"
for plabeller, plist in zip(pixel_labeller, result):
plabeller.add(plist)
result.clear()
# Create shoeboxes from pixel list
return pixel_list_to_reflection_table(
imageset,
pixel_labeller,
filter_spots=self.filter_spots,
min_spot_size=self.min_spot_size,
max_spot_size=self.max_spot_size,
write_hot_pixel_mask=self.write_hot_pixel_mask,
)
def _find_spots_2d_no_shoeboxes(self, imageset):
'''
Find the spots in the imageset
:param imageset: The imageset to process
:return: The list of spot shoeboxes
'''
from dials.array_family import flex
from dxtbx.imageset import ImageSweep
from dials.model.data import PixelListLabeller
from dials.util.mp import batch_multi_node_parallel_map
from math import floor, ceil
import platform
# Change the number of processors if necessary
mp_nproc = self.mp_nproc
mp_njobs = self.mp_njobs
if (mp_nproc > 1 or mp_njobs > 1) and platform.system(
) == "Windows": # platform.system() forks which is bad for MPI, so don't use it unless nproc > 1
logger.warn("")
logger.warn("*" * 80)
logger.warn(
"Multiprocessing is not available on windows. Setting nproc = 1, njobs = 1"
)
logger.warn("*" * 80)
logger.warn("")
mp_nproc = 1
mp_njobs = 1
if mp_nproc * mp_njobs > len(imageset):
mp_nproc = min(mp_nproc, len(imageset))
mp_njobs = int(ceil(len(imageset) / mp_nproc))
mp_method = self.mp_method
mp_chunksize = self.mp_chunksize
import libtbx
if mp_chunksize == libtbx.Auto:
mp_chunksize = self._compute_chunksize(len(imageset),
mp_njobs * mp_nproc,
self.min_chunksize)
logger.info("Setting chunksize=%i" % mp_chunksize)
len_by_nproc = int(floor(len(imageset) / (mp_njobs * mp_nproc)))
if mp_chunksize > len_by_nproc:
mp_chunksize = len_by_nproc
assert mp_nproc > 0, "Invalid number of processors"
assert mp_njobs > 0, "Invalid number of jobs"
assert mp_njobs == 1 or mp_method is not None, "Invalid cluster method"
assert mp_chunksize > 0, "Invalid chunk size"
# The extract pixels function
function = ExtractPixelsFromImage2DNoShoeboxes(
imageset=imageset,
threshold_function=self.threshold_function,
mask=self.mask,
max_strong_pixel_fraction=self.max_strong_pixel_fraction,
compute_mean_background=self.compute_mean_background,
region_of_interest=self.region_of_interest,
min_spot_size=self.min_spot_size,
max_spot_size=self.max_spot_size,
filter_spots=self.filter_spots)
# The indices to iterate over
indices = list(range(len(imageset)))
# The resulting reflections
reflections = flex.reflection_table()
# Do the processing
logger.info('Extracting strong spots from images')
if mp_njobs > 1:
logger.info(
' Using %s with %d parallel job(s) and %d processes per node\n'
% (mp_method, mp_njobs, mp_nproc))
else:
logger.info(' Using multiprocessing with %d parallel job(s)\n' %
(mp_nproc))
if mp_nproc > 1 or mp_njobs > 1:
def process_output(result):
for message in result[1]:
logger.log(message.levelno, message.msg)
reflections.extend(result[0][0])
result[0][0] = None
batch_multi_node_parallel_map(
func=ExtractSpotsParallelTask(function),
iterable=indices,
nproc=mp_nproc,
njobs=mp_njobs,
cluster_method=mp_method,
chunksize=mp_chunksize,
callback=process_output)
else:
for task in indices:
reflections.extend(function(task)[0])
# Return the reflections
return reflections, None
