def run(self, all_experiments, all_reflections):
""" Load all the data using MPI """
from dxtbx.model.experiment_list import ExperimentList
from dials.array_family import flex
# Both must be none or not none
test = [all_experiments is None, all_reflections is None].count(True)
assert test in [0, 2]
if test == 2:
all_experiments = ExperimentList()
all_reflections = flex.reflection_table()
starting_expts_count = starting_refls_count = 0
else:
starting_expts_count = len(all_experiments)
starting_refls_count = len(all_reflections)
self.logger.log(
"Initial number of experiments: %d; Initial number of reflections: %d"
% (starting_expts_count, starting_refls_count))
# Generate and send a list of file paths to each worker
if self.mpi_helper.rank == 0:
file_list = self.get_list()
self.logger.log(
"Built an input list of %d json/pickle file pairs" %
(len(file_list)))
self.params.input.path = None # Rank 0 has already parsed the input parameters
per_rank_file_list = file_load_calculator(self.params, file_list, self.logger).\
calculate_file_load(available_rank_count = self.mpi_helper.size)
self.logger.log(
'Transmitting a list of %d lists of json/pickle file pairs' %
(len(per_rank_file_list)))
transmitted = per_rank_file_list
else:
transmitted = None
self.logger.log_step_time("BROADCAST_FILE_LIST")
transmitted = self.mpi_helper.comm.bcast(transmitted, root=0)
new_file_list = transmitted[
self.mpi_helper.
rank] if self.mpi_helper.rank < len(transmitted) else None
self.logger.log_step_time("BROADCAST_FILE_LIST", True)
# Load the data
self.logger.log_step_time("LOAD")
if new_file_list is not None:
self.logger.log("Received a list of %d json/pickle file pairs" %
len(new_file_list))
for experiments_filename, reflections_filename in new_file_list:
self.logger.log("Reading %s %s" %
(experiments_filename, reflections_filename))
experiments = ExperimentListFactory.from_json_file(
experiments_filename, check_format=False)
reflections = flex.reflection_table.from_file(
reflections_filename)
self.logger.log("Data read, prepping")
if 'intensity.sum.value' in reflections:
reflections[
'intensity.sum.value.unmodified'] = reflections[
'intensity.sum.value'] * 1
if 'intensity.sum.variance' in reflections:
reflections[
'intensity.sum.variance.unmodified'] = reflections[
'intensity.sum.variance'] * 1
new_ids = flex.int(len(reflections), -1)
new_identifiers = flex.std_string(len(reflections))
eid = reflections.experiment_identifiers()
for k in eid.keys():
del eid[k]
for experiment_id, experiment in enumerate(experiments):
# select reflections of the current experiment
refls_sel = reflections['id'] == experiment_id
if refls_sel.count(True) == 0: continue
if experiment.identifier is None or len(
experiment.identifier) == 0:
experiment.identifier = create_experiment_identifier(
experiment, experiments_filename, experiment_id)
if not self.params.input.keep_imagesets:
experiment.imageset = None
all_experiments.append(experiment)
# Reflection experiment 'id' is unique within this rank; 'exp_id' (i.e. experiment identifier) is unique globally
new_identifiers.set_selected(refls_sel,
experiment.identifier)
new_id = len(all_experiments) - 1
eid[new_id] = experiment.identifier
new_ids.set_selected(refls_sel, new_id)
assert (new_ids < 0
).count(True) == 0, "Not all reflections accounted for"
reflections['id'] = new_ids
reflections['exp_id'] = new_identifiers
all_reflections.extend(reflections)
else:
self.logger.log("Received a list of 0 json/pickle file pairs")
self.logger.log_step_time("LOAD", True)
self.logger.log('Read %d experiments consisting of %d reflections' %
(len(all_experiments) - starting_expts_count,
len(all_reflections) - starting_refls_count))
self.logger.log("Memory usage: %d MB" % get_memory_usage())
all_reflections = self.prune_reflection_table_keys(all_reflections)
# Do we have any data?
from xfel.merging.application.utils.data_counter import data_counter
data_counter(self.params).count(all_experiments, all_reflections)
return all_experiments, all_reflections
def run(self, all_experiments, all_reflections):
""" Load all the data using MPI """
from dxtbx.model.experiment_list import ExperimentList
from dials.array_family import flex
# Both must be none or not none
test = [all_experiments is None, all_reflections is None].count(True)
assert test in [0, 2]
if test == 2:
all_experiments = ExperimentList()
all_reflections = flex.reflection_table()
starting_expts_count = starting_refls_count = 0
else:
starting_expts_count = len(all_experiments)
starting_refls_count = len(all_reflections)
self.logger.log(
"Initial number of experiments: %d; Initial number of reflections: %d"
% (starting_expts_count, starting_refls_count))
# Generate and send a list of file paths to each worker
if self.mpi_helper.rank == 0:
file_list = self.get_list()
self.logger.log(
"Built an input list of %d json/pickle file pairs" %
(len(file_list)))
self.params.input.path = None # the input is already parsed
per_rank_file_list = file_load_calculator(self.params, file_list, self.logger).\
calculate_file_load(available_rank_count = self.mpi_helper.size)
self.logger.log(
'Transmitting a list of %d lists of json/pickle file pairs' %
(len(per_rank_file_list)))
transmitted = per_rank_file_list
else:
transmitted = None
self.logger.log_step_time("BROADCAST_FILE_LIST")
transmitted = self.mpi_helper.comm.bcast(transmitted, root=0)
new_file_list = transmitted[self.mpi_helper.rank]
self.logger.log("Received a list of %d json/pickle file pairs" %
len(new_file_list))
self.logger.log_step_time("BROADCAST_FILE_LIST", True)
# Load the data
self.logger.log_step_time("LOAD")
for experiments_filename, reflections_filename in new_file_list:
experiments = ExperimentListFactory.from_json_file(
experiments_filename, check_format=False)
reflections = flex.reflection_table.from_file(reflections_filename)
for experiment_id, experiment in enumerate(experiments):
if experiment.identifier is None or len(
experiment.identifier) == 0:
experiment.identifier = create_experiment_identifier(
experiment, experiments_filename, experiment_id)
all_experiments.append(experiment)
refls = reflections.select(reflections['id'] == experiment_id)
refls['exp_id'] = flex.std_string(len(refls),
experiment.identifier)
all_reflections.extend(refls)
self.logger.log_step_time("LOAD", True)
self.logger.log('Read %d experiments consisting of %d reflections' %
(len(all_experiments) - starting_expts_count,
len(all_reflections) - starting_refls_count))
self.logger.log(get_memory_usage())
# Count the loaded data
data_counter(self.params).count(all_experiments, all_reflections)
return all_experiments, all_reflections
def run(self, all_experiments, all_reflections):
""" Load all the data using MPI """
from dxtbx.model.experiment_list import ExperimentList
from dials.array_family import flex
# Both must be none or not none
test = [all_experiments is None, all_reflections is None].count(True)
assert test in [0, 2]
if test == 2:
all_experiments = ExperimentList()
all_reflections = flex.reflection_table()
starting_expts_count = starting_refls_count = 0
else:
starting_expts_count = len(all_experiments)
starting_refls_count = len(all_reflections)
self.logger.log(
"Initial number of experiments: %d; Initial number of reflections: %d"
% (starting_expts_count, starting_refls_count))
# Generate and send a list of file paths to each worker
if self.mpi_helper.rank == 0:
file_list = self.get_list()
self.logger.log(
"Built an input list of %d json/pickle file pairs" %
(len(file_list)))
self.params.input.path = None # Rank 0 has already parsed the input parameters
per_rank_file_list = file_load_calculator(self.params, file_list, self.logger).\
calculate_file_load(available_rank_count = self.mpi_helper.size)
self.logger.log(
'Transmitting a list of %d lists of json/pickle file pairs' %
(len(per_rank_file_list)))
transmitted = per_rank_file_list
else:
transmitted = None
self.logger.log_step_time("BROADCAST_FILE_LIST")
transmitted = self.mpi_helper.comm.bcast(transmitted, root=0)
new_file_list = transmitted[
self.mpi_helper.
rank] if self.mpi_helper.rank < len(transmitted) else None
self.logger.log_step_time("BROADCAST_FILE_LIST", True)
# Load the data
self.logger.log_step_time("LOAD")
if new_file_list is not None:
self.logger.log("Received a list of %d json/pickle file pairs" %
len(new_file_list))
for experiments_filename, reflections_filename in new_file_list:
experiments = ExperimentListFactory.from_json_file(
experiments_filename, check_format=False)
reflections = flex.reflection_table.from_file(
reflections_filename)
for experiment_id, experiment in enumerate(experiments):
if experiment.identifier is None or len(
experiment.identifier) == 0:
experiment.identifier = create_experiment_identifier(
experiment, experiments_filename, experiment_id)
all_experiments.append(experiment)
#experiment.identifier = "%d"%(len(all_experiments) - 1)
# select reflections of the current experiment
refls = reflections.select(
reflections['id'] == experiment_id)
# Reflection experiment 'id' is supposed to be unique within this rank; 'exp_id' (i.e. experiment identifier) is supposed to be unique globally
#refls['id'] = flex.size_t(len(refls), len(all_experiments)-1)
refls['exp_id'] = flex.std_string(len(refls),
experiment.identifier)
all_reflections.extend(refls)
else:
self.logger.log("Received a list of 0 json/pickle file pairs")
self.logger.log_step_time("LOAD", True)
self.logger.log('Read %d experiments consisting of %d reflections' %
(len(all_experiments) - starting_expts_count,
len(all_reflections) - starting_refls_count))
self.logger.log("Memory usage: %d MB" % get_memory_usage())
from xfel.merging.application.reflection_table_utils import reflection_table_utils
all_reflections = reflection_table_utils.prune_reflection_table_keys(
reflections=all_reflections,
keys_to_keep=[
'intensity.sum.value', 'intensity.sum.variance',
'miller_index', 'miller_index_asymmetric', 'exp_id', 's1'
])
self.logger.log("Pruned reflection table")
self.logger.log("Memory usage: %d MB" % get_memory_usage())
# Do we have any data?
from xfel.merging.application.utils.data_counter import data_counter
data_counter(self.params).count(all_experiments, all_reflections)
return all_experiments, all_reflections
def run(self, all_experiments, all_reflections):
""" Load all the data using MPI """
from dxtbx.model.experiment_list import ExperimentList
from dials.array_family import flex
# Both must be none or not none
test = [all_experiments is None, all_reflections is None].count(True)
assert test in [0, 2]
if test == 2:
all_experiments = ExperimentList()
all_reflections = flex.reflection_table()
starting_expts_count = starting_refls_count = 0
else:
starting_expts_count = len(all_experiments)
starting_refls_count = len(all_reflections)
self.logger.log(
"Initial number of experiments: %d; Initial number of reflections: %d"
% (starting_expts_count, starting_refls_count))
# Generate and send a list of file paths to each worker
if self.mpi_helper.rank == 0:
file_list = self.get_list()
self.logger.log(
"Built an input list of %d json/pickle file pairs" %
(len(file_list)))
self.params.input.path = None # Rank 0 has already parsed the input parameters
# optionally write a file list mapping to disk, useful in post processing if save_experiments_and_reflections=True
file_id_from_names = None
if self.params.output.expanded_bookkeeping:
apath = lambda x: os.path.abspath(x)
file_names_from_id = {
i_f: tuple(map(apath, exp_ref_pair))
for i_f, exp_ref_pair in enumerate(file_list)
}
with open(
os.path.join(self.params.output.output_dir,
"file_list_map.json"), "w") as o:
json.dump(file_names_from_id, o)
file_id_from_names = {
tuple(map(apath, exp_ref_pair)): i_f
for i_f, exp_ref_pair in enumerate(file_list)
}
per_rank_file_list = file_load_calculator(self.params, file_list, self.logger).\
calculate_file_load(available_rank_count = self.mpi_helper.size)
self.logger.log(
'Transmitting a list of %d lists of json/pickle file pairs' %
(len(per_rank_file_list)))
transmitted = per_rank_file_list, file_id_from_names
else:
transmitted = None
self.logger.log_step_time("BROADCAST_FILE_LIST")
new_file_list, file_names_mapping = self.mpi_helper.comm.bcast(
transmitted, root=0)
new_file_list = new_file_list[
self.mpi_helper.
rank] if self.mpi_helper.rank < len(new_file_list) else None
self.logger.log_step_time("BROADCAST_FILE_LIST", True)
# Load the data
self.logger.log_step_time("LOAD")
if new_file_list is not None:
self.logger.log("Received a list of %d json/pickle file pairs" %
len(new_file_list))
for experiments_filename, reflections_filename in new_file_list:
self.logger.log("Reading %s %s" %
(experiments_filename, reflections_filename))
experiments = ExperimentListFactory.from_json_file(
experiments_filename,
check_format=self.params.input.read_image_headers)
reflections = flex.reflection_table.from_file(
reflections_filename)
if self.params.output.expanded_bookkeeping:
# NOTE: these are un-prunable
reflections["input_refl_index"] = flex.int(
list(range(len(reflections))))
reflections["orig_exp_id"] = reflections['id']
assert file_names_mapping is not None
exp_ref_pair = os.path.abspath(
experiments_filename), os.path.abspath(
reflections_filename)
this_refl_fileMappings = [
file_names_mapping[exp_ref_pair]
] * len(reflections)
reflections["file_list_mapping"] = flex.int(
this_refl_fileMappings)
self.logger.log("Data read, prepping")
if 'intensity.sum.value' in reflections:
reflections[
'intensity.sum.value.unmodified'] = reflections[
'intensity.sum.value'] * 1
if 'intensity.sum.variance' in reflections:
reflections[
'intensity.sum.variance.unmodified'] = reflections[
'intensity.sum.variance'] * 1
new_ids = flex.int(len(reflections), -1)
new_identifiers = flex.std_string(len(reflections))
eid = reflections.experiment_identifiers()
for k in eid.keys():
del eid[k]
if self.params.output.expanded_bookkeeping:
preGen_experiment_identifiers(experiments,
experiments_filename)
for experiment_id, experiment in enumerate(experiments):
# select reflections of the current experiment
refls_sel = reflections['id'] == experiment_id
if refls_sel.count(True) == 0: continue
if experiment.identifier is None or len(
experiment.identifier) == 0:
experiment.identifier = create_experiment_identifier(
experiment, experiments_filename, experiment_id)
if not self.params.input.keep_imagesets:
experiment.imageset = None
all_experiments.append(experiment)
# Reflection experiment 'id' is unique within this rank; 'exp_id' (i.e. experiment identifier) is unique globally
new_identifiers.set_selected(refls_sel,
experiment.identifier)
new_id = len(all_experiments) - 1
eid[new_id] = experiment.identifier
new_ids.set_selected(refls_sel, new_id)
assert (new_ids < 0
).count(True) == 0, "Not all reflections accounted for"
reflections['id'] = new_ids
reflections['exp_id'] = new_identifiers
all_reflections.extend(reflections)
else:
self.logger.log("Received a list of 0 json/pickle file pairs")
self.logger.log_step_time("LOAD", True)
self.logger.log('Read %d experiments consisting of %d reflections' %
(len(all_experiments) - starting_expts_count,
len(all_reflections) - starting_refls_count))
self.logger.log("Memory usage: %d MB" % get_memory_usage())
all_reflections = self.prune_reflection_table_keys(all_reflections)
# Do we have any data?
from xfel.merging.application.utils.data_counter import data_counter
data_counter(self.params).count(all_experiments, all_reflections)
return all_experiments, all_reflections
def run(self, all_experiments, all_reflections):
""" Load all the data using MPI """
from dxtbx.model.experiment_list import ExperimentList
from dials.array_family import flex
# Both must be none or not none
test = [all_experiments is None, all_reflections is None].count(True)
assert test in [0, 2]
if test == 2:
all_experiments = ExperimentList()
all_reflections = flex.reflection_table()
starting_expts_count = starting_refls_count = 0
else:
starting_expts_count = len(all_experiments)
starting_refls_count = len(all_reflections)
self.logger.log(
"Initial number of experiments: %d; Initial number of reflections: %d"
% (starting_expts_count, starting_refls_count))
# Generate and send a list of file paths to each worker
if self.mpi_helper.rank == 0:
file_list = self.get_list()
self.logger.log(
"Built an input list of %d json/pickle file pairs" %
(len(file_list)))
self.params.input.path = None # Rank 0 has already parsed the input parameters
per_rank_file_list = file_load_calculator(self.params, file_list, self.logger).\
calculate_file_load(available_rank_count = self.mpi_helper.size)
self.logger.log(
'Transmitting a list of %d lists of json/pickle file pairs' %
(len(per_rank_file_list)))
transmitted = per_rank_file_list
else:
transmitted = None
self.logger.log_step_time("BROADCAST_FILE_LIST")
transmitted = self.mpi_helper.comm.bcast(transmitted, root=0)
new_file_list = transmitted[
self.mpi_helper.
rank] if self.mpi_helper.rank < len(transmitted) else None
self.logger.log_step_time("BROADCAST_FILE_LIST", True)
# Load the data
self.logger.log_step_time("LOAD")
if new_file_list is not None:
self.logger.log("Received a list of %d json/pickle file pairs" %
len(new_file_list))
for experiments_filename, reflections_filename in new_file_list:
experiments = ExperimentListFactory.from_json_file(
experiments_filename, check_format=False)
reflections = flex.reflection_table.from_file(
reflections_filename)
# NOTE: had to use slicing below because it selection no longer works...
reflections.sort("id")
unique_refl_ids = set(reflections['id'])
assert len(unique_refl_ids) == len(
experiments
), "refl table and experiment list should contain data on same experiment " # TODO: decide if this is true
assert min(
reflections["id"]
) >= 0, "No more -1 in the id column, ideally it should be the numerical index of experiment, but beware that this is not enforced anywhere in the upstream code base"
if 'intensity.sum.value' in reflections:
reflections[
'intensity.sum.value.unmodified'] = reflections[
'intensity.sum.value'] * 1
if 'intensity.sum.variance' in reflections:
reflections[
'intensity.sum.variance.unmodified'] = reflections[
'intensity.sum.variance'] * 1
for experiment_id, experiment in enumerate(experiments):
if experiment.identifier is None or len(
experiment.identifier) == 0:
experiment.identifier = create_experiment_identifier(
experiment, experiments_filename, experiment_id)
all_experiments.append(experiment)
# select reflections of the current experiment
# FIXME the selection was broke for me, it raised
# RuntimeError: boost::bad_get: failed value get using boost::get
#refls = reflections.select(reflections['id'] == experiment_id)
# NOTE: this is a hack due to the broken expereimnt_id selection above
exp_id_pos = np.where(
reflections['id'] == experiment_id)[0]
assert exp_id_pos.size, "no refls in this experiment" # NOTE: maybe we can relax this assertion ?
refls = reflections[exp_id_pos[0]:exp_id_pos[-1] + 1]
#FIXME: how will this work if reading in multiple composite mode experiment jsons?
# Reflection experiment 'id' is supposed to be unique within this rank; 'exp_id' (i.e. experiment identifier) is supposed to be unique globally
refls['exp_id'] = flex.std_string(len(refls),
experiment.identifier)
new_id = 0
if len(all_reflections) > 0:
new_id = max(all_reflections['id']) + 1
# FIXME: it is hard to interperet that a function call returning a changeable property
eid = refls.experiment_identifiers()
for k in eid.keys():
del eid[k]
eid[new_id] = experiment.identifier
refls['id'] = flex.int(len(refls), new_id)
all_reflections.extend(refls)
else:
self.logger.log("Received a list of 0 json/pickle file pairs")
self.logger.log_step_time("LOAD", True)
self.logger.log('Read %d experiments consisting of %d reflections' %
(len(all_experiments) - starting_expts_count,
len(all_reflections) - starting_refls_count))
self.logger.log("Memory usage: %d MB" % get_memory_usage())
from xfel.merging.application.reflection_table_utils import reflection_table_utils
all_reflections = reflection_table_utils.prune_reflection_table_keys(
reflections=all_reflections,
keys_to_keep=[
'intensity.sum.value', 'intensity.sum.variance',
'miller_index', 'miller_index_asymmetric', 'exp_id', 's1',
'intensity.sum.value.unmodified',
'intensity.sum.variance.unmodified'
])
self.logger.log("Pruned reflection table")
self.logger.log("Memory usage: %d MB" % get_memory_usage())
# Do we have any data?
from xfel.merging.application.utils.data_counter import data_counter
data_counter(self.params).count(all_experiments, all_reflections)
return all_experiments, all_reflections
