def get_file_list(self):
# get json/pickle file list
loader = file_loader(self.params)
file_list = loader.filename_lister()
print("Discovered %d input json/pickle file pairs" % len(file_list))
return file_list
def get_list(self):
# example showing what reading all the data into a single experiment list/
# reflection table would look like
loader = file_loader(self.params)
file_list = list(loader.filepair_generator())
return file_list
def get_file_list(self):
# get json/pickle file list
loader = file_loader(self.params)
file_list = loader.filename_lister()
if debug_file_load_calculator:
self.log_write("\nDiscovered %d input json/pickle file pairs"%len(file_list))
print("\nDiscovered %d input json/pickle file pairs"%len(file_list))
return file_list
def load_data(self):
from xfel.merging.application.input.file_loader import file_loader
from dxtbx.model.experiment_list import ExperimentList
from dials.array_family import flex
all_experiments = ExperimentList()
all_reflections = flex.reflection_table()
# example showing what reading all the data into a single experiment list/
# reflection table would look like
loader = file_loader(self.params)
for experiments_filename, reflections_filename in loader.filepair_generator():
experiments, reflections = loader.load_data(experiments_filename, reflections_filename)
for experiment_id, experiment in enumerate(experiments):
all_experiments.append(experiment)
refls = reflections.select(reflections['id'] == experiment_id)
refls['id'] = flex.int(len(refls), len(all_experiments)-1)
all_reflections.extend(refls)
all_reflections.sort('miller_index_asymmetric')
return all_experiments, all_reflections
def load_data(self, file_list):
from dxtbx.model.experiment_list import ExperimentList
from dials.array_family import flex
all_experiments = ExperimentList()
all_reflections = flex.reflection_table()
loader = file_loader(self.params)
for experiments_filename, reflections_filename in file_list:
experiments, reflections = loader.load_data(experiments_filename, reflections_filename)
for experiment_id, experiment in enumerate(experiments):
all_experiments.append(experiment)
refls = reflections.select(reflections['id'] == experiment_id)
if len(refls) > 0:
refls['id'] = flex.int(len(refls), len(all_experiments)-1)
all_reflections.extend(refls)
return all_experiments, all_reflections
def load_data(self, file_list):
from dxtbx.model.experiment_list import ExperimentList
from dials.array_family import flex
all_experiments = ExperimentList()
all_reflections = flex.reflection_table()
# example showing what reading all the data into a single experiment list/
# reflection table would look like
loader = file_loader(self.params)
for experiments_filename, reflections_filename in file_list:
experiments, reflections = loader.load_data(
experiments_filename, reflections_filename)
for experiment_id, experiment in enumerate(experiments):
all_experiments.append(experiment)
refls = reflections.select(reflections['id'] == experiment_id)
refls['id'] = flex.int(len(refls), len(all_experiments) - 1)
all_reflections.extend(refls)
return all_experiments, all_reflections
def run(self, comm, timing=True):
# create timing table
self.do_timing = timing
if self.do_timing:
self.timing_table = dict()
# get rank and rank count
self.rank = comm.Get_rank()
self.rank_count = comm.Get_size()
self.log_step_time("TOTAL")
# prepare a list of input params and data files to be transmitted to all worker ranks
transmitted = {}
if self.rank == 0:
self.log_step_time("PARSE_INPUT_PARAMS")
# read and parse phil
self.initialize()
self.validate()
# initialize output paths; doing it here for rank 0 only - to be able to log timing, etc., before every rank initializes the paths
if self.do_timing:
self.mpi_timing_file_path = self.params.output.output_dir + '/timing_%06d_%06d.out'%(self.rank_count, self.rank)
self.mpi_log_file_path = self.params.output.output_dir + '/rank_%06d_%06d.out'%(self.rank_count, self.rank)
self.merged_reflections_file_path = self.params.output.output_dir + '/merge.out'
self.log_step_time("PARSE_INPUT_PARAMS", True)
###########################
# GET JSON/PICKLE FILE LIST
self.log_step_time("LIST_FILES")
loader = file_loader(self.params)
file_list = loader.filename_lister()
self.log_step_time("CALCULATE_FILE_LOAD")
load_calculator = file_load_calculator(self.params, file_list)
rank_files = load_calculator.calculate_file_load(self.rank_count)
total_file_pairs = 0
for key, value in rank_files.items():
total_file_pairs += len(value)
self.mpi_log_write("\nRank 0 generated a list of %d file items for %d ranks"%(total_file_pairs,len(rank_files)))
self.log_step_time("CALCULATE_FILE_LOAD", True)
self.log_step_time("LIST_FILES", True)
# prepare for transmitting the job to all ranks
self.mpi_log_write("\nRank 0 transmitting file list of length %d"%(len(file_list)))
transmitted = dict(params = self.params, options = self.options, rank_files = rank_files)
#########################################
# BROADCAST WORK FROM RANK 0 TO ALL RANKS
comm.barrier()
self.log_step_time("BROADCAST")
transmitted = comm.bcast(transmitted, root = 0)
self.params = transmitted['params']
self.options = transmitted['options']
new_file_list = transmitted['rank_files'][self.rank]
if self.do_timing:
self.mpi_timing_file_path = self.params.output.output_dir + '/timing_%06d_%06d.out'%(self.rank_count, self.rank)
self.mpi_log_file_path = self.params.output.output_dir + '/rank_%06d_%06d.out'%(self.rank_count, self.rank)
self.mpi_log_write ("\nRank %d received a file list of %d json-pickle file pairs" % (self.rank, len(new_file_list)))
comm.barrier()
self.log_step_time("BROADCAST", True)
################################################################################################
# EACH RANK: SET UP HKL CHUNKS TO BE USED FOR DISTRIBUTING LOADED REFLECTIONS AND FOR ALL-TO-ALL
self.log_step_time("SETUP_HKL_CHUNKS")
self.setup_hkl_chunks() # even if a rank got no files to load, it still has to participate in all-to-all
comm.barrier()
self.log_step_time("SETUP_HKL_CHUNKS", True)
# initialize data loading statistics
rank_experiment_count = 0
rank_image_count = 0
rank_reflection_count = 0
if len(new_file_list) > 0:
self.mpi_log_write("\nRank %d: first file to load is: %s"%(self.rank, str(new_file_list[0])))
######################
# EACH RANK: LOAD DATA
self.log_step_time("LOAD")
experiments, reflections = self.load_data(new_file_list)
self.mpi_log_write ('\nRank %d has read %d experiments consisting of %d reflections'%(self.rank, len(experiments), len(reflections)))
self.mpi_log_write("\nRANK %d: %s"%(self.rank, get_memory_usage()))
comm.barrier()
self.log_step_time("LOAD", True)
###############################################################################
# EACH RANK: GET THE TOTAL NUMBER OF LOADED EXPERIMENTS, IMAGES AND REFLECTIONS
self.log_step_time("LOAD_STATISTICS")
rank_experiment_count = len(experiments)
# count number of images
all_imgs = []
for iset in experiments.imagesets():
all_imgs.extend(iset.paths())
rank_image_count = len(set(all_imgs))
rank_reflection_count = len(reflections)
comm.barrier()
self.log_step_time("LOAD_STATISTICS", True)
#####################################################
# EACH RANK: ADD A COLUMN WITH SYMMETRY-REDUCED HKL's
self.log_step_time("ADD_ASU_HKL_COLUMN")
#reflections = self.add_asu_miller_indices_column(experiments, reflections)
self.add_asu_miller_indices_column(experiments, reflections)
comm.barrier()
self.log_step_time("ADD_ASU_HKL_COLUMN", True)
#########################################################################
# EACH RANK: PRUNE REFLECTION COLUMNS, WHICH ARE NOT RELEVANT FOR MERGING
self.log_step_time("PRUNE")
self.prune_reflection_columns(reflections=reflections)
comm.barrier()
self.log_step_time("PRUNE", True)
else:
reflections = distribute_reflection_table()
self.mpi_log_write ("\nRank %d received no data" % self.rank)
comm.barrier()
comm.barrier()
comm.barrier()
comm.barrier()
########################################
# MPI-REDUCE ALL DATA LOADING STATISTICS
self.log_step_time("REDUCE_LOAD_STATS")
total_experiment_count = comm.reduce(rank_experiment_count, MPI.SUM, 0)
total_image_count = comm.reduce(rank_image_count, MPI.SUM, 0)
total_reflection_count = comm.reduce(rank_reflection_count, MPI.SUM, 0)
max_reflection_count = comm.reduce(rank_reflection_count, MPI.MAX, 0)
min_reflection_count = comm.reduce(rank_reflection_count, MPI.MIN, 0)
########################################################################################
# RANK 0: LOG ALL DATA LOADING STATISTICS, INITIALIZE THE TOTAL MERGING REFLECTION TABLE
if self.rank == 0:
self.mpi_log_write('\nAll ranks have read %d experiments'%total_experiment_count)
self.mpi_log_write('\nAll ranks have read %d images'%total_image_count)
self.mpi_log_write('\nAll ranks have read %d reflections'%total_reflection_count)
self.mpi_log_write('\nThe maximum number of reflections loaded per rank is: %d reflections'%max_reflection_count)
self.mpi_log_write('\nThe minimum number of reflections loaded per rank is: %d reflections'%min_reflection_count)
comm.barrier()
self.log_step_time("REDUCE_LOAD_STATS", True)
#######################################################
# EACH RANK: DISTRIBUTE REFLECTIONS OVER ALL HKL CHUNKS
self.log_step_time("DISTRIBUTE")
self.distribute_reflections_over_hkl_chunks(reflections=reflections)
comm.barrier()
self.log_step_time("DISTRIBUTE", True)
#############################################################
# EACH RANK: GET A REFLECTION TABLE FOR MERGING BY ALL-TO-ALL
if self.params.parallel.a2a == 1:
alltoall_reflections = self.get_reflections_from_alltoall(comm=comm)
else: # if encountered alltoall memory problem, do alltoall on chunk slices
alltoall_reflections = self.get_reflections_from_alltoall_sliced(comm=comm, number_of_slices=self.params.parallel.a2a)
#####################################################
# EACH RANK: SORT THE REFLECTION TABLE BEFORE MERGING
self.log_step_time("SORT")
self.mpi_log_write("\nRank %d sorting consolidated reflection table..."%self.rank)
if len(alltoall_reflections) > 0:
alltoall_reflections.sort('miller_index_asymmetric')
comm.barrier()
self.log_step_time("SORT", True)
####################################################
# EACH RANK: DO STATISTICS ON REFLECTION INTENSITIES
self.log_step_time("AVERAGE")
self.mpi_log_write("\nRank %d doing intensity statistics..."%self.rank)
all_rank_merged_reflections = merging_reflection_table()
if len(alltoall_reflections) > 0:
for hkl_reflection_table in self.get_next_hkl_reflection_table(reflections=alltoall_reflections):
intensity_stats = self.calc_reflection_intensity_stats(reflections=hkl_reflection_table)
all_rank_merged_reflections.append({'miller_index': hkl_reflection_table[0].get('miller_index_asymmetric'),
'intensity': intensity_stats['average'],
'esd' : intensity_stats['esd'],
'rmsd' : intensity_stats['rmsd'],
'multiplicity': intensity_stats['multiplicity']})
self.mpi_log_write ("\nRank %d merged intensities for %d HKLs"%(self.rank, all_rank_merged_reflections.size()))
comm.barrier()
self.log_step_time("AVERAGE", True)
####################################################
# EACH RANK: SEND MERGED REFLECTION TABLES TO RANK 0
self.log_step_time("GATHER")
if self.rank != 0:
self.mpi_log_write("\nRank %d executing MPI gathering of all reflection tables at rank 0..."%self.rank)
all_merged_reflection_tables = comm.gather(all_rank_merged_reflections, root = 0)
comm.barrier()
self.log_step_time("GATHER", True)
####################################
# RANK 0: DO FINAL MERGING OF TABLES
if self.rank == 0:
self.log_step_time("MERGE")
final_merged_reflection_table = merging_reflection_table()
self.mpi_log_write ("\nRank 0 doing final merging of reflection tables received from all ranks...")
for table in all_merged_reflection_tables:
final_merged_reflection_table.extend(table)
self.mpi_log_write("\nRank 0 total merged HKLs: {}".format(final_merged_reflection_table.size()))
self.log_step_time("MERGE", True)
# write the final merged reflection table out to an ASCII file
self.log_step_time("WRITE")
self.output_merged_reflections(final_merged_reflection_table)
self.log_step_time("WRITE", True)
comm.barrier()
self.log_step_time("TOTAL", True)
MPI.Finalize()
return