def parallel_map (
func,
iterable,
params=None,
processes=1,
method="multiprocessing",
qsub_command=None,
asynchronous=True,
callback=None,
preserve_order=True,
preserve_exception_message=False,
use_manager=True) :
if method == 'mpi' and processes <= 2:
method = 'multiprocessing'
if method == 'mpi':
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank, size = misc.get_mpi_rank_and_size()
assert size == processes
assert not preserve_order
results = []
# use a client/server approach to be sure every process is busy as much as possible
# only do this if there are more than 2 processes, as one process will be a server
if rank == 0:
# server process
for task_id in xrange(len(iterable)):
# a client process will indicate it's ready by sending its rank
request = comm.recv(source=MPI.ANY_SOURCE)
comm.send(task_id,dest=request)
# send a stop command to each process
for rankreq in xrange(1,size):
#rankreq = comm.recv(source=MPI.ANY_SOURCE)
comm.send('endjob',dest=rankreq)
for proc_id in xrange(1,size):
res = comm.recv(source=proc_id)
try:
results.extend(res)
except TypeError:
results.extend(comm.recv(source=proc_id))
return results
else:
# client process
results = []
while True:
# inform the server this process is ready for an event
comm.send(rank,dest=0)
task_id = comm.recv(source=0)
if task_id == 'endjob': break
results.append(func(iterable[task_id]))
comm.send(results,dest=0)
else:
return easy_mp_parallel_map(func, iterable, params, processes, method,
qsub_command, asynchronous, callback,
preserve_order, preserve_exception_message,
use_manager)
def run(self):
self.args, self.phil_args = parse_command_args(self.iver,
self.help_message).parse_known_args()
# Check for type of input
if self.args.path == None: # No input
parse_command_args(self.iver, self.help_message).print_help()
if self.args.default: # Write out default params and exit
help_out, txt_out = inp.print_params()
print '\n{:-^70}\n'.format('IOTA Parameters')
print help_out
inp.write_defaults(os.path.abspath(os.path.curdir), txt_out)
misc.iota_exit(self.iver)
else: # If input exists, check type
carg = os.path.abspath(self.args.path)
if os.path.exists(carg):
# If user provided a parameter file
if os.path.isfile(carg) and os.path.basename(carg).endswith('.param'):
msg = ''
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'file')
# If user provided a list of input files
elif os.path.isfile(carg) and os.path.basename(carg).endswith('.lst'):
msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg)
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'auto', self.now)
# If user provided a single filepath
elif os.path.isfile(carg) and not os.path.basename(carg).endswith('.lst'):
msg = "\nIOTA will run in SINGLE-FILE mode using {}:\n".format(carg)
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'auto', self.now)
# If user provided a data folder
elif os.path.isdir(carg):
msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg)
self.params, self.txt_out = inp.process_input(self.args,
self.phil_args,
carg, 'auto', self.now)
# If user provided gibberish
else:
print self.logo
print "ERROR: Invalid input! Need parameter filename or data folder."
misc.iota_exit(self.iver)
# Identify indexing / integration program
if self.params.advanced.integrate_with == 'cctbx':
prg = " with CCTBX.XFEL\n"
elif self.params.advanced.integrate_with == 'dials':
prg = " with DIALS\n"
self.logo += prg
print self.logo
print '\n{}\n'.format(self.now)
if msg != '':
print msg
# Check for -l option, output list of input files and exit
if self.args.list:
list_file = os.path.abspath("{}/input.lst".format(os.curdir))
print '\nIOTA will run in LIST INPUT ONLY mode'
print 'Input list in {} \n\n'.format(list_file)
with open(list_file, "w") as lf:
for i, input_file in enumerate(input_list, 1):
print "{}: {}".format(i, input_file)
lf.write('{}\n'.format(input_file))
print '\nExiting...\n\n'
misc.iota_exit(self.iver)
if self.args.analyze != None:
self.analyze_prior_results('{:003d}'.format(int(self.args.analyze)))
misc.iota_exit(self.iver)
if self.params.mp_method == 'mpi':
rank, size = misc.get_mpi_rank_and_size()
self.master_process = rank == 0
else:
self.master_process = True
# Call function to read input folder structure (or input file) and
# generate list of image file paths
if self.params.cctbx.selection.select_only.flag_on:
self.gs_img_objects = self.make_int_object_list()
self.input_list = [i.conv_img for i in self.gs_img_objects]
else:
self.input_list = self.make_input_list()
# If fewer images than requested processors are supplied, set the number of
# processors to the number of images
if self.params.n_processors > len(self.input_list):
self.params.n_processors = len(self.input_list)
# Generate base folder paths
self.conv_base = misc.set_base_dir('converted_pickles')
self.int_base = misc.set_base_dir('integration')
self.obj_base = os.path.join(self.int_base, 'image_objects')
self.fin_base = os.path.join(self.int_base, 'final')
self.tmp_base = os.path.join(self.int_base, 'tmp')
if self.params.analysis.viz != 'None' or\
self.params.analysis.heatmap != 'None' or\
self.params.analysis.charts:
self.viz_base = os.path.join(self.int_base, 'visualization')
else:
self.viz_base = None
# Generate base folders
os.makedirs(self.int_base)
os.makedirs(self.obj_base)
os.makedirs(self.fin_base)
os.makedirs(self.tmp_base)
# Determine input base
self.input_base = os.path.abspath(os.path.dirname(os.path.commonprefix(self.input_list)))
# Initialize main log
self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log'))
# Log starting info
misc.main_log(self.logfile, '{:=^100} \n'.format(' IOTA MAIN LOG '))
misc.main_log(self.logfile, '{:-^100} \n'.format(' SETTINGS FOR THIS RUN '))
misc.main_log(self.logfile, self.txt_out)
misc.main_log(self.logfile, '{:-^100} \n\n'
''.format(' TARGET FILE ({}) CONTENTS '
''.format(self.params.target)))
with open(self.params.target, 'r') as phil_file:
phil_file_contents = phil_file.read()
misc.main_log(self.logfile, phil_file_contents)
