def parallel_map ( func, iterable, params=None, processes=1, method="multiprocessing", qsub_command=None, asynchronous=True, callback=None, preserve_order=True, preserve_exception_message=False, use_manager=True) : if method == 'mpi' and processes <= 2: method = 'multiprocessing' if method == 'mpi': from mpi4py import MPI comm = MPI.COMM_WORLD rank, size = misc.get_mpi_rank_and_size() assert size == processes assert not preserve_order results = [] # use a client/server approach to be sure every process is busy as much as possible # only do this if there are more than 2 processes, as one process will be a server if rank == 0: # server process for task_id in xrange(len(iterable)): # a client process will indicate it's ready by sending its rank request = comm.recv(source=MPI.ANY_SOURCE) comm.send(task_id,dest=request) # send a stop command to each process for rankreq in xrange(1,size): #rankreq = comm.recv(source=MPI.ANY_SOURCE) comm.send('endjob',dest=rankreq) for proc_id in xrange(1,size): res = comm.recv(source=proc_id) try: results.extend(res) except TypeError: results.extend(comm.recv(source=proc_id)) return results else: # client process results = [] while True: # inform the server this process is ready for an event comm.send(rank,dest=0) task_id = comm.recv(source=0) if task_id == 'endjob': break results.append(func(iterable[task_id])) comm.send(results,dest=0) else: return easy_mp_parallel_map(func, iterable, params, processes, method, qsub_command, asynchronous, callback, preserve_order, preserve_exception_message, use_manager)
def run(self): self.args, self.phil_args = parse_command_args(self.iver, self.help_message).parse_known_args() # Check for type of input if self.args.path == None: # No input parse_command_args(self.iver, self.help_message).print_help() if self.args.default: # Write out default params and exit help_out, txt_out = inp.print_params() print '\n{:-^70}\n'.format('IOTA Parameters') print help_out inp.write_defaults(os.path.abspath(os.path.curdir), txt_out) misc.iota_exit(self.iver) else: # If input exists, check type carg = os.path.abspath(self.args.path) if os.path.exists(carg): # If user provided a parameter file if os.path.isfile(carg) and os.path.basename(carg).endswith('.param'): msg = '' self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'file') # If user provided a list of input files elif os.path.isfile(carg) and os.path.basename(carg).endswith('.lst'): msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg) self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'auto', self.now) # If user provided a single filepath elif os.path.isfile(carg) and not os.path.basename(carg).endswith('.lst'): msg = "\nIOTA will run in SINGLE-FILE mode using {}:\n".format(carg) self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'auto', self.now) # If user provided a data folder elif os.path.isdir(carg): msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg) self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'auto', self.now) # If user provided gibberish else: print self.logo print "ERROR: Invalid input! Need parameter filename or data folder." misc.iota_exit(self.iver) # Identify indexing / integration program if self.params.advanced.integrate_with == 'cctbx': prg = " with CCTBX.XFEL\n" elif self.params.advanced.integrate_with == 'dials': prg = " with DIALS\n" self.logo += prg print self.logo print '\n{}\n'.format(self.now) if msg != '': print msg # Check for -l option, output list of input files and exit if self.args.list: list_file = os.path.abspath("{}/input.lst".format(os.curdir)) print '\nIOTA will run in LIST INPUT ONLY mode' print 'Input list in {} \n\n'.format(list_file) with open(list_file, "w") as lf: for i, input_file in enumerate(input_list, 1): print "{}: {}".format(i, input_file) lf.write('{}\n'.format(input_file)) print '\nExiting...\n\n' misc.iota_exit(self.iver) if self.args.analyze != None: self.analyze_prior_results('{:003d}'.format(int(self.args.analyze))) misc.iota_exit(self.iver) if self.params.mp_method == 'mpi': rank, size = misc.get_mpi_rank_and_size() self.master_process = rank == 0 else: self.master_process = True # Call function to read input folder structure (or input file) and # generate list of image file paths if self.params.cctbx.selection.select_only.flag_on: self.gs_img_objects = self.make_int_object_list() self.input_list = [i.conv_img for i in self.gs_img_objects] else: self.input_list = self.make_input_list() # If fewer images than requested processors are supplied, set the number of # processors to the number of images if self.params.n_processors > len(self.input_list): self.params.n_processors = len(self.input_list) # Generate base folder paths self.conv_base = misc.set_base_dir('converted_pickles') self.int_base = misc.set_base_dir('integration') self.obj_base = os.path.join(self.int_base, 'image_objects') self.fin_base = os.path.join(self.int_base, 'final') self.tmp_base = os.path.join(self.int_base, 'tmp') if self.params.analysis.viz != 'None' or\ self.params.analysis.heatmap != 'None' or\ self.params.analysis.charts: self.viz_base = os.path.join(self.int_base, 'visualization') else: self.viz_base = None # Generate base folders os.makedirs(self.int_base) os.makedirs(self.obj_base) os.makedirs(self.fin_base) os.makedirs(self.tmp_base) # Determine input base self.input_base = os.path.abspath(os.path.dirname(os.path.commonprefix(self.input_list))) # Initialize main log self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log')) # Log starting info misc.main_log(self.logfile, '{:=^100} \n'.format(' IOTA MAIN LOG ')) misc.main_log(self.logfile, '{:-^100} \n'.format(' SETTINGS FOR THIS RUN ')) misc.main_log(self.logfile, self.txt_out) misc.main_log(self.logfile, '{:-^100} \n\n' ''.format(' TARGET FILE ({}) CONTENTS ' ''.format(self.params.target))) with open(self.params.target, 'r') as phil_file: phil_file_contents = phil_file.read() misc.main_log(self.logfile, phil_file_contents)