def main(match_funcs = None, num_parallel = 1): if not match_funcs: fs_list = fsop.get_all_fs() match_funcs = fsop.get_matched_ops(fs_list, "*") if num_parallel > 1: pool = mp.Pool(num_parallel) results = [pool.apply_async(match_each, args=(x,)) for x in match_funcs] while 1: if len(results) == 0: break print "> Unfinished jobs : ", len(results) print next_results = [] for res in results: try: print res.get(False), except mp.TimeoutError: next_results.append(res) time.sleep(2) results = next_results for funcs in match_funcs: match_each(funcs) return
def __generate_fops(self): all_pickles = glob.glob(join(self.log_d, "*.p")) if self.check_all: for pickle in all_pickles: fops = pickle[len(self.log_d) + 9:-2] yield [fops] else: for fops in fsop.get_matched_ops(self.fs, *self.args): fops = self.__check_validity(all_pickles, fops) if fops == None: continue yield fops """
def __generate_fops(self): all_pickles = glob.glob( join(self.log_d, "*.p") ) if self.check_all: for pickle in all_pickles: fops = pickle[len(self.log_d)+9:-2] yield [fops] else: for fops in fsop.get_matched_ops(self.fs, *self.args): fops = self.__check_validity(all_pickles, fops) if fops == None: continue yield fops """
def main(match_funcs = None, num_parallel = 1): if not match_funcs: fs_list = fsop.get_all_fs() match_funcs = fsop.get_matched_ops(fs_list, "*") if num_parallel > 1: pool = mp.Pool(num_parallel) results = [pool.apply_async(match_each, args=(x,)) for x in match_funcs] pool.close() pool.join() for res in results: try: print res.get(False) except: print("") else: for funcs in match_funcs: print match_each(funcs) return
def main(match_funcs=None, num_parallel=1): if not match_funcs: fs_list = fsop.get_all_fs() match_funcs = fsop.get_matched_ops(fs_list, "*") if num_parallel > 1: pool = mp.Pool(num_parallel) results = [ pool.apply_async(match_each, args=(x, )) for x in match_funcs ] pool.close() pool.join() for res in results: try: print res.get(False) except: print("") else: for funcs in match_funcs: print match_each(funcs) return