def queryProcess(phone_md5_list): manager = Manager() q = manager.list() jobs = PROCESS_NUM pool = Pool(jobs) filestep = len(phone_md5_list) / jobs + 1 posBegin = 0 posEnd = posBegin + filestep processList = [] #创建十个进程,item:进程id , list:每个线程处理的文件大小,q:线程公共变量,收集十个进程统计的数据 for item in range(0, jobs): p = pool.apply_async(query, args=(item, phone_md5_list[posBegin:posEnd], q)) posBegin = posEnd posEnd = posBegin + filestep processList.append(p) while 1: try: time.sleep(1) status = True for item in processList: status = status and item.ready() if status: break except KeyboardInterrupt: pool.terminate() pool.wait(None) break for item in processList: item.wait() pool.close() pool.join() del pool return q
def register_images(temp_dir, temp_folder_raw, temp_folder_reg, ref_img, cfg): assert cfg.kp_algo.lower() in [ "surf", "orb" ], "Keypoint algorithm '{}' not supported".format(cfg.kp_algo) p_bar = tqdm(total=len(os.listdir(temp_folder_raw)), disable=not cfg.verbose, desc="Register frames".center(30)) p = Pool(cfg.num_workers).map_async( functools.partial(_register_image, ref_image=ref_img, raw_img_folder=temp_folder_raw, temp_folder=temp_dir, hessian_threshold=cfg.hessian_threshold, jpg_quality=cfg.jpg_quality, kp_algo=cfg.kp_algo), os.listdir(temp_folder_raw)) while True: a = len(os.listdir(temp_folder_raw)) b = len(os.listdir(temp_folder_reg)) p_bar.n = (a - (a - b)) p_bar.refresh() if a == b: break time.sleep(1) p_bar.close() p.wait() [os.remove(temp_folder_raw + file) for file in os.listdir(temp_folder_raw)]
def main(): args = get_args() start = time() with open(args.output, 'w') as f: wids = [line.strip() for line in open(args.input).readlines()[:args.number]] mapped = Pool(processes=8).map_async(identify_worker, wids) mapped.wait() print >> f, '\n'.join([x.encode('utf-8') for x in mapped.get()]) end = time() total = end - start print '%d seconds elapsed' % total
def main(repo_name): processingDirs = [] for corpus_dir in os.listdir("../"+repo_name+"_repo"): # if not corpus_dir == "Z057Ml": # "S002Mg": # continue processCorpusDir(repo_name, corpus_dir) pool = Pool(10) for corpus_dir in processingDirs: try: result = pool.apply_async(processCorpusDir, args=(corpus_dir,)) except KeyboardInterrupt: print ("Caught KeyboardInterrupt, terminating workers") pool.terminate() pool.wait() break pool.close() pool.join()
lang = details.get('lang') #print url #doc_ids = ListDocIdsService().get_value(wid) doc_ids = map( lambda x: x.split('_')[1], filter( lambda y: '_' in y, #ListDocIdsService().get_value(wid)))[:100] ListDocIdsService().get_value(wid))) #pprint(doc_ids); sys.exit(0) #for n in range(0, len(doc_ids), step): ##for n in range(0, 20, step): # print 'n = %d' % n # doc_ids_subset = doc_ids[n:n+step] r = Pool(processes=8).map_async(get_fields, chunks(doc_ids, step)) r.wait() pprint(r.get()) print '*' * 80 #for k in r.get(): # DEBUG # print k fields = [] m = map(lambda x: fields.extend(x), r.get()) #pprint(fields) indexed = dict(fields) pprint(indexed) # DEBUG #for doc_id in doc_ids_to_heads: # entity_response = doc_ids_to_entities.get( # doc_id, {'titles': [], 'redirects': {}}) # doc_ids_combined[doc_id] = map(preprocess,
res = [(sum(lnls) + lnp) / ini['temperature'], lnp] + lnls + derivs return tuple(res) ### Import additional modules for parallel computing if requested pool = None # Multithreading parallel computing via python-native multiprocessing module if (ini['parallel'][0] == 'multiprocessing') & (not ini['debug_mode']): from multiprocessing import Pool pool = Pool(int(ini['parallel'][1])) # number of threads chosen by user # MPI parallel computing via external schwimmbad module elif (ini['parallel'][0] == 'MPI') & (not ini['debug_mode']): from schwimmbad import MPIPool pool = MPIPool() if not pool.is_master(): # Necessary bit for MPI pool.wait() sys.exit(0) ### Conveninent MCMC settings variables n_dim = len(ini['var_par']) n_steps = ini['n_steps'] thin_by = ini['thin_by'] n_walkers = ini['n_walkers'] ### Randomize initial walkers positions according to "start" & "width" columns in ini file p0_start = [par[2] for par in ini['var_par']] std_start = [par[5] for par in ini['var_par']] p_start = np.zeros((n_walkers, n_dim)) for i in range(n_dim):
if details is not None: url = details.get('url') lang = details.get('lang') #print url #doc_ids = ListDocIdsService().get_value(wid) doc_ids = map(lambda x: x.split('_')[1], filter(lambda y: '_' in y, #ListDocIdsService().get_value(wid)))[:100] ListDocIdsService().get_value(wid))) #pprint(doc_ids); sys.exit(0) #for n in range(0, len(doc_ids), step): ##for n in range(0, 20, step): # print 'n = %d' % n # doc_ids_subset = doc_ids[n:n+step] r = Pool(processes=8).map_async(get_fields, chunks(doc_ids, step)) r.wait() pprint(r.get()) print '*'*80 #for k in r.get(): # DEBUG # print k fields = [] m = map(lambda x: fields.extend(x), r.get()) #pprint(fields) indexed = dict(fields) pprint(indexed) # DEBUG #for doc_id in doc_ids_to_heads: # entity_response = doc_ids_to_entities.get( # doc_id, {'titles': [], 'redirects': {}}) # doc_ids_combined[doc_id] = map(preprocess,