Exemple #1
0
def queryProcess(phone_md5_list):
    manager = Manager()
    q = manager.list()
    jobs = PROCESS_NUM
    pool = Pool(jobs)
    filestep = len(phone_md5_list) / jobs + 1
    posBegin = 0
    posEnd = posBegin + filestep
    processList = []
    #创建十个进程,item:进程id , list:每个线程处理的文件大小,q:线程公共变量,收集十个进程统计的数据
    for item in range(0, jobs):
        p = pool.apply_async(query,
                             args=(item, phone_md5_list[posBegin:posEnd], q))
        posBegin = posEnd
        posEnd = posBegin + filestep
        processList.append(p)
    while 1:
        try:
            time.sleep(1)
            status = True
            for item in processList:
                status = status and item.ready()
            if status:
                break
        except KeyboardInterrupt:
            pool.terminate()
            pool.wait(None)
            break

    for item in processList:
        item.wait()
    pool.close()
    pool.join()
    del pool
    return q
def register_images(temp_dir, temp_folder_raw, temp_folder_reg, ref_img, cfg):
    assert cfg.kp_algo.lower() in [
        "surf", "orb"
    ], "Keypoint algorithm '{}' not supported".format(cfg.kp_algo)
    p_bar = tqdm(total=len(os.listdir(temp_folder_raw)),
                 disable=not cfg.verbose,
                 desc="Register frames".center(30))
    p = Pool(cfg.num_workers).map_async(
        functools.partial(_register_image,
                          ref_image=ref_img,
                          raw_img_folder=temp_folder_raw,
                          temp_folder=temp_dir,
                          hessian_threshold=cfg.hessian_threshold,
                          jpg_quality=cfg.jpg_quality,
                          kp_algo=cfg.kp_algo), os.listdir(temp_folder_raw))
    while True:
        a = len(os.listdir(temp_folder_raw))
        b = len(os.listdir(temp_folder_reg))
        p_bar.n = (a - (a - b))
        p_bar.refresh()
        if a == b:
            break
        time.sleep(1)
    p_bar.close()
    p.wait()

    [os.remove(temp_folder_raw + file) for file in os.listdir(temp_folder_raw)]
Exemple #3
0
def main():
    args = get_args()

    start = time()
    with open(args.output, 'w') as f:
        wids = [line.strip() for line in
                open(args.input).readlines()[:args.number]]
        mapped = Pool(processes=8).map_async(identify_worker, wids)
        mapped.wait()
        print >> f, '\n'.join([x.encode('utf-8') for x in mapped.get()])
    end = time()
    total = end - start
    print '%d seconds elapsed' % total
Exemple #4
0
def main(repo_name):
    processingDirs = []
    for corpus_dir in os.listdir("../"+repo_name+"_repo"):
        # if not corpus_dir == "Z057Ml": # "S002Mg":
        #     continue
        processCorpusDir(repo_name, corpus_dir)
    pool = Pool(10)

    for corpus_dir in processingDirs:
        try:
            result = pool.apply_async(processCorpusDir, args=(corpus_dir,))
        except KeyboardInterrupt:
            print ("Caught KeyboardInterrupt, terminating workers")
            pool.terminate()
            pool.wait()
            break
    pool.close()
    pool.join()
Exemple #5
0
    lang = details.get('lang')
    #print url
    #doc_ids = ListDocIdsService().get_value(wid)
    doc_ids = map(
        lambda x: x.split('_')[1],
        filter(
            lambda y: '_' in y,
            #ListDocIdsService().get_value(wid)))[:100]
            ListDocIdsService().get_value(wid)))
    #pprint(doc_ids); sys.exit(0)
    #for n in range(0, len(doc_ids), step):
    ##for n in range(0, 20, step):
    #    print 'n = %d' % n
    #    doc_ids_subset = doc_ids[n:n+step]
    r = Pool(processes=8).map_async(get_fields, chunks(doc_ids, step))
    r.wait()
    pprint(r.get())
    print '*' * 80
    #for k in r.get():  # DEBUG
    #    print k
    fields = []
    m = map(lambda x: fields.extend(x), r.get())
    #pprint(fields)
    indexed = dict(fields)

pprint(indexed)  # DEBUG

#for doc_id in doc_ids_to_heads:
#    entity_response = doc_ids_to_entities.get(
#        doc_id, {'titles': [], 'redirects': {}})
#    doc_ids_combined[doc_id] = map(preprocess,
Exemple #6
0
    res = [(sum(lnls) + lnp) / ini['temperature'], lnp] + lnls + derivs
    return tuple(res)


### Import additional modules for parallel computing if requested
pool = None
# Multithreading parallel computing via python-native multiprocessing module
if (ini['parallel'][0] == 'multiprocessing') & (not ini['debug_mode']):
    from multiprocessing import Pool
    pool = Pool(int(ini['parallel'][1])) # number of threads chosen by user
# MPI parallel computing via external schwimmbad module
elif (ini['parallel'][0] == 'MPI') & (not ini['debug_mode']):
    from schwimmbad import MPIPool
    pool = MPIPool()
    if not pool.is_master(): # Necessary bit for MPI
        pool.wait()
        sys.exit(0)


### Conveninent MCMC settings variables
n_dim = len(ini['var_par'])
n_steps = ini['n_steps']
thin_by = ini['thin_by']
n_walkers = ini['n_walkers']


### Randomize initial walkers positions according to "start" & "width" columns in ini file
p0_start = [par[2] for par in ini['var_par']]
std_start = [par[5] for par in ini['var_par']]
p_start = np.zeros((n_walkers, n_dim))
for i in range(n_dim):
Exemple #7
0
if details is not None:
    url = details.get('url')
    lang = details.get('lang')
    #print url
    #doc_ids = ListDocIdsService().get_value(wid)
    doc_ids = map(lambda x: x.split('_')[1],
                  filter(lambda y: '_' in y,
                         #ListDocIdsService().get_value(wid)))[:100]
                         ListDocIdsService().get_value(wid)))
    #pprint(doc_ids); sys.exit(0)
    #for n in range(0, len(doc_ids), step):
    ##for n in range(0, 20, step):
    #    print 'n = %d' % n
    #    doc_ids_subset = doc_ids[n:n+step]
    r = Pool(processes=8).map_async(get_fields, chunks(doc_ids, step))
    r.wait()
    pprint(r.get())
    print '*'*80
    #for k in r.get():  # DEBUG
    #    print k
    fields = []
    m = map(lambda x: fields.extend(x), r.get())
    #pprint(fields)
    indexed = dict(fields)

pprint(indexed)  # DEBUG

#for doc_id in doc_ids_to_heads:
#    entity_response = doc_ids_to_entities.get(
#        doc_id, {'titles': [], 'redirects': {}})
#    doc_ids_combined[doc_id] = map(preprocess,