def _launch_pnn_workers_st(left, diff, pnn, pcli, rcli): """ Launch workers for the pnn worker single thread (remove and updates) """ pool = multiprocessing.Pool(1) workers = {} for dataset in diff['to_remove'] + diff['to_update']: if left == 0: break workers[dataset] = pool.apply_async( block_sync, (), {'pnn': pnn, 'rds': dataset, 'pcli': pcli, 'rcli': rcli} ) left -= 1 pool.close() return workers
def _launch_pnn_workers(conf, diff, pnn, pcli, rcli): """ Launch workers for the pnn worker """ size = int(conf['pool']) chunck = int(conf['chunck']) pool = multiprocessing.Pool(size) workers = {} for dataset in diff['missing']: if chunck == 0: break workers[dataset] = pool.apply_async( block_sync, (), {'pnn': pnn, 'rds': dataset, 'pcli': pcli, 'rcli': rcli} ) chunck -= 1 pool.close() return workers