def test_resolve_distributed_mode_slurm3(): args = argparse.Namespace( multiprocessing_distributed=True, dist_world_size=None, dist_rank=None, ngpu=1, local_rank=None, dist_launcher="slurm", dist_backend="nccl", dist_init_method="env://", dist_master_addr=None, dist_master_port=10000, ) env = dict( SLURM_PROCID="0", SLURM_NTASKS="1", SLURM_STEP_NUM_NODES="1", SLURM_STEP_NODELIST="localhost", SLURM_NODEID="0", CUDA_VISIBLE_DEVICES="0,1", ) e = ProcessPoolExecutor(max_workers=2) with unittest.mock.patch.dict("os.environ", dict(env, SLURM_LOCALID="0")): resolve_distributed_mode(args) option = build_dataclass(DistributedOption, args) fn = e.submit(option.init) with unittest.mock.patch.dict("os.environ", dict(env, SLURM_LOCALID="0")): option2 = build_dataclass(DistributedOption, args) fn2 = e.submit(option2.init) fn.result() fn2.result()
class PendantFeaturesService: @inject def __init__(self, default_params_factory: PendantFeaturesParamsFactory) -> None: self._executor = ProcessPoolExecutor(max_workers=1) self._default_params_factory = default_params_factory def extract( self, image: np.ndarray, params: Optional[PendantFeaturesParams] = None, *, labels: bool = False, ) -> asyncio.Future: if params is None: params = self._default_params_factory.create() cfut = self._executor.submit( extract_pendant_features, image, params.drop_region, params.needle_region, thresh1=params.thresh1, thresh2=params.thresh2, labels=labels, ) fut = asyncio.wrap_future(cfut, loop=asyncio.get_event_loop()) return fut def destroy(self) -> None: self._executor.shutdown()
class MultiCommand: def __init__( self, commands: Sequence[Union[List[str], str]], config: Dict[str, Any], abort_reasons: Optional[List[str]] = None, ): self.processes: List[Future[Dict[str, Any]]] = [] self.commands = commands self.config = config self.abort_reasons = abort_reasons self.process_pool_executor = ProcessPoolExecutor( max_workers=len(commands) if config["parallel"] else 1) def run(self) -> List[Dict[str, Any]]: for command in self.commands: if isinstance(command, str): command = [command] logger.debug(f'Spawning "{command}"') process = self.process_pool_executor.submit( retry_process, command, self.config, self.abort_reasons) self.processes += [process] # result() is blocking. The function will return when all processes are done return [process.result() for process in self.processes]
class ConanFeaturesService: @inject def __init__(self, default_params_factory: ConanParamsFactory) -> None: self._executor = ProcessPoolExecutor(max_workers=1) self._default_params_factory = default_params_factory def extract(self, image: np.ndarray, params: Optional[ConanFeaturesParams] = None, *, labels: bool = False) -> asyncio.Future: params = params or self._default_params_factory.create() params_dict = { 'baseline': params.baseline, 'inverted': params.inverted, 'thresh': params.thresh, 'roi': params.roi, 'labels': labels, } cfut = self._executor.submit(extract_contact_angle_features, image, **params_dict) fut = asyncio.wrap_future(cfut, loop=asyncio.get_event_loop()) return fut def destroy(self) -> None: self._executor.shutdown()
def submit_job(database_api, dataset_api, email, dataset, job_name, job_type, params): global executor is_serve = os.environ.get(CIRRO_SERVE) == 'true' if executor is None: max_workers = int( os.environ.get(CIRRO_MAX_WORKERS, '2' if is_serve else '1')) if max_workers > 0: import multiprocessing from concurrent.futures.process import ProcessPoolExecutor from concurrent.futures.thread import ThreadPoolExecutor executor = ProcessPoolExecutor( max_workers=max_workers, mp_context=multiprocessing.get_context('spawn') ) if is_serve else ThreadPoolExecutor(max_workers=max_workers) job_id = database_api.create_job(email=email, dataset_id=dataset['id'], job_name=job_name, job_type=job_type, params=params) if executor is not None: future = executor.submit(run_job, email, job_id, job_type, dataset, params, database_api if not is_serve else None, dataset_api if not is_serve else None) future.add_done_callback(done_callback) job_id_2_future[job_id] = future else: run_job(email, job_id, job_type, dataset, params, database_api if not is_serve else None, dataset_api if not is_serve else None) return job_id
def test(path, run_id, runs): # load parser and data handler parser = CYKParser.load(path) data_handler = DataHandler(config.test_set, run_id, runs) # parse sentences in parallel executor = ProcessPoolExecutor(config.processes) futures = [ executor.submit(parse_tree, parser, sent, run_id) for sent in data_handler.generator() ] # following code is to track progress kwargs = { 'total': len(futures), 'unit': 'nap', 'unit_scale': True, 'leave': True } for _ in tqdm(as_completed(futures), **kwargs): pass for future in futures: if future.exception() is not None: print(future.exception()) # stitch files if number of runs is 1 if runs == 1: stitch_files() print("Done parsing")
def _schedule_user_task(self, executor: ProcessPoolExecutor, future_set: dict, task: TaskInfo) -> Future: task.start_task() future = executor.submit(task.method, *task.args, **task.kwargs) future.add_done_callback( lambda x: (self._user_task_done_callback(x, future_set, task))) future_set[future] = task return future
def main(): pool = ProcessPoolExecutor(max_workers=int(multiprocessing.cpu_count() * 0.5)) # pool = ProcessPoolExecutor(max_workers=1) pages_2 = [{ 'id': "p1", 'type': 'STACK', 'constraint': None }, { 'id': "p2", 'type': 'STACK', 'constraint': None }] pages_3 = [{ 'id': "p1", 'type': 'STACK', 'constraint': None }, { 'id': "p2", 'type': 'STACK', 'constraint': None }, { 'id': "p3", 'type': 'STACK', 'constraint': None }] base_constraints = [ { "type": "EDGES_TO_SUB_ARC_ON_PAGES", "arguments": ["0", "1"], # the outer terminals "modifier": ["p1", "p2"] # the pages }, # inner terminals are after one outer terminal and before the other { "type": "NODES_PREDECESSOR", "arguments": ["0"], "modifier": ["2", "3"], }, { "type": "NODES_PREDECESSOR", "arguments": ["2", "3"], "modifier": ["1"], } ] set_printing(False) onlyfiles = [ os.path.join('500-random-planar-3-trees', f) for f in listdir(path='500-random-planar-3-trees') if isfile(os.path.join('500-random-planar-3-trees', f)) ] for file in onlyfiles: with open(file, mode="r") as f: graph_str = f.read() future = pool.submit(do_experiment, base_constraints, pages_2, pages_3, graph_str) future.add_done_callback(callback) pool.shutdown(wait=True)
def main(): pool = ProcessPoolExecutor(max_workers=int(multiprocessing.cpu_count() - 2)) # pool = ProcessPoolExecutor(max_workers=1) pages = [{ 'id': "p1", 'type': 'STACK', 'constraint': None }, { 'id': "p2", 'type': 'STACK', 'constraint': None }, { 'id': "p3", 'type': 'STACK', 'constraint': None }] base_constraints = [ { "type": "EDGES_TO_SUB_ARC_ON_PAGES", "arguments": ["0", "1"], # the outer terminals "modifier": ["p1", "p2"] # the pages }, # inner terminals are after one outer terminal and before the other { "type": "NODES_PREDECESSOR", "arguments": ["0"], "modifier": ["2", "3"], }, { "type": "NODES_PREDECESSOR", "arguments": ["2", "3"], "modifier": ["1"], } ] set_printing(False) experiments = [ # (graph_generation.random_planar_gh, 300, [15, 10]), (graph_generation.random_planar, 1000, [110]), # (graph_generation.spine_graph, 1, list(range(10, 350, 1))) ] with open("results_random_planar_110.json", mode="a") as f: def callback(my_future: Future): if not my_future.done() or my_future.cancelled(): return result: ExResult = my_future.result() print(simplejson.dumps(result), file=f) for graph_gen, number_of_runs, num_node_list in experiments: for num_nodes in num_node_list: for i in range(number_of_runs): future = pool.submit(do_experiment, base_constraints, graph_gen, i, num_nodes, pages) future.add_done_callback(callback) pool.shutdown(wait=True)
def show(qtd): jsons = list() for i in range(int(qtd)): executor = ProcessPoolExecutor(max_workers=2) task1 = executor.submit(send) jsons.append(task1.result()) return {"arquivos": jsons}
class ProcessesPool(PoolInterface): def __init__(self, amount_of_entities): super().__init__(amount_of_entities) self.pool = ProcessPoolExecutor(self.max_amount) def shutdown(self, timeout): self.pool.close() def execute(self, funct, socket): return self.pool.submit(funct, socket[0], socket[1])
def main(): parser = argparse.ArgumentParser() parser.add_argument('-v', '--verbose', action='store_true', default=False, help='print stats') parser.add_argument('-o', '--output', type=str, help='path to CSV output') parser.add_argument('--publishers-per-client', type=int, default=7) parser.add_argument('--publish-interval', type=int, default=100, help='publish interval in ms') parser.add_argument('--clients-per-group', type=int, default=10) parser.add_argument('--enable-ack', action='store_true') args = parser.parse_args(sys.argv[1:]) if args.output: chdir(args.output) common_kwargs = { 'publishers_per_client': args.publishers_per_client, 'publish_interval': args.publish_interval, 'clients_per_group': args.clients_per_group, 'verbose': args.verbose, 'enable_ack': args.enable_ack, } scenario_configs = [ { **common_kwargs, 'name': 'baseline', }, { **common_kwargs, 'name': 'vivaldi', 'use_vivaldi': True, }, ] def run_scenario(**kwargs): EmmaScenario(**kwargs).run() if args.verbose: print('running emma scenarios sequentially') for config in scenario_configs: run_scenario(**config) else: print('running emma scenarios in separate processes') executor = ProcessPoolExecutor(len(scenario_configs)) for config in scenario_configs: executor.submit(run_scenario, **config)
def run(executor: ProcessPoolExecutor, num_of_experiments: int, objective_function_pointer: Any, spawn_boundaries: List[List[float]], objective_function_goal_point: array, maximum_iterations: int, swarm_size: int = 40, isClan: bool = False, number_of_clans: int = 4, c1: float = 2.0, c2: float = 2.0, adaptivePSO: bool = False, eis: Tuple[Tuple[GlobalLocalCoefficientTypes, float or None], Tuple[ControlFactorTypes, float or None]] = ((GlobalLocalCoefficientTypes.NONE, None), (ControlFactorTypes.NONE, None)), search_and_velocity_boundaries: List[List[float]] = None, wt: WallTypes = WallTypes.NONE ): experiments_precisions_per_experiment = [] experiments_iterations_per_experiment = [] experiments_average_iteration_cpu_times_per_experiment = [] experiments_cpu_times_per_experiment = [] experiments_array = [] for experiment in range(num_of_experiments): function_task = executor.submit(scripts.experiments.experiment.experiment, objective_function=objective_function_pointer, spawn_boundaries=spawn_boundaries, objective_function_goal_point=objective_function_goal_point, maximum_iterations=maximum_iterations, swarm_size=swarm_size, isClan=isClan, number_of_clans=number_of_clans, c1=c1, c2=c2, adaptivePSO=adaptivePSO, eis=eis, search_and_velocity_boundaries=search_and_velocity_boundaries, wt=wt ) experiments_array.append(function_task) for task in experiments_array: precision, iterations, experiment_average_iteration_cpu_time, experiment_total_cpu_time = task.result() experiments_precisions_per_experiment.append(precision) experiments_iterations_per_experiment.append(iterations) experiments_average_iteration_cpu_times_per_experiment.append(experiment_average_iteration_cpu_time) experiments_cpu_times_per_experiment.append(experiment_total_cpu_time) mean_experiment_precision = mean(experiments_precisions_per_experiment) std_experiment_precision = std(experiments_precisions_per_experiment) median_experiment_precision = median(experiments_precisions_per_experiment) best_experiment_precision = min(experiments_precisions_per_experiment) mean_experiment_iterations = mean(experiments_iterations_per_experiment) mean_experiments_average_iteration_cpu_times = mean(experiments_average_iteration_cpu_times_per_experiment) mean_experiment_cpu_time = mean(experiments_cpu_times_per_experiment) return {"Precision mean": mean_experiment_precision, "Precision std": std_experiment_precision, "Precision median": median_experiment_precision, "Best precision": best_experiment_precision, "Mean iterations per experiment": mean_experiment_iterations, "Mean iteration CPU time": mean_experiments_average_iteration_cpu_times, "Mean experiment CPU time": mean_experiment_cpu_time}
def runTournamentSizeTests(): size = 1000 selectors = [ TournamentSelector(150), TournamentSelector(350), TournamentSelector(500) ] crossoverThreshold = 0.2 mutationProbability = 0.1 # internal params crossoverProbability = 0.2 mutationStrength = 1 calculator = createDefaultLossCalculator() subDirectory = 'tournamentSize' os.makedirs(os.path.join(path, subDirectory), exist_ok=True) file = open(os.path.join(path, subDirectory, 'results.txt'), 'w+') board = loadFromFile('textTests/zad1.txt') file.write(f'Parametry testow:\n' f'Rozmiar populacji: {size}\n' f'Prawdopodobienstwo krzyzowania: {(1 - crossoverThreshold)}\n' f'Prawdopodobienstwo mutacji: {(1 - mutationProbability)}\n\n' f'Wyniki testow:\nParametr:\tBest:\tWorst:\tAvg:\tStdDev:\n') for selector in selectors: bestSolutionLosses = [] worstSolutionLosses = [] file.write(f'{selector}\t') pool = ProcessPoolExecutor(max_workers=10) futures = [] for i in range(10): futures.append( pool.submit( immediatelyCall, GeneticAlgorithm(board, size, calculator, selector, crossoverThreshold, crossoverProbability, mutationProbability, mutationStrength))) result = wait(futures, return_when=ALL_COMPLETED) for index, value in enumerate(result[0]): (best, worst, _, image) = value.result() bestSolutionLosses.append(best) worstSolutionLosses.append(worst) image.save( os.path.join(path, subDirectory, f'{selector}-test-{index}.png')) image.close() best = min(bestSolutionLosses) worst = max(worstSolutionLosses) avg = statistics.mean(bestSolutionLosses) stdDev = statistics.stdev(bestSolutionLosses) file.write(f'{best}\t{worst}\t{avg}\t{stdDev}\n') file.close()
def lyricSpider(user_id): print("======= 开始爬 歌词 信息 ===========") startTime = datetime.datetime.now() print(startTime.strftime('%Y-%m-%d %H:%M:%S')) # 所有歌手数量 try: musics_num = sql.get_music_num(user_id) except: print("用户未开启权限,程序结束") sys.exit(0) print("musics :", len(musics_num), "start") batch = math.ceil(musics_num.get('num') / 34.0) pool = ProcessPoolExecutor(3) for index in range(0, batch): pool.submit(saveLyricBatch, user_id, index) pool.shutdown(wait=True) print("======= 结束爬 歌词 信息 ===========") endTime = datetime.datetime.now() print(endTime.strftime('%Y-%m-%d %H:%M:%S')) print("耗时:", (endTime - startTime).seconds, "秒")
def create_process(func: Callable, *args: Any, **kwargs: Any) -> Future: ''' Calls a function in its own process :param func: The function to be called :param args: The function arguments :param kwargs: The function keyword arguments :return: The created Future object, from which we can call 'result()' to get the function return value. ''' tp = ProcessPoolExecutor(max_workers=1) future = tp.submit(func, *args, **kwargs) create_thread(tp.shutdown, wait=True) # Necessary since wait=False is broken return future
class DelayedFileList: def __init__(self, filename): self.__loader = ProcessPoolExecutor(4) self.__filehandle = open( filename, "r", encoding="iso-8859-1" ) # :type self.__filehandle: typing.TextIO self.__lines = 0 self.__thousand_offsets = [0] line = self.__filehandle.readline() while line: if line.strip(): self.__lines += 1 if self.__lines % 1000 == 0: self.__thousand_offsets.append( self.__filehandle.tell()) line = self.__filehandle.readline() self.__filehandle.seek(0) def __len__(self): return self.__lines def __getitem__(self, i): if i < 0 or i >= len(self): raise IndexError("Out of bounds") thousand_offset = self.__thousand_offsets[i // 1000] remainder = i % 1000 self.__filehandle.seek(thousand_offset) while remainder > 0: self.__filehandle.readline() remainder -= 1 name, url = [ x.strip() for x in self.__filehandle.readline().strip().split("\t") ] class_name = name.split("_")[0] pre_image = Image(name, class_name, url, None, None) submitted_task = self.__loader.submit(load_image, pre_image.url) def deferred_load(): return submitted_task.result() return Image(name, class_name, url, load_annotation_for_image(pre_image), deferred_load) def close(self): self.__loader.shutdown() self.__filehandle.close()
def thead_collect(): results = MONGO.select('albums', {'collected': {'$ne': 1}}, limit=200) # thead_pool = ThreadPoolExecutor(5) thead_pool = ProcessPoolExecutor(5) while results: for result in results: baby = CollectBaby(result) # magic() thead_pool.submit(baby.collect) # baby.collect() results = MONGO.select('albums', { '_id': { '$gt': result.get('_id') }, 'collected': { '$ne': 1 } }, limit=200) thead_pool.shutdown(wait=True)
async def test_endpoint(): print(f"main process: {os.getpid()}") START_TIME = time.time() STOP_TIME = START_TIME + 2 pool = ProcessPoolExecutor(max_workers=3) futures = [ pool.submit(simple_routine, [1]), pool.submit(simple_routine, [1]), pool.submit(simple_routine, [10]), ] results = [] for fut in futures: remains = max(STOP_TIME - time.time(), 0) try: results.append(fut.get(timeout = remains)) except: results.append("not done") # terminate the entire pool pool.shutdown(wait=False) print("exiting at: ", int(time.time() - START_TIME)) return "True"
def get_linkpred_scores(lp_data, weighted, predictor_indices=None, include_train=False, parallel_version=False): predictor_indices = predictor_indices or range(len(all_predictors)) predictors = [all_predictors[i] for i in predictor_indices] predictor_names = [all_predictor_names[i] for i in predictor_indices] A_train = lp_data['A_train'] A_test = lp_data['A_test'] A_test_pos = lp_data['A_test_pos'] A_test_neg = lp_data['A_test_neg'] G_train = nx.from_scipy_sparse_matrix(A_train) if include_train: pairs = list(itertools.combinations(range(A_train.shape[0]), 2)) else: test_pairs = list(zip(*triu(A_test_pos + A_test_neg).nonzero())) pairs = test_pairs if not include_train else test_pairs + list( zip(*triu(A_train).nonzero())) scores = {} if not parallel_version: for i, predictor in tqdm(enumerate(predictors), 'Predictor: '): predictor_name = predictor_names[i] abbr, lp_scores = perform_one_lp(predictor_name, predictor, G_train, pairs, weighted) scores[abbr] = lp_scores else: num_predictors = len(predictors) max_workers = min(num_predictors, multiprocessing.cpu_count()) pool = ProcessPoolExecutor(max_workers=max_workers) process_list = [] for i, predictor in enumerate(predictors): predictor_name = predictor_names[i] process_list.append( pool.submit(perform_one_lp, predictor_name, predictor, G_train, pairs, weighted)) print('{} of {} processes scheduled ({})'.format( len(process_list), num_predictors, predictor_name)) for p in as_completed(process_list): abbr, lp_scores = p.result() scores[abbr] = lp_scores print('{} of {} processes completed'.format( len(scores), len(process_list))) pool.shutdown(wait=True) scores_df = pd.DataFrame(scores) return scores_df
class ProcessExecutor(Executor): slug = 'executor:process' name = "Executor: Process" awaiter_dict = { 'as_completed': as_completed, } def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.executor = ProcessPoolExecutor(max_workers=self.workers) def submit(self, fn: Callable, *args, **kwargs): return self.executor.submit(fn, *args, **kwargs) def shutdown(self, wait=True): self.executor.shutdown(wait)
def main(args, device, num_available_devices): model_path = Path(args.model) root_dir = Path(args.root_dir) image_paths = [ file_name for file_name in root_dir.glob('**/*') if is_image(file_name) ] analyzed_images = [] ctx = multiprocessing.get_context('forkserver') executor = ProcessPoolExecutor(max_workers=num_available_devices, mp_context=ctx, initializer=init_process, initargs=(model_path, not args.no_split, device)) try: with executor: current_jobs = [] for i, image_path in enumerate(image_paths): submitted_job = executor.submit( consumer, image_path, str(image_path.relative_to(root_dir))) current_jobs.append(submitted_job) for job in tqdm(as_completed(current_jobs), total=len(current_jobs)): try: result = job.result() analyzed_images.append(result) except Exception as e: print(f"Could not process {str(image_path)}, reason: {e}") traceback.print_exc(file=sys.stdout) except KeyboardInterrupt: pass with (root_dir / 'handwriting_analysis.json').open('w') as f: json.dump(analyzed_images, f, indent='\t') num_has_handwriting = len( [im for im in analyzed_images if im['has_handwriting']]) print( f"Handwriting to no handwriting ratio: {num_has_handwriting / len(analyzed_images)}" )
def runner(self): process_pool = ProcessPoolExecutor(max_workers=4) futures = dict() for url in self.urls: future = process_pool.submit(self.get_web_content, url)#造子弹 futures[future] = url#用url给子弹标上号然后放入子弹箱 for future in concurrent.futures.as_completed(futures):#取子弹 try: url = futures[future]#得到子弹的标号 crawl_result = future.result() urlretrieve(url = crawl_result['img_src'],filename = 'F:\\CSGO\\csitem\\' + crawl_result["image_name"] + '.png') #数据库的插入 #self.cursor.execute("INSERT INTO csgo_item (type,item_type,item_name,price,img,exterior,quality,rarity,sales) VALUES ('%s','%s','%s','%s','%s','%s','%s','%s','%s')" # % (crawl_result["csgotype"],crawl_result["item_type"],crawl_result["name_text"],crawl_result["price"],crawl_result["image_name"],crawl_result["exterior"],crawl_result["quality"],crawl_result["rarity"],crawl_result["sales"])) #self.conn.commit() except: print("线程取出出现错误") continue print('Finished!')
def computeAllPredictions(self, X: pd.DataFrame): if self.numProcesses == 1 or len(self.models) == 1: return [model.predict(X) for model in self.models] predictionFutures = [] executor = ProcessPoolExecutor(max_workers=self.numProcesses) predictors = [ VectorModelWithSeparateFeatureGeneration(model) for model in self.models ] for predictor in predictors: predictFinaliser = predictor.predictStart(X) frameinfo = getframeinfo(currentframe()) PickleFailureDebugger.logFailureIfEnabled( predictFinaliser, contextInfo= f"Submitting {predictFinaliser} in {frameinfo.filename}:{frameinfo.lineno}" ) predictionFutures.append(executor.submit(predictFinaliser.execute)) return [ predictionFuture.result() for predictionFuture in predictionFutures ]
def main(): pool = ProcessPoolExecutor(max_workers=int(multiprocessing.cpu_count() * 0.25)) pages = [{ 'id': "p1", 'type': 'STACK', 'constraint': None }, { 'id': "p2", 'type': 'STACK', 'constraint': None }] base_constraints = [ { "type": "EDGES_TO_SUB_ARC_ON_PAGES", "arguments": ["0", "1"], # the outer terminals "modifier": ["p1", "p2"] # the pages }, # inner terminals are after one outer terminal and before the other { "type": "NODES_PREDECESSOR", "arguments": ["0"], "modifier": ["2", "3"], }, { "type": "NODES_PREDECESSOR", "arguments": ["2", "3"], "modifier": ["1"], } ] set_printing(False) with open("results_random_planar_110.json", mode="r") as f: for line in f: future = pool.submit(do_experiment, base_constraints, pages, line) future.add_done_callback(callback) pool.shutdown(wait=True)
def _fit(self, X: pd.DataFrame, Y: pd.DataFrame): if self.numProcesses == 1 or len(self.models) == 1: for model in self.models: model.fit(X, Y) return fittedModelFutures = [] executor = ProcessPoolExecutor(max_workers=self.numProcesses) fitters = [ VectorModelWithSeparateFeatureGeneration(model) for model in self.models ] for fitter in fitters: intermediateStep = fitter.fitStart(X, Y) frameinfo = getframeinfo(currentframe()) PickleFailureDebugger.logFailureIfEnabled( intermediateStep, contextInfo= f"Submitting {fitter} in {frameinfo.filename}:{frameinfo.lineno}" ) fittedModelFutures.append(executor.submit( intermediateStep.execute)) for i, fittedModelFuture in enumerate(fittedModelFutures): self.models[i] = fitters[i].fitEnd(fittedModelFuture.result())
def main(): start = time.time() args = parse_args() wwise_dir = Path(args.wwise_dir) out_dir = Path(args.out_dir) out_dir.mkdir(exist_ok=True) id_to_filename_path = out_dir / ID_TO_FILENAME manager = mp.Manager() queue = manager.Queue() id_queue = manager.Queue() setup_logging(queue) target_handlers = logbook.NestedSetup([ logbook.NullHandler(), logbook.StreamHandler(sys.stdout, level=logbook.INFO, bubble=True), logbook.FileHandler("extract.log", mode="w", level=logbook.INFO, bubble=True), ]) sub = MultiProcessingSubscriber(queue) controller = sub.dispatch_in_background(target_handlers) quickbms_log_lock = manager.Lock() quickbms_log = out_dir / "quickbms.log" try: id_to_filename_path.unlink() logbook.info("removed old {id_file}", id_file=id_to_filename_path.absolute()) except FileNotFoundError: pass logbook.info("QuickBMS log: '{qlog}'", qlog=quickbms_log.absolute()) try: quickbms_log.unlink() logbook.info("removed old {f}", f=quickbms_log.absolute()) except FileNotFoundError: pass id_to_filename_path.touch() logbook.info("writing old ID -> new filename info in '{id_file}'", id_file=id_to_filename_path.absolute()) id_to_filename_p = mp.Process(target=id_to_filename_worker, args=(id_queue, id_to_filename_path, queue)) id_to_filename_p.start() logbook.info("processing audio files in '{wd}'", wd=wwise_dir.absolute()) fut2func = {} # Parse .bnk files and metadata. with ProcessPoolExecutor(max_workers=MAX_WORKERS) as executor: fut2func[executor.submit(parse_banks_metadata, wwise_dir, queue)] = parse_banks_metadata fut2func[executor.submit(decode_banks, wwise_dir, out_dir, quickbms_log, quickbms_log_lock, queue)] = decode_banks memory_bnk_meta_file2metadata = {} streamed_bnk_meta_file2metadata = {} orig_bnk2decode_info = {} for completed_fut in futures.as_completed(fut2func): if fut2func[completed_fut] == parse_banks_metadata: result = completed_fut.result() memory_bnk_meta_file2metadata = result[0] streamed_bnk_meta_file2metadata = result[1] elif fut2func[completed_fut] == decode_banks: orig_bnk2decode_info = completed_fut.result() if len(memory_bnk_meta_file2metadata) != len(orig_bnk2decode_info): logbook.warning( "Amount of Bank and metadata files " "do not match ({first}) != {second})", first=len(orig_bnk2decode_info), second=len(memory_bnk_meta_file2metadata)) s1 = memory_bnk_meta_file2metadata.keys() s2 = set([key.stem for key in orig_bnk2decode_info]) to_del = [] diff = s2.difference(s1) for d in diff: # TODO: expensive! for key in orig_bnk2decode_info: if key.stem == d: logbook.warn("ignoring {f}", f=str(key)) to_del.append(key) for td in to_del: del orig_bnk2decode_info[td] wem_src2wem_dst = {} # Move .wem files to out_dir in correct places. with ProcessPoolExecutor(max_workers=MAX_WORKERS) as executor: for bnk_meta_file, meta in streamed_bnk_meta_file2metadata.items(): for m in meta: src_dir = bnk_meta_file.parent src = src_dir / Path(m.generated_audio_file) if src.exists(): wwise_path = Path(m.wwise_object_path) dst = out_dir / wwise_path.relative_to( wwise_path.anchor).with_suffix(".wem") executor.submit(copy, src, dst, queue, id_queue) wem_src2wem_dst[src] = dst else: logbook.warning( "found references to {src} in metadata, but " "the file cannot be found in wwise_dir", src=src) decoded_file2metas = {} for orig_bnk_file, decode_info in orig_bnk2decode_info.items(): orig_bnk_file = orig_bnk_file.stem meta = memory_bnk_meta_file2metadata[orig_bnk_file] if len(decode_info) != len(meta): logbook.error( "decode_info and meta length mismatch: " "{len1} != {len2} for bnk: '{bnk}'", len1=len(decode_info), len2=len(meta), bnk=orig_bnk_file) # print(decode_info) # print(meta) continue # raise ValueError(f"decode_info and meta length mismatch " # f"{len(decode_info)} != {len(meta)}") for m, (decoded_stem, decoded_size) in zip(meta, decode_info.items()): if m.data_size != decoded_size: # raise ValueError(f"{m.data_size} != {decoded_size}") logbook.error( "metadata size and decoded data size length mismatch: " "{len1} != {len2}", len1=m.data_size, len2=decoded_size) continue decoded_file2metas[decoded_stem] = m fs = [] # Move output from decoding .bnk files to correct places in out_dir. executor = ProcessPoolExecutor(max_workers=MAX_WORKERS) for decoded_file, meta in decoded_file2metas.items(): src = out_dir / f"{decoded_file}.bin" wwise_path = Path(meta.wwise_object_path) dst = out_dir / wwise_path.relative_to( wwise_path.anchor).with_suffix(".bin") fs.append(executor.submit(move, src, dst, queue, id_queue)) futures.wait(fs, return_when=futures.ALL_COMPLETED) fs = [] # Convert all .wem and .bin files to .ogg. executor = ProcessPoolExecutor(max_workers=MAX_WORKERS) for bin_file in out_dir.rglob("*.bin"): fs.append(executor.submit(ww2ogg, bin_file, queue)) for wem_file in out_dir.rglob("*.wem"): fs.append(executor.submit(ww2ogg, wem_file, queue)) futures.wait(fs, return_when=futures.ALL_COMPLETED) done_wems_stems = set([ws.stem for ws in wem_src2wem_dst.keys()]) source_wems = [w for w in wwise_dir.rglob("*.wem")] source_wems_stems = set([w.stem for w in source_wems]) wem_diff = source_wems_stems.difference(done_wems_stems) if wem_diff: logbook.warn("failed to determine filename for " "{num} files", num=len(wem_diff)) for ws in source_wems: if str(ws.stem) in wem_diff: logbook.info("couldn't determine filename for: {ws}", ws=ws) copy_seq(ws, out_dir, id_to_filename_queue=None) # Convert leftovers. leftovers_fs = [] with ProcessPoolExecutor(max_workers=MAX_WORKERS) as executor: for wem_file in out_dir.rglob("*.wem"): leftovers_fs.append(executor.submit(ww2ogg, wem_file, queue)) futures.wait(leftovers_fs, return_when=futures.ALL_COMPLETED) id_queue.put(SENTINEL) id_to_filename_p.join() secs = time.time() - start logbook.info("finished successfully in {secs:.2f} seconds", secs=secs) controller.stop()
class ConcurrentMapData(Node): def __init__(self, ds, num_concurrent=1, map_func=None, executor_type=None, buffer_factor=4, none_on_exception=False, initializer=None, initargs=()): if num_concurrent == 1: _logger.warn("You need num_concurrent > 1 to obtain acceleration.") assert num_concurrent > 0 if num_concurrent >= os.cpu_count(): _logger.warn("num_concurrent is too large") assert map_func is not None assert buffer_factor > 1 self._ds = ds self._num_concurrent = num_concurrent self._fn = map_func self._exe_type = executor_type self._num_submit = int(buffer_factor * self._num_concurrent) self._none_on_exception = none_on_exception if self._exe_type == "thread": self._exe = ThreadPoolExecutor(max_workers=self._num_concurrent, initializer=initializer, initargs=initargs) elif self._exe_type == "process": self._exe = ProcessPoolExecutor(max_workers=self._num_concurrent, initializer=initializer, initargs=initargs) else: raise RuntimeError( "Unknown executor type, must be 'thread' or 'process'") self._exhausted = False def __iter__(self): def try_fill_pending(pending: set, iterator): while not self._exhausted: if len(pending) >= self._num_submit: break try: dp = next(iterator) future = self._exe.submit(self._fn, dp) pending.add(future) except StopIteration: self._exhausted = True break done = set() pending = set() it = iter(self._ds) while True: try_fill_pending(pending, it) if len(pending) == 0: break # raise StopIteration else: done, pending = concurrent.futures.wait( pending, return_when=concurrent.futures.FIRST_COMPLETED) for d in done: if d is not None: try: yield d.result() except Exception as e: _logger.error(repr(e)) if self._none_on_exception: yield None
# win32api.keybd_event(86, 0, 0, 0) # v # win32api.keybd_event(86, 0, win32con.KEYEVENTF_KEYUP, 0) # win32api.keybd_event(17, 0, win32con.KEYEVENTF_KEYUP, 0) # # alt s # win32api.keybd_event(18, 0, 0, 0) # Alt # win32api.keybd_event(83, 0, 0, 0) # s # win32api.keybd_event(18, 0, win32con.KEYEVENTF_KEYUP, 0) # win32api.keybd_event(83, 0, win32con.KEYEVENTF_KEYUP, 0) time.sleep(5) if __name__ == '__main__': q = Manager().Queue() # 多进程队列 executor = ProcessPoolExecutor(max_workers=10) # 进程池 # 九位玩家分别对应一个子进程 task_1 = executor.submit(subprocess_event_producer, BYISHIN_64BIT, BYISHIN_32BIT, q) task_2 = executor.submit(subprocess_event_producer, LEEROY_64BIT, LEEROY_32BIT, q) task_3 = executor.submit(subprocess_event_producer, NEKO_64BIT, NEKO_32BIT, q) task_4 = executor.submit(subprocess_event_producer, ORI_64BIT, ORI_32BIT, q) task_5 = executor.submit(subprocess_event_producer, ASIIMOV_64BIT, ASIIMOV_32BIT, q) task_6 = executor.submit(subprocess_event_producer, SAKANA_64BIT, SAKANA_32BIT, q) task_7 = executor.submit(subprocess_event_producer, DIDIDI_64BIT, DIDIDI_32BIT, q) task_8 = executor.submit(subprocess_event_producer, NEVEROWNED_64BIT, NEVEROWNED_32BIT, q) task_9 = executor.submit(subprocess_event_producer, RABBIT_64BIT,
import requests from concurrent.futures.process import ProcessPoolExecutor from utils import ts M = 5000 executor = ProcessPoolExecutor(16) def job(): # print(f'job on thread {threading.current_thread().name}') r = requests.get('http://localhost:2233/status') return len(r.json()['comment']) # future = executor.submit(job) futures = [] start = ts() for i in range(M): futures.append(executor.submit(job)) result = sum(f.result() for f in futures) print(f'check: 12={result/M}') delta = ts() - start print(f'{M} requests in {delta:.3f}s ({M / delta:.0f}RPS)')
def do_test3(workers): param = {"max_workers": workers} loop = asyncio.new_event_loop() lock = threading.Lock() tresult = [] presult = [] cresult = [] pre_input1 = input_generator(workers, 0) pre_input2 = input_generator(workers, max(pre_input1)) pre_input3 = input_generator(workers, max(pre_input2)) def result_checker(list, lock, fut): with lock: try: list.append(fut.result()) except Exception as e: list.append(e) texec = ThreadPoolExecutor(**param) pexec = ProcessPoolExecutor(**param) cexec = CoroutinePoolExecutor(**param, loop=loop) tstart = round(time.time()+1) input1 = [tstart + i for i in pre_input1] input2 = [tstart + i for i in pre_input2] input3 = [tstart + i for i in pre_input3] for x in input1: future = texec.submit(wake_at, x) future.add_done_callback( functools.partial(result_checker, tresult, lock)) result_iter = texec.map(wake_at, input2) for x in input3: future = texec.submit(wake_at, x) future.add_done_callback( functools.partial(result_checker, tresult, lock)) for x in result_iter: with lock: tresult.append(x) texec.shutdown(True) pstart = round(time.time() + _start_warm_up) input1 = [pstart + i for i in pre_input1] input2 = [pstart + i for i in pre_input2] input3 = [pstart + i for i in pre_input3] for x in input1: future = pexec.submit(wake_at, x) future.add_done_callback( functools.partial(result_checker, presult, lock)) result_iter = pexec.map(wake_at, input2) for x in input3: future = pexec.submit(wake_at, x) future.add_done_callback( functools.partial(result_checker, presult, lock)) for x in result_iter: with lock: presult.append(x) pexec.shutdown(True) cstart = round(time.time() + _start_warm_up) input1 = [cstart + i for i in pre_input1] input2 = [cstart + i for i in pre_input2] input3 = [cstart + i for i in pre_input3] async def async_main(): for x in input1: future = cexec.submit(async_wake_at, x) future.add_done_callback( functools.partial(result_checker, cresult, lock)) result_iter = cexec.map(async_wake_at, input2) for x in input3: future = cexec.submit(async_wake_at, x) future.add_done_callback( functools.partial(result_checker, cresult, lock)) async for x in result_iter: with lock: cresult.append(x) await cexec.shutdown(False) loop.run_until_complete(async_main()) try: loop.run_until_complete(cexec.shutdown(True)) texec.shutdown(True) pexec.shutdown(True) finally: loop.close() tresult = [round((x - tstart) / _precision) for x in tresult] presult = [round((x - pstart) / _precision) for x in presult] cresult = [round((x - cstart) / _precision) for x in cresult] result = True for (t, p, c) in zip(tresult, presult, cresult): result = result and (t == p) if not result: print(tresult) print(presult) print(cresult) print(t,p,c) assert False result = result and (p == c) if not result: print(tresult) print(presult) print(cresult) print(t, p, c) assert False result = result and (c == t) if not result: print(tresult) print(presult) print(cresult) print(t, p, c) assert False return result
def do_test1(workers): param = {"max_workers": workers} start = round(time.time() + _start_warm_up) input = input_generator(workers, start) loop = asyncio.new_event_loop() lock = threading.Lock() tresult = [] presult = [] cresult = [] def result_checker(list, lock, fut): with lock: try: list.append(fut.result()) except Exception as e: list.append(e) texec = ThreadPoolExecutor(**param) pexec = ProcessPoolExecutor(**param) cexec = CoroutinePoolExecutor(**param, loop=loop) for x in input: future = texec.submit(wake_at, x) future.add_done_callback( functools.partial(result_checker, tresult, lock)) future = pexec.submit(wake_at, x) future.add_done_callback( functools.partial(result_checker, presult, lock)) future = cexec.submit(async_wake_at, x) future.add_done_callback( functools.partial(result_checker, cresult, lock)) texec.shutdown(False) pexec.shutdown(False) loop.run_until_complete(cexec.shutdown(False)) try: loop.run_until_complete(cexec.shutdown(True)) texec.shutdown(True) pexec.shutdown(True) finally: loop.close() tresult = [round((x - start) / _precision) for x in tresult] presult = [round((x - start) / _precision) for x in presult] cresult = [round((x - start) / _precision) for x in cresult] result = True for (t, p, c) in zip(tresult, presult, cresult): result = result and (t == p) if not result: print(tresult) print(presult) print(cresult) print(t, p, c) assert False result = result and (p == c) if not result: print(tresult) print(presult) print(cresult) print(t, p, c) assert False result = result and (c == t) if not result: print(tresult) print(presult) print(cresult) print(t, p, c) assert False return result