def prepare_env_and_recheck(input_dir, out_dir, target, num_jobs, config_file): if not common.check_if_dir_exists(input_dir): common.print_and_exit("Can't use input directory") common.check_dir_and_create(out_dir) run_gen.gen_test_makefile_and_copy(out_dir, config_file) run_gen.dump_testing_sets(target) run_gen.print_compilers_version(target) lock = multiprocessing.Lock() task_queue = multiprocessing.JoinableQueue() process_dir(input_dir, task_queue) failed_queue = multiprocessing.SimpleQueue() passed_queue = multiprocessing.SimpleQueue() task_threads = [0] * num_jobs for num in range(num_jobs): task_threads[num] = \ multiprocessing.Process(target=recheck, args=(num, lock, task_queue, failed_queue, passed_queue, target, out_dir)) task_threads[num].start() task_queue.join() task_queue.close() for num in range(num_jobs): task_threads[num].join()
def __init__(self): self.queue = multiprocessing.SimpleQueue() self.result_queue = multiprocessing.SimpleQueue() self.process = multiprocessing.Process(target=_start_and_run_window, args=(self.queue, self.result_queue)) self.process.start()
def _simulate_in_parallel(env, game, nn_att, nn_def, n_processes: int, n_episodes: int, save_dir: str = None, summary_writer=None): """ Run simulations in parallel processes. """ worker_processes = [] simulation_request_queue = multiprocessing.SimpleQueue() attacker_reward_queue = multiprocessing.SimpleQueue() defender_reward_queue = multiprocessing.SimpleQueue() # Set-up all the processes. for _ in range(n_processes): worker_processes += [SimulationWorker( simulation_request_queue, defender_reward_queue, attacker_reward_queue, nn_att, nn_def, settings.get_attacker_strategy_dir(), settings.get_defender_strategy_dir())] worker_processes[-1].start() # Request all simulations. for _ in range(n_episodes): simulation_request_queue.put(CloudpickleWrapper(game)) # Send sentinel values to tell processes to cleanly shutdown (1 per worker). for _ in range(n_processes): simulation_request_queue.put(None) for process in worker_processes: process.join() # Aggregate results. attacker_rewards = np.zeros([n_episodes]) defender_rewards = np.zeros([n_episodes]) for episode_i in range(n_episodes): attacker_rewards[episode_i] = attacker_reward_queue.get() defender_rewards[episode_i] = defender_reward_queue.get() return attacker_rewards, defender_rewards
def spinup_server(manager_config): reg_queue = multiprocessing.SimpleQueue() worker_msg_queue = multiprocessing.SimpleQueue() trainer_in_queue = multiprocessing.SimpleQueue() trainer_out_queue = multiprocessing.SimpleQueue() manager_client = Server( manager_config, { "register": reg_queue, "worker": worker_msg_queue, "trainer": trainer_in_queue }) p1 = multiprocessing.Process(target=mqtt_process, args=(manager_client, )) p2 = multiprocessing.Process(target=manager_process, args=(manager_client, worker_msg_queue, reg_queue, trainer_in_queue, trainer_out_queue)) p3 = multiprocessing.Process( target=trainer_process, args=( # config, queue for new sessions, model, environment manager_config, trainer_out_queue, trainer_in_queue, None, None)) jobs = [p1, p2, p3] for j in jobs: j.start()
def __init__(self, loader): self.loader = loader self.dataset = loader.dataset self.batch_size = loader.batch_size self.collate_fn = loader.collate_fn self.sampler = loader.sampler self.num_workers = loader.num_workers self.done_event = threading.Event() self.samples_remaining = len(self.sampler) self.sample_iter = iter(self.sampler) if self.num_workers > 0: self.index_queue = multiprocessing.SimpleQueue() self.data_queue = multiprocessing.SimpleQueue() self.batches_outstanding = 0 self.shutdown = False self.send_idx = 0 self.rcvd_idx = 0 self.reorder_dict = {} self.workers = [ multiprocessing.Process( target=_worker_loop, args=(self.dataset, self.index_queue, self.data_queue, self.collate_fn)) for _ in range(self.num_workers)] for w in self.workers: w.daemon = True # ensure that the worker exits on process exit w.start() # prime the prefetch loop for _ in range(2 * self.num_workers): self._put_indices()
def run(self, result): # We use SimpleQueues because they are more predictable. # They do the necessary IO directly, without using a # helper thread. result_queue = multiprocessing.SimpleQueue() status_queue = multiprocessing.SimpleQueue() worker_param_queue = multiprocessing.SimpleQueue() # Prepopulate the worker param queue with server connection # information. for _ in range(self.num_workers): worker_param_queue.put((self.server_conn, self.postgres_dsn)) result_thread = threading.Thread(name='test-monitor', target=monitor_thread, args=(result_queue, result), daemon=True) result_thread.start() initargs = (status_queue, worker_param_queue, result_queue) pool = multiprocessing.Pool(self.num_workers, initializer=mproc_fixes.WorkerScope( init_worker, shutdown_worker), initargs=initargs) # Wait for all workers to initialize. for _ in range(self.num_workers): status_queue.get() with pool: ar = pool.map_async(_run_test, iter(self.tests), chunksize=1) while True: try: ar.get(timeout=0.1) except multiprocessing.TimeoutError: if self.stop_requested: break else: continue else: break # Post the terminal message to the queue so that # test-monitor can stop. result_queue.put((None, None, None)) # Give the test-monitor thread some time to # process the queue messages. If something # goes wrong, the thread will be forcibly # joined by a timeout. result_thread.join(timeout=3) # Wait for pool to shutdown, this includes test teardowns. pool.join() return result
def __init__(self, port=DEFAULT_PORT,authkey=DEFAULT_AUTHKEY,timeout=DEFAULT_TIMEOUT,output=DEFAULT_OUTPUT): self.port = port self.timeout = timeout self.output=output self.authkey=authkey self.job_q = multiprocessing.SimpleQueue() self.result_q = multiprocessing.SimpleQueue() self._manager = self._create_manager() self.commands=[]
def run(self, nprocs): #create if (self.runner_output == "file"): pathlib.Path(self.runner_dir).mkdir(parents=True,exist_ok=True) #properties of this blockrunner hosttags={"hostname": socket.gethostname(),"multirunnerid": self.blockrunnerid, "nprocs": nprocs} #inter-block shared objects job_q = multiprocessing.SimpleQueue() result_q = multiprocessing.SimpleQueue() idle_q = multiprocessing.SimpleQueue() job_count = multiprocessing.Value('i',0) result_count = multiprocessing.Value('i',0) job_outofstock = multiprocessing.Value('i',False) #start the retriever workers = [] p_retriever = multiprocessing.Process( target=self._commandretriever, args=(job_q, idle_q, job_count, job_outofstock, nprocs)) try: p_retriever.start() #start workers for i in range(nprocs): p = multiprocessing.Process( target=self._runner, args=(job_q, result_q,idle_q,i)) p.start() workers.append(p) #comunicate to the master that we are starting self.remote_result_q.put({"tag":"start"}) #send back results while True: oos=job_outofstock.value #save it now because it could change during next "if condition" evaluation if (result_count.value < job_count.value): result=result_q.get() result.update(hosttags) self.remote_result_q.put(result) result_count.value+=1 #I'm the only one to increment this elif (oos): break else: time.sleep(1) except Exception: pass finally: #just in case for p in workers: p.terminate() p_retriever.terminate()
def __init__(self, loader): self.dataset = loader.dataset self.collate_fn = loader.collate_fn self.batch_sampler = loader.batch_sampler self.num_workers = loader.num_workers self.pin_memory = loader.pin_memory self.done_event = threading.Event() self.worker_init_fn = loader.worker_init_fn self.worker_init_args = loader.worker_init_args self.worker_init_kwargs = loader.worker_init_kwargs self.sample_iter = iter(self.batch_sampler) if self.num_workers > 0: self.index_queue = multiprocessing.SimpleQueue() self.data_queue = multiprocessing.SimpleQueue() self.batches_outstanding = 0 self.shutdown = False self.send_idx = 0 self.rcvd_idx = 0 self.reorder_dict = {} self.seeds = loader.gen_seeds() self.workers = [ multiprocessing.Process( target=_worker_loop_seed, args=(i, self.dataset, self.index_queue, self.data_queue, self.collate_fn, self.seeds[i], self.worker_init_fn, self.worker_init_args[i], self.worker_init_kwargs[i])) for i in range(self.num_workers) ] for w in self.workers: w.daemon = True # ensure that the worker exits on process exit w.start() if self.pin_memory: in_data = self.data_queue self.data_queue = queue.Queue() self.pin_thread = threading.Thread(target=_pin_memory_loop, args=(in_data, self.data_queue, self.done_event)) self.pin_thread.daemon = True self.pin_thread.start() # prime the prefetch loop for _ in range(2 * self.num_workers): self._put_indices() else: if self.worker_init_fn is not None: self.worker_init_fn(-1, *self.worker_init_args, **self.worker_init_kwargs)
def _after_prelude(self): inputQ = multiprocessing.SimpleQueue() outputQ = multiprocessing.SimpleQueue() inputP = multiprocessing.Process(target=_queue_read_json_lines, args=(inputQ, sys.stdin)) inputP.start() outputP = multiprocessing.Process(target=_queue_dump_json_lines, args=(outputQ, sys.stdout)) outputP.start() for jsIn in _queue_iterate(inputQ): jsOut = self.each(jsIn) outputQ.put(jsOut)
def mpAPrimes(self, limit): limit += 1 cutoffs = [1] for i in range(1, self.threads): cutoffs.append(int(sqrt(i * (limit**2) / self.threads))) cutoffs.append(limit) resultQueue = multiprocessing.SimpleQueue() printQueue = multiprocessing.SimpleQueue() printThread = threading.Thread(target=self.printThread, args=(printQueue, ), name='printThread') printThread.start() for i in range(1, self.threads): printQueue.put('cutoff ' + str(i) + ': ' + str(cutoffs[i])) jobs = [] for i in range(self.threads): jobs.append( multiprocessing.Process(target=self.aprime_process, args=(cutoffs[i], cutoffs[i + 1], resultQueue, printQueue), name='p' + str(i))) for job in jobs: job.start() aPrimeList = [] for i in range(self.threads): aPrimeList += resultQueue.get() printQueue.put('got result ' + str(i)) for job in jobs: job.join() printQueue.put(aPrimeList) aPrimeList.sort() antiprimes = [] most = 0 for num in aPrimeList: factors = self.num_factors(num) if factors > most: most = factors antiprimes.append(num) printQueue.put(num) printQueue.put(antiprimes) printQueue.put('stop') printThread.join() return antiprimes
def start_training(self, feeding_q, gpu_device): """ gpu_device = -1 for CPU """ if self._verbose: print("start AE on device:", gpu_device) self._exit_msg = self.__hash__() # not super-safe but safe enough self._q = feeding_q self._serialized_net_file = mp.SimpleQueue() self._eval_score = mp.SimpleQueue() self._train_process = mp.Process(target=self._train, args=(gpu_device, )) self._train_process.start()
def __init__(self, env, n_actors, extras): self.env = env self.n_actors = n_actors self.proc_objs = [] for proc_index in range(n_actors): in_q = mp.SimpleQueue() #mp.Queue() out_q = mp.SimpleQueue() #mp.Queue() args = (proc_index, env, in_q, out_q, extras) proc = mp.Process(target=actor_process_main, args=args) self.proc_objs.append((proc, in_q, out_q)) self._started = False self._joined = False
def main(argv=None): log_queue = mp.Queue() with launch_logging_thread(log_queue), set_up_logging(log_queue): prog_args = parse_args(argv) access_queue = mp.SimpleQueue() scandir_queue = mp.SimpleQueue() access_process_args = prog_args, log_queue, access_queue, scandir_queue access_process = mp.Process(target=access_main, args=access_process_args) access_queue.put([prog_args.root_dir]) access_process.start() scandir_main(access_queue, scandir_queue) access_process.join()
def __init__(self, worker_class, initargs=(), worker_count=None, batch_size=None, wrap_exception=False): worker_count = worker_count or os.cpu_count() or 1 use_fastqueue = True if use_fastqueue: self._task_queue = FastQueue() self._result_queue = FastQueue() self._quick_put = self._task_queue.put self._quick_get = self._result_queue.get else: self._task_queue = multiprocessing.SimpleQueue() self._result_queue = multiprocessing.SimpleQueue() self._quick_put = self._task_queue._writer.send self._quick_get = self._result_queue._reader.recv self._batch_size = batch_size self._callback = None self._error_callback = None self._listener = None main_module = sys.modules['__main__'] is_profiling = os.path.basename( main_module.__file__) in ['profile.py', 'cProfile.py'] self._pool = [] for i in range(worker_count): worker_params = _WorkerParams(i, self._task_queue, self._result_queue, worker_class, initargs, wrap_exception=wrap_exception, is_profiling=is_profiling) w = multiprocessing.Process(target=worker_func, args=(worker_params, )) w.name = w.name.replace('Process', 'PoolWorker') w.daemon = True w.start() self._pool.append(w) self._result_handler = threading.Thread( target=WorkerPool._handleResults, args=(self, )) self._result_handler.daemon = True self._result_handler.start() self._closed = False
def run(self, result): # We use SimpleQueues because they are more predictable. # The do the necessary IO directly, without using a # helper thread. result_queue = multiprocessing.SimpleQueue() worker_param_queue = multiprocessing.SimpleQueue() # Prepopulate the worker param queue with server connection # information. for server_conn in self.server_conns: worker_param_queue.put(server_conn) result_thread = threading.Thread(name='test-monitor', target=monitor_thread, args=(result_queue, result), daemon=True) result_thread.start() initargs = (worker_param_queue, result_queue) pool = multiprocessing.Pool(self.num_workers, initializer=init_worker, initargs=initargs) with pool: ar = pool.map_async(_run_test, iter(self.tests), chunksize=1) while True: try: ar.get(timeout=0.1) except multiprocessing.TimeoutError: if self.stop_requested: pool.terminate() break continue else: break # Post the terminal message to the queue so that # test-monitor can stop. result_queue.put((None, None, None)) # Give the test-monitor thread some time to # process the queue messages. If something # goes wrong, the thread will be forcibly # joined by a timeout. result_thread.join(timeout=3) return result
def main(args): # Build the pipeline. procs = [] # Read from input. q_games = mp.SimpleQueue() if args: prefix = args.pop(0) print("Reading from chunkfiles {}".format(prefix)) procs.append( mp.Process(target=disk_fetch_games, args=(q_games, prefix))) else: print("Reading from MongoDB") #procs.append(mp.Process(target=fake_fetch_games, args=(q_games, 20))) procs.append( mp.Process(target=mongo_fetch_games, args=(q_games, 275000))) # Split into train/test q_test = mp.SimpleQueue() q_train = mp.SimpleQueue() procs.append( mp.Process(target=split_train_test, args=(q_games, q_train, q_test))) # Convert v1 to v2 format and shuffle, writing 8192 moves per chunk. q_write_train = mp.SimpleQueue() q_write_test = mp.SimpleQueue() # Shuffle buffer is ~ 2.2GB of RAM with 2^20 (~1e6) entries. A game is ~500 moves, so # there's ~2000 games in the shuffle buffer. Selecting 8k moves gives an expected # number of ~4 moves from the same game in a given chunk file. # # The output files are in parse.py via another 1e6 sized shuffle buffer. At 8192 moves # per chunk, there's ~ 128 chunks in the shuffle buffer. With a batch size of 4096, # the expected max number of moves from the same game in the batch is < 1.14 procs.append( mp.Process(target=chunk_parser, args=(q_train, q_write_train, 1 << 20, 8192))) procs.append( mp.Process(target=chunk_parser, args=(q_test, q_write_test, 1 << 16, 8192))) # Write to output files procs.append( mp.Process(target=chunk_writer, args=(q_write_train, NameSrc('train_')))) procs.append( mp.Process(target=chunk_writer, args=(q_write_test, NameSrc('test_')))) # Start all the child processes running. for p in procs: p.start() # Wait for everything to finish. for p in procs: p.join()
def run(self): yield messages.StartedMessage.get() if self.runnable.config.get("nrunner.max_parallel_tasks", 1) != 1: yield messages.FinishedMessage.get( 'cancel', fail_reason="parallel run is not" " allowed for vt tests") else: try: queue = multiprocessing.SimpleQueue() vt_test = VirtTest(queue, self.runnable) process = multiprocessing.Process(target=vt_test.runTest) process.start() while True: time.sleep(nrunner.RUNNER_RUN_CHECK_INTERVAL) if queue.empty(): yield messages.RunningMessage.get() else: message = queue.get() yield message if message.get('status') == 'finished': break except Exception: yield messages.StderrMessage.get(traceback.format_exc()) yield messages.FinishedMessage.get('error')
def train(self, episodes=1000): """ Lower level training orchestration method. Written in the generator style. Intended to be used with "for update in train(...):" """ # create the requested number of background agents and runners: worker_agents = self.agent.clone(num = self.workers) runners = [ Runner(agent=agent, ix=(ix + 1), train=True) for ix, agent in enumerate(worker_agents) ] # we're going to communicate the workers' updates via the thread safe queue: queue = mp.SimpleQueue() # if we've not been given a number of episodes: assume the process is going to be interrupted with the keyboard interrupt once the user (us) decides so: if episodes is None: print('Starting out an infinite training process') # create the actual background processes, making their entry be the train_one method: processes = [ mp.Process(target=self.train_one, args=(runners[ix - 1], queue, episodes, ix)) for ix in range(1, self.workers + 1) ] # run those processes: for process in processes: process.start() try: # what follows is a rather naive implementation of listening to workers updates. it works though for our purposes: while any([ process.is_alive() for process in processes ]): results = queue.get() yield results except Exception as e: logger.error(str(e))
def run(self): if not self.runnable.uri: yield {'status': 'error', 'output': 'uri is required but was not given'} return queue = multiprocessing.SimpleQueue() process = multiprocessing.Process(target=self._run, args=(self.runnable.uri, queue)) time_start = time.time() time_start_sent = False process.start() last_status = None while queue.empty(): time.sleep(RUNNER_RUN_CHECK_INTERVAL) now = time.time() if last_status is None or now > last_status + RUNNER_RUN_STATUS_INTERVAL: last_status = now if not time_start_sent: time_start_sent = True yield {'status': 'running', 'time_start': time_start} yield {'status': 'running'} yield queue.get()
def run(self): if not self.runnable.uri: yield { 'status': 'finished', 'result': 'error', 'output': 'uri is required but was not given' } return queue = multiprocessing.SimpleQueue() process = multiprocessing.Process(target=self._run, args=(self.runnable.uri, queue)) time_start = time.time() time_start_sent = False process.start() most_current_execution_state_time = None while queue.empty(): time.sleep(nrunner.RUNNER_RUN_CHECK_INTERVAL) now = time.time() if most_current_execution_state_time is not None: next_execution_state_mark = ( most_current_execution_state_time + nrunner.RUNNER_RUN_STATUS_INTERVAL) if (most_current_execution_state_time is None or now > next_execution_state_mark): most_current_execution_state_time = now if not time_start_sent: time_start_sent = True yield {'status': 'running', 'time_start': time_start} yield {'status': 'running'} yield queue.get()
def multi_proc_run(num_proc, fun, fun_args=(), fun_kwargs=None): """Runs a function in a multi-proc setting (unless num_proc == 1).""" # There is no need for multi-proc in the single-proc case fun_kwargs = fun_kwargs if fun_kwargs else {} if num_proc == 1: fun(*fun_args, **fun_kwargs) return # Handle errors from training subprocesses error_queue = multiprocessing.SimpleQueue() error_handler = ErrorHandler(error_queue) # Run each training subprocess ps = [] for i in range(num_proc): p_i = multiprocessing.Process(target=run, args=(i, num_proc, error_queue, fun, fun_args, fun_kwargs)) ps.append(p_i) p_i.start() error_handler.add_child(p_i.pid) # Wait for each subprocess to finish for p in ps: p.join()
def run_amplifiers(filename, phase_settings): num_amplifiers = len(phase_settings) amplifiers = [] # initialise buffers with phase settings buffers = [] for idx in range(0,num_amplifiers): buf = multiprocessing.SimpleQueue() buf.put(phase_settings[idx]) buffers.append(buf) for idx in range(0,num_amplifiers): amplifier_name = chr(ord('A') + idx) input_buf = buffers[idx] if idx == num_amplifiers - 1: # output of last amplifier is fed back to first amplifier output_buf = buffers[0] else: output_buf = buffers[idx + 1] amplifiers.append(run_amplifier(filename, amplifier_name, input_buf, output_buf)) # start input buffers[0].put(0) for a in amplifiers: a.join() #print("Amplifiers done") return buffers[0].get()
def test_wrap(self): none = "" stdout_log = os.path.join(self.test_dir, "stdout.log") stderr_log = os.path.join(self.test_dir, "stderr.log") redirs = [ (none, none), (none, stderr_log), (stdout_log, none), (stdout_log, stderr_log), ] for stdout_redir, stderr_redir in redirs: queue = multiprocessing.SimpleQueue() _wrap( local_rank=0, fn=echo1, args={0: ("hello", )}, envs={0: { "RANK": "0" }}, stdout_redirects={0: stdout_redir}, stderr_redirects={0: stderr_redir}, ret_vals={0: queue}, ) self.assertEqual("hello_0", queue.get()) if stdout_redir: self.assert_in_file(["hello stdout from 0"], stdout_log) if stderr_redir: self.assert_in_file(["hello stderr from 0"], stderr_log)
def run(self): if not self.runnable.uri: yield messages.FinishedMessage.get('error', fail_reason='uri is required ' 'but was not given') return queue = multiprocessing.SimpleQueue() process = multiprocessing.Process(target=self._run, args=(self.runnable.uri, queue)) process.start() yield messages.StartedMessage.get() most_current_execution_state_time = None while queue.empty(): time.sleep(nrunner.RUNNER_RUN_CHECK_INTERVAL) now = time.monotonic() if most_current_execution_state_time is not None: next_execution_state_mark = (most_current_execution_state_time + nrunner.RUNNER_RUN_STATUS_INTERVAL) if (most_current_execution_state_time is None or now > next_execution_state_mark): most_current_execution_state_time = now yield messages.RunningMessage.get() status = queue.get() yield messages.StdoutMessage.get(status['stdout']) yield messages.StderrMessage.get(status['stderr']) yield messages.FinishedMessage.get(status['result'])
def run(self, runnable): # pylint: disable=W0201 self.runnable = runnable file_name, suite_name, test_name = self._uri_to_file_suite_test() if not all([file_name, suite_name, test_name]): yield messages.FinishedMessage.get('error', fail_reason='Invalid URI given') return queue = multiprocessing.SimpleQueue() process = multiprocessing.Process(target=self._run, args=(file_name, suite_name, test_name, queue)) process.start() yield messages.StartedMessage.get() most_current_execution_state_time = None while queue.empty(): time.sleep(RUNNER_RUN_CHECK_INTERVAL) now = time.monotonic() if most_current_execution_state_time is not None: next_execution_state_mark = ( most_current_execution_state_time + RUNNER_RUN_STATUS_INTERVAL) if (most_current_execution_state_time is None or now > next_execution_state_mark): most_current_execution_state_time = now yield messages.RunningMessage.get() status = queue.get() yield messages.StdoutMessage.get(status['stdout']) yield messages.StderrMessage.get(status['stderr']) yield messages.FinishedMessage.get(status['result'])
def run(self): if not self.module_class_method: error_msg = ("Invalid URI: could not be converted to an unittest " "dotted name.") yield self.prepare_status('finished', {'result': 'error', 'output': error_msg}) return queue = multiprocessing.SimpleQueue() process = multiprocessing.Process(target=self._run_unittest, args=(self.module_path, self.module_class_method, queue)) process.start() yield self.prepare_status('started') most_current_execution_state_time = None while queue.empty(): time.sleep(RUNNER_RUN_CHECK_INTERVAL) now = time.monotonic() if most_current_execution_state_time is not None: next_execution_state_mark = (most_current_execution_state_time + RUNNER_RUN_STATUS_INTERVAL) if (most_current_execution_state_time is None or now > next_execution_state_mark): most_current_execution_state_time = now yield self.prepare_status('running') status = queue.get() yield self.prepare_status('running', {'type': 'stdout', 'log': status.pop('output').encode()}) status['time'] = time.monotonic() yield status
def test_wrap_bad(self): none = "" stdout_log = os.path.join(self.test_dir, "stdout.log") stderr_log = os.path.join(self.test_dir, "stderr.log") redirs = [ (none, none), (none, stderr_log), (stdout_log, none), (stdout_log, stderr_log), ] for stdout_redir, stderr_redir in redirs: queue = multiprocessing.SimpleQueue() worker_finished_event_mock = mock.Mock() _wrap( local_rank=0, fn=echo1, args={0: ("hello",)}, envs={0: {"RANK": "0"}}, stdout_redirects={0: stdout_redir}, stderr_redirects={0: stderr_redir}, ret_vals={0: queue}, queue_finished_reading_event=worker_finished_event_mock, ) self.assertEqual("hello_0", queue.get()) if stdout_redir: self.assert_in_file(["hello stdout from 0"], stdout_log) if stderr_redir: self.assert_in_file(["hello stderr from 0"], stderr_log) worker_finished_event_mock.wait.assert_called_once()
def run(self): if not self.runnable.uri: error_msg = 'uri is required but was not given' yield self.prepare_status('finished', {'result': 'error', 'output': error_msg}) return queue = multiprocessing.SimpleQueue() process = multiprocessing.Process(target=self._run_unittest, args=(self.runnable.uri, queue)) process.start() yield self.prepare_status('started') most_current_execution_state_time = None while queue.empty(): time.sleep(RUNNER_RUN_CHECK_INTERVAL) now = time.monotonic() if most_current_execution_state_time is not None: next_execution_state_mark = (most_current_execution_state_time + RUNNER_RUN_STATUS_INTERVAL) if (most_current_execution_state_time is None or now > next_execution_state_mark): most_current_execution_state_time = now yield self.prepare_status('running') status = queue.get() status['time'] = time.monotonic() yield status
def __init__(self, data_structure, processes, scan_function, init_args, _mp_init_function): """ Init the scanner """ assert isinstance(data_structure, world.DataSet) self.data_structure = data_structure self.list_files_to_scan = data_structure._get_list() self.processes = processes self.scan_function = scan_function # Queue used by processes to pass results self.queue = multiprocessing.SimpleQueue() init_args.update({'queue': self.queue}) # NOTE TO SELF: initargs doesn't handle kwargs, only args! # Pass a dict with all the args self.pool = multiprocessing.Pool(processes=processes, initializer=_mp_init_function, initargs=(init_args, )) # Recommended time to sleep between polls for results self.SCAN_START_SLEEP_TIME = 0.001 self.SCAN_MIN_SLEEP_TIME = 1e-6 self.SCAN_MAX_SLEEP_TIME = 0.1 self.scan_sleep_time = self.SCAN_START_SLEEP_TIME self.queries_without_results = 0 self.last_time = time() self.MIN_QUERY_NUM = 1 self.MAX_QUERY_NUM = 5 # Holds a friendly string with the name of the last file scanned self._str_last_scanned = None