class MultiProcessRunner(BaseRunner): JOIN_TIMEOUT = 60 def __init__(self, target, **kwargs): super(MultiProcessRunner, self).__init__(target, **kwargs) self.process = None # type: Process @capture_monitor_exception def start(self): self.process = Process(target=self.target, kwargs=self.kwargs) self.process.start() @capture_monitor_exception def stop(self): if self.process and self.is_alive(): self.process.terminate() self.process.join(MultiProcessRunner.JOIN_TIMEOUT) if self.process.is_alive(): self.process.kill() @capture_monitor_exception def heartbeat(self): # do we want to do something here? pass @capture_monitor_exception def is_alive(self): return self.process.is_alive() def __str__(self): s = super(MultiProcessRunner, self).__str__() return f"{s}({self.process})"
def func(*args, **kwargs): deltas, before, after = [], 0, 0 sys.stdout.write(" " * 147) for item in iterator: if isinstance(item, tuple): args = list(item) + list(args) item = item[0] else: args = [item] + args delta = after - before eta = 0 if delta > 0: deltas.append(delta) eta = (length - item) * (sum(deltas) / len(deltas)) percentage = 100 * item / length p = Process(target=draw_progress_bar, args=(eta, percentage, f"{item + 1} of {length}...")) before = time.time() p.start() function(*args, **kwargs) p.terminate() after = time.time() p = Process(target=draw_progress_bar, args=(0, 100, "Ready!")) p.start() time.sleep(0.1) p.terminate() print()
class AIOProcess: """ Execute a coroutine on a separate process """ def __init__(self, coroutine: Callable = None, *args, daemon: bool = False, target_override: Callable = None, **kwargs): if not asyncio.iscoroutinefunction(coroutine): raise ValueError("target must be a coroutine function") self.aio_process = Process(target=target_override or partial(AIOProcess.run_async, coroutine), args=args, kwargs=kwargs, daemon=daemon) @staticmethod def run_async(coroutine: Callable, *args, **kwargs): try: loop = uvloop.new_event_loop() asyncio.set_event_loop(loop) result = loop.run_until_complete(coroutine(*args, **kwargs)) return result except BaseException: log.exception(f"aio process {os.getpid()} failed") raise def start(self): self.aio_process.start() async def join(self, timeout=None): if not self.is_alive() and self.exit_code is None: raise ValueError("must start process before joining") if timeout is not None: return await asyncio.wait_for(self.join(), timeout) while self.exit_code is None: await asyncio.sleep(0.005) @property def pid(self): return self.aio_process.pid @property def daemon(self): return self.aio_process.daemon @property def exit_code(self): return self.aio_process.exitcode def is_alive(self): return self.aio_process.is_alive() def terminate(self): self.aio_process.terminate()
def process_file(file_path_input, file_path_output, i): if os.path.exists(file_path_output): return print('{} {} Already done'.format(now(), file_path_output)) input_file = open(file_path_input, 'r') output_file = open(file_path_output, 'a') # every worker create separate file for a input file writer = csv.writer(output_file, delimiter='\t') # for every file we create seperate driver (100 URLs) driver = webdriver.PhantomJS(executable_path=path_to_phantomjs) for line in input_file: splited = line.split('\t') # property_type = splited[0] url = splited[1] print('{} Process={} Current url: {}'.format(now(), i, url)) # start process for getting microformat properties temp_queue = Queue() # p = Process(target=get_microformat_properties_by_type, args=(url, property_type, temp_queue, i)) p = Process(target=get_element_features, args=(url, driver, temp_queue, i)) print("{} {} Process={} {} {}".format(now(), i, "Started: ", "feature extraction", url)) p.start() event_features = temp_queue.get(timeout=TIME_OUT_FEATURE) # try: # pass # except Empty: # print("{} {} Process={} {} {}".format(now(), i, "Timed out on: ", "feature extraction", url)) if p.is_alive(): p.terminate() print("Event features:" + str(event_features)) if event_features is not None: print("{} Process={} Got properties for {}".format( now(), i, url)) # start process for feature extraction and writing to separate file # p_event_features = Process(target=get_event_features_and_write, # args=(event_features, driver, writer, i, output_file)) # p_event_features = Process(target=write_element_features, args=(event_features, writer, i, output_file)) p.start() # start_with_timeout(p_event_features, TIME_OUT_LOAD, "feature writing", url, i) if p_event_features.is_alive(): p.terminate() driver.service.process.send_signal(signal.SIGTERM) driver.quit() return 'done'
def dashboard(): # This fixture guarantees the proper termination of all spawned subprocesses # after the tests. dashboard = Process(target=run_detection) dashboard.start() yield for child in psutil.Process(dashboard.pid).children(recursive=True): child.kill() dashboard.terminate() dashboard.join()
def serve(routing: dict, open_browser=False, timeout=None, filename=''): p = Process(target=WebServer.serve_and_browse, args=( routing, open_browser, filename, )) p.start() wait_for_server_seconds = timeout logging.info("Waiting for server %d seconds" % wait_for_server_seconds) time.sleep(wait_for_server_seconds) p.terminate()
def main(): logging.info("Main thread started") maxValue = 3 worker = Process(target=work, args=("working", maxValue), daemon=True, name="Worker") worker.start() time.sleep(5) # if the process is running , stop it if worker.is_alive: worker.terminate() # pretty dangerous worker.join() logging.info(f"Main thread finished {worker.exitcode}")
def main(): print('主进程开始') p = Process(target=test, args=(1, )) p.start() print(p.is_alive()) print('主进程结束') print(p.is_alive()) print(p.is_alive()) print(p.is_alive()) print(p.is_alive()) print(p.is_alive()) print('子进程名字:', p.name) print('子进程pid:', p.pid) p.terminate() p.join(20)
def setup_and_teardown_flask_app(app: Flask, host: str, port: int): """ Manages setup of provided flask app on given `host` and `port` and its teardown. As for setup process following things are done: * `/health` endpoint is added to provided flask app, * app is launched in separate process, * function waits for flask app to fully launch - to do this it repetitively checks `/health` endpoint if it will return status code 200. Example use of this function in fixture: >>> with setup_and_teardown_flask_app(Flask(__name__), "localhost", 10000): >>> yield :param app: app to launch :param host: host on which to launch app :param port: port on which to launch app """ def wait_for_flask_app_to_be_accessible(): timeout = 1 end_time = datetime.now() + timedelta(seconds=timeout) response = requests.Response() response.status_code = HTTP_404_NOT_FOUND while response.status_code != HTTP_200_OK and datetime.now( ) < end_time: with contextlib.suppress(requests.exceptions.ConnectionError): response = requests.request( "POST", "http://{}:{}/health".format(host, port)) time.sleep(0.01) fail_message = "Timeout expired: failed to start mock REST API in {} seconds".format( timeout) assert response.status_code == HTTP_200_OK, fail_message app.route("/health", methods=["POST"])(lambda: "OK") process = Process(target=app.run, args=(host, port)) process.start() wait_for_flask_app_to_be_accessible() yield process.terminate() process.join()
def main(): logging.info(f'App started') max = 2 worker = Process(target=work, args=['Working', max], daemon=True, name='Worker') worker.start() time.sleep(5) # if the process is running, stop it if worker.is_alive(): worker.terminate() # kill the process with SIGTERM worker.join() # exicode == 0 is good # anything else is an error logging.info(f'App finished: {worker.exitcode}')
def run_server(launcher, wait_before_entering=0, verbose=False, **kwargs): """Context manager to launch server on entry, and shut it down on exit""" from warnings import warn clog = conditional_logger(verbose) server = None try: server = Process(target=launcher, kwargs=kwargs) clog(f'Starting server...') server.start() clog(f'... server started.') sleep(wait_before_entering) yield server finally: if server is not None and server.is_alive(): clog(f'Terminating server...') server.terminate() clog(f'... server terminated')
def test_gunicorn_uvicorn(): options = { "bind": "%s:%s" % ("127.0.0.1", "8000"), "workers": 1, "worker_class": "uvicorn.workers.UvicornWorker", "log_level": "debug", } gunicorn_uvicorn_server = StandaloneApplication(handler_app, options) process = Process(target=gunicorn_uvicorn_server.run) process.start() time.sleep(0.1) response = requests.get("http://127.0.0.1:8000") assert response.status_code == 200 assert response.content == b"Hello" process.terminate() # needed timeout for travis or the port wont get released fast enough # and will "block" subsequent tests time.sleep(1)
def process_file(file_path_input, file_path_output, i): if os.path.exists(file_path_output): return print('{} {} Already done'.format(now(), file_path_output)) input_file = open(file_path_input, 'r') output_file = open(file_path_output, 'a') # every worker create separate file for a input file writer = csv.writer(output_file, delimiter='\t') # for every file we create seperate driver (100 URLs) driver = webdriver.PhantomJS(executable_path=path_to_phantomjs) for line in input_file: splited = line.split('\t') property_type = splited[0] url = splited[1] print('{} Process={} Current url: {}'.format(now(), i, url)) # start process for getting microformat properties temp_queue = Queue() p = Process(target=get_microformat_properties_by_type, args=(url, property_type, temp_queue, i)) start_with_timeout(p, TIME_OUT_LOAD, "loading", url, i) event_properties = temp_queue.get() if not temp_queue.empty() else None if p.is_alive(): p.terminate() if event_properties is not None: print("{} Process={} Got properties for {}".format( now(), i, url)) # start process for feature extraction and writing to separate file p_event_features = Process(target=get_event_features_and_write, args=(event_properties, driver, writer, i, output_file)) start_with_timeout(p_event_features, TIME_OUT_FEATURE, "feature extraction", url, i) if p_event_features.is_alive(): p.terminate() return 'done'
def main(): logging.info('Started') max = 2 worker = Process(target=work, args=['Working', max], daemon=True, name='Super Mario') worker.start() time.sleep(5) #if the process is running, stop it if worker.is_alive(): worker.terminate() worker.join() #exitcode == 0 #Anything else is an error logging.info(f'Finished: {worker.exitcode}')
def main(): name = process.current_process().name logging.info(f'{name} started') #Setup the process address = 'localhost' #127.0.0.1 port = 2823 # above 1024 password = b'password' p = Process(target=proc, args=[address, port, password], daemon=True, name="Worker") p.start() logging.info(f'{name} waiting on the worker...') time.sleep(1) #Connect to the process dest = (address, port) conn = Client(dest, authkey=password) #Command loop while True: command = input('\r\nEnter a command or type quit:\r\n').strip() logging.info(f'{name} command: {command}') conn.send(command) if command == 'quit': break #Cleanup and shutdown if p.is_alive: logging.info(f'{name} terminating worker') conn.close() time.sleep(1) p.terminate() p.join() logging.info(f'{name} finished')
class CreateProcess: """A context manager to launch a parallel process and close it on exit. """ def __init__( self, proc_func: Callable, process_name=None, wait_before_entering=2, verbose=False, args=(), **kwargs, ): """ Essentially, this context manager will call ``` proc_func(*args, **kwargs) ``` in an independent process. :param proc_func: A function that will be launched in the process :param process_name: The name of the process. :param wait_before_entering: A pause (in seconds) before returning from the enter phase. (in case the outside should wait before assuming everything is ready) :param verbose: If True, will print some info on the starting/stoping of the process :param args: args that will be given as arguments to the proc_func call :param kwargs: The kwargs that will be given as arguments to the proc_func call The following should print 'Hello console!' in the console. >>> with CreateProcess(print, verbose=True, args=('Hello console!',)) as p: ... print("-------> Hello module!") # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE Starting process: print... ... print process started. -------> Hello module! ... print process terminated """ self.proc_func = proc_func self.process_name = process_name or getattr(proc_func, '__name__', '') self.wait_before_entering = float(wait_before_entering) self.verbose = verbose self.args = args self.kwargs = kwargs self.clog = conditional_logger(verbose) self.process = None self.exception_info = None def process_is_running(self): return self.process is not None and self.process.is_alive() def __enter__(self): self.process = Process( target=self.proc_func, args=self.args, kwargs=self.kwargs, name=self.process_name, ) self.clog(f'Starting process: {self.process_name}...') try: self.process.start() if self.process_is_running(): self.clog(f'... {self.process_name} process started.') sleep(self.wait_before_entering) return self else: raise RuntimeError('Process is not running') except Exception: raise RuntimeError( f'Something went wrong when trying to launch process {self.process_name}' ) def __exit__(self, exc_type, exc_val, exc_tb): if self.process is not None and self.process.is_alive(): self.clog(f'Terminating process: {self.process_name}...') self.process.terminate() self.clog(f'... {self.process_name} process terminated') if exc_type is not None: self.exception_info = dict( exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb )
class SubprocEnvWorker(EnvWorker): """Subprocess worker used in SubprocVectorEnv and ShmemVectorEnv.""" def __init__( self, env_fn: Callable[[], gym.Env], share_memory: bool = False ) -> None: super().__init__(env_fn) self.parent_remote, self.child_remote = Pipe() self.share_memory = share_memory self.buffer: Optional[Union[dict, tuple, ShArray]] = None if self.share_memory: dummy = env_fn() obs_space = dummy.observation_space dummy.close() del dummy self.buffer = _setup_buf(obs_space) args = ( self.parent_remote, self.child_remote, CloudpickleWrapper(env_fn), self.buffer, ) self.process = Process(target=_worker, args=args, daemon=True) self.process.start() self.child_remote.close() def __getattr__(self, key: str) -> Any: self.parent_remote.send(["getattr", key]) return self.parent_remote.recv() def _decode_obs(self) -> Union[dict, tuple, np.ndarray]: def decode_obs( buffer: Optional[Union[dict, tuple, ShArray]] ) -> Union[dict, tuple, np.ndarray]: if isinstance(buffer, ShArray): return buffer.get() elif isinstance(buffer, tuple): return tuple([decode_obs(b) for b in buffer]) elif isinstance(buffer, dict): return {k: decode_obs(v) for k, v in buffer.items()} else: raise NotImplementedError return decode_obs(self.buffer) def reset(self) -> Any: self.parent_remote.send(["reset", None]) obs = self.parent_remote.recv() if self.share_memory: obs = self._decode_obs() return obs @staticmethod def wait( # type: ignore workers: List["SubprocEnvWorker"], wait_num: int, timeout: Optional[float] = None, ) -> List["SubprocEnvWorker"]: remain_conns = conns = [x.parent_remote for x in workers] ready_conns: List[connection.Connection] = [] remain_time, t1 = timeout, time.time() while len(remain_conns) > 0 and len(ready_conns) < wait_num: if timeout: remain_time = timeout - (time.time() - t1) if remain_time <= 0: break # connection.wait hangs if the list is empty new_ready_conns = connection.wait( remain_conns, timeout=remain_time) ready_conns.extend(new_ready_conns) # type: ignore remain_conns = [ conn for conn in remain_conns if conn not in ready_conns] return [workers[conns.index(con)] for con in ready_conns] def send_action(self, action: np.ndarray) -> None: self.parent_remote.send(["step", action]) def get_result( self, ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: obs, rew, done, info = self.parent_remote.recv() if self.share_memory: obs = self._decode_obs() return obs, rew, done, info def seed(self, seed: Optional[int] = None) -> Optional[List[int]]: self.parent_remote.send(["seed", seed]) return self.parent_remote.recv() def render(self, **kwargs: Any) -> Any: self.parent_remote.send(["render", kwargs]) return self.parent_remote.recv() def close_env(self) -> None: try: self.parent_remote.send(["close", None]) # mp may be deleted so it may raise AttributeError self.parent_remote.recv() self.process.join() except (BrokenPipeError, EOFError, AttributeError): pass # ensure the subproc is terminated self.process.terminate()
class AsyncMCTS(MoveChooser): """ Implementation of Monte Carlo Tree Search that uses the other player's time to continue thinking. This is achieved using multiprocessing, and a Pipe for transferring data to and from the worker process. """ def __init__(self, GameClass, starting_position, time_limit=3, network=None, c=np.sqrt(2), d=1, threads=1): """ Either: If network is provided, threads must be 1. If network is not provided, then threads will be used for leaf parallelization """ super().__init__(GameClass, starting_position) if network is not None and threads != 1: raise Exception('Threads != 1 with Network != None') self.parent_pipe, worker_pipe = Pipe() self.worker_process = Process(target=self.loop_func, args=(GameClass, starting_position, time_limit, network, c, d, threads, worker_pipe)) def start(self): self.worker_process.start() def report_user_move(self, user_chosen_move): """ Reports the given user chosen move to the worker thread. This allows the search tree to be narrowed. :param user_chosen_move: """ self.parent_pipe.send(user_chosen_move) self.position = user_chosen_move def choose_move(self, return_distribution=False): """ Instructs the worker thread to decide on an optimal move. The worker thread will then continue thinking for time_limit, and then return a list of its chosen moves. If multiple states are passed through before the ai's turn is completed, then they will be the contents of the list. Otherwise the list will have a single state. :return: The moves chosen by monte carlo tree search. """ self.parent_pipe.send(None) chosen_positions = self.parent_pipe.recv() self.position = chosen_positions[-1][0] return chosen_positions if return_distribution else [ position for position, _ in chosen_positions ] def terminate(self): self.worker_process.terminate() self.worker_process.join() @staticmethod def loop_func(GameClass, position, time_limit, network, c, d, threads, worker_pipe): if network is None: pool = Pool(threads) if threads > 1 else None root = RolloutNode(position, parent=None, GameClass=GameClass, c=c, rollout_batch_size=threads, pool=pool, verbose=True) else: network.initialize() root = HeuristicNode(position, None, GameClass, network, c, d, verbose=True) while True: best_node = root.choose_expansion_node() if best_node is not None: best_node.expand() if root.children is not None and worker_pipe.poll(): user_chosen_position = worker_pipe.recv() if user_chosen_position is not None: # an updated position has been received so we can truncate the tree for child in root.children: if np.all(child.position == user_chosen_position): root = child root.parent = None break else: print(user_chosen_position) raise Exception('Invalid user chosen move!') if GameClass.is_over(root.position): print('Game Over in Async MCTS: ', GameClass.get_winner(root.position)) return else: # this move chooser has been requested to decide on a move via the choose_move function start_time = time() while time() - start_time < time_limit: best_node = root.choose_expansion_node() # best_node will be None if the tree is fully expanded if best_node is None: break best_node.expand() is_ai_player_1 = GameClass.is_player_1_turn(root.position) chosen_positions = [] print( f'MCTS choosing move based on {root.count_expansions()} expansions!' ) # choose moves as long as it is still the ai's turn while GameClass.is_player_1_turn( root.position) == is_ai_player_1: if root.children is None: best_node = root.choose_expansion_node() if best_node is not None: best_node.expand() root, distribution = root.choose_best_node( return_probability_distribution=True, optimal=True) chosen_positions.append((root.position, distribution)) print('Expected outcome: ', root.get_evaluation()) root.parent = None # delete references to the parent and siblings worker_pipe.send(chosen_positions) if GameClass.is_over(root.position): print('Game Over in Async MCTS: ', GameClass.get_winner(root.position)) return def reset(self): raise NotImplementedError('')
class TestOrchestrator(BaseTestCase): def setUp(self): self.test_directory = os.path.dirname(os.path.realpath(__file__)) self.test_config = os.path.join(self.test_directory, "packager_config.cfg") self.packager_config = PackagerConfig(self.test_config) """Spin up a server in a seperate process""" def start_test_server(): app = Orchestrator(self.packager_config) from werkzeug.serving import run_simple run_simple('127.0.0.1', 5000, app) self.p = Process(target=start_test_server, daemon=True) self.p.start() def tearDown(self): """Make sure we tear down the process properly at the end""" self.p.terminate() def testOrchestratorAcceptingInputCorrectly(self): @exponential_back_off def wait_for_process(): return requests.get("http://127.0.0.1:5000/health") wait_seconds = 1 print("await process spin up for %d seconds" % wait_seconds) time.sleep(wait_seconds) wait_for_process() with open(self.get_file('code'), 'r') as code, \ open(self.get_file('tests'), 'r') as test: language = 'python' question_name = 'foo' code_payload = code.read() test_payload = test.read() payload = { 'question_name': question_name, 'language': language, 'code': code_payload, 'test': test_payload } # http:// required, # requests library needs protocol scheme to connect got_resp_raw = requests.post('http://127.0.0.1:5000/submit', data=payload) got_resp = json.loads(got_resp_raw.text) exp_data = { 'language': language, 'code': code_payload, 'test': test_payload, 'question_name': question_name } exp_response_message = 'not tested for in this testcase' exp_resp = { 'execution_result': exp_response_message, 'got_data': exp_data } self.assertEqual(exp_resp['got_data'], got_resp['got_data']) # we just need to check if the result contains success for testcase, test_result in got_resp['execution_result'].items(): self.assertTrue(test_result['success'], 'testcase {} failure with message {}' .format(testcase, test_result['message']))
class AsyncIterativeDeepening(MoveChooser): def __init__(self, GameClass, starting_position=None, time_limit=3): super(AsyncIterativeDeepening, self).__init__(GameClass, starting_position) self.root = DeepeningNode(GameClass, starting_position) self.time_limit = time_limit self.parent_pipe, worker_pipe = Pipe() self.worker_process = Process(target=self.loop_func, args=(GameClass, starting_position, worker_pipe)) self.receipt_backlog = 0 def start(self): self.worker_process.start() def terminate(self): self.worker_process.terminate() self.worker_process.join() @staticmethod def loop_func(GameClass, starting_position, worker_pipe): root = DeepeningNode(GameClass, starting_position) while True: root.deepen() print(root.get_depth()) worker_pipe.send(root.children[0].state) while worker_pipe.poll(): chosen_position = worker_pipe.recv() if root.children is None: root.deepen() for child in root.children: if np.all(chosen_position == child.state): root = child break else: raise ValueError('Invalid move!') worker_pipe.send(None) # send acknowledgement receipt def report_user_move(self, user_chosen_position): # notify process that position was chosen self.parent_pipe.send(user_chosen_position) self.receipt_backlog += 1 self.position = user_chosen_position def reset(self): raise NotImplementedError def choose_move(self, return_distribution=False): if return_distribution: raise NotImplementedError start_time = time() while self.receipt_backlog > 0: while self.parent_pipe.recv() is not None: pass self.receipt_backlog -= 1 remaining_time = self.time_limit - (time() - start_time) if remaining_time > 0: sleep(remaining_time) self.position = self.parent_pipe.recv() while self.parent_pipe.poll(): self.position = self.parent_pipe.recv() # notify process that position was chosen self.parent_pipe.send(self.position) self.receipt_backlog += 1 return [self.position]
import os, time, random # 写数据进程执行的代码: def write(q): print('Process to write: %s' % os.getpid()) for value in ['A', 'B', 'C']: print('Put %s to queue...' % value) q.put(value) time.sleep(random.random()) # 读数据进程执行的代码: def read(q): print('Process to read: %s' % os.getpid()) while True: value = q.get(True) print('Get %s from queue.' % value) if __name__=='__main__': # 父进程创建Queue,并传给各个子进程: q = Queue() pw = Process(target=write, args=(q,)) pr = Process(target=read, args=(q,)) # 启动子进程pw,写入: pw.start() # 启动子进程pr,读取: pr.start() # 等待pw结束: pw.join() # pr进程里是死循环,无法等待其结束,只能强行终止: pr.terminate()
def run_in_parallel( target: Callable[[Any], Any], target_name: str, data_set: Dict[str, Any], table_name: str, ): return_values = [] processes = [] parent_connections = [] chunked_kwargs = _get_chunked_list(dataset=data_set, tablename=table_name) logger.info( f"Starting up {len(chunked_kwargs)} process(es) for [{target_name}]") try: for chunk_index, target_kwargs in enumerate(chunked_kwargs): parent_connection, child_connection = Pipe() parent_connections.append(parent_connection) process = Process( target=__chunk_wrapper, kwargs=dict( target=target, target_kwargs=target_kwargs, target_name=target_name, connection=child_connection, chunk_index=chunk_index, chunk_count=len(chunked_kwargs), ), daemon=True, ) process.start() processes.append(process) for connection in parent_connections: response_data = connection.recv() if isinstance(response_data, Exception): raise response_data return_values.append(response_data) for process in processes: process.join() except Exception as error: logger.exception(f"Error when running {len(processes)} process(es)" f" for {target_name}") raise error finally: for process in processes: process.terminate() logger.info( f"Successfully executed {len(chunked_kwargs)} process(es) for [{target_name}]" ) return return_values
class ProcessExecution(OutputExecution): def __init__(self, target, args): self.target = target self.args = args self.output_queue = Queue(maxsize=2048) self._process: Union[Process, None] = None self._status = None self._stopped: bool = False self._interrupted: bool = False self._output_observers = [] @property def is_async(self) -> bool: return False def execute(self) -> ExecutionState: if not self._stopped and not self._interrupted: self._process = Process(target=self._run) self._process.start() output_reader = Thread(target=self._read_output, name='Output-Reader', daemon=True) output_reader.start() self._process.join() self.output_queue.put_nowait(_QueueStop()) output_reader.join(timeout=1) self.output_queue.close() if self._process.exitcode == 0: return ExecutionState.COMPLETED if self._stopped: return ExecutionState.STOPPED if self._interrupted: return ExecutionState.INTERRUPTED raise ExecutionError( "Process returned non-zero code " + str(self._process.exitcode), ExecutionState.FAILED) def _run(self): with self._capture_stdout(): try: self.target(*self.args) except: for line in traceback.format_exception(*sys.exc_info()): self.output_queue.put_nowait(line) raise @contextmanager def _capture_stdout(self): import sys original_stdout = sys.stdout original_stderr = sys.stderr stdout_writer = _CapturingWriter(original_stdout, self.output_queue) stderr_writer = _CapturingWriter(original_stderr, self.output_queue) sys.stdout = stdout_writer sys.stderr = stderr_writer try: yield finally: sys.stdout = original_stdout sys.stderr = original_stderr @property def status(self): return self._status def stop(self): self._stopped = True if self._process: self._process.terminate() def interrupt(self): self._interrupted = True if self._process: self._process.terminate() def add_output_observer(self, observer): self._output_observers.append(observer) def remove_output_observer(self, observer): self._output_observers.remove(observer) def _read_output(self): while True: output_text = self.output_queue.get() if isinstance(output_text, _QueueStop): break self._status = output_text self._notify_output_observers(output_text) def _notify_output_observers(self, output): for observer in self._output_observers: # noinspection PyBroadException try: if isinstance(observer, ExecutionOutputObserver): observer.output_update(output) elif callable(observer): observer(output) else: log.warning( "event=[unsupported_output_observer] observer=[%s]", observer) except BaseException: log.exception("event=[state_observer_exception]")