def start(): builder = cluster_crawler.ClusterCrawler() start_block_id = int(sys.argv[1]) block_id = start_block_id process = None while builder.crawl_block(block_id): print("Block %d crawled" % block_id) if block_id - start_block_id > 0 and (block_id - start_block_id) % settings.block_crawling_limit == 0: builder.network_graph.check_integrity() while process is not None and process.is_alive(): print("Waiting for insertion thread to complete...") process.join() if process is not None and process.exitcode > 0 : #error raise Exception("Errorcode %d in DB Sync Thread, aborting" % process.exitcode) process = Process(target=builder.network_graph.synchronize_mongo_db) process.start() builder.network_graph = cluster_network.ClusterNetwork(settings.db_server, settings.db_port) #Starting a new graph while other graph data is inserted. builder.connect_to_bitcoind_rpc() if process is not None and not process.is_alive() and process.exitcode > 0 : #error raise Exception("Errorcode %d in DB Sync Thread, aborting" % process.exitcode) block_id+=1 #Finished Crawling, Flushing to DB. #Waiting for any previous DB Sync while process is not None and process.is_alive(): print("Waiting for insertion thread to complete...") process.join() #Sync the rest process = Process(target=builder.network_graph.synchronize_mongo_db) process.start() process.join()
class MultiProcessRunner(BaseRunner): JOIN_TIMEOUT = 60 def __init__(self, target, **kwargs): super(MultiProcessRunner, self).__init__(target, **kwargs) self.process = None # type: Process @capture_monitor_exception def start(self): self.process = Process(target=self.target, kwargs=self.kwargs) self.process.start() @capture_monitor_exception def stop(self): if self.process and self.is_alive(): self.process.terminate() self.process.join(MultiProcessRunner.JOIN_TIMEOUT) if self.process.is_alive(): self.process.kill() @capture_monitor_exception def heartbeat(self): # do we want to do something here? pass @capture_monitor_exception def is_alive(self): return self.process.is_alive() def __str__(self): s = super(MultiProcessRunner, self).__str__() return f"{s}({self.process})"
def process_file(file_path_input, file_path_output, i): if os.path.exists(file_path_output): return print('{} {} Already done'.format(now(), file_path_output)) input_file = open(file_path_input, 'r') output_file = open(file_path_output, 'a') # every worker create separate file for a input file writer = csv.writer(output_file, delimiter='\t') # for every file we create seperate driver (100 URLs) driver = webdriver.PhantomJS(executable_path=path_to_phantomjs) for line in input_file: splited = line.split('\t') # property_type = splited[0] url = splited[1] print('{} Process={} Current url: {}'.format(now(), i, url)) # start process for getting microformat properties temp_queue = Queue() # p = Process(target=get_microformat_properties_by_type, args=(url, property_type, temp_queue, i)) p = Process(target=get_element_features, args=(url, driver, temp_queue, i)) print("{} {} Process={} {} {}".format(now(), i, "Started: ", "feature extraction", url)) p.start() event_features = temp_queue.get(timeout=TIME_OUT_FEATURE) # try: # pass # except Empty: # print("{} {} Process={} {} {}".format(now(), i, "Timed out on: ", "feature extraction", url)) if p.is_alive(): p.terminate() print("Event features:" + str(event_features)) if event_features is not None: print("{} Process={} Got properties for {}".format( now(), i, url)) # start process for feature extraction and writing to separate file # p_event_features = Process(target=get_event_features_and_write, # args=(event_features, driver, writer, i, output_file)) # p_event_features = Process(target=write_element_features, args=(event_features, writer, i, output_file)) p.start() # start_with_timeout(p_event_features, TIME_OUT_LOAD, "feature writing", url, i) if p_event_features.is_alive(): p.terminate() driver.service.process.send_signal(signal.SIGTERM) driver.quit() return 'done'
def main(): print('主进程开始') p = Process(target=test, args=(1, )) p.start() print(p.is_alive()) print('主进程结束') print(p.is_alive()) print(p.is_alive()) print(p.is_alive()) print(p.is_alive()) print(p.is_alive()) print('子进程名字:', p.name) print('子进程pid:', p.pid) p.terminate() p.join(20)
class AIOProcess: """ Execute a coroutine on a separate process """ def __init__(self, coroutine: Callable = None, *args, daemon: bool = False, target_override: Callable = None, **kwargs): if not asyncio.iscoroutinefunction(coroutine): raise ValueError("target must be a coroutine function") self.aio_process = Process(target=target_override or partial(AIOProcess.run_async, coroutine), args=args, kwargs=kwargs, daemon=daemon) @staticmethod def run_async(coroutine: Callable, *args, **kwargs): try: loop = uvloop.new_event_loop() asyncio.set_event_loop(loop) result = loop.run_until_complete(coroutine(*args, **kwargs)) return result except BaseException: log.exception(f"aio process {os.getpid()} failed") raise def start(self): self.aio_process.start() async def join(self, timeout=None): if not self.is_alive() and self.exit_code is None: raise ValueError("must start process before joining") if timeout is not None: return await asyncio.wait_for(self.join(), timeout) while self.exit_code is None: await asyncio.sleep(0.005) @property def pid(self): return self.aio_process.pid @property def daemon(self): return self.aio_process.daemon @property def exit_code(self): return self.aio_process.exitcode def is_alive(self): return self.aio_process.is_alive() def terminate(self): self.aio_process.terminate()
def start(): builder = cluster_crawler.ClusterCrawler() start_block_id = int(sys.argv[1]) block_id = start_block_id process = None while builder.crawl_block(block_id): if settings.debug or block_id % 100 == 0: print("Block %d crawled" % block_id) if block_id - start_block_id > 0 and ( block_id - start_block_id) % settings.block_crawling_limit == 0: builder.network_graph.check_integrity() while process is not None and process.is_alive(): print("Waiting for insertion thread to complete...") process.join() if process is not None and process.exitcode > 0: #error raise Exception("Errorcode %d in DB Sync Thread, aborting" % process.exitcode) process = Process( target=builder.network_graph.synchronize_mongo_db) process.start() builder.network_graph = cluster_network.ClusterNetwork( settings.db_server, settings.db_port ) #Starting a new graph while other graph data is inserted. builder.connect_to_bitcoind_rpc() if process is not None and not process.is_alive( ) and process.exitcode > 0: #error raise Exception("Errorcode %d in DB Sync Thread, aborting" % process.exitcode) block_id += 1 #Finished Crawling, Flushing to DB. #Waiting for any previous DB Sync while process is not None and process.is_alive(): print("Waiting for insertion thread to complete...") process.join() #Sync the rest print("Inserting into the DB") process = Process(target=builder.network_graph.synchronize_mongo_db) process.start() process.join()
def process_file(file_path_input, file_path_output, i): if os.path.exists(file_path_output): return print('{} {} Already done'.format(now(), file_path_output)) input_file = open(file_path_input, 'r') output_file = open(file_path_output, 'a') # every worker create separate file for a input file writer = csv.writer(output_file, delimiter='\t') # for every file we create seperate driver (100 URLs) driver = webdriver.PhantomJS(executable_path=path_to_phantomjs) for line in input_file: splited = line.split('\t') property_type = splited[0] url = splited[1] print('{} Process={} Current url: {}'.format(now(), i, url)) # start process for getting microformat properties temp_queue = Queue() p = Process(target=get_microformat_properties_by_type, args=(url, property_type, temp_queue, i)) start_with_timeout(p, TIME_OUT_LOAD, "loading", url, i) event_properties = temp_queue.get() if not temp_queue.empty() else None if p.is_alive(): p.terminate() if event_properties is not None: print("{} Process={} Got properties for {}".format( now(), i, url)) # start process for feature extraction and writing to separate file p_event_features = Process(target=get_event_features_and_write, args=(event_properties, driver, writer, i, output_file)) start_with_timeout(p_event_features, TIME_OUT_FEATURE, "feature extraction", url, i) if p_event_features.is_alive(): p.terminate() return 'done'
def start(): builder = GraphBuilder() start_block_id = int(sys.argv[1]) block_id = start_block_id process = None try: while builder.crawl_block(block_id): print("Block %d crawled" % block_id) if block_id - start_block_id > 0 and (block_id - start_block_id) % Settings.block_crawling_limit == 0: builder.network_graph.check_integrity() while process is not None and process.is_alive(): print("Waiting for insertion thread to complete...") process.join() if process is not None and process.exitcode > 0 : #error raise Exception("Errorcode %d in DB Sync Thread, aborting" % process.exitcode) process = Process(target=builder.network_graph.synchronize_mongo_db) process.start() builder.network_graph = NetworkGraph.Network(Settings.db_server,Settings.db_port) #Starting a new graph while other graph data is inserted. if process is not None and not process.is_alive() and process.exitcode > 0 : #error raise Exception("Errorcode %d in DB Sync Thread, aborting" % process.exitcode) block_id+=1 #Finished Crawling, Flushing to DB. #Waiting for any previous DB Sync while process is not None and process.is_alive(): print("Waiting for insertion thread to complete...") process.join() #Sync the rest process = Process(target=builder.network_graph.synchronize_mongo_db) process.start() process.join() #DONE! #For Debugging purpose except: input("An exception will rise ") raise
def start_up_app(self): self.show_splash_screen() # load db in separate process process_startup = Process(target=LucteriosRefreshAll) process_startup.start() while process_startup.is_alive(): # print('updating') self.splash.update() self.remove_splash_screen()
def start_up_app(self): self.show_splash_screen() try: # load db in separate process process_startup = Process(target=LucteriosRefreshAll) process_startup.start() while process_startup.is_alive(): # print('updating') self.splash.update() finally: self.remove_splash_screen()
def test_keyboard_interrupt_on_linux(self) -> None: """ - Keyboard interrupt should reach to all descendant processes. - Keyboard interrupt should shutdown ProcessTaskPoolExecutor gracefully. """ process = Process(target=self.report_raises_keyboard_interrupt) process.start() LocalSocket.receive() time.sleep(SECOND_SLEEP_FOR_TEST_SHORT) self.simulate_ctrl_c_in_posix(process) assert LocalSocket.receive() == "Test succeed" process.join() assert process.exitcode == 0 assert not process.is_alive()
def main(): logging.info(f'App started') max = 2 worker = Process(target=work, args=['Working', max], daemon=True, name='Worker') worker.start() time.sleep(5) # if the process is running, stop it if worker.is_alive(): worker.terminate() # kill the process with SIGTERM worker.join() # exicode == 0 is good # anything else is an error logging.info(f'App finished: {worker.exitcode}')
def run_server(launcher, wait_before_entering=0, verbose=False, **kwargs): """Context manager to launch server on entry, and shut it down on exit""" from warnings import warn clog = conditional_logger(verbose) server = None try: server = Process(target=launcher, kwargs=kwargs) clog(f'Starting server...') server.start() clog(f'... server started.') sleep(wait_before_entering) yield server finally: if server is not None and server.is_alive(): clog(f'Terminating server...') server.terminate() clog(f'... server terminated')
def main(): logging.info('Started') max = 2 worker = Process(target=work, args=['Working', max], daemon=True, name='Super Mario') worker.start() time.sleep(5) #if the process is running, stop it if worker.is_alive(): worker.terminate() worker.join() #exitcode == 0 #Anything else is an error logging.info(f'Finished: {worker.exitcode}')
def run_load_test_clients(clients_count: int): shared_rows_counts = multiprocessing.Manager().dict() last_failed_rows_count = [0] def log_rows_counts(): current_rows_counts = dict(shared_rows_counts) inserted_rows_count = sum( map( lambda x: x[1], filter(lambda x: x[0].startswith("inserted"), current_rows_counts.items()))) failed_rows_count = sum( map( lambda x: x[1], filter(lambda x: x[0].startswith("failed"), current_rows_counts.items()))) log_message = f"Inserted {inserted_rows_count} rows, failed to insert {failed_rows_count} rows" if failed_rows_count % 1000 > last_failed_rows_count[0]: logging.warning(log_message) last_failed_rows_count[0] = failed_rows_count else: logging.info(log_message) clients = [] for client_number in range(clients_count): client = Process(target=main, args=(True, client_number, shared_rows_counts)) client.start() clients.append(client) alive_clients = clients while len(alive_clients) > 0: sleep(3) log_rows_counts() alive_clients = list(filter(lambda client: client.is_alive(), clients))
class TestRemoteSession(TestCase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.server_process = None """:type: Process""" def setUp(self): self.test_file = TestFile(settings.test_file_size) self.original_data = self.test_file.get_content() self.ftp_root = self.test_file.path self.remote_filename = uuid.uuid4().hex tuned_server, self.port = server.get_tuned_server(self.ftp_root) def server_func(): tuned_server.serve_forever(handle_exit=True) self.server_process = Process(target=server_func) """:type: Process""" self.server_process.start() def get_connected_client_in_binary_mode(self): client = Client() client.connect('localhost', self.port) client.login(settings.ftp_user, settings.ftp_pass) client.type('I') # Binary mode ('I'mage) return client def tearDown(self): while self.server_process.is_alive(): # посылаем SIGINT процессу, pyftpd умеет его обрабатывать и закрывается корректно os.kill(self.server_process.pid, signal.SIGINT) self.server_process.join(timeout=1) unlink(self.test_file.full_filename) def test_receive_file(self): """ тест на получение заранее сгенерированных и сохраненных на диске данных клиентом через подключение к локальному серверу. """ client = self.get_connected_client_in_binary_mode() code, rest, data = client.retr(self.test_file.filename) q_code, q_rest = client.quit() self.assertEqual(code, 226) self.assertTrue(self.original_data == data) def test_send_file(self): """ Тест на отправку файла клиентом на локальный сервер и сравнение содержимого файла на диске с отправленными данными """ client = self.get_connected_client_in_binary_mode() client.stor(self.remote_filename, self.original_data) with open(os.path.join(self.ftp_root, self.remote_filename), 'rb') as f: stored_file_content = f.read() code, rest = client.quit() self.assertTrue(self.original_data == stored_file_content)
class CreateProcess: """A context manager to launch a parallel process and close it on exit. """ def __init__( self, proc_func: Callable, process_name=None, wait_before_entering=2, verbose=False, args=(), **kwargs, ): """ Essentially, this context manager will call ``` proc_func(*args, **kwargs) ``` in an independent process. :param proc_func: A function that will be launched in the process :param process_name: The name of the process. :param wait_before_entering: A pause (in seconds) before returning from the enter phase. (in case the outside should wait before assuming everything is ready) :param verbose: If True, will print some info on the starting/stoping of the process :param args: args that will be given as arguments to the proc_func call :param kwargs: The kwargs that will be given as arguments to the proc_func call The following should print 'Hello console!' in the console. >>> with CreateProcess(print, verbose=True, args=('Hello console!',)) as p: ... print("-------> Hello module!") # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE Starting process: print... ... print process started. -------> Hello module! ... print process terminated """ self.proc_func = proc_func self.process_name = process_name or getattr(proc_func, '__name__', '') self.wait_before_entering = float(wait_before_entering) self.verbose = verbose self.args = args self.kwargs = kwargs self.clog = conditional_logger(verbose) self.process = None self.exception_info = None def process_is_running(self): return self.process is not None and self.process.is_alive() def __enter__(self): self.process = Process( target=self.proc_func, args=self.args, kwargs=self.kwargs, name=self.process_name, ) self.clog(f'Starting process: {self.process_name}...') try: self.process.start() if self.process_is_running(): self.clog(f'... {self.process_name} process started.') sleep(self.wait_before_entering) return self else: raise RuntimeError('Process is not running') except Exception: raise RuntimeError( f'Something went wrong when trying to launch process {self.process_name}' ) def __exit__(self, exc_type, exc_val, exc_tb): if self.process is not None and self.process.is_alive(): self.clog(f'Terminating process: {self.process_name}...') self.process.terminate() self.clog(f'... {self.process_name} process terminated') if exc_type is not None: self.exception_info = dict( exc_type=exc_type, exc_val=exc_val, exc_tb=exc_tb )
def child_2(interval): print('子进程(%s)开始执行, 父进程为(%s)' % (os.getpid(), os.getppid())) t_start = time.time() time.sleep(interval) t_end = time.time() print('子进程(%s)执行时间为"%0.2f"秒' % (os.getpid(), t_end - t_start)) if __name__ == '__main__': print('------父进程开始执行-------') print('父进程PID: %s' % os.getpid()) p1 = Process(target=child_1, args=(1, )) p2 = Process(target=child_2, name='mrsoft', args=(2, )) p1.start() p2.start() print('p1.is_alive=%s' % p1.is_alive()) print('p2.is_alive=%s' % p2.is_alive()) print('p1.name=%s' % p1.name) print('p1.pid=%s' % p1.pid) print('p2.name=%s' % p2.name) print('p2.pid=%s' % p2.pid) print('------等待子进程-------') p1.join() p2.join() print('------父进程执行结束--------')
userData['playlist'], vnc, config.SQS_ENDPOINT, screenshotDir, runnerStats, processStates, )) p.start() processes.append(p) except: runnerStats[STAT_ERROR] += 1 console.exception() leftProcesses = [] for p in processes: if p.is_alive(): leftProcesses.append(p) else: p.join() processes = leftProcesses if showInfo: showStats(len(processes), systemStats, runnerStats) if len(processes) == 0: break except KeyboardInterrupt: shutdown(processes) break except: console.exception() shutdown(processes) break