class MultiProcessRunner(BaseRunner): JOIN_TIMEOUT = 60 def __init__(self, target, **kwargs): super(MultiProcessRunner, self).__init__(target, **kwargs) self.process = None # type: Process @capture_monitor_exception def start(self): self.process = Process(target=self.target, kwargs=self.kwargs) self.process.start() @capture_monitor_exception def stop(self): if self.process and self.is_alive(): self.process.terminate() self.process.join(MultiProcessRunner.JOIN_TIMEOUT) if self.process.is_alive(): self.process.kill() @capture_monitor_exception def heartbeat(self): # do we want to do something here? pass @capture_monitor_exception def is_alive(self): return self.process.is_alive() def __str__(self): s = super(MultiProcessRunner, self).__str__() return f"{s}({self.process})"
def start(): builder = cluster_crawler.ClusterCrawler() start_block_id = int(sys.argv[1]) block_id = start_block_id process = None while builder.crawl_block(block_id): print("Block %d crawled" % block_id) if block_id - start_block_id > 0 and (block_id - start_block_id) % settings.block_crawling_limit == 0: builder.network_graph.check_integrity() while process is not None and process.is_alive(): print("Waiting for insertion thread to complete...") process.join() if process is not None and process.exitcode > 0 : #error raise Exception("Errorcode %d in DB Sync Thread, aborting" % process.exitcode) process = Process(target=builder.network_graph.synchronize_mongo_db) process.start() builder.network_graph = cluster_network.ClusterNetwork(settings.db_server, settings.db_port) #Starting a new graph while other graph data is inserted. builder.connect_to_bitcoind_rpc() if process is not None and not process.is_alive() and process.exitcode > 0 : #error raise Exception("Errorcode %d in DB Sync Thread, aborting" % process.exitcode) block_id+=1 #Finished Crawling, Flushing to DB. #Waiting for any previous DB Sync while process is not None and process.is_alive(): print("Waiting for insertion thread to complete...") process.join() #Sync the rest process = Process(target=builder.network_graph.synchronize_mongo_db) process.start() process.join()
def test_disk_locking_processes(block_maker, storage_factory, redis_hostname): def visit(storage, root): ds = block_maker.first_ds(first_constant=2, ids_arg=3) cached = ds >> Apply(image=sleeper(0.1)) >> CacheToDisk( root, storage, JsonSerializer(), 'image') for i in ds.ids: assert ds.image(i) == cached.image(i) assert ds.image(i) == cached.image(i) for _ in range(5): with tempfile.TemporaryDirectory() as temp, storage_factory( ) as temp_storage: temp = Path(temp) / 'cache' init_storage(temp, algorithm={ 'name': 'blake2b', 'digest_size': 64 }, levels=[1, 31, 32], locker={ 'name': 'RedisLocker', 'args': [redis_hostname], 'kwargs': { 'prefix': 'connectome.tests', 'expire': 10 } }) th = Process(target=visit, args=(temp_storage, temp)) th.start() visit(temp_storage, temp) th.join()
def test_agent() -> None: stop_ev = Event() conn, agent_conn = Pipe() waypoints = [(-80.0, -65.0, 3.0), (-45.0, -66.0, 3.0), (-33.0, -65.0, 3.0), (-33.0, -65.0, 0.3)] aut = Agent(uid=1, motion=MotionHectorQuad("/drone1"), waypoints=waypoints) p = Process(target=run_as_process, kwargs={ "aut": aut, "conn": agent_conn, "stop_ev": stop_ev }) p.start() try: for i in range(10): if conn.poll(1.0): act = conn.recv() print(act) if act[0] == "request": reply = input("> ") conn.send(("reply", { "uid": aut.uid, "acquired": Contract() })) else: print("Response timeout") finally: stop_ev.set() # Stop all automatons p.join() conn.close() agent_conn.close()
def test_contract_manager() -> None: aut = AirspaceManager() stop_ev = Event() conn, manager_conn = Pipe() p = Process(target=run_as_process, kwargs={"aut": aut, "conn": manager_conn, "stop_ev": stop_ev}) p.start() try: uid = 0 for i in range(10): if conn.poll(1.0): act = conn.recv() print(act) elif i % 3 != 2: uid = i % 5 target = Contract.from_stamped_rectangles([ (0.0, Rectangle(mins=[0, 0, 0], maxes=[1, 1, 0.5])), (0.5, Rectangle(mins=[0, 0.5, 0], maxes=[2, 3, 0.5])), (1.0, Rectangle(mins=[0.5, 0.5, 1.0], maxes=[1.5, 1.5, 1.5])) ]) conn.send(("request", {"uid": uid, "target": target})) else: releasable = Contract.from_stamped_rectangles([ (0.0, Rectangle(mins=[0, 0, 0], maxes=[1, 1, 0.5])), (0.5, Rectangle(mins=[0, 0.5, 0], maxes=[2, 2, 0.5])) ]) print("Agent " + str(uid) + " release > " + str(releasable)) conn.send(("release", {"uid": uid, "releasable": releasable})) finally: stop_ev.set() # Stop all automatons p.join() conn.close() manager_conn.close()
def proof_of_work(self, last_block): """ Simple Proof of Work Algorithm: - Find a number p' such that hash(pp') contains leading 4 zeroes - Where p is the previous proof, and p' is the new proof :param last_block: <dict> last Block :return: <int> """ last_proof = last_block['proof'] last_hash = self.hash(last_block) # Flags flags = Value(c_bool, False) proof_result = Value(c_int, 0) # Proof proof1 = 0 proof2 = 75001 proof3 = 150001 proof4 = 225001 proof5 = 300001 start_time = time.time() p1 = Process(target=self.find_proof, args=(flags, 0, last_proof, proof1, last_hash, proof_result, 0, 75001)) p2 = Process(target=self.find_proof, args=(flags, 1, last_proof, proof2, last_hash, proof_result, 75001, 150001)) p3 = Process(target=self.find_proof, args=(flags, 2, last_proof, proof3, last_hash, proof_result, 150001, 225001)) p4 = Process(target=self.find_proof, args=(flags, 3, last_proof, proof4, last_hash, proof_result, 225001, 300001)) p5 = Process(target=self.find_proof, args=(flags, 4, last_proof, proof5, last_hash, proof_result, 300001, sys.maxsize)) p1.start() p2.start() p3.start() p4.start() p5.start() p1.join() p2.join() p3.join() p4.join() p5.join() end_time = time.time() print('TIMETIMETIMETIME : ', end_time - start_time) print('proof_result : ', proof_result.value) return proof_result.value
def compute_hub_SALSA(inlinks, inlinks_count, outlinks, outlinks_count, rootSet): print("Size of rootset is " + str(len(rootSet))) # computing hub scores salsa_hub_scores = dict() countBaseSetWithOutlinks = 0 for page_id in rootSet: if page_id in outlinks: if len(outlinks[page_id]) > 0: countBaseSetWithOutlinks += 1 print(countBaseSetWithOutlinks) for page_id in rootSet: if page_id in outlinks: if len(outlinks[page_id]) > 0: salsa_hub_scores[page_id] = 1 / countBaseSetWithOutlinks else: salsa_hub_scores[page_id] = 0 else: salsa_hub_scores[page_id] = 0 for iter in range(0, 3): print(iter) global temp_salsa_hub_scores count = 0 page_list = list() procs = list() for page_id in rootSet: page_list.append(page_id) count += 1 if count % 100 == 0: print(count) proc = Process(target=multi_scoring, args=( page_list, inlinks, inlinks_count, outlinks, outlinks_count, temp_salsa_hub_scores, salsa_hub_scores, )) page_list = list() procs.append(proc) proc.start() print("Total processes " + str(len(procs))) for proc in procs: proc.join() for page_id in temp_salsa_hub_scores: salsa_hub_scores[page_id] = temp_salsa_hub_scores[page_id] temp_salsa_hub_scores = dict() return salsa_hub_scores
def dashboard(): # This fixture guarantees the proper termination of all spawned subprocesses # after the tests. dashboard = Process(target=run_detection) dashboard.start() yield for child in psutil.Process(dashboard.pid).children(recursive=True): child.kill() dashboard.terminate() dashboard.join()
def main(): size = os.path.getsize("dict.txt") half_size = size // 2 p1 = Process(target=read_part1, args=("dict.txt", half_size)) p2 = Process(target=read_part2, args=("dict.txt", half_size)) p1.start() p2.start() p1.join() p2.join()
def run(): # while True: for name in ['jd','sn']: #对应存储再config中的JSON配置文件 custom_settings = get_config(name) # 爬取使用的 Spider 名称 spider = custom_settings.get('spider', 'aCrawler') project_settings = get_project_settings() settings = dict(project_settings.copy()) # 合并配置 settings.update(custom_settings.get('settings')) process = Process(target=crawl, kwargs={'settings': settings, 'spider': spider, 'name': name, 'custom_settings': custom_settings}) process.start() process.join()
def run(): while True: for name in ['sina', 'xinhua']: #对应存储再config中的JSON配置文件 custom_settings = get_config(name) # 爬取使用的 Spider 名称 spider = custom_settings.get('spider', 'sina') project_settings = get_project_settings() settings = dict(project_settings.copy()) # 合并配置 settings.update(custom_settings.get('settings')) process = Process(target=crawl, kwargs={'settings': settings, 'spider': spider, 'name': name, 'custom_settings': custom_settings}) process.start() process.join() time.sleep(60 * 60 * 24) #一天运行一次
def test_keyboard_interrupt_on_linux(self) -> None: """ - Keyboard interrupt should reach to all descendant processes. - Keyboard interrupt should shutdown ProcessTaskPoolExecutor gracefully. """ process = Process(target=self.report_raises_keyboard_interrupt) process.start() LocalSocket.receive() time.sleep(SECOND_SLEEP_FOR_TEST_SHORT) self.simulate_ctrl_c_in_posix(process) assert LocalSocket.receive() == "Test succeed" process.join() assert process.exitcode == 0 assert not process.is_alive()
def main(): print('主进程开始') p = Process(target=test, args=(1, )) p.start() print(p.is_alive()) print('主进程结束') print(p.is_alive()) print(p.is_alive()) print(p.is_alive()) print(p.is_alive()) print(p.is_alive()) print('子进程名字:', p.name) print('子进程pid:', p.pid) p.terminate() p.join(20)
def main(): logging.info("Main thread started") maxValue = 3 worker = Process(target=work, args=("working", maxValue), daemon=True, name="Worker") worker.start() time.sleep(5) # if the process is running , stop it if worker.is_alive: worker.terminate() # pretty dangerous worker.join() logging.info(f"Main thread finished {worker.exitcode}")
def test_parallel_read_processes(storage_factory, redis_hostname): def job(): storage.load(lambda x: time.sleep(1), key) with storage_factory({'name': 'RedisLocker', 'args': [redis_hostname], 'kwargs': {'prefix': 'connectome.tests', 'expire': 10}}) as storage: key = storage.store(__file__) start = time.time() th = Process(target=job) th.start() job() th.join() stop = time.time() assert stop - start < 1.5
def setup_and_teardown_flask_app(app: Flask, host: str, port: int): """ Manages setup of provided flask app on given `host` and `port` and its teardown. As for setup process following things are done: * `/health` endpoint is added to provided flask app, * app is launched in separate process, * function waits for flask app to fully launch - to do this it repetitively checks `/health` endpoint if it will return status code 200. Example use of this function in fixture: >>> with setup_and_teardown_flask_app(Flask(__name__), "localhost", 10000): >>> yield :param app: app to launch :param host: host on which to launch app :param port: port on which to launch app """ def wait_for_flask_app_to_be_accessible(): timeout = 1 end_time = datetime.now() + timedelta(seconds=timeout) response = requests.Response() response.status_code = HTTP_404_NOT_FOUND while response.status_code != HTTP_200_OK and datetime.now( ) < end_time: with contextlib.suppress(requests.exceptions.ConnectionError): response = requests.request( "POST", "http://{}:{}/health".format(host, port)) time.sleep(0.01) fail_message = "Timeout expired: failed to start mock REST API in {} seconds".format( timeout) assert response.status_code == HTTP_200_OK, fail_message app.route("/health", methods=["POST"])(lambda: "OK") process = Process(target=app.run, args=(host, port)) process.start() wait_for_flask_app_to_be_accessible() yield process.terminate() process.join()
def do_scraping_from_sitemaps(country: str): """ Launch scrapping for each of the hotels """ sitemaps_paths = get_all_sitemaps_paths() l = len(sitemaps_paths) for i, sitemaps_path in enumerate(sitemaps_paths): sitemap_urls = get_sitemap_urls(sitemaps_path=sitemaps_path, country=country) if sitemap_urls: numpy = np.array(sitemap_urls) urls_splitted = np.array_split(numpy, 2) p = Process(target=init_scrap, args=(tuple([urls_splitted[0]]))) p.start() p2 = Process(target=init_scrap, args=(tuple([urls_splitted[1]]))) p2.start() p.join() p2.join()
def main(): logging.info(f'App started') max = 2 worker = Process(target=work, args=['Working', max], daemon=True, name='Worker') worker.start() time.sleep(5) # if the process is running, stop it if worker.is_alive(): worker.terminate() # kill the process with SIGTERM worker.join() # exicode == 0 is good # anything else is an error logging.info(f'App finished: {worker.exitcode}')
def start(): builder = cluster_crawler.ClusterCrawler() start_block_id = int(sys.argv[1]) block_id = start_block_id process = None while builder.crawl_block(block_id): if settings.debug or block_id % 100 == 0: print("Block %d crawled" % block_id) if block_id - start_block_id > 0 and ( block_id - start_block_id) % settings.block_crawling_limit == 0: builder.network_graph.check_integrity() while process is not None and process.is_alive(): print("Waiting for insertion thread to complete...") process.join() if process is not None and process.exitcode > 0: #error raise Exception("Errorcode %d in DB Sync Thread, aborting" % process.exitcode) process = Process( target=builder.network_graph.synchronize_mongo_db) process.start() builder.network_graph = cluster_network.ClusterNetwork( settings.db_server, settings.db_port ) #Starting a new graph while other graph data is inserted. builder.connect_to_bitcoind_rpc() if process is not None and not process.is_alive( ) and process.exitcode > 0: #error raise Exception("Errorcode %d in DB Sync Thread, aborting" % process.exitcode) block_id += 1 #Finished Crawling, Flushing to DB. #Waiting for any previous DB Sync while process is not None and process.is_alive(): print("Waiting for insertion thread to complete...") process.join() #Sync the rest print("Inserting into the DB") process = Process(target=builder.network_graph.synchronize_mongo_db) process.start() process.join()
def get_data_from_sn(self, search_list, database_name): if database_name == 'neo': sn_node = graph.find_one("SN", "name", "SOCIALNETWORKS") twitter_node = graph.merge_one("Twitter", "name", "TWITTER") sn_has_twitter = Relationship(sn_node, "HAS", twitter_node) graph.create_unique(sn_has_twitter) twitter_searcher = TwitterSearcher() twitter_searcher_process = Process(target=twitter_searcher.run_twitter_search, args=(search_list, database_name,)) twitter_streamer = TwitterStreamer() twitter_streamer_process = Process(target=twitter_streamer.run_twitter_stream, args=(search_list, database_name,)) twitter_searcher_process.start() twitter_streamer_process.start() twitter_streamer_process.join() twitter_searcher_process.join()
def parser(final_link, file_name): def crawl(): crawler = CrawlerProcess(settings={ "FEEDS": { file_name: { "format": "jl", "encoding": "utf-8" }, }, }, ) crawler.crawl(WildberriesSpider, start_urls=[final_link]) crawler.start() process = Process(target=crawl) process.start() process.join() return file_name
def generate_settings(base_requirement: Requirements, heterogeneity_score: Callable[ [Requirements, Requirements], float], steps: int = 5, arch_steps: int = 5, percentage: float = 1, folder: str = './data', cores: int = None) -> None: """Saves the settings in the given folder""" probs_for_archs = generate_arch_probs(arch_steps) old_probs_per_arch = generate_probabilities(steps) if cores is None: cores = multiprocessing.cpu_count() now = datetime.datetime.now() now = now.strftime('%Y_%m_%d_%H_%M_%S') folder = f'{folder}/{now}_archsteps-{arch_steps}_steps-{steps}_percentage-{percentage}' Path(folder).mkdir(parents=True, exist_ok=True) old_probs_per_arch = filter_invalid_settings(old_probs_per_arch) probs_per_arch = defaultdict(list) for arch, old_probs in old_probs_per_arch.items(): for values in old_probs.values(): probs_per_arch[arch].append( choose_attribute_settings(values, percentage)) split = np.array_split(probs_for_archs, cores) ps = [] for i in range(cores): if len(split[i]) > 0: # process_arches(split[i], probs_per_arch, base_requirement, heterogeneity_score, folder,) p = Process(target=process_arches, args=( split[i], probs_per_arch, base_requirement, heterogeneity_score, folder, )) p.start() ps.append(p) for p in ps: p.join()
def start(): builder = GraphBuilder() start_block_id = int(sys.argv[1]) block_id = start_block_id process = None try: while builder.crawl_block(block_id): print("Block %d crawled" % block_id) if block_id - start_block_id > 0 and (block_id - start_block_id) % Settings.block_crawling_limit == 0: builder.network_graph.check_integrity() while process is not None and process.is_alive(): print("Waiting for insertion thread to complete...") process.join() if process is not None and process.exitcode > 0 : #error raise Exception("Errorcode %d in DB Sync Thread, aborting" % process.exitcode) process = Process(target=builder.network_graph.synchronize_mongo_db) process.start() builder.network_graph = NetworkGraph.Network(Settings.db_server,Settings.db_port) #Starting a new graph while other graph data is inserted. if process is not None and not process.is_alive() and process.exitcode > 0 : #error raise Exception("Errorcode %d in DB Sync Thread, aborting" % process.exitcode) block_id+=1 #Finished Crawling, Flushing to DB. #Waiting for any previous DB Sync while process is not None and process.is_alive(): print("Waiting for insertion thread to complete...") process.join() #Sync the rest process = Process(target=builder.network_graph.synchronize_mongo_db) process.start() process.join() #DONE! #For Debugging purpose except: input("An exception will rise ") raise
def main(): logging.info('Started') max = 2 worker = Process(target=work, args=['Working', max], daemon=True, name='Super Mario') worker.start() time.sleep(5) #if the process is running, stop it if worker.is_alive(): worker.terminate() worker.join() #exitcode == 0 #Anything else is an error logging.info(f'Finished: {worker.exitcode}')
def get_data_from_sn(self, search_list, database_name): if database_name == 'neo': sn_node = graph.find_one("SN", "name", "SOCIALNETWORKS") facebook_node = graph.merge_one("Facebook", "name", "FACEBOOK") sn_has_facebook = Relationship(sn_node, "HAS", facebook_node) graph.create_unique(sn_has_facebook) event_searcher = EventSearcher() event_process = Process(target=event_searcher.run_facebook_events, args=(search_list, database_name,)) group_searcher = GroupSearcher() group_process = Process(target=group_searcher.run_facebook_groups, args=(search_list, database_name,)) page_searcher = PageSearcher() page_process = Process(target=page_searcher.run_facebook_pages, args=(search_list, database_name,)) event_process.start() group_process.start() page_process.start() event_process.join() group_process.join() page_process.join()
class Temp_API_Server: def __init__(self): self.port = random_port() self.proc = None def __enter__(self): self.start_server() return self def __exit__(self, exc_type, exc_val, exc_tb): self.stop_server() def http_GET(self, path=''): try: full_url = urljoin(self.server_url(), path) return GET_json(full_url) except: return None def server_running(self): return self.proc.is_alive() def start_server(self): self.proc = Process(target=run_server, args=[self.port]) self.proc.start() return self.wait_for_server_ok() def stop_server(self): self.proc.kill() self.proc.join() def server_url(self): return f"http://127.0.0.1:{self.port}" def wait_for_server_ok(self, max_attempts=20, wait_interval=0.1): for i in range(0, max_attempts): status = self.http_GET('/health') if status: return status wait_for(wait_interval)
def run_all(self, nof_processes: int): procs = [] if len(self.scenarios) == 0: return if len(self.scenarios) < nof_processes: nof_processes = len(self.scenarios) width = math.ceil(len(self.scenarios) / float(nof_processes)) for i in range(0, nof_processes): log_file = os.path.join(self.output_dir, "experiment_log_{}{}.log".format(self.prefix, i)) data_file = os.path.join(self.output_dir, "experiment_data_{}{}.log".format(self.prefix, i)) l = i * width u = min((i + 1) * width, len(self.scenarios)) if nof_processes == 1: ExperimentRunner._run_scenarios(self.scenarios, data_file, log_file) else: part = self.scenarios[l:u] p = Process(target=ExperimentRunner._run_scenarios, args=(part, data_file, log_file)) procs.append(p) p.start() for p in procs: p.join()
def main(): name = process.current_process().name logging.info(f'{name} started') #Setup the process address = 'localhost' #127.0.0.1 port = 2823 # above 1024 password = b'password' p = Process(target=proc, args=[address, port, password], daemon=True, name="Worker") p.start() logging.info(f'{name} waiting on the worker...') time.sleep(1) #Connect to the process dest = (address, port) conn = Client(dest, authkey=password) #Command loop while True: command = input('\r\nEnter a command or type quit:\r\n').strip() logging.info(f'{name} command: {command}') conn.send(command) if command == 'quit': break #Cleanup and shutdown if p.is_alive: logging.info(f'{name} terminating worker') conn.close() time.sleep(1) p.terminate() p.join() logging.info(f'{name} finished')
def automatizar_descargas(): book = open_workbook("seed.xlsx") sheet = book.sheet_by_index(0) # If your data is on sheet 1 columna_nombres = [] for row in range(1, sheet.nrows): # start from 1, to leave out row 0 columna_nombres.append(" ".join( str(sheet.row_values(row) [0]).upper().strip().split())) # extract from zero col lista_personas_no_encontradas = manager.list( ) # <-- can be shared between processes. lista_personas_encontradas = manager.list( ) # <-- can be shared between processes. columnas_de_nombres = chunkIt(columna_nombres, 1) processes = [] for nombres in columnas_de_nombres: p = Process(target=descargar_archivos_persona, args=(nombres, lista_personas_no_encontradas, lista_personas_encontradas)) # Passing the list p.start() processes.append(p) for p in processes: p.join() with xlsxwriter.Workbook('NO_ENCONTRADAS.xlsx') as workbook: worksheet = workbook.add_worksheet() worksheet.write_column('A1', lista_personas_no_encontradas) with xlsxwriter.Workbook('ENCONTRADAS.xlsx') as workbook: worksheet = workbook.add_worksheet() worksheet.write_column('A1', lista_personas_encontradas) # done print("+++++++++++++++++ Descargas terminadas!!!!!")
def _run_repo_main(_args): p = Process(target=repo.main, args=(_args,)) p.start() p.join()
def build_in_subprocess(exec_file_path: Path, output_dir: Path): process = Process(target=build, args=(exec_file_path, output_dir), kwargs=dict(reload=True)) process.start() process.join()
class SubprocEnvWorker(EnvWorker): """Subprocess worker used in SubprocVectorEnv and ShmemVectorEnv.""" def __init__( self, env_fn: Callable[[], gym.Env], share_memory: bool = False ) -> None: super().__init__(env_fn) self.parent_remote, self.child_remote = Pipe() self.share_memory = share_memory self.buffer: Optional[Union[dict, tuple, ShArray]] = None if self.share_memory: dummy = env_fn() obs_space = dummy.observation_space dummy.close() del dummy self.buffer = _setup_buf(obs_space) args = ( self.parent_remote, self.child_remote, CloudpickleWrapper(env_fn), self.buffer, ) self.process = Process(target=_worker, args=args, daemon=True) self.process.start() self.child_remote.close() def __getattr__(self, key: str) -> Any: self.parent_remote.send(["getattr", key]) return self.parent_remote.recv() def _decode_obs(self) -> Union[dict, tuple, np.ndarray]: def decode_obs( buffer: Optional[Union[dict, tuple, ShArray]] ) -> Union[dict, tuple, np.ndarray]: if isinstance(buffer, ShArray): return buffer.get() elif isinstance(buffer, tuple): return tuple([decode_obs(b) for b in buffer]) elif isinstance(buffer, dict): return {k: decode_obs(v) for k, v in buffer.items()} else: raise NotImplementedError return decode_obs(self.buffer) def reset(self) -> Any: self.parent_remote.send(["reset", None]) obs = self.parent_remote.recv() if self.share_memory: obs = self._decode_obs() return obs @staticmethod def wait( # type: ignore workers: List["SubprocEnvWorker"], wait_num: int, timeout: Optional[float] = None, ) -> List["SubprocEnvWorker"]: remain_conns = conns = [x.parent_remote for x in workers] ready_conns: List[connection.Connection] = [] remain_time, t1 = timeout, time.time() while len(remain_conns) > 0 and len(ready_conns) < wait_num: if timeout: remain_time = timeout - (time.time() - t1) if remain_time <= 0: break # connection.wait hangs if the list is empty new_ready_conns = connection.wait( remain_conns, timeout=remain_time) ready_conns.extend(new_ready_conns) # type: ignore remain_conns = [ conn for conn in remain_conns if conn not in ready_conns] return [workers[conns.index(con)] for con in ready_conns] def send_action(self, action: np.ndarray) -> None: self.parent_remote.send(["step", action]) def get_result( self, ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: obs, rew, done, info = self.parent_remote.recv() if self.share_memory: obs = self._decode_obs() return obs, rew, done, info def seed(self, seed: Optional[int] = None) -> Optional[List[int]]: self.parent_remote.send(["seed", seed]) return self.parent_remote.recv() def render(self, **kwargs: Any) -> Any: self.parent_remote.send(["render", kwargs]) return self.parent_remote.recv() def close_env(self) -> None: try: self.parent_remote.send(["close", None]) # mp may be deleted so it may raise AttributeError self.parent_remote.recv() self.process.join() except (BrokenPipeError, EOFError, AttributeError): pass # ensure the subproc is terminated self.process.terminate()
if (l == ord('q')): break maxInd = np.argmax(predictions) NNState = IndexToState[maxInd] m['NeuralNetworkState'] = NNState if __name__ == '__main__': print("starting") GPIO.setmode(GPIO.BCM) GPIO.setup(GPIO_TRIGGER, GPIO.OUT) GPIO.setup(GPIO_ECHO, GPIO.IN) m = mp.Manager().dict() t = mp.Value('d', 0) m['NeuralNetworkState'] = NeuralNetWorkRead.Stop m['distance'] = 0 m['confidence'] = 1 m['Start'] = 0 m['NetworkDone'] = 0 picar.setup() p1 = Process(target=RobotProccess, args=(m, )) p2 = Process(target=DistanceProccess, args=(m, )) p3 = Process(target=GetNeuralNetworkResponseProccess, args=(m, )) p1.start() p2.start() p3.start() p1.join() p2.join() p3.join()
class AsyncMCTS(MoveChooser): """ Implementation of Monte Carlo Tree Search that uses the other player's time to continue thinking. This is achieved using multiprocessing, and a Pipe for transferring data to and from the worker process. """ def __init__(self, GameClass, starting_position, time_limit=3, network=None, c=np.sqrt(2), d=1, threads=1): """ Either: If network is provided, threads must be 1. If network is not provided, then threads will be used for leaf parallelization """ super().__init__(GameClass, starting_position) if network is not None and threads != 1: raise Exception('Threads != 1 with Network != None') self.parent_pipe, worker_pipe = Pipe() self.worker_process = Process(target=self.loop_func, args=(GameClass, starting_position, time_limit, network, c, d, threads, worker_pipe)) def start(self): self.worker_process.start() def report_user_move(self, user_chosen_move): """ Reports the given user chosen move to the worker thread. This allows the search tree to be narrowed. :param user_chosen_move: """ self.parent_pipe.send(user_chosen_move) self.position = user_chosen_move def choose_move(self, return_distribution=False): """ Instructs the worker thread to decide on an optimal move. The worker thread will then continue thinking for time_limit, and then return a list of its chosen moves. If multiple states are passed through before the ai's turn is completed, then they will be the contents of the list. Otherwise the list will have a single state. :return: The moves chosen by monte carlo tree search. """ self.parent_pipe.send(None) chosen_positions = self.parent_pipe.recv() self.position = chosen_positions[-1][0] return chosen_positions if return_distribution else [ position for position, _ in chosen_positions ] def terminate(self): self.worker_process.terminate() self.worker_process.join() @staticmethod def loop_func(GameClass, position, time_limit, network, c, d, threads, worker_pipe): if network is None: pool = Pool(threads) if threads > 1 else None root = RolloutNode(position, parent=None, GameClass=GameClass, c=c, rollout_batch_size=threads, pool=pool, verbose=True) else: network.initialize() root = HeuristicNode(position, None, GameClass, network, c, d, verbose=True) while True: best_node = root.choose_expansion_node() if best_node is not None: best_node.expand() if root.children is not None and worker_pipe.poll(): user_chosen_position = worker_pipe.recv() if user_chosen_position is not None: # an updated position has been received so we can truncate the tree for child in root.children: if np.all(child.position == user_chosen_position): root = child root.parent = None break else: print(user_chosen_position) raise Exception('Invalid user chosen move!') if GameClass.is_over(root.position): print('Game Over in Async MCTS: ', GameClass.get_winner(root.position)) return else: # this move chooser has been requested to decide on a move via the choose_move function start_time = time() while time() - start_time < time_limit: best_node = root.choose_expansion_node() # best_node will be None if the tree is fully expanded if best_node is None: break best_node.expand() is_ai_player_1 = GameClass.is_player_1_turn(root.position) chosen_positions = [] print( f'MCTS choosing move based on {root.count_expansions()} expansions!' ) # choose moves as long as it is still the ai's turn while GameClass.is_player_1_turn( root.position) == is_ai_player_1: if root.children is None: best_node = root.choose_expansion_node() if best_node is not None: best_node.expand() root, distribution = root.choose_best_node( return_probability_distribution=True, optimal=True) chosen_positions.append((root.position, distribution)) print('Expected outcome: ', root.get_evaluation()) root.parent = None # delete references to the parent and siblings worker_pipe.send(chosen_positions) if GameClass.is_over(root.position): print('Game Over in Async MCTS: ', GameClass.get_winner(root.position)) return def reset(self): raise NotImplementedError('')
def show_info_on(a_topic): ''' Postconditions: 1. a_topic is on the monitor 2. The name of the module executing this is on the monitor 3. If available, the ID of the process parent to this is on the monitor 4. The ID of the process executing this is on the monitor ''' print('Information on ' + a_topic) # 1. print('Name of module executing this: ', __name__) # 2. if hasattr(os, 'getppid'): # 3: if available on this OS print('Parent process: ', os.getppid()) print('ID of process executing this: ', os.getpid(), '\n') # 4. def say_hello(name): ''' Postconditions: 1. = Postconditions of get_info_on('Say-hello process') 2. "Hello <name>" is on the monitor ''' show_info_on('Say-hello process') print('Hello ', name) if __name__ == '__main__': ''' Postconditions: 1. = Postconditions of get_info_on (this) main process 2. = Postconditions of get_info_on a new process executing say_hello('Hugh Person') ''' show_info_on('main line') p = Process(target=say_hello, args=('Hugh Person',)) p.start() p.join()
from multiprocessing.queues import Queue import os, time, random # 写数据进程执行的代码: def write(q): print('Process to write: %s' % os.getpid()) for value in ['A', 'B', 'C']: print('Put %s to queue...' % value) q.put(value) time.sleep(random.random()) # 读数据进程执行的代码: def read(q): print('Process to read: %s' % os.getpid()) while True: value = q.get(True) print('Get %s from queue.' % value) if __name__=='__main__': # 父进程创建Queue,并传给各个子进程: q = Queue() pw = Process(target=write, args=(q,)) pr = Process(target=read, args=(q,)) # 启动子进程pw,写入: pw.start() # 启动子进程pr,读取: pr.start() # 等待pw结束: pw.join() # pr进程里是死循环,无法等待其结束,只能强行终止: pr.terminate()
def process_paid_order(self, order): self.logger.info(order.order_number+" - process") self.ioc.new_order_service().process_paid_order(order) p = Process(target=self.inform_customer, args=(self.ioc, order,)) p.start() p.join()
# get break down of how many rows per process size = int(n / 4) # take time start_time = time.time() # create each process p0 = Process(target=subMatrixSmallestCount, args=(matrix, 0, size, n, Smallest, TotalCount, 1)) p1 = Process(target=subMatrixSmallestCount, args=(matrix, 1, size, n, Smallest, TotalCount, 2)) p2 = Process(target=subMatrixSmallestCount, args=(matrix, 2, size, n, Smallest, TotalCount, 3)) p3 = Process(target=subMatrixSmallestCount, args=(matrix, 3, size, n, Smallest, TotalCount, 4)) # start processes p0.start() p1.start() p2.start() p3.start() # wait for processes to finish p0.join() p1.join() p2.join() p3.join() print('Smallest value: {}'.format(Smallest.value)) print('Count of that value: {}'.format(TotalCount.value)) # print time print('Total time: {}s'.format (time.time() - start_time))
class TestRemoteSession(TestCase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.server_process = None """:type: Process""" def setUp(self): self.test_file = TestFile(settings.test_file_size) self.original_data = self.test_file.get_content() self.ftp_root = self.test_file.path self.remote_filename = uuid.uuid4().hex tuned_server, self.port = server.get_tuned_server(self.ftp_root) def server_func(): tuned_server.serve_forever(handle_exit=True) self.server_process = Process(target=server_func) """:type: Process""" self.server_process.start() def get_connected_client_in_binary_mode(self): client = Client() client.connect('localhost', self.port) client.login(settings.ftp_user, settings.ftp_pass) client.type('I') # Binary mode ('I'mage) return client def tearDown(self): while self.server_process.is_alive(): # посылаем SIGINT процессу, pyftpd умеет его обрабатывать и закрывается корректно os.kill(self.server_process.pid, signal.SIGINT) self.server_process.join(timeout=1) unlink(self.test_file.full_filename) def test_receive_file(self): """ тест на получение заранее сгенерированных и сохраненных на диске данных клиентом через подключение к локальному серверу. """ client = self.get_connected_client_in_binary_mode() code, rest, data = client.retr(self.test_file.filename) q_code, q_rest = client.quit() self.assertEqual(code, 226) self.assertTrue(self.original_data == data) def test_send_file(self): """ Тест на отправку файла клиентом на локальный сервер и сравнение содержимого файла на диске с отправленными данными """ client = self.get_connected_client_in_binary_mode() client.stor(self.remote_filename, self.original_data) with open(os.path.join(self.ftp_root, self.remote_filename), 'rb') as f: stored_file_content = f.read() code, rest = client.quit() self.assertTrue(self.original_data == stored_file_content)