def main(): smm = SharedMemoryManager() smm.start() ls = smm.ShareableList(range(2000)) with Pool(4) as p: print(*list(p.imap_unordered(f, ls)), sep='\n') # print(p.map(f, [2, 3, 4, 5, 6])) # lock # print(p.map(f, [2, 3, 4, 5, 6])) smm.shutdown()
def lif_feed_forward_benchmark(parameters: BenchmarkParameters): shared = SharedMemoryManager() shared.start() params = list(parameters._asdict().values()) shared_list = shared.ShareableList(params) run(["python3", __file__, shared_list.shm.name], stderr=STDOUT) duration = shared_list[0] shared_list.shm.close() shared.shutdown() return duration
def multiprocess(self, n, nthread=8): """ 多进程 并发 :return: """ print('Parent process %s.' % os.getpid()) p = Pool(nthread) # 进程池, 和系统申请 nthread 个进程 smm = SharedMemoryManager() #TODO: pyrhon3.8+ 才有 smm.start() # Start the process that manages the shared memory blocks cache_list = smm.ShareableList([0] * n) # 限制了可被存储在其中的值只能是 int, float, bool, str (每条数据小于10M), bytes (每条数据小于10M)以及 None 这些内置类型。 # 它另一个显著区别于内置 list 类型的地方在于它的长度无法修改(比如,没有 append, insert 等操作) # 且不支持通过切片操作动态创建新的 ShareableList 实例。 shm_a = smm.SharedMemory(size=n) # shm_a.buf[:] = bytearray([0]*n) # shm_a.buf[:] = [0] * n print('shm_a id in main process: {} '.format(id(shm_a))) # 主进程的内存空间 和 子进程的内存空间 的考察 self.global_array = [0] * n print('array id in main process: {} '.format(id(self.global_array))) self.global_string = 'abc' print('string id in main process: {} '.format(id(self.global_string))) self.global_int = 10 print('int id in main process: {} '.format(id(self.global_int))) for i in range(n): # p.apply_async(task, args=(cache_name,i)) # apply_async 异步取回结果 p.apply_async(self.task, args=(cache_list, shm_a, i)) print('Waiting for all subprocesses done...') p.close() p.join() print('All subprocesses done.') smm.shutdown() return cache_list, shm_a
def main(): # This is the number of values that the writer will send to the reader items_to_send = random.randint(100000, 1000000) smm = SharedMemoryManager() smm.start() # Create a ShareableList to be used between the processes size = BUFFER_SIZE + 2 shared_list = smm.ShareableList(range(size)) for i in range(0, size): shared_list[i] = -1 # TODO - Create any lock(s) or semaphore(s) that you feel you need lock = mp.Lock() sem = mp.Semaphore(BUFFER_SIZE) # TODO - create reader and writer processes items_sent = 0 items_read = 0 p1 = mp.Process(target=write, args=(shared_list, items_sent, items_to_send, sem, lock)) p2 = mp.Process(target=read, args=(shared_list, items_read, items_to_send, sem, lock)) # TODO - Start the processes and wait for them to finish p1.start() p2.start() p1.join() p2.join() print(f'{items_to_send} sent by the writer') # TODO - Display the number of numbers/items received by the reader. print(f'{shared_list[-1]} read by the reader') smm.shutdown()
def main(): # This is the number of values that the writer will send to the reader items_to_send = random.randint(10000, 100000) smm = SharedMemoryManager() smm.start() # TODO - Create a ShareableList to be used between the processes # TODO - Create any lock(s) or semaphore(s) that you feel you need # TODO - create reader and writer processes # TODO - Start the processes and wait for them to finish print(f'{items_to_send} values sent') # TODO - Display the number of numbers/items received by the reader. # Can not use "items_to_send", must be a value collected # by the reader processes. # print(f'{<your variable>} values received') smm.shutdown()
class ParallelSimulation(Simulation): """ Parallel simulation of Barnes-Hut algorithm realised using shared memory. """ def __init__(self, positions: np.ndarray, velocities: np.ndarray, masses: np.ndarray, params: Namespace): super().__init__(positions, velocities, masses, params) self._theta = params.theta self._init_memory(positions, velocities, masses) self._init_workers() atexit.register(self._cleanup) def _init_memory(self, positions: np.ndarray, velocities: np.ndarray, masses: np.ndarray): """ Prepares shared memory arrays. """ # setup process that sets up shared memory self._memory_manager = SharedMemoryManager() self._memory_manager.start() max_nodes = self.bodies + 64 # create shared memory buffers self._positions_shm = self._memory_manager.SharedMemory(positions.nbytes) self._velocities_shm = self._memory_manager.SharedMemory(velocities.nbytes) self._accelerations_shm = self._memory_manager.SharedMemory(velocities.nbytes) self._masses_shm = self._memory_manager.SharedMemory(masses.nbytes) self._nodes_positions_shm = self._memory_manager.SharedMemory(np.empty((max_nodes, 3), np.float).nbytes) self._nodes_masses_shm = self._memory_manager.SharedMemory(np.empty((max_nodes, ), np.float).nbytes) self._nodes_sizes_shm = self._memory_manager.SharedMemory(np.empty((max_nodes, ), np.float).nbytes) self._nodes_children_types_shm = self._memory_manager.SharedMemory(np.empty((max_nodes, 8), np.int).nbytes) self._nodes_children_ids_shm = self._memory_manager.SharedMemory(np.empty((max_nodes, 8), np.int).nbytes) # setup NumPy arrays self._data = SharedData( time_step=self.time_step, theta=self._theta, gravitational_constant=self.gravitational_constant, softening=self.softening, nodes_count=Value('i', 0), positions=np.ndarray((self.bodies, 3), dtype=np.float, buffer=self._positions_shm.buf), velocities=np.ndarray((self.bodies, 3), dtype=np.float, buffer=self._velocities_shm.buf), accelerations=np.ndarray((self.bodies, 3), dtype=np.float, buffer=self._accelerations_shm.buf), masses=np.ndarray((self.bodies, ), dtype=np.float, buffer=self._masses_shm.buf), nodes_positions=np.ndarray((max_nodes, 3), dtype=np.float, buffer=self._nodes_positions_shm.buf), nodes_masses=np.ndarray((max_nodes, ), dtype=np.float, buffer=self._nodes_masses_shm.buf), nodes_sizes=np.ndarray((max_nodes, ), dtype=np.float, buffer=self._nodes_sizes_shm.buf), nodes_children_types=np.ndarray((max_nodes, 8), dtype=np.int, buffer=self._nodes_children_types_shm.buf), nodes_children_ids=np.ndarray((max_nodes, 8), dtype=np.int, buffer=self._nodes_children_ids_shm.buf) ) # copy data into shared arrays self._data.positions[:] = positions[:] self._data.velocities[:] = velocities[:] self._data.masses[:] = masses[:] def _init_workers(self): """ Prepares pool of workers. """ self._pool = Pool( processes=self._params.processes, initializer=worker.initialize, initargs=(self._data, ) ) def _cleanup(self): """ Cleans up shared memory and pool of workers. """ self._pool.terminate() self._memory_manager.shutdown() print('Memory manager was shut down.') def simulate(self) -> Iterable[Tuple[np.ndarray, np.ndarray, np.ndarray]]: """ Runs parallel implementation of Barnes-Hut simulation. """ while True: self._build_octree() self._update_accelerations() self._update_positions() yield self._data.positions, self._data.velocities, self._data.accelerations def _build_octree(self): """ Builds octree used in Barnes-Hut. """ global_coords_min = np.repeat(np.min(self._data.positions), 3) global_coords_max = np.repeat(np.max(self._data.positions), 3) global_coords_mid = (global_coords_min + global_coords_max) / 2 # manually build first node self._data.nodes_count.value = 1 self._data.nodes_positions[0] = np.average(self._data.positions, axis=0, weights=self._data.masses) self._data.nodes_masses[0] = np.sum(self._data.masses) self._data.nodes_sizes[0] = global_coords_max[0] - global_coords_min[0] # calculate base octant for each body bodies_base_octant = np.sum((self._data.positions > global_coords_mid) * [1, 2, 4], axis=1) tasks_targets = [] tasks_args = [] # build second layer of nodes and collect tasks for octant in range(8): coords_min, coords_max = octant_coords(global_coords_min, global_coords_max, octant) coords_mid = (coords_min + coords_max) / 2 # get indices of bodies in this octant octant_bodies = np.argwhere(bodies_base_octant == octant).flatten() # if node is empty or has one body handle it separately if octant_bodies.size == 0: self._data.nodes_children_types[0, octant] = OCTANT_EMPTY continue if octant_bodies.size == 1: self._data.nodes_children_types[0, octant] = OCTANT_BODY self._data.nodes_children_ids[0, octant] = octant_bodies[0] continue # create node node_id = self._data.nodes_count.value self._data.nodes_count.value = node_id + 1 self._data.nodes_children_types[0, octant] = OCTANT_NODE self._data.nodes_children_ids[0, octant] = node_id self._data.nodes_positions[node_id] = np.average(self._data.positions[octant_bodies], axis=0, weights=self._data.masses[octant_bodies]) self._data.nodes_masses[node_id] = np.sum(self._data.masses[octant_bodies]) self._data.nodes_sizes[node_id] = coords_max[0] - coords_min[0] # split bodies into sub octants bodies_sub_octant = np.sum((self._data.positions[octant_bodies] > coords_mid) * [1, 2, 4], axis=1) # create tasks for i in range(8): tasks_targets.append((node_id, i)) tasks_args.append(( octant_bodies[bodies_sub_octant == i], *octant_coords(coords_min, coords_max, i) )) # run tasks results = self._pool.starmap(worker.build_octree_branch, tasks_args) # update references in nodes for (node_id, i), (sub_node_type, sub_node_id) in zip(tasks_targets, results): self._data.nodes_children_types[node_id, i] = sub_node_type self._data.nodes_children_ids[node_id, i] = sub_node_id def _update_accelerations(self): """ Calculates accelerations of the bodies. """ if self.bodies < 2: return self._pool.map(worker.update_acceleration, range(self.bodies)) def _update_positions(self): """ Calculates positions of the bodies. """ self._pool.map(worker.update_position, range(self.bodies)) @property def positions(self) -> np.ndarray: return self._data.positions @property def velocities(self) -> np.ndarray: return self._data.velocities @property def masses(self) -> np.ndarray: return self._data.masses @property def accelerations(self) -> np.ndarray: return self._data.accelerations
class DataManager: def __init__(self, obs=1000, config=None): self._datasets = dict() self.smm = SharedMemoryManager() self.smm.start() self.result = Manager().list() self.conns = dict() self.obs = obs self.config = config def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.shutdown() def shutdown(self): self.smm.shutdown() for conn in self.conns.values(): conn.close() def _add_to_shared_memory(self, nparray: recarray) -> SharedMemory: """Internal function to copy an array into shared memory. Parameters ---------- nparray : recarray The array to be copied into shared memory. Returns ------- SharedMemoryName The shared memory object. """ shm = self.smm.SharedMemory(nparray.nbytes) array = recarray(shape=nparray.shape, dtype=nparray.dtype, buf=shm.buf) copyto(array, nparray) return shm def _download_dataset(self, dataset: Dataset) -> recarray: """Internal function to download the dataset. Parameters ---------- dataset : Dataset Dataset information including source, library, table, vars, etc. Returns ------- recarray `numpy.recarry` of the downloaded dataset. """ # TODO: generic login data for different data sources if dataset.source == 'wrds': usr = self.config.get('wrds_username') pwd = self.config.get('wrds_password') # If there exists a connection for the data source, use it! if (conn := self.conns.get(dataset.source, None)) is None: module = import_module(f'frds.data.{dataset.source}') conn = module.Connection(usr=usr, pwd=pwd) self.conns.update({dataset.source: conn}) df = conn.get_table(library=dataset.library, table=dataset.table, columns=dataset.vars, date_cols=dataset.date_vars, obs=self.obs) assert isinstance(df, DataFrame) return df.to_records(index=False)
class ParallelTransformStep(ProcessingStep[TPayload]): def __init__( self, function: Callable[[Message[TPayload]], TTransformed], next_step: ProcessingStep[TTransformed], processes: int, max_batch_size: int, max_batch_time: float, input_block_size: int, output_block_size: int, metrics: MetricsBackend, ) -> None: self.__transform_function = function self.__next_step = next_step self.__max_batch_size = max_batch_size self.__max_batch_time = max_batch_time self.__shared_memory_manager = SharedMemoryManager() self.__shared_memory_manager.start() self.__pool = Pool( processes, initializer=parallel_transform_worker_initializer, context=multiprocessing.get_context("spawn"), ) self.__input_blocks = [ self.__shared_memory_manager.SharedMemory(input_block_size) for _ in range(processes) ] self.__output_blocks = [ self.__shared_memory_manager.SharedMemory(output_block_size) for _ in range(processes) ] self.__batch_builder: Optional[BatchBuilder[TPayload]] = None self.__results: Deque[Tuple[MessageBatch[TPayload], AsyncResult[Tuple[ int, MessageBatch[TTransformed]]], ]] = deque() self.__metrics = metrics self.__batches_in_progress = Gauge(metrics, "batches_in_progress") self.__pool_waiting_time: Optional[float] = None self.__closed = False def handle_sigchld(signum: int, frame: Any) -> None: # Terminates the consumer if any child process of the # consumer is terminated. # This is meant to detect the unexpected termination of # multiprocessor pool workers. if not self.__closed: self.__metrics.increment("sigchld.detected") raise ChildProcessTerminated() signal.signal(signal.SIGCHLD, handle_sigchld) def __submit_batch(self) -> None: assert self.__batch_builder is not None batch = self.__batch_builder.build() logger.debug("Submitting %r to %r...", batch, self.__pool) self.__results.append(( batch, self.__pool.apply_async( parallel_transform_worker_apply, (self.__transform_function, batch, self.__output_blocks.pop()), ), )) self.__batches_in_progress.increment() self.__metrics.timing("batch.size.msg", len(batch)) self.__metrics.timing("batch.size.bytes", batch.get_content_size()) self.__batch_builder = None def __check_for_results(self, timeout: Optional[float] = None) -> None: input_batch, result = self.__results[0] # If this call is being made in a context where it is intended to be # nonblocking, checking if the result is ready (rather than trying to # retrieve the result itself) avoids costly synchronization. if timeout == 0 and not result.ready(): # ``multiprocessing.TimeoutError`` (rather than builtin # ``TimeoutError``) maintains consistency with ``AsyncResult.get``. raise multiprocessing.TimeoutError() i, output_batch = result.get(timeout=timeout) # TODO: This does not handle rejections from the next step! for message in output_batch: self.__next_step.poll() self.__next_step.submit(message) if i != len(input_batch): logger.warning( "Received incomplete batch (%0.2f%% complete), resubmitting...", i / len(input_batch) * 100, ) # TODO: This reserializes all the ``SerializedMessage`` data prior # to the processed index even though the values at those indices # will never be unpacked. It probably makes sense to remove that # data from the batch to avoid unnecessary serialization overhead. self.__results[0] = ( input_batch, self.__pool.apply_async( parallel_transform_worker_apply, ( self.__transform_function, input_batch, output_batch.block, i, ), ), ) return logger.debug("Completed %r, reclaiming blocks...", input_batch) self.__input_blocks.append(input_batch.block) self.__output_blocks.append(output_batch.block) self.__batches_in_progress.decrement() del self.__results[0] def poll(self) -> None: self.__next_step.poll() while self.__results: try: self.__check_for_results(timeout=0) except multiprocessing.TimeoutError: if self.__pool_waiting_time is None: self.__pool_waiting_time = time.time() else: current_time = time.time() if current_time - self.__pool_waiting_time > LOG_THRESHOLD_TIME: logger.warning( "Waited on the process pool longer than %d seconds. Waiting for %d results. Pool: %r", LOG_THRESHOLD_TIME, len(self.__results), self.__pool, ) self.__pool_waiting_time = current_time break else: self.__pool_waiting_time = None if self.__batch_builder is not None and self.__batch_builder.ready(): self.__submit_batch() def __reset_batch_builder(self) -> None: try: input_block = self.__input_blocks.pop() except IndexError as e: raise MessageRejected("no available input blocks") from e self.__batch_builder = BatchBuilder( MessageBatch(input_block), self.__max_batch_size, self.__max_batch_time, ) def submit(self, message: Message[TPayload]) -> None: assert not self.__closed if self.__batch_builder is None: self.__reset_batch_builder() assert self.__batch_builder is not None try: self.__batch_builder.append(message) except ValueTooLarge as e: logger.debug("Caught %r, closing batch and retrying...", e) self.__submit_batch() # This may raise ``MessageRejected`` (if all of the shared memory # is in use) and create backpressure. self.__reset_batch_builder() assert self.__batch_builder is not None # If this raises ``ValueTooLarge``, that means that the input block # size is too small (smaller than the Kafka payload limit without # compression.) self.__batch_builder.append(message) def close(self) -> None: self.__closed = True if self.__batch_builder is not None and len(self.__batch_builder) > 0: self.__submit_batch() def terminate(self) -> None: self.__closed = True logger.debug("Terminating %r...", self.__pool) self.__pool.terminate() logger.debug("Shutting down %r...", self.__shared_memory_manager) self.__shared_memory_manager.shutdown() logger.debug("Terminating %r...", self.__next_step) self.__next_step.terminate() def join(self, timeout: Optional[float] = None) -> None: deadline = time.time() + timeout if timeout is not None else None logger.debug("Waiting for %s batches...", len(self.__results)) while self.__results: self.__check_for_results( timeout=max(deadline - time.time(), 0) if deadline is not None else None) self.__pool.close() logger.debug("Waiting for %s...", self.__pool) # ``Pool.join`` doesn't accept a timeout (?!) but this really shouldn't # block for any significant amount of time unless something really went # wrong (i.e. we lost track of a task) self.__pool.join() self.__shared_memory_manager.shutdown() self.__next_step.close() self.__next_step.join( timeout=max(deadline - time.time(), 0) if deadline is not None else None)
def _batched_train_test(self): #batch_size = len(self.GPU_LIST)*self.model_per_gpu smm = SharedMemoryManager() smm.start() X_shared = _to_shared_mem(smm, self.X) w_shared = _to_shared_mem(smm, self.w) delta_shared = _to_shared_mem(smm, self.delta) with Manager() as manager: param_dict_mngd = manager.dict({ 'param_dicts_train': self.param_dicts_train, 'param_dicts_test': self.param_dicts_test, 'data_idx': self.data_idx, 'X_shared_name': X_shared.name, 'X_shape': self.X.shape, 'X_dtype': self.X.dtype, 'w_shared_name': w_shared.name, 'w_shape': self.w.shape, 'w_dtype': self.w.dtype, 'delta_shared_name': delta_shared.name, 'delta_shape': self.delta.shape, 'delta_dtype': self.delta.dtype, 'batch_size': self.batch_size, 'batch_idx': None, 'test_block_size': self.test_block_size }) rslt_mngd = manager.list([0] * len(self.param_dicts_test)) for batch_idx in tqdm(range( int(len(self.param_dicts_train) / self.batch_size) + 1), desc="batched cross validation"): param_dict_mngd['batch_idx'] = batch_idx ''' _run_batch_process(param_dict_mngd, rslt_mngd) print (rslt_mngd) raise ''' p = mp.Process(target=_run_batch_process, args=(param_dict_mngd, rslt_mngd)) p.start() p.join() p.terminate() smm.shutdown() self.rslts = list(rslt_mngd)
class DataManager(Singleton): """DataManager loads data from sources and manages the shared memory""" def __init__(self, obs=-1, config=None): self._datasets = dict() self.smm = SharedMemoryManager() self.smm.start() self.conns = dict() self.obs = obs self.config = config def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.shutdown() def shutdown(self): """Shutdown the shared memory manager and all data connections""" self.smm.shutdown() for conn in self.conns.values(): conn.close() def _add_to_shared_memory(self, nparray: np.recarray) -> SharedMemory: """Internal function to copy an array into shared memory. Parameters ---------- nparray : np.recarray The array to be copied into shared memory. Returns ------- SharedMemory The shared memory object. """ shm = self.smm.SharedMemory(nparray.nbytes) array = np.recarray(nparray.shape, dtype=nparray.dtype, buf=shm.buf) np.copyto(array, nparray) return shm def _download_dataset(self, dataset: Dataset) -> pd.DataFrame: """Internal function to download the dataset. Parameters ---------- dataset : Dataset Dataset information including source, library, table, vars, etc. Returns ------- pd.DataFrame DataFrame of the downloaded dataset. """ # TODO: generic login data for different data sources if dataset.source == "wrds": usr = self.config.get("wrds_username") pwd = self.config.get("wrds_password") # If there exists a connection for the data source, use it! if (conn := self.conns.get(dataset.source, None)) is None: module = import_module(f"frds.data.{dataset.source}") conn = module.Connection(usr=usr, pwd=pwd) self.conns.update({dataset.source: conn}) return conn.get_table( library=dataset.library, table=dataset.table, columns=dataset.vars, date_cols=dataset.date_vars, obs=self.obs, )
a[1] = "Proceso 1" print('\n hola soy el proceso 1 \n' + 'Imprimiendo la lista compartida \n' + str(a)) def workerProceso2(): #Obtenemos la lista por su nombre b = ShareableList(name=lc.shm.name) #Agregamos a la lista un string b[2] = "Proceso 2" print('\n hola soy el proceso 2 \n' + 'Imprimiendo la lista compartida: \n' + str(b)) #Definimos los procesos p1 = Process(target=workerProceso1) p2 = Process(target=workerProceso2) p1.start() p2.start() p1.join() p2.join() #Imprimimos la lista para ver si los cambios han sido efectivamente aplicados. print('\n Imprimiendo la lista para ver si se efectuaron los cambios: \n' + str(lc)) #Damos de baja el gestor de bloques de memoria compartida smm.shutdown()
class MonitorClient(Thread): """Client for napari shared memory monitor. Napari launches us. We get config information from our NAPARI_MON_CLIENT environment variable. That contains a port number to connect to. We connect our SharedMemoryManager to that port. We get these resources from the manager: 1) shutdown_event() If this is set napari is exiting. Ususally it exists so fast we get at ConnectionResetError exception instead of see this was set. We have no clean way to exit the SocketIO server yet. 2) command_queue() We put command onto this queue for napari to execute. 3) data() Data from napari's monitor.add() command. """ def __init__(self, config: dict, client_name="?"): super().__init__() assert config self.config = config self.client_name = client_name self.running = True self.napari_data = None LOGGER.info("Starting MonitorClient process %s", os.getpid()) _log_env() server_port = config['server_port'] LOGGER.info("Connecting to port %d...", server_port) # Right now we just need to magically know these callback names, # maybe we can come up with a better way. napari_api = ['shutdown_event', 'command_queue', 'data'] for name in napari_api: SharedMemoryManager.register(name) # Connect to napari's shared memory. self._manager = SharedMemoryManager( address=('localhost', config['server_port']), authkey=str.encode('napari'), ) self._manager.connect() # Get the shared resources. self._shared = SharedResources( self._manager.shutdown_event(), self._manager.command_queue(), self._manager.data(), ) # Start our thread so we can poll napari. self.start() def run(self) -> None: """Check shared memory for new data.""" LOGGER.info("MonitorClient thread is running...") while True: if not self._poll(): LOGGER.info("Exiting...") break time.sleep(POLL_INTERVAL_MS / 1000) # webmon checks this and stops/exits. self.running = False def _poll(self) -> bool: """See if there is now information in shared mem.""" # LOGGER.info("Poll...") try: if self._shared.shutdown.is_set(): # We sometimes do see the shutdown event was set. But usually # we just get ConnectionResetError, because napari is exiting. LOGGER.info("Shutdown event was set.") return False # Stop polling except ConnectionResetError: LOGGER.info("ConnectionResetError.") return False # Stop polling # Do we need to copy here? self.napari_data = { "tile_config": self._shared.data.get('tile_config'), "tile_state": self._shared.data.get('tile_state'), } if DUMP_DATA_FROM_NAPARI: pretty_str = json.dumps(self.napari_data, indent=4) LOGGER.info("New data from napari: %s", pretty_str) return True # Keep polling def post_command(self, command) -> None: """Send new command to napari. """ LOGGER.info(f"Posting command {command}") try: self._shared.commands.put(command) except ConnectionRefusedError: self._log("ConnectionRefusedError") def stop(self) -> None: """Call on shutdown. TODO_MON: no one calls this yet?""" self._manager.shutdown()
def parallel_eval_expr(n, ops, goal, timeout=1, proc_cnt=16): q, q_type = make_tasks(n, ops) task_cnt = len(q) task_ranges = calc_task_ranges(proc_cnt, q) smm = SharedMemoryManager() smm.start() task_mem, tq = make_shared_mem(smm, task_cnt, q_type) tq[:] = q[:] del q res_mem, res = make_shared_mem(smm, task_cnt, np.int8) res[:] = -2 # initial value indicating that result is not being calculated start_time_mem, start_time = make_shared_mem(smm, proc_cnt, np.float_) status_mem, status = make_shared_mem(smm, proc_cnt, np.int64) status[:] = ProcStatus.IDLE cur_idx_mem, cur_idx = make_shared_mem(smm, proc_cnt, np.int64) cur_idx[:] = -1 processes = [None] * proc_cnt restarts = [0] * proc_cnt cur_t = time.time() while not all(status[i] == ProcStatus.DONE for i in range(proc_cnt)): print_status(res, status, cur_idx, restarts, proc_cnt) try: for i, p in enumerate(processes): if p is None: continue if status[i] == ProcStatus.DONE: continue if (cur_t - start_time[i]) > timeout: p.kill() if cur_idx[i] < 0: continue res[cur_idx[i]] = -1 for i, p in enumerate(processes): if p is not None and p.is_alive(): continue if status[i] == ProcStatus.DONE: try: p.close() except Exception: pass processes[i] = None continue try: p.close() except Exception: pass is_new_proc = cur_idx[i] == -1 idx = task_ranges[i][0] if is_new_proc else cur_idx[i] end_idx = task_ranges[i][1] restarts[i] += 1 processes[i] = Process( target=eval_f, args=( task_mem.name, res_mem.name, status_mem.name, start_time_mem.name, cur_idx_mem.name, i, # skipping the one that takes too long idx + (0 if is_new_proc else 1), end_idx, goal, task_cnt, q_type)) processes[i].start() except Exception: import traceback traceback.print_exc() cur_t = time.time() time.sleep(timeout) print_status(res, status, cur_idx, restarts, proc_cnt) for p in processes: if p is None: continue p.join() p.close() for pf in tq[np.argwhere(res == 1)].flatten(): pf = pf.decode('ascii') print(eval_psfx(pf), '==', postfix_to_infix(pf), pf) smm.shutdown()