def test01_bgsave_stress(self): n_reads = 50000 n_creations = 50000 n_updates = n_creations / 10 n_deletions = n_creations / 2 conn = self.env.getConnection() graphs[0].query("CREATE INDEX FOR (n:Node) ON (n.v)") pool = Pool(nodes=5) t1 = pool.apipe(create_nodes, graphs[0], n_creations) t2 = pool.apipe(delete_nodes, graphs[1], n_deletions) t3 = pool.apipe(read_nodes, graphs[2], n_reads) t4 = pool.apipe(update_nodes, graphs[3], n_updates) t5 = pool.apipe(BGSAVE_loop, self.env, conn, 10000) # wait for processes to join t1.wait() t2.wait() t3.wait() t4.wait() t5.wait() # make sure we did not crashed conn.ping() conn.close()
def test01_bgsave_stress(self): n_nodes = 1000 n_iterations = 10 conn = self.env.getConnection() graphs[0].query("CREATE INDEX ON :Node(val)") pool = Pool(nodes=5) t1 = pool.apipe(create_nodes, graphs[0], n_iterations, n_nodes) t2 = pool.apipe(delete_nodes, graphs[1], n_iterations, n_nodes / 2) t3 = pool.apipe(read_nodes, graphs[2], n_iterations) t4 = pool.apipe(update_nodes, graphs[3], n_iterations) t5 = pool.apipe(BGSAVE_loop, self.env, conn, 3) # wait for processes to join t1.wait() t2.wait() t3.wait() t4.wait() t5.wait() # make sure we did not crashed conn.ping() conn.close()
def reachability(model, from_state, goal, max_length=2000, on_start=None, on_reach=None, max_repeat=10000, n_workers=1): if isinstance(model, pypint.Model): model = pypint_to_model(model) if isinstance(goal, list) or isinstance(goal, dict): goal = Goal(goal) if isinstance(from_state, list): if from_state: if isinstance(from_state[0], str): from_state = dict([(e, 1) for e in from_state]) elif isinstance(from_state[0], tuple): from_state = dict(from_state) from_state = complete_state(from_state, model) trace = Trace(from_state) if on_start is not None: next_subgoal = goal.subgoals[0] on_start(model, trace, next_subgoal) if n_workers == 1: for n_repeat in range(max_repeat): reached, trace = _reach(copy.copy(model), from_state, goal, max_length, on_start, on_reach) if reached is True: return reached, trace else: pool = ProcessPool(n_workers) processes = set([]) n_repeat = 0 while n_repeat < max_repeat and n_repeat < n_workers: processes.add( pool.apipe(_reach, copy.copy(model), from_state, goal, max_length, on_start, on_reach)) n_repeat += 1 reached = pypint.Inconc while reached is not True and n_repeat < max_repeat: for process in processes: if process.ready(): reached, trace = process.get() processes.remove(process) if reached is True: return reached, trace else: processes.add( pool.apipe(_reach, copy.copy(model), from_state, goal, max_length, on_start, on_reach)) n_repeat += 1 break return reached, trace
def multi_process(data_path, time_list): for time in time_list[:]: # print(time) base_path = arrow.get(time['ini']).format('YYYYMMDDHH') # --预报数据处理 gefs_fcst = GEFSFcst(data_path['gefs_fcst'], time, base_path) p = ProcessPool(7) for n in range(21): # gefs_fcst.download(n) p.apipe(download, gefs_fcst, n) p.close() p.join() p.clear()
def extract_all(self, merge_file, locate, time): extract_path = self.data_path + 'extract_{}/'.format( time.format('YYYYMMDDHH')) try: os.makedirs(extract_path) except OSError: pass finally: p = ProcessPool(16) for lat, lon in locate[:]: p.apipe(self.extract_point, merge_file, extract_path, lat, lon) p.close() p.join() p.clear() os.remove(merge_file)
def test_07_concurrent_write_rename(self): # Test setup - validate that graph exists and possible results are None graphs[0].query("MATCH (n) RETURN n") pool = Pool(nodes=1) redis_con = self.env.getConnection() new_graph = GRAPH_ID + "2" # Create new empty graph with id GRAPH_ID + "2" redis_con.execute_command("GRAPH.QUERY", new_graph, """MATCH (n) return n""", "--compact") heavy_write_query = """UNWIND(range(0,999999)) as x CREATE(n)""" writer = pool.apipe(thread_run_query, graphs[0], heavy_write_query) redis_con.rename(GRAPH_ID, new_graph) writer.wait() # Possible scenarios: # 1. Rename is done before query is sent. The name in the graph context is new_graph, so when upon commit, when trying to open new_graph key, it will encounter an empty key since new_graph is not a valid key. # Note: As from https://github.com/RedisGraph/RedisGraph/pull/820 this may not be valid since the rename event handler might actually rename the graph key, before the query execution. # 2. Rename is done during query executing, so when commiting and comparing stored graph context name (GRAPH_ID) to the retrived value graph context name (new_graph), the identifiers are not the same, since new_graph value is now stored at GRAPH_ID value. possible_exceptions = [ "Encountered different graph value when opened key " + GRAPH_ID, "Encountered an empty key when opened key " + new_graph ] result = writer.get() if isinstance(result, str): self.env.assertContains(result, possible_exceptions) else: self.env.assertEquals(1000000, result.nodes_created)
def test_08_concurrent_write_replace(self): # Test setup - validate that graph exists and possible results are None self.graph.query("MATCH (n) RETURN n") pool = Pool(nodes=1) heavy_write_query = """UNWIND(range(0,999999)) as x CREATE(n) RETURN count(n)""" writer = pool.apipe(thread_run_query, heavy_write_query, None) set_result = self.conn.set(GRAPH_ID, "1") writer.wait() possible_exceptions = [ "Encountered a non-graph value type when opened key " + GRAPH_ID, "WRONGTYPE Operation against a key holding the wrong kind of value" ] result = writer.get() if isinstance(result, str): # If the SET command attempted to execute while the CREATE query was running, # an exception should have been issued. self.env.assertContains(result, possible_exceptions) else: # Otherwise, both the CREATE query and the SET command should have succeeded. self.env.assertEquals(1000000, result.result_set[0][0]) self.env.assertEquals(set_result, True) # Delete the key self.conn.delete(GRAPH_ID)
def map_reduce_multicore( f: tp.Callable[..., ResultType], reduction: tp.Callable[[ResultType, ResultType], ResultType], initial_value: tp.Optional[ResultType] = None, args_list: tp.Optional[tp.Sequence[tp.Sequence]] = None, kwargs_list: tp.Optional[tp.Sequence[tp.Dict[str, tp.Any]]] = None, number_of_batches: tp.Optional[int] = None, multiprocessing_pool_type: MultiprocessingPoolType = MultiprocessingPoolType.default()) \ -> ResultType: if number_of_batches is None: if args_list is not None: number_of_batches = len(args_list) elif kwargs_list is not None: number_of_batches = len(kwargs_list) else: raise ValueError('Number_of_batches must be defined if ' 'both args_list and kwargs_list are empty') if args_list is None: args_list = number_of_batches * [list()] if kwargs_list is None: kwargs_list = number_of_batches * [dict()] result = initial_value if multiprocessing_pool_type == MultiprocessingPoolType.LOKY: from concurrent.futures import as_completed from loky import get_reusable_executor executor = \ get_reusable_executor(timeout=None, context='loky') futures = [ executor.submit(f, *args, **kwargs) for args, kwargs in zip(args_list, kwargs_list) ] result_from_future = lambda x: x.result() elif multiprocessing_pool_type == MultiprocessingPoolType.PATHOS: from pathos.pools import ProcessPool pool = ProcessPool() futures = [ pool.apipe(f, *args, **kwargs) for args, kwargs in zip(args_list, kwargs_list) ] result_from_future = lambda x: x.get() else: raise ValueError( f'Multiprocessing pool type {multiprocessing_pool_type} not supported' ) for future in futures: result = reduce_with_none(result, result_from_future(future), reduction) return result
def test02_write_only_workload(self): pool = Pool(nodes=3) n_creations = 20000 n_node_deletions = 10000 n_edge_deletions = 10000 self.env.start() # invoke queries t1 = pool.apipe(merge_nodes_and_edges, graphs[0], n_creations) t2 = pool.apipe(delete_nodes, graphs[1], n_node_deletions) t3 = pool.apipe(delete_edges, graphs[2], n_edge_deletions) # wait for processes to join t1.wait() t2.wait() t3.wait() # make sure we did not crash conn = self.env.getConnection() conn.ping() conn.close()
def test_06_concurrent_write_delete(self): # Test setup - validate that graph exists and possible results are None self.graph.query("MATCH (n) RETURN n") pool = Pool(nodes=1) heavy_write_query = """UNWIND(range(0,999999)) as x CREATE(n) RETURN count(n)""" writer = pool.apipe(thread_run_query, heavy_write_query, None) self.conn.delete(GRAPH_ID) writer.wait() possible_exceptions = ["Encountered different graph value when opened key " + GRAPH_ID, "Encountered an empty key when opened key " + GRAPH_ID] result = writer.get() if isinstance(result, str): self.env.assertContains(result, possible_exceptions) else: self.env.assertEquals(1000000, result["result_set"][0][0])
class ComputeDevicePool: def __init__(self, compute_devices: tp.Optional[tp.Iterable[tp.Union[int, ComputeDevice]]] = None, compute_device_filter: tp.Optional[ComputeDeviceFilter] = exclude_intel_devices, multiprocessing_pool_type: MultiprocessingPoolType = MultiprocessingPoolType.default()) \ -> None: """ This method constructs a compute device pool from a collection of individual devices. :param compute_devices: a collection of device ids or compute devices :param compute_device_filter: provide a predicate used to filter devices to include in the pool :param multiprocessing_pool_type: the type of multi-processing pool (see class MultiprocessingPoolType) """ if compute_devices is None: compute_devices = ComputeDeviceManager.get_compute_devices() self._compute_devices \ = _get_set_of_compute_devices_from_iterable(compute_devices) if compute_device_filter is not None: compute_devices = \ filter(compute_device_filter, [compute_device for compute_device in self._compute_devices]) self._compute_devices = frozenset(compute_devices) if exclude_intel_devices: compute_devices = \ filter(lambda x: 'intel' not in x.name.lower(), [compute_device for compute_device in self._compute_devices]) self._compute_devices = frozenset(compute_devices) # ctx = multiprocessing.get_context("spawn") # self._executor = ProcessPoolExecutor(max_workers=self._n_gpus, # mp_context=ctx) if multiprocessing_pool_type == MultiprocessingPoolType.LOKY: from loky import get_reusable_executor, wait self._executor = get_reusable_executor( max_workers=self.number_of_devices, timeout=None, context='loky') futures = [ self._executor.submit(_init_gpu_in_process, device_id=compute_device.id) for compute_device in self._compute_devices ] wait(futures) [future.result() for future in futures] elif multiprocessing_pool_type == MultiprocessingPoolType.PATHOS: from pathos.pools import ProcessPool self._executor = ProcessPool(nodes=self.number_of_devices) futures = [ self._executor.apipe(_init_gpu_in_process, device_id=compute_device.id) for compute_device in self._compute_devices ] for future in futures: while not future.ready(): pass else: raise ValueError( f'Multiprocessing pool type {multiprocessing_pool_type} not supported' ) self._multiprocessing_pool_type = multiprocessing_pool_type @property def compute_devices(self) -> tp.FrozenSet[ComputeDevice]: return self._compute_devices @property def number_of_devices(self) -> int: return len(self.compute_devices) @property def multiprocessing_pool_type(self) -> MultiprocessingPoolType: return self._multiprocessing_pool_type def sync(self): for compute_device in self._compute_devices: compute_device.sync() def map_reduce( self, f: tp.Callable[..., ResultType], reduction: tp.Callable[[ResultType, ResultType], ResultType], initial_value: ResultType, host_to_device_transfer_function: tp.Optional[ParameterTransferFunction] = None, device_to_host_transfer_function: tp.Optional[tp.Callable[[ResultType], ResultType]] = None, args_list: tp.Optional[tp.Sequence[tp.Sequence]] = None, kwargs_list: tp.Optional[tp.Sequence[tp.Dict[str, tp.Any]]] = None, number_of_batches: tp.Optional[int] = None) \ -> ResultType: """ This method evaluates the function 'f' on elements of 'args_list' and 'kwargs_list' in parallel on multiple devices and performs the reduction by calling the function 'reduction' on the result and the result of the reductions so far to eventually produce one final result of type 'ResultType'. The reduce step is performed from the left and results are being processed in the same order as they appear in `args_list` and `kwargs_list`. Input data to the function f must initially reside in host memory and the user must provide functions 'host_to_device_transfer_function' and 'device_to_host_transfer_function' to transfer the data to and results from device memory respectively. If the arguments for each run of 'f' are identical and they have already been applied to the function that is passed then 'args_list' and 'kwargs_list' may both be None but the argument 'number_of_batches' must be specified so the method knows how many times to run the function 'f'. Args: f: The map function to be evaluated over elements of 'args_list' and 'kwargs_list'. reduction: The reduction to be performed on the results of 'f'. This is done on the host (not the device). initial_value: The initial value of the reduction (i.e. the neutral element). host_to_device_transfer_function: A function that transfers elements of args_list and kwargs_list from host memory to device memory. device_to_host_transfer_function: A function that transfers results from device to host memory. args_list: A sequence of sequences of positional arguments. kwargs_list: A sequence of dictionaries of keyword arguments. number_of_batches: The number of function evaluations is required if 'args_list' and 'kwargs_list' are both empty. """ args_list, kwargs_list, number_of_batches = \ _extract_arguments_and_number_of_batches( args_list=args_list, kwargs_list=kwargs_list, number_of_batches=number_of_batches) def synced_f(index, *args, **kwargs) -> ResultType: if host_to_device_transfer_function is not None: args, kwargs = host_to_device_transfer_function( *args, **kwargs) sync() result = f(*args, **kwargs) if device_to_host_transfer_function is not None: result = device_to_host_transfer_function(result) sync() return index, result results = [] if self.multiprocessing_pool_type == MultiprocessingPoolType.LOKY: from loky import as_completed futures = [ self._executor.submit(synced_f, i, *args, **kwargs) for i, (args, kwargs) in enumerate(zip(args_list, kwargs_list)) ] for future in as_completed(futures): results.append(future.result()) # result = reduction(result, future.result()) elif self.multiprocessing_pool_type == MultiprocessingPoolType.PATHOS: futures = [ self._executor.apipe(synced_f, i, *args, **kwargs) for i, (args, kwargs) in enumerate(zip(args_list, kwargs_list)) ] for future in futures: results.append(future.get()) # result = reduction(result, future.get()) else: raise ValueError( f'Multiprocessing pool type {self.multiprocessing_pool_type} not supported' ) results = sorted(results, key=lambda x: x[0]) results = [result[1] for result in results] result = initial_value for new_result in results: result = reduction(result, new_result) return result def map_combine(self, f: tp.Callable[..., ResultType], combination: tp.Callable[[tp.Iterable[ResultType]], ResultType], host_to_device_transfer_function: tp. Optional[ParameterTransferFunction] = None, device_to_host_transfer_function: tp.Optional[tp.Callable[ [ResultType], ResultType]] = None, args_list: tp.Optional[tp.Sequence[tp.Sequence]] = None, kwargs_list: tp.Optional[tp.Sequence[tp.Dict[ str, tp.Any]]] = None, number_of_batches: tp.Optional[int] = None) -> ResultType: """ This method evaluates the function `f` on elements of `args_list` and `kwargs_list` in parallel on multiple devices and aggregates results in a single step by calling the function `combination` with a list of all results. Results provided to `combination` are in the same order as they appear in `args_list` and `kwargs_list`. Input data to the function f must initially reside in host memory and the user must provide functions 'host_to_device_transfer_function' and 'device_to_host_transfer_function' to transfer the data to and results from device memory respectively. If the arguments for each run of 'f' are identical and they have already been applied to the function that is passed then 'args_list' and 'kwargs_list' may both be None but the argument 'number_of_batches' must be specified so the method knows how many times to run the function 'f'. Args: f: The map function to be evaluated over elements of 'args_list' and 'kwargs_list'. combination: A function that aggregates a list of all results in a single step host_to_device_transfer_function: A function that transfers elements of args_list and kwargs_list from host memory to device memory. device_to_host_transfer_function: A function that transfers results from device to host memory. args_list: A sequence of sequences of positional arguments. kwargs_list: A sequence of dictionaries of keyword arguments. number_of_batches: The number of function evaluations is required if 'args_list' and 'kwargs_list' are both empty. """ args_list, kwargs_list, number_of_batches = \ _extract_arguments_and_number_of_batches( args_list=args_list, kwargs_list=kwargs_list, number_of_batches=number_of_batches) def synced_f(index, *args, **kwargs) -> ResultType: if host_to_device_transfer_function is not None: args, kwargs = host_to_device_transfer_function( *args, **kwargs) sync() result = f(*args, **kwargs) if device_to_host_transfer_function is not None: result = device_to_host_transfer_function(result) sync() return index, result results = [] if self.multiprocessing_pool_type == MultiprocessingPoolType.LOKY: from loky import as_completed futures = [ self._executor.submit(synced_f, i, *args, **kwargs) for i, (args, kwargs) in enumerate(zip(args_list, kwargs_list)) ] for future in as_completed(futures): results.append(future.result()) elif self.multiprocessing_pool_type == MultiprocessingPoolType.PATHOS: futures = [ self._executor.apipe(synced_f, i, *args, **kwargs) for i, (args, kwargs) in enumerate(zip(args_list, kwargs_list)) ] for future in futures: results.append(future.get()) else: raise ValueError( f'Multiprocessing pool type {self.multiprocessing_pool_type} not supported' ) results = sorted(results, key=lambda x: x[0]) results = [result[1] for result in results] return combination(results)
Msub = np.sum(SH_M[iChild]) Mrem = SH_M1_GM_wChild[i] - Msub if (Mrem >= 10**11) and (Mrem < 10**12.5): GAlist[i] = SH_id1_GM_wChild[i] return GAlist Nproc = 20 IDs_split = np.array_split(np.arange(SH_id1_GM_wChild.size), Nproc) pool = ProcessPool(Nproc) result = {} for i in range(Nproc): result['res{}'.format(i)] = pool.apipe(level1, SH_id1_GM_wChild, SH_M1_GM_wChild, SH_pid, SH_M, IDs_split[i]) ans = {} GAlist = np.concatenate( [result['res{}'.format(i)].get() for i in np.arange(Nproc)]) #for i in range(Nproc): #ans['ans{}'.format(i)] = result['res{}'.format(i)].get() GA1b = GAlist[GAlist > 0] print('No. of GA at lev1 w children:', len(GA1b)) GA1 = np.concatenate((GA1a, GA1b))
def map_combine_multicore( f: tp.Callable[..., ResultType], combination: tp.Callable[[tp.Iterable[ResultType]], ResultType], args_list: tp.Optional[tp.Sequence[tp.Sequence]] = None, kwargs_list: tp.Optional[tp.Sequence[tp.Dict[str, tp.Any]]] = None, number_of_batches: tp.Optional[int] = None, multiprocessing_pool_type: MultiprocessingPoolType = MultiprocessingPoolType.default()) \ -> ResultType: """ This function evaluates the function `f` on elements of `args_list` and `kwargs_list` in parallel on multiple cpu cores and aggregates results in a single step by calling the function `combination` with a list of all results. Results provided to `combination` are in the same order as they appear in `args_list` and `kwargs_list`. If the arguments for each run of 'f' are identical and they have already been applied to the function that is passed then 'args_list' and 'kwargs_list' may both be None but the argument 'number_of_batches' must be specified so the method knows how many times to run the function 'f'. Args: f: The map function to be evaluated over elements of 'args_list' and 'kwargs_list'. combination: A function that aggregates a list of all results in a single step args_list: A sequence of sequences of positional arguments. kwargs_list: A sequence of dictionaries of keyword arguments. number_of_batches: The number of function evaluations is required if 'args_list' and 'kwargs_list' are both empty. multiprocessing_pool_type: the type of multi-processing pool (see class MultiprocessingPoolType) """ args_list, kwargs_list, number_of_batches = \ _extract_arguments_and_number_of_batches( args_list=args_list, kwargs_list=kwargs_list, number_of_batches=number_of_batches) def wrapped_f(index, *args, **kwargs) -> ResultType: return index, f(*args, **kwargs) results = [] if multiprocessing_pool_type == MultiprocessingPoolType.LOKY: from concurrent.futures import as_completed from loky import get_reusable_executor executor = \ get_reusable_executor(timeout=None, context='loky') futures = [executor.submit(wrapped_f, i, *args, **kwargs) for i, (args, kwargs) in enumerate(zip(args_list, kwargs_list))] for future in as_completed(futures): results.append(future.result()) elif multiprocessing_pool_type == MultiprocessingPoolType.PATHOS: from pathos.pools import ProcessPool pool = ProcessPool() futures = [pool.apipe(wrapped_f, i, *args, **kwargs) for i, (args, kwargs) in enumerate(zip(args_list, kwargs_list))] for future in futures: results.append(future.get()) else: raise ValueError(f'Multiprocessing pool type {multiprocessing_pool_type} not supported') results = sorted(results, key=lambda x: x[0]) results = [result[1] for result in results] # print(f'len(results)={len(results)}') # for result in results: # print(result.shape) return combination(results)
def map_reduce_multicore( f: tp.Callable[..., ResultType], reduction: tp.Callable[[ResultType, ResultType], ResultType], initial_value: ResultType, args_list: tp.Optional[tp.Sequence[tp.Sequence]] = None, kwargs_list: tp.Optional[tp.Sequence[tp.Dict[str, tp.Any]]] = None, number_of_batches: tp.Optional[int] = None, multiprocessing_pool_type: MultiprocessingPoolType = MultiprocessingPoolType.default()) \ -> ResultType: """ This function evaluates the function 'f' on elements of 'args_list' and 'kwargs_list' in parallel on multiple cpu cores and performs the reduction by calling the function 'reduction' on the result and the result of the reductions so far to eventually produce one final result of type 'ResultType'. The reduce step is performed from the left and results are being processed in the same order as they appear in `args_list` and `kwargs_list`. If the arguments for each run of 'f' are identical and they have already been applied to the function that is passed then 'args_list' and 'kwargs_list' may both be None but the argument 'number_of_batches' must be specified so the method knows how many times to run the function 'f'. Args: f: The map function to be evaluated over elements of 'args_list' and 'kwargs_list'. reduction: The reduction to be performed on the results of 'f'. This is done on the host (not the device). initial_value: The initial value of the reduction (i.e. the neutral element). args_list: A sequence of sequences of positional arguments. kwargs_list: A sequence of dictionaries of keyword arguments. number_of_batches: The number of function evaluations is required if 'args_list' and 'kwargs_list' are both empty. multiprocessing_pool_type: the type of multi-processing pool (see class MultiprocessingPoolType) """ args_list, kwargs_list, number_of_batches = \ _extract_arguments_and_number_of_batches( args_list=args_list, kwargs_list=kwargs_list, number_of_batches=number_of_batches) def wrapped_f(index, *args, **kwargs) -> ResultType: return index, f(*args, **kwargs) if multiprocessing_pool_type == MultiprocessingPoolType.LOKY: from concurrent.futures import as_completed from loky import get_reusable_executor executor = \ get_reusable_executor(timeout=None, context='loky') futures = [executor.submit(wrapped_f, i, *args, **kwargs) for i, (args, kwargs) in enumerate(zip(args_list, kwargs_list))] result_from_future = lambda x: x.result() elif multiprocessing_pool_type == MultiprocessingPoolType.PATHOS: from pathos.pools import ProcessPool pool = ProcessPool() futures = [pool.apipe(wrapped_f, i, *args, **kwargs) for i, (args, kwargs) in enumerate(zip(args_list, kwargs_list))] result_from_future = lambda x: x.get() else: raise ValueError(f'Multiprocessing pool type {multiprocessing_pool_type} not supported') results = [result_from_future(future) for future in futures] results = sorted(results, key=lambda x: x[0]) results = [result[1] for result in results] result = initial_value for new_result in results: result = reduction(result, new_result) return result
class MainLoop(metaclass=ABCMeta): """ Defines the logical loop of a game, running MAX_FPS times per second, sending the inputs to the HumanControllerWrapper, and the game updates to the BotControllerWrappers. """ def __init__(self, api: API): """ Instantiates the logical loop Args: api: The game to run in this loop """ self.api = api self.game = api.game self.game.addCustomMoveFunc = self._addCustomMove self._currentTurnTaken = False self._screen = None self._state = CONTINUE # The game must go on at start self._eventsToSend = {} # type: Dict[ControllerWrapper, List[Event]] self.wrappersConnection = { } # type: Dict[ControllerWrapper, PipeConnection] self.wrappersInfoConnection = { } # type: Dict[ControllerWrapper, PipeConnection] self.wrappers = {} # type: Dict[ControllerWrapper, Unit] self._unitsMoves = {} # type: Dict[Unit, Tuple[Path, Queue]] self._moveDescriptors = {} # type: Dict[Path, MoveDescriptor] self._otherMoves = {} # type: Dict[Unit, Path] self._killSent = { } # Used to maintain the fact that the kill event has been sent self.executor = None self._prepared = False self._frames = 0 # -------------------- PUBLIC METHODS -------------------- # def run(self, max_fps: int = MAX_FPS) -> Union[None, Tuple[Unit, ...]]: """ Launch the game and its logical loop Args: max_fps: The maximum frame per seconds of the game Returns: a tuple containing all the winning players, or an empty tuple in case of draw, or None if the game was closed by the user """ pygame.init() clock = pygame.time.Clock() assert self.game.board.graphics is not None try: self._screen = pygame.display.set_mode( self.game.board.graphics.size, DOUBLEBUF) except pygame.error: # No video device pass if not self._prepared: self._prepareLoop() for player_number in self.api.getPlayerNumbers(): self._sendEventsToController(player_number) while self._state != END: clock.tick(max_fps) self._frames += 1 self._handleInputs() if self._state == FINISH: self.executor.terminate() self._prepared = False return None elif self._state != PAUSE: self._state = self._checkGameState() if self._state == CONTINUE: self._getNextMoveFromControllerWrapperIfAvailable() self._handlePendingMoves() self._refreshScreen() self.executor.terminate() self._prepared = False return self.game.winningPlayers def addUnit(self, unit: Unit, wrapper: ControllerWrapper, tile_id: TileIdentifier, initial_action: MoveDescriptor = None, team: int = -1) -> None: """ Adds a unit to the game, located on the tile corresponding to the the given tile id and controlled by the given controller Args: unit: The unit to add to the game wrapper: The linker of that unit tile_id: The identifier of the tile it will be placed on initial_action: The initial action of the unit team: The number of the team this player is in (-1 = no team) """ is_controlled = wrapper is not None self.game.addUnit(unit, team, tile_id, is_avatar=is_controlled) if is_controlled: self._addControllerWrapper(wrapper, unit) if self._mustSendInitialWakeEvent(initial_action, unit): self._eventsToSend[wrapper].append(WakeEvent()) self._unitsMoves[unit] = (None, Queue()) tile = self.game.board.getTileById(tile_id) resize_unit(unit, self.game.board) unit.moveTo(tile.center) if initial_action is not None: unit.setLastAction(initial_action) self._handleEvent(unit, initial_action, wrapper.controller.playerNumber, force=True) def getWrapperFromPlayerNumber(self, player_number: int): """ Retrieves the wrapper from the given player number Args: player_number: The number representing the player for which we want the wrapper Returns: The wrapper that wraps the controller of the given player """ found = None for wrapper in self.wrappersConnection: if wrapper.controller.playerNumber == player_number: found = wrapper break return found def pause(self) -> None: """ Change the state of the game to "PAUSE" """ self._state = PAUSE print(self._frames, "frames") def resume(self) -> None: """ Resume the game """ self._state = CONTINUE # -------------------- PROTECTED METHODS -------------------- # def _refreshScreen(self) -> None: """ Update the visual state of the game """ try: if self._screen is None: raise pygame.error("No Video device") self.game.board.draw(self._screen) drawn_units = [] for unit in self.wrappers.values(): if unit.isAlive(): unit.draw(self._screen) drawn_units.append(unit) for unit in self.game.unitsLocation: if unit.isAlive() and unit not in drawn_units: unit.draw(self._screen) pygame.display.flip() except pygame.error: # No video device pass def _handleInputs(self) -> None: """ Handles all the user input (mouse and keyboard) """ try: events_got = pygame.event.get() except pygame.error: # No video device events_got = [] for event in events_got: if event.type == QUIT: self._state = FINISH elif event.type == KEYDOWN: if event.key == K_ESCAPE: if self._state == CONTINUE: self.pause() elif self._state == PAUSE: self.resume() else: self._dispatchInputToHumanControllers(event.key) elif event.type == MOUSEBUTTONDOWN: self._dispatchMouseEventToHumanControllers(event.pos) elif event.type == MOUSEBUTTONUP: self._dispatchMouseEventToHumanControllers(None, click_up=True) def _addMove(self, unit: Unit, move: Path) -> None: """ Adds a move (cancelling the pending moves) Args: unit: The unit for which add a move move: The move to add for the given controller """ if self._unitsMoves[unit][0] is not None: self._cancelCurrentMoves(unit) fifo = self._unitsMoves[unit][1] # type: Queue fifo.put(move) def _addCustomMove(self, unit: Unit, move: Path, event: MoveDescriptor) -> None: """ Adds a move that is NOT PERFORMED BY A CONTROLLER Args: unit: The unit that will be moved move: The move that will be performed """ if unit not in self._otherMoves or self._otherMoves[unit] is None: self._otherMoves[unit] = move self._moveDescriptors[move] = event def _cancelCurrentMoves(self, unit: Unit) -> None: """ Cancel the current movement if there is one and remove all the other pending movements. Args: unit: The unit for which cancel the movements """ if unit in self._unitsMoves: move_tuple = self._unitsMoves[unit] fifo = move_tuple[1] # type: Queue last_move = move_tuple[0] # type: Path new_fifo = Queue() if last_move is not None: last_move.stop() while True: try: move = fifo.get_nowait() del self._moveDescriptors[move] except Empty: break self._unitsMoves[unit] = (last_move, new_fifo) def _dispatchInputToHumanControllers(self, input_key) -> None: """ Handles keyboard events and send them to Human Controllers to trigger actions if needed Args: input_key: The key pressed on the keyboard """ for linker in self.wrappers: # type: HumanControllerWrapper if issubclass(type(linker), HumanControllerWrapper): self._getPipeConnection(linker).send( self.game.createKeyboardEvent( self._getUnitFromControllerWrapper(linker), input_key)) def _dispatchMouseEventToHumanControllers(self, pixel: Optional[Coordinates], click_up=False) -> None: """ Handles mouse events and send them to Human Controllers to trigger actions if needed Args: pixel: The pixel clicked click_up: True if the button was released, False if the button was pressed """ tile = None if pixel is not None: tile = self.game.board.getTileByPixel(pixel) self._previouslyClickedTile = tile mouse_state = pygame.mouse.get_pressed() for linker in self.wrappers: # type: ControllerWrapper if issubclass(type(linker), HumanControllerWrapper): tile_id = None if tile is not None: tile_id = tile.identifier self._getPipeConnection(linker).send( self.game.createMouseEvent( self._getUnitFromControllerWrapper(linker), pixel, mouse_state, click_up, tile_id)) def _getNextMoveFromControllerWrapperIfAvailable(self) -> None: """ Gets event from the controllers and dispatch them to the right method """ for current_wrapper in self.wrappersConnection: # type: ControllerWrapper pipe_conn = self._getPipeConnection(current_wrapper) if pipe_conn.poll(): move = pipe_conn.recv() if self._mustRetrieveNextMove(current_wrapper): self._handleEvent(self.wrappers[current_wrapper], move, current_wrapper.controller.playerNumber) def _handlePendingMoves(self) -> None: """ Get the next move to be performed and perform its next step """ moved_units = [] completed_moves = { } # type: Dict[Unit, Tuple[TileIdentifier, MoveDescriptor]] just_started = {} # type: Dict[int, MoveDescriptor] illegal_moves = [] # type: List[Unit] impossible_moves = {} # type: List[Unit] self._handleOtherMoves(completed_moves, illegal_moves, impossible_moves, just_started, moved_units) self._handleMoves(completed_moves, illegal_moves, impossible_moves, just_started, moved_units) self._updateFromMoves(completed_moves, illegal_moves, impossible_moves, just_started) def _updateFromMoves(self, completed_moves, illegal_moves, impossible_moves, just_started): players_to_be_sent_messages = [] for unit, (tile_id, move_descriptor) in completed_moves.items(): self.game.updateGameState(unit, tile_id, move_descriptor) for player_number, move_descriptor in just_started.items(): controller_unit = self.game.getControllerUnitForNumber( player_number) if controller_unit is not None: controller_unit.setCurrentAction(move_descriptor) self._addMessageToSendToAll(player_number, move_descriptor) players_to_be_sent_messages = self._getPlayerNumbersToWhichSendEvents( ) for player_number in players_to_be_sent_messages: self._sendEventsToController(player_number) for unit in illegal_moves: self.game.unitsLocation[ unit] = self.game.board.OUT_OF_BOARD_TILE.identifier self._killUnit(unit, self.getWrapperFromPlayerNumber(unit.playerNumber)) # self.game.checkIfFinished() self._cancelCurrentMoves(unit) for unit in impossible_moves: self._cancelCurrentMoves(unit) self.game.checkIfFinished() def _handleMoves(self, completed_moves, illegal_moves, impossible_moves, just_started, moved_units): for wrapper in self.wrappers: # type: ControllerWrapper unit = self._getUnitFromControllerWrapper(wrapper) if unit not in moved_units: # Two moves on the same unit cannot be performed at the same time... if not unit.isAlive() and (unit not in self._killSent or not self._killSent[unit]): self.wrappersInfoConnection[wrapper].send( SpecialEvent(flag=SpecialEvent.UNIT_KILLED)) self._killSent[unit] = True current_move = self._getNextMoveForUnitIfAvailable(unit) if current_move is not None: move_state = self._performNextStepOfMove( current_move.unit, current_move) self._fillMoveStructures(completed_moves, just_started, illegal_moves, impossible_moves, current_move, move_state) def _handleOtherMoves(self, completed_moves, illegal_moves, impossible_moves, just_started, moved_units): for unit in self._otherMoves: # type: Unit move = self._otherMoves[unit] if move is not None: move_state = self._performNextStepOfMove(move.unit, move) if move_state != MOVE_FAILED: moved_units.append(move.unit) if move.finished(): self._otherMoves[unit] = None self._fillMoveStructures(completed_moves, just_started, illegal_moves, impossible_moves, move, move_state) def _fillMoveStructures(self, completed_moves: Dict[Unit, Tuple[TileIdentifier, MoveDescriptor]], just_started: Dict[int, MoveDescriptor], illegal_moves: List[Unit], impossible_moves: List[Unit], move: Path, move_state: int): """ Takes a move's state and the data structures of the performed moves in this iteration an fill them following the state's value Args: completed_moves: The dict containing the units that completed a move along with their new tile_id just_started: The dict containing the number of the units that started a move, along with the descriptor of the started move illegal_moves: The list containing all the units that performed an illegal move this iteration impossible_moves: The list containing all the units that performed an impossible move this iteration move: The performed move move_state: The state of the performed move """ if move_state == MOVE_COMPLETED_AND_JUST_STARTED: completed_moves[move.unit] = (move.reachedTileIdentifier, self._moveDescriptors[move]) just_started[move.unit.playerNumber] = self._moveDescriptors[move] elif move_state == MOVE_COMPLETED: completed_moves[move.unit] = (move.reachedTileIdentifier, self._moveDescriptors[move]) elif move_state == MOVE_JUST_STARTED: just_started[move.unit.playerNumber] = self._moveDescriptors[move] elif move_state == MOVE_ILLEGAL: illegal_moves.append(move.unit) elif move_state == MOVE_IMPOSSIBLE: impossible_moves.append(move.unit) def _addMessageToSendToAll(self, moved_unit_number: int, move_descriptor: MoveDescriptor): """ Adds a message to the message queue of each ControllerWrapper Args: moved_unit_number: The number representing the unit that moved move_descriptor: The descriptor of the performed move """ controlled_unit = self.game.getControllerUnitForNumber( moved_unit_number) if controlled_unit is None: controlled_unit_number = moved_unit_number else: controlled_unit_number = controlled_unit.playerNumber for wrapper in self._eventsToSend: self._eventsToSend[wrapper].append( BotEvent(controlled_unit_number, move_descriptor)) @staticmethod def _performNextStepOfMove(unit: Unit, current_move: Path) -> int: """ Perform the next step of the given move on the given unit Args: unit: The unit that performs the move current_move: The current move to perform Returns: A couple of booleans. The first indicating that the move has been completed and the second indicating that the move has just started """ if unit.isAlive(): if current_move is not None: try: just_started, move_completed, tile_id = current_move.performNextMove( ) if just_started and move_completed: return MOVE_COMPLETED_AND_JUST_STARTED elif move_completed: # A new tile has been reached by the movement return MOVE_COMPLETED elif just_started: return MOVE_JUST_STARTED return MOVE_IN_PROGRESS except IllegalMove: return MOVE_ILLEGAL except ImpossibleMove: return MOVE_IMPOSSIBLE else: if current_move is not None: current_move.stop(cancel_post_action=True) return MOVE_FAILED def _getNextMoveForUnitIfAvailable(self, unit: Unit) -> Union[Path, None]: """ Checks if a move is available for the given controller, and if so, returns it Args: unit: The given Returns: The next move if it is available, and None otherwise """ moves = self._unitsMoves[unit] current_move = moves[0] # type: Path if current_move is None or current_move.finished(): if current_move is not None: if isinstance(current_move, Path): self._reactToFinishedMove() del self._moveDescriptors[current_move] try: move = moves[1].get_nowait() # type: Path self._unitsMoves[unit] = (move, moves[1]) current_move = move except Empty: self._unitsMoves[unit] = (None, moves[1]) current_move = None return current_move def _checkGameState(self) -> int: """ Checks if the game is finished Returns: 0 = CONTINUE; 2 = END """ if self.game.isFinished(): self.winningPlayers = self.game.winningPlayers return END return CONTINUE def _handleEvent(self, unit: Unit, event: MoveDescriptor, player_number: int, force: bool = False) -> None: """ The goal of this method is to handle the given event for the given unit Args: unit: The unit that sent the event through its linker event: The event sent by the controller """ try: move = self.api.createMoveForDescriptor( unit, event, force=force) # may raise: UnfeasibleMoveException self._currentTurnTaken = True self._moveDescriptors[move] = event self._addMove(unit, move) except UnfeasibleMoveException: self._sendEventsToController(player_number, event=WakeEvent()) def _getPipeConnection(self, linker: ControllerWrapper) -> PipeConnection: """ Args: linker: The linker for which we want the pipe connection Returns: The pipe connection to send and receive game updates """ return self.wrappersConnection[linker] def _getUnitFromControllerWrapper(self, linker: ControllerWrapper) -> Unit: """ Args: linker: The linker for which we want the unit Returns: The unit for the given linker """ return self.wrappers[linker] def _sendEventsToController(self, player_number: int, event: Event = None): player_wrapper = self.getWrapperFromPlayerNumber(player_number) pipe_conn = self._getPipeConnection(player_wrapper) if event is None: events = self._eventsToSend[player_wrapper] if len(events) > 0: event = MultipleEvents(events) if event is not None: pipe_conn.send(event) self._eventsToSend[player_wrapper] = [] def _informBotOnPerformedMove(self, moved_unit_number: int, move_descriptor: MoveDescriptor) -> None: """ Update the game state of the bot controllers Args: moved_unit_number: The number representing the unit that moved and caused the update move_descriptor: The move that caused the update """ for wrapper in self.wrappers: if issubclass(type(wrapper), BotControllerWrapper): pipe_conn = self._getPipeConnection(wrapper) pipe_conn.send(BotEvent(moved_unit_number, move_descriptor)) def _killUnit(self, unit: Unit, linker: ControllerWrapper) -> None: """ Kills the given unit and tells its linker Args: unit: The unit to kill linker: The linker, to which tell that the unit is dead """ unit.kill() if not unit.isAlive(): self.wrappersInfoConnection[linker].send( SpecialEvent(flag=SpecialEvent.UNIT_KILLED)) def _addCollaborationPipes(self, linker: BotControllerWrapper) -> None: """ Adds the collaboration pipes between the given linker and its teammate's Args: linker: The linker to connect with its teammate """ for teammate in self.game.teams[self.game.unitsTeam[ self.wrappers[linker]]]: if teammate is not self.wrappers[linker]: teammate_linker = None # type: BotControllerWrapper for other_linker in self.wrappers: if self.wrappers[other_linker] is teammate: teammate_linker = other_linker break pipe1, pipe2 = Pipe() linker.addCollaborationPipe( teammate_linker.controller.playerNumber, pipe1) teammate_linker.addCollaborationPipe( linker.controller.playerNumber, pipe2) def _prepareLoop(self) -> None: """ Launches the processes of the AIs """ self.executor = Pool(len(self.wrappers)) try: self.executor.apipe(lambda: None) except ValueError: self.executor.restart() for wrapper in self.wrappers: if isinstance(wrapper, BotControllerWrapper): wrapper.controller.gameState = self.game.copy() self.executor.apipe(wrapper.run) for wrapper in self.wrappers: pipe = self._getPipeConnection(wrapper) event = pipe.recv( ) # Waiting for the processes to launch correctly assert (isinstance(event, ReadyEvent)) self._prepared = True def _addControllerWrapper(self, wrapper: ControllerWrapper, unit: Unit) -> None: """ Adds the linker to the loop, creating the pipe connections Args: wrapper: The linker to add unit: The unit, linked by this linker """ self.wrappers[wrapper] = unit parent_conn, child_conn = Pipe() parent_info_conn, child_info_conn = Pipe() self.wrappersConnection[wrapper] = parent_conn self.wrappersInfoConnection[wrapper] = parent_info_conn self._eventsToSend[wrapper] = [] wrapper.setMainPipe(child_conn) wrapper.setGameInfoPipe(child_info_conn) if isinstance(wrapper, BotControllerWrapper): self._addCollaborationPipes(wrapper) @abstractmethod def _mustSendInitialWakeEvent(self, initial_action: MoveDescriptor, unit: Unit) -> bool: pass @abstractmethod def _mustRetrieveNextMove(self, current_wrapper: ControllerWrapper) -> bool: pass @abstractmethod def _getPlayerNumbersToWhichSendEvents(self) -> List[int]: pass @abstractmethod def _reactToFinishedMove(self): pass