def test01_bgsave_stress(self):
n_reads = 50000
n_creations = 50000
n_updates = n_creations / 10
n_deletions = n_creations / 2
conn = self.env.getConnection()
graphs[0].query("CREATE INDEX FOR (n:Node) ON (n.v)")
pool = Pool(nodes=5)
t1 = pool.apipe(create_nodes, graphs[0], n_creations)
t2 = pool.apipe(delete_nodes, graphs[1], n_deletions)
t3 = pool.apipe(read_nodes, graphs[2], n_reads)
t4 = pool.apipe(update_nodes, graphs[3], n_updates)
t5 = pool.apipe(BGSAVE_loop, self.env, conn, 10000)
# wait for processes to join
t1.wait()
t2.wait()
t3.wait()
t4.wait()
t5.wait()
# make sure we did not crashed
conn.ping()
conn.close()
def test01_bgsave_stress(self):
n_nodes = 1000
n_iterations = 10
conn = self.env.getConnection()
graphs[0].query("CREATE INDEX ON :Node(val)")
pool = Pool(nodes=5)
t1 = pool.apipe(create_nodes, graphs[0], n_iterations, n_nodes)
t2 = pool.apipe(delete_nodes, graphs[1], n_iterations, n_nodes / 2)
t3 = pool.apipe(read_nodes, graphs[2], n_iterations)
t4 = pool.apipe(update_nodes, graphs[3], n_iterations)
t5 = pool.apipe(BGSAVE_loop, self.env, conn, 3)
# wait for processes to join
t1.wait()
t2.wait()
t3.wait()
t4.wait()
t5.wait()
# make sure we did not crashed
conn.ping()
conn.close()
def reachability(model,
from_state,
goal,
max_length=2000,
on_start=None,
on_reach=None,
max_repeat=10000,
n_workers=1):
if isinstance(model, pypint.Model):
model = pypint_to_model(model)
if isinstance(goal, list) or isinstance(goal, dict):
goal = Goal(goal)
if isinstance(from_state, list):
if from_state:
if isinstance(from_state[0], str):
from_state = dict([(e, 1) for e in from_state])
elif isinstance(from_state[0], tuple):
from_state = dict(from_state)
from_state = complete_state(from_state, model)
trace = Trace(from_state)
if on_start is not None:
next_subgoal = goal.subgoals[0]
on_start(model, trace, next_subgoal)
if n_workers == 1:
for n_repeat in range(max_repeat):
reached, trace = _reach(copy.copy(model), from_state, goal,
max_length, on_start, on_reach)
if reached is True:
return reached, trace
else:
pool = ProcessPool(n_workers)
processes = set([])
n_repeat = 0
while n_repeat < max_repeat and n_repeat < n_workers:
processes.add(
pool.apipe(_reach, copy.copy(model), from_state, goal,
max_length, on_start, on_reach))
n_repeat += 1
reached = pypint.Inconc
while reached is not True and n_repeat < max_repeat:
for process in processes:
if process.ready():
reached, trace = process.get()
processes.remove(process)
if reached is True:
return reached, trace
else:
processes.add(
pool.apipe(_reach, copy.copy(model), from_state,
goal, max_length, on_start, on_reach))
n_repeat += 1
break
return reached, trace
def multi_process(data_path, time_list):
for time in time_list[:]:
# print(time)
base_path = arrow.get(time['ini']).format('YYYYMMDDHH')
# --预报数据处理
gefs_fcst = GEFSFcst(data_path['gefs_fcst'], time, base_path)
p = ProcessPool(7)
for n in range(21):
# gefs_fcst.download(n)
p.apipe(download, gefs_fcst, n)
p.close()
p.join()
p.clear()
def extract_all(self, merge_file, locate, time):
extract_path = self.data_path + 'extract_{}/'.format(
time.format('YYYYMMDDHH'))
try:
os.makedirs(extract_path)
except OSError:
pass
finally:
p = ProcessPool(16)
for lat, lon in locate[:]:
p.apipe(self.extract_point, merge_file, extract_path, lat, lon)
p.close()
p.join()
p.clear()
os.remove(merge_file)
def test_07_concurrent_write_rename(self):
# Test setup - validate that graph exists and possible results are None
graphs[0].query("MATCH (n) RETURN n")
pool = Pool(nodes=1)
redis_con = self.env.getConnection()
new_graph = GRAPH_ID + "2"
# Create new empty graph with id GRAPH_ID + "2"
redis_con.execute_command("GRAPH.QUERY", new_graph,
"""MATCH (n) return n""", "--compact")
heavy_write_query = """UNWIND(range(0,999999)) as x CREATE(n)"""
writer = pool.apipe(thread_run_query, graphs[0], heavy_write_query)
redis_con.rename(GRAPH_ID, new_graph)
writer.wait()
# Possible scenarios:
# 1. Rename is done before query is sent. The name in the graph context is new_graph, so when upon commit, when trying to open new_graph key, it will encounter an empty key since new_graph is not a valid key.
# Note: As from https://github.com/RedisGraph/RedisGraph/pull/820 this may not be valid since the rename event handler might actually rename the graph key, before the query execution.
# 2. Rename is done during query executing, so when commiting and comparing stored graph context name (GRAPH_ID) to the retrived value graph context name (new_graph), the identifiers are not the same, since new_graph value is now stored at GRAPH_ID value.
possible_exceptions = [
"Encountered different graph value when opened key " + GRAPH_ID,
"Encountered an empty key when opened key " + new_graph
]
result = writer.get()
if isinstance(result, str):
self.env.assertContains(result, possible_exceptions)
else:
self.env.assertEquals(1000000, result.nodes_created)
def test_08_concurrent_write_replace(self):
# Test setup - validate that graph exists and possible results are None
self.graph.query("MATCH (n) RETURN n")
pool = Pool(nodes=1)
heavy_write_query = """UNWIND(range(0,999999)) as x CREATE(n) RETURN count(n)"""
writer = pool.apipe(thread_run_query, heavy_write_query, None)
set_result = self.conn.set(GRAPH_ID, "1")
writer.wait()
possible_exceptions = [
"Encountered a non-graph value type when opened key " + GRAPH_ID,
"WRONGTYPE Operation against a key holding the wrong kind of value"
]
result = writer.get()
if isinstance(result, str):
# If the SET command attempted to execute while the CREATE query was running,
# an exception should have been issued.
self.env.assertContains(result, possible_exceptions)
else:
# Otherwise, both the CREATE query and the SET command should have succeeded.
self.env.assertEquals(1000000, result.result_set[0][0])
self.env.assertEquals(set_result, True)
# Delete the key
self.conn.delete(GRAPH_ID)
def map_reduce_multicore(
f: tp.Callable[..., ResultType],
reduction: tp.Callable[[ResultType, ResultType], ResultType],
initial_value: tp.Optional[ResultType] = None,
args_list: tp.Optional[tp.Sequence[tp.Sequence]] = None,
kwargs_list: tp.Optional[tp.Sequence[tp.Dict[str, tp.Any]]] = None,
number_of_batches: tp.Optional[int] = None,
multiprocessing_pool_type: MultiprocessingPoolType = MultiprocessingPoolType.default()) \
-> ResultType:
if number_of_batches is None:
if args_list is not None:
number_of_batches = len(args_list)
elif kwargs_list is not None:
number_of_batches = len(kwargs_list)
else:
raise ValueError('Number_of_batches must be defined if '
'both args_list and kwargs_list are empty')
if args_list is None:
args_list = number_of_batches * [list()]
if kwargs_list is None:
kwargs_list = number_of_batches * [dict()]
result = initial_value
if multiprocessing_pool_type == MultiprocessingPoolType.LOKY:
from concurrent.futures import as_completed
from loky import get_reusable_executor
executor = \
get_reusable_executor(timeout=None,
context='loky')
futures = [
executor.submit(f, *args, **kwargs)
for args, kwargs in zip(args_list, kwargs_list)
]
result_from_future = lambda x: x.result()
elif multiprocessing_pool_type == MultiprocessingPoolType.PATHOS:
from pathos.pools import ProcessPool
pool = ProcessPool()
futures = [
pool.apipe(f, *args, **kwargs)
for args, kwargs in zip(args_list, kwargs_list)
]
result_from_future = lambda x: x.get()
else:
raise ValueError(
f'Multiprocessing pool type {multiprocessing_pool_type} not supported'
)
for future in futures:
result = reduce_with_none(result, result_from_future(future),
reduction)
return result
def test02_write_only_workload(self):
pool = Pool(nodes=3)
n_creations = 20000
n_node_deletions = 10000
n_edge_deletions = 10000
self.env.start()
# invoke queries
t1 = pool.apipe(merge_nodes_and_edges, graphs[0], n_creations)
t2 = pool.apipe(delete_nodes, graphs[1], n_node_deletions)
t3 = pool.apipe(delete_edges, graphs[2], n_edge_deletions)
# wait for processes to join
t1.wait()
t2.wait()
t3.wait()
# make sure we did not crash
conn = self.env.getConnection()
conn.ping()
conn.close()
def test_06_concurrent_write_delete(self):
# Test setup - validate that graph exists and possible results are None
self.graph.query("MATCH (n) RETURN n")
pool = Pool(nodes=1)
heavy_write_query = """UNWIND(range(0,999999)) as x CREATE(n) RETURN count(n)"""
writer = pool.apipe(thread_run_query, heavy_write_query, None)
self.conn.delete(GRAPH_ID)
writer.wait()
possible_exceptions = ["Encountered different graph value when opened key " + GRAPH_ID,
"Encountered an empty key when opened key " + GRAPH_ID]
result = writer.get()
if isinstance(result, str):
self.env.assertContains(result, possible_exceptions)
else:
self.env.assertEquals(1000000, result["result_set"][0][0])
class ComputeDevicePool:
def __init__(self,
compute_devices:
tp.Optional[tp.Iterable[tp.Union[int, ComputeDevice]]] = None,
compute_device_filter:
tp.Optional[ComputeDeviceFilter] = exclude_intel_devices,
multiprocessing_pool_type: MultiprocessingPoolType = MultiprocessingPoolType.default()) \
-> None:
"""
This method constructs a compute device pool from a collection of
individual devices.
:param compute_devices: a collection of device ids or compute devices
:param compute_device_filter: provide a predicate used to filter devices
to include in the pool
:param multiprocessing_pool_type: the type of multi-processing pool
(see class MultiprocessingPoolType)
"""
if compute_devices is None:
compute_devices = ComputeDeviceManager.get_compute_devices()
self._compute_devices \
= _get_set_of_compute_devices_from_iterable(compute_devices)
if compute_device_filter is not None:
compute_devices = \
filter(compute_device_filter,
[compute_device
for compute_device
in self._compute_devices])
self._compute_devices = frozenset(compute_devices)
if exclude_intel_devices:
compute_devices = \
filter(lambda x: 'intel' not in x.name.lower(),
[compute_device
for compute_device
in self._compute_devices])
self._compute_devices = frozenset(compute_devices)
# ctx = multiprocessing.get_context("spawn")
# self._executor = ProcessPoolExecutor(max_workers=self._n_gpus,
# mp_context=ctx)
if multiprocessing_pool_type == MultiprocessingPoolType.LOKY:
from loky import get_reusable_executor, wait
self._executor = get_reusable_executor(
max_workers=self.number_of_devices,
timeout=None,
context='loky')
futures = [
self._executor.submit(_init_gpu_in_process,
device_id=compute_device.id)
for compute_device in self._compute_devices
]
wait(futures)
[future.result() for future in futures]
elif multiprocessing_pool_type == MultiprocessingPoolType.PATHOS:
from pathos.pools import ProcessPool
self._executor = ProcessPool(nodes=self.number_of_devices)
futures = [
self._executor.apipe(_init_gpu_in_process,
device_id=compute_device.id)
for compute_device in self._compute_devices
]
for future in futures:
while not future.ready():
pass
else:
raise ValueError(
f'Multiprocessing pool type {multiprocessing_pool_type} not supported'
)
self._multiprocessing_pool_type = multiprocessing_pool_type
@property
def compute_devices(self) -> tp.FrozenSet[ComputeDevice]:
return self._compute_devices
@property
def number_of_devices(self) -> int:
return len(self.compute_devices)
@property
def multiprocessing_pool_type(self) -> MultiprocessingPoolType:
return self._multiprocessing_pool_type
def sync(self):
for compute_device in self._compute_devices:
compute_device.sync()
def map_reduce(
self,
f: tp.Callable[..., ResultType],
reduction: tp.Callable[[ResultType, ResultType], ResultType],
initial_value: ResultType,
host_to_device_transfer_function:
tp.Optional[ParameterTransferFunction] = None,
device_to_host_transfer_function:
tp.Optional[tp.Callable[[ResultType], ResultType]] = None,
args_list: tp.Optional[tp.Sequence[tp.Sequence]] = None,
kwargs_list: tp.Optional[tp.Sequence[tp.Dict[str, tp.Any]]] = None,
number_of_batches: tp.Optional[int] = None) \
-> ResultType:
"""
This method evaluates the function 'f' on elements of 'args_list' and
'kwargs_list' in parallel on multiple devices and performs the reduction
by calling the function 'reduction' on the result and the result of the
reductions so far to eventually produce one final result of type
'ResultType'. The reduce step is performed from the left and results are
being processed in the same order as they appear in `args_list` and
`kwargs_list`.
Input data to the function f must initially reside in host memory and
the user must provide functions 'host_to_device_transfer_function' and
'device_to_host_transfer_function' to transfer the data to and results
from device memory respectively.
If the arguments for each run of 'f' are identical and they have already
been applied to the function that is passed then 'args_list' and
'kwargs_list' may both be None but the argument 'number_of_batches' must
be specified so the method knows how many times to run the function 'f'.
Args:
f: The map function to be evaluated over elements of 'args_list' and
'kwargs_list'.
reduction: The reduction to be performed on the results of 'f'.
This is done on the host (not the device).
initial_value: The initial value of the reduction
(i.e. the neutral element).
host_to_device_transfer_function:
A function that transfers elements of args_list and kwargs_list
from host memory to device memory.
device_to_host_transfer_function:
A function that transfers results from device to host memory.
args_list: A sequence of sequences of positional arguments.
kwargs_list: A sequence of dictionaries of keyword arguments.
number_of_batches:
The number of function evaluations is required if 'args_list'
and 'kwargs_list' are both empty.
"""
args_list, kwargs_list, number_of_batches = \
_extract_arguments_and_number_of_batches(
args_list=args_list,
kwargs_list=kwargs_list,
number_of_batches=number_of_batches)
def synced_f(index, *args, **kwargs) -> ResultType:
if host_to_device_transfer_function is not None:
args, kwargs = host_to_device_transfer_function(
*args, **kwargs)
sync()
result = f(*args, **kwargs)
if device_to_host_transfer_function is not None:
result = device_to_host_transfer_function(result)
sync()
return index, result
results = []
if self.multiprocessing_pool_type == MultiprocessingPoolType.LOKY:
from loky import as_completed
futures = [
self._executor.submit(synced_f, i, *args, **kwargs)
for i, (args, kwargs) in enumerate(zip(args_list, kwargs_list))
]
for future in as_completed(futures):
results.append(future.result())
# result = reduction(result, future.result())
elif self.multiprocessing_pool_type == MultiprocessingPoolType.PATHOS:
futures = [
self._executor.apipe(synced_f, i, *args, **kwargs)
for i, (args, kwargs) in enumerate(zip(args_list, kwargs_list))
]
for future in futures:
results.append(future.get())
# result = reduction(result, future.get())
else:
raise ValueError(
f'Multiprocessing pool type {self.multiprocessing_pool_type} not supported'
)
results = sorted(results, key=lambda x: x[0])
results = [result[1] for result in results]
result = initial_value
for new_result in results:
result = reduction(result, new_result)
return result
def map_combine(self,
f: tp.Callable[..., ResultType],
combination: tp.Callable[[tp.Iterable[ResultType]],
ResultType],
host_to_device_transfer_function: tp.
Optional[ParameterTransferFunction] = None,
device_to_host_transfer_function: tp.Optional[tp.Callable[
[ResultType], ResultType]] = None,
args_list: tp.Optional[tp.Sequence[tp.Sequence]] = None,
kwargs_list: tp.Optional[tp.Sequence[tp.Dict[
str, tp.Any]]] = None,
number_of_batches: tp.Optional[int] = None) -> ResultType:
"""
This method evaluates the function `f` on elements of `args_list` and
`kwargs_list` in parallel on multiple devices and aggregates results
in a single step by calling the function `combination` with a list of all
results. Results provided to `combination` are in the same order as
they appear in `args_list` and `kwargs_list`.
Input data to the function f must initially reside in host memory and
the user must provide functions 'host_to_device_transfer_function' and
'device_to_host_transfer_function' to transfer the data to and results
from device memory respectively.
If the arguments for each run of 'f' are identical and they have already
been applied to the function that is passed then 'args_list' and
'kwargs_list' may both be None but the argument 'number_of_batches' must
be specified so the method knows how many times to run the function 'f'.
Args:
f: The map function to be evaluated over elements of 'args_list' and
'kwargs_list'.
combination:
A function that aggregates a list of all results in a single step
host_to_device_transfer_function:
A function that transfers elements of args_list and kwargs_list
from host memory to device memory.
device_to_host_transfer_function:
A function that transfers results from device to host memory.
args_list: A sequence of sequences of positional arguments.
kwargs_list: A sequence of dictionaries of keyword arguments.
number_of_batches:
The number of function evaluations is required if 'args_list'
and 'kwargs_list' are both empty.
"""
args_list, kwargs_list, number_of_batches = \
_extract_arguments_and_number_of_batches(
args_list=args_list,
kwargs_list=kwargs_list,
number_of_batches=number_of_batches)
def synced_f(index, *args, **kwargs) -> ResultType:
if host_to_device_transfer_function is not None:
args, kwargs = host_to_device_transfer_function(
*args, **kwargs)
sync()
result = f(*args, **kwargs)
if device_to_host_transfer_function is not None:
result = device_to_host_transfer_function(result)
sync()
return index, result
results = []
if self.multiprocessing_pool_type == MultiprocessingPoolType.LOKY:
from loky import as_completed
futures = [
self._executor.submit(synced_f, i, *args, **kwargs)
for i, (args, kwargs) in enumerate(zip(args_list, kwargs_list))
]
for future in as_completed(futures):
results.append(future.result())
elif self.multiprocessing_pool_type == MultiprocessingPoolType.PATHOS:
futures = [
self._executor.apipe(synced_f, i, *args, **kwargs)
for i, (args, kwargs) in enumerate(zip(args_list, kwargs_list))
]
for future in futures:
results.append(future.get())
else:
raise ValueError(
f'Multiprocessing pool type {self.multiprocessing_pool_type} not supported'
)
results = sorted(results, key=lambda x: x[0])
results = [result[1] for result in results]
return combination(results)
Msub = np.sum(SH_M[iChild])
Mrem = SH_M1_GM_wChild[i] - Msub
if (Mrem >= 10**11) and (Mrem < 10**12.5):
GAlist[i] = SH_id1_GM_wChild[i]
return GAlist
Nproc = 20
IDs_split = np.array_split(np.arange(SH_id1_GM_wChild.size), Nproc)
pool = ProcessPool(Nproc)
result = {}
for i in range(Nproc):
result['res{}'.format(i)] = pool.apipe(level1, SH_id1_GM_wChild,
SH_M1_GM_wChild, SH_pid, SH_M,
IDs_split[i])
ans = {}
GAlist = np.concatenate(
[result['res{}'.format(i)].get() for i in np.arange(Nproc)])
#for i in range(Nproc):
#ans['ans{}'.format(i)] = result['res{}'.format(i)].get()
GA1b = GAlist[GAlist > 0]
print('No. of GA at lev1 w children:', len(GA1b))
GA1 = np.concatenate((GA1a, GA1b))
def map_combine_multicore(
f: tp.Callable[..., ResultType],
combination: tp.Callable[[tp.Iterable[ResultType]], ResultType],
args_list: tp.Optional[tp.Sequence[tp.Sequence]] = None,
kwargs_list: tp.Optional[tp.Sequence[tp.Dict[str, tp.Any]]] = None,
number_of_batches: tp.Optional[int] = None,
multiprocessing_pool_type: MultiprocessingPoolType = MultiprocessingPoolType.default()) \
-> ResultType:
"""
This function evaluates the function `f` on elements of `args_list` and
`kwargs_list` in parallel on multiple cpu cores and aggregates results
in a single step by calling the function `combination` with a list of all
results. Results provided to `combination` are in the same order as
they appear in `args_list` and `kwargs_list`.
If the arguments for each run of 'f' are identical and they have already
been applied to the function that is passed then 'args_list' and
'kwargs_list' may both be None but the argument 'number_of_batches' must
be specified so the method knows how many times to run the function 'f'.
Args:
f: The map function to be evaluated over elements of 'args_list' and
'kwargs_list'.
combination: A function that aggregates a list of all results in a single step
args_list: A sequence of sequences of positional arguments.
kwargs_list: A sequence of dictionaries of keyword arguments.
number_of_batches:
The number of function evaluations is required if 'args_list'
and 'kwargs_list' are both empty.
multiprocessing_pool_type:
the type of multi-processing pool (see class MultiprocessingPoolType)
"""
args_list, kwargs_list, number_of_batches = \
_extract_arguments_and_number_of_batches(
args_list=args_list,
kwargs_list=kwargs_list,
number_of_batches=number_of_batches)
def wrapped_f(index, *args, **kwargs) -> ResultType:
return index, f(*args, **kwargs)
results = []
if multiprocessing_pool_type == MultiprocessingPoolType.LOKY:
from concurrent.futures import as_completed
from loky import get_reusable_executor
executor = \
get_reusable_executor(timeout=None,
context='loky')
futures = [executor.submit(wrapped_f, i, *args, **kwargs)
for i, (args, kwargs)
in enumerate(zip(args_list, kwargs_list))]
for future in as_completed(futures):
results.append(future.result())
elif multiprocessing_pool_type == MultiprocessingPoolType.PATHOS:
from pathos.pools import ProcessPool
pool = ProcessPool()
futures = [pool.apipe(wrapped_f, i, *args, **kwargs)
for i, (args, kwargs)
in enumerate(zip(args_list, kwargs_list))]
for future in futures:
results.append(future.get())
else:
raise ValueError(f'Multiprocessing pool type {multiprocessing_pool_type} not supported')
results = sorted(results, key=lambda x: x[0])
results = [result[1] for result in results]
# print(f'len(results)={len(results)}')
# for result in results:
# print(result.shape)
return combination(results)
def map_reduce_multicore(
f: tp.Callable[..., ResultType],
reduction: tp.Callable[[ResultType, ResultType], ResultType],
initial_value: ResultType,
args_list: tp.Optional[tp.Sequence[tp.Sequence]] = None,
kwargs_list: tp.Optional[tp.Sequence[tp.Dict[str, tp.Any]]] = None,
number_of_batches: tp.Optional[int] = None,
multiprocessing_pool_type: MultiprocessingPoolType = MultiprocessingPoolType.default()) \
-> ResultType:
"""
This function evaluates the function 'f' on elements of 'args_list' and
'kwargs_list' in parallel on multiple cpu cores and performs the reduction
by calling the function 'reduction' on the result and the result of the
reductions so far to eventually produce one final result of type
'ResultType'. The reduce step is performed from the left and results are
being processed in the same order as they appear in `args_list` and
`kwargs_list`.
If the arguments for each run of 'f' are identical and they have already
been applied to the function that is passed then 'args_list' and
'kwargs_list' may both be None but the argument 'number_of_batches' must
be specified so the method knows how many times to run the function 'f'.
Args:
f: The map function to be evaluated over elements of 'args_list' and
'kwargs_list'.
reduction: The reduction to be performed on the results of 'f'.
This is done on the host (not the device).
initial_value: The initial value of the reduction
(i.e. the neutral element).
args_list: A sequence of sequences of positional arguments.
kwargs_list: A sequence of dictionaries of keyword arguments.
number_of_batches:
The number of function evaluations is required if 'args_list'
and 'kwargs_list' are both empty.
multiprocessing_pool_type:
the type of multi-processing pool (see class MultiprocessingPoolType)
"""
args_list, kwargs_list, number_of_batches = \
_extract_arguments_and_number_of_batches(
args_list=args_list,
kwargs_list=kwargs_list,
number_of_batches=number_of_batches)
def wrapped_f(index, *args, **kwargs) -> ResultType:
return index, f(*args, **kwargs)
if multiprocessing_pool_type == MultiprocessingPoolType.LOKY:
from concurrent.futures import as_completed
from loky import get_reusable_executor
executor = \
get_reusable_executor(timeout=None,
context='loky')
futures = [executor.submit(wrapped_f, i, *args, **kwargs)
for i, (args, kwargs)
in enumerate(zip(args_list, kwargs_list))]
result_from_future = lambda x: x.result()
elif multiprocessing_pool_type == MultiprocessingPoolType.PATHOS:
from pathos.pools import ProcessPool
pool = ProcessPool()
futures = [pool.apipe(wrapped_f, i, *args, **kwargs)
for i, (args, kwargs)
in enumerate(zip(args_list, kwargs_list))]
result_from_future = lambda x: x.get()
else:
raise ValueError(f'Multiprocessing pool type {multiprocessing_pool_type} not supported')
results = [result_from_future(future) for future in futures]
results = sorted(results, key=lambda x: x[0])
results = [result[1] for result in results]
result = initial_value
for new_result in results:
result = reduction(result, new_result)
return result
class MainLoop(metaclass=ABCMeta):
"""
Defines the logical loop of a game, running MAX_FPS times per second, sending the inputs to the HumanControllerWrapper, and the
game updates to the BotControllerWrappers.
"""
def __init__(self, api: API):
"""
Instantiates the logical loop
Args:
api: The game to run in this loop
"""
self.api = api
self.game = api.game
self.game.addCustomMoveFunc = self._addCustomMove
self._currentTurnTaken = False
self._screen = None
self._state = CONTINUE # The game must go on at start
self._eventsToSend = {} # type: Dict[ControllerWrapper, List[Event]]
self.wrappersConnection = {
} # type: Dict[ControllerWrapper, PipeConnection]
self.wrappersInfoConnection = {
} # type: Dict[ControllerWrapper, PipeConnection]
self.wrappers = {} # type: Dict[ControllerWrapper, Unit]
self._unitsMoves = {} # type: Dict[Unit, Tuple[Path, Queue]]
self._moveDescriptors = {} # type: Dict[Path, MoveDescriptor]
self._otherMoves = {} # type: Dict[Unit, Path]
self._killSent = {
} # Used to maintain the fact that the kill event has been sent
self.executor = None
self._prepared = False
self._frames = 0
# -------------------- PUBLIC METHODS -------------------- #
def run(self, max_fps: int = MAX_FPS) -> Union[None, Tuple[Unit, ...]]:
"""
Launch the game and its logical loop
Args:
max_fps: The maximum frame per seconds of the game
Returns:
a tuple containing all the winning players, or an empty tuple in case of draw,
or None if the game was closed by the user
"""
pygame.init()
clock = pygame.time.Clock()
assert self.game.board.graphics is not None
try:
self._screen = pygame.display.set_mode(
self.game.board.graphics.size, DOUBLEBUF)
except pygame.error: # No video device
pass
if not self._prepared:
self._prepareLoop()
for player_number in self.api.getPlayerNumbers():
self._sendEventsToController(player_number)
while self._state != END:
clock.tick(max_fps)
self._frames += 1
self._handleInputs()
if self._state == FINISH:
self.executor.terminate()
self._prepared = False
return None
elif self._state != PAUSE:
self._state = self._checkGameState()
if self._state == CONTINUE:
self._getNextMoveFromControllerWrapperIfAvailable()
self._handlePendingMoves()
self._refreshScreen()
self.executor.terminate()
self._prepared = False
return self.game.winningPlayers
def addUnit(self,
unit: Unit,
wrapper: ControllerWrapper,
tile_id: TileIdentifier,
initial_action: MoveDescriptor = None,
team: int = -1) -> None:
"""
Adds a unit to the game, located on the tile corresponding
to the the given tile id and controlled by the given controller
Args:
unit: The unit to add to the game
wrapper: The linker of that unit
tile_id: The identifier of the tile it will be placed on
initial_action: The initial action of the unit
team: The number of the team this player is in (-1 = no team)
"""
is_controlled = wrapper is not None
self.game.addUnit(unit, team, tile_id, is_avatar=is_controlled)
if is_controlled:
self._addControllerWrapper(wrapper, unit)
if self._mustSendInitialWakeEvent(initial_action, unit):
self._eventsToSend[wrapper].append(WakeEvent())
self._unitsMoves[unit] = (None, Queue())
tile = self.game.board.getTileById(tile_id)
resize_unit(unit, self.game.board)
unit.moveTo(tile.center)
if initial_action is not None:
unit.setLastAction(initial_action)
self._handleEvent(unit,
initial_action,
wrapper.controller.playerNumber,
force=True)
def getWrapperFromPlayerNumber(self, player_number: int):
"""
Retrieves the wrapper from the given player number
Args:
player_number: The number representing the player for which we want the wrapper
Returns: The wrapper that wraps the controller of the given player
"""
found = None
for wrapper in self.wrappersConnection:
if wrapper.controller.playerNumber == player_number:
found = wrapper
break
return found
def pause(self) -> None:
"""
Change the state of the game to "PAUSE"
"""
self._state = PAUSE
print(self._frames, "frames")
def resume(self) -> None:
"""
Resume the game
"""
self._state = CONTINUE
# -------------------- PROTECTED METHODS -------------------- #
def _refreshScreen(self) -> None:
"""
Update the visual state of the game
"""
try:
if self._screen is None:
raise pygame.error("No Video device")
self.game.board.draw(self._screen)
drawn_units = []
for unit in self.wrappers.values():
if unit.isAlive():
unit.draw(self._screen)
drawn_units.append(unit)
for unit in self.game.unitsLocation:
if unit.isAlive() and unit not in drawn_units:
unit.draw(self._screen)
pygame.display.flip()
except pygame.error: # No video device
pass
def _handleInputs(self) -> None:
"""
Handles all the user input (mouse and keyboard)
"""
try:
events_got = pygame.event.get()
except pygame.error: # No video device
events_got = []
for event in events_got:
if event.type == QUIT:
self._state = FINISH
elif event.type == KEYDOWN:
if event.key == K_ESCAPE:
if self._state == CONTINUE:
self.pause()
elif self._state == PAUSE:
self.resume()
else:
self._dispatchInputToHumanControllers(event.key)
elif event.type == MOUSEBUTTONDOWN:
self._dispatchMouseEventToHumanControllers(event.pos)
elif event.type == MOUSEBUTTONUP:
self._dispatchMouseEventToHumanControllers(None, click_up=True)
def _addMove(self, unit: Unit, move: Path) -> None:
"""
Adds a move (cancelling the pending moves)
Args:
unit: The unit for which add a move
move: The move to add for the given controller
"""
if self._unitsMoves[unit][0] is not None:
self._cancelCurrentMoves(unit)
fifo = self._unitsMoves[unit][1] # type: Queue
fifo.put(move)
def _addCustomMove(self, unit: Unit, move: Path,
event: MoveDescriptor) -> None:
"""
Adds a move that is NOT PERFORMED BY A CONTROLLER
Args:
unit: The unit that will be moved
move: The move that will be performed
"""
if unit not in self._otherMoves or self._otherMoves[unit] is None:
self._otherMoves[unit] = move
self._moveDescriptors[move] = event
def _cancelCurrentMoves(self, unit: Unit) -> None:
"""
Cancel the current movement if there is one and remove all the other pending movements.
Args:
unit: The unit for which cancel the movements
"""
if unit in self._unitsMoves:
move_tuple = self._unitsMoves[unit]
fifo = move_tuple[1] # type: Queue
last_move = move_tuple[0] # type: Path
new_fifo = Queue()
if last_move is not None:
last_move.stop()
while True:
try:
move = fifo.get_nowait()
del self._moveDescriptors[move]
except Empty:
break
self._unitsMoves[unit] = (last_move, new_fifo)
def _dispatchInputToHumanControllers(self, input_key) -> None:
"""
Handles keyboard events and send them to Human Controllers to trigger actions if needed
Args:
input_key: The key pressed on the keyboard
"""
for linker in self.wrappers: # type: HumanControllerWrapper
if issubclass(type(linker), HumanControllerWrapper):
self._getPipeConnection(linker).send(
self.game.createKeyboardEvent(
self._getUnitFromControllerWrapper(linker), input_key))
def _dispatchMouseEventToHumanControllers(self,
pixel: Optional[Coordinates],
click_up=False) -> None:
"""
Handles mouse events and send them to Human Controllers to trigger actions if needed
Args:
pixel: The pixel clicked
click_up: True if the button was released, False if the button was pressed
"""
tile = None
if pixel is not None:
tile = self.game.board.getTileByPixel(pixel)
self._previouslyClickedTile = tile
mouse_state = pygame.mouse.get_pressed()
for linker in self.wrappers: # type: ControllerWrapper
if issubclass(type(linker), HumanControllerWrapper):
tile_id = None
if tile is not None:
tile_id = tile.identifier
self._getPipeConnection(linker).send(
self.game.createMouseEvent(
self._getUnitFromControllerWrapper(linker), pixel,
mouse_state, click_up, tile_id))
def _getNextMoveFromControllerWrapperIfAvailable(self) -> None:
"""
Gets event from the controllers and dispatch them to the right method
"""
for current_wrapper in self.wrappersConnection: # type: ControllerWrapper
pipe_conn = self._getPipeConnection(current_wrapper)
if pipe_conn.poll():
move = pipe_conn.recv()
if self._mustRetrieveNextMove(current_wrapper):
self._handleEvent(self.wrappers[current_wrapper], move,
current_wrapper.controller.playerNumber)
def _handlePendingMoves(self) -> None:
"""
Get the next move to be performed and perform its next step
"""
moved_units = []
completed_moves = {
} # type: Dict[Unit, Tuple[TileIdentifier, MoveDescriptor]]
just_started = {} # type: Dict[int, MoveDescriptor]
illegal_moves = [] # type: List[Unit]
impossible_moves = {} # type: List[Unit]
self._handleOtherMoves(completed_moves, illegal_moves,
impossible_moves, just_started, moved_units)
self._handleMoves(completed_moves, illegal_moves, impossible_moves,
just_started, moved_units)
self._updateFromMoves(completed_moves, illegal_moves, impossible_moves,
just_started)
def _updateFromMoves(self, completed_moves, illegal_moves,
impossible_moves, just_started):
players_to_be_sent_messages = []
for unit, (tile_id, move_descriptor) in completed_moves.items():
self.game.updateGameState(unit, tile_id, move_descriptor)
for player_number, move_descriptor in just_started.items():
controller_unit = self.game.getControllerUnitForNumber(
player_number)
if controller_unit is not None:
controller_unit.setCurrentAction(move_descriptor)
self._addMessageToSendToAll(player_number, move_descriptor)
players_to_be_sent_messages = self._getPlayerNumbersToWhichSendEvents(
)
for player_number in players_to_be_sent_messages:
self._sendEventsToController(player_number)
for unit in illegal_moves:
self.game.unitsLocation[
unit] = self.game.board.OUT_OF_BOARD_TILE.identifier
self._killUnit(unit,
self.getWrapperFromPlayerNumber(unit.playerNumber))
# self.game.checkIfFinished()
self._cancelCurrentMoves(unit)
for unit in impossible_moves:
self._cancelCurrentMoves(unit)
self.game.checkIfFinished()
def _handleMoves(self, completed_moves, illegal_moves, impossible_moves,
just_started, moved_units):
for wrapper in self.wrappers: # type: ControllerWrapper
unit = self._getUnitFromControllerWrapper(wrapper)
if unit not in moved_units: # Two moves on the same unit cannot be performed at the same time...
if not unit.isAlive() and (unit not in self._killSent
or not self._killSent[unit]):
self.wrappersInfoConnection[wrapper].send(
SpecialEvent(flag=SpecialEvent.UNIT_KILLED))
self._killSent[unit] = True
current_move = self._getNextMoveForUnitIfAvailable(unit)
if current_move is not None:
move_state = self._performNextStepOfMove(
current_move.unit, current_move)
self._fillMoveStructures(completed_moves, just_started,
illegal_moves, impossible_moves,
current_move, move_state)
def _handleOtherMoves(self, completed_moves, illegal_moves,
impossible_moves, just_started, moved_units):
for unit in self._otherMoves: # type: Unit
move = self._otherMoves[unit]
if move is not None:
move_state = self._performNextStepOfMove(move.unit, move)
if move_state != MOVE_FAILED:
moved_units.append(move.unit)
if move.finished():
self._otherMoves[unit] = None
self._fillMoveStructures(completed_moves, just_started,
illegal_moves, impossible_moves, move,
move_state)
def _fillMoveStructures(self, completed_moves: Dict[Unit,
Tuple[TileIdentifier,
MoveDescriptor]],
just_started: Dict[int, MoveDescriptor],
illegal_moves: List[Unit],
impossible_moves: List[Unit], move: Path,
move_state: int):
"""
Takes a move's state and the data structures of the performed moves in this iteration an fill them
following the state's value
Args:
completed_moves: The dict containing the units that completed a move along with their new tile_id
just_started:
The dict containing the number of the units that started a move,
along with the descriptor of the started move
illegal_moves: The list containing all the units that performed an illegal move this iteration
impossible_moves: The list containing all the units that performed an impossible move this iteration
move: The performed move
move_state: The state of the performed move
"""
if move_state == MOVE_COMPLETED_AND_JUST_STARTED:
completed_moves[move.unit] = (move.reachedTileIdentifier,
self._moveDescriptors[move])
just_started[move.unit.playerNumber] = self._moveDescriptors[move]
elif move_state == MOVE_COMPLETED:
completed_moves[move.unit] = (move.reachedTileIdentifier,
self._moveDescriptors[move])
elif move_state == MOVE_JUST_STARTED:
just_started[move.unit.playerNumber] = self._moveDescriptors[move]
elif move_state == MOVE_ILLEGAL:
illegal_moves.append(move.unit)
elif move_state == MOVE_IMPOSSIBLE:
impossible_moves.append(move.unit)
def _addMessageToSendToAll(self, moved_unit_number: int,
move_descriptor: MoveDescriptor):
"""
Adds a message to the message queue of each ControllerWrapper
Args:
moved_unit_number: The number representing the unit that moved
move_descriptor: The descriptor of the performed move
"""
controlled_unit = self.game.getControllerUnitForNumber(
moved_unit_number)
if controlled_unit is None:
controlled_unit_number = moved_unit_number
else:
controlled_unit_number = controlled_unit.playerNumber
for wrapper in self._eventsToSend:
self._eventsToSend[wrapper].append(
BotEvent(controlled_unit_number, move_descriptor))
@staticmethod
def _performNextStepOfMove(unit: Unit, current_move: Path) -> int:
"""
Perform the next step of the given move on the given unit
Args:
unit: The unit that performs the move
current_move: The current move to perform
Returns:
A couple of booleans. The first indicating that the move has been completed and the second indicating that
the move has just started
"""
if unit.isAlive():
if current_move is not None:
try:
just_started, move_completed, tile_id = current_move.performNextMove(
)
if just_started and move_completed:
return MOVE_COMPLETED_AND_JUST_STARTED
elif move_completed: # A new tile has been reached by the movement
return MOVE_COMPLETED
elif just_started:
return MOVE_JUST_STARTED
return MOVE_IN_PROGRESS
except IllegalMove:
return MOVE_ILLEGAL
except ImpossibleMove:
return MOVE_IMPOSSIBLE
else:
if current_move is not None:
current_move.stop(cancel_post_action=True)
return MOVE_FAILED
def _getNextMoveForUnitIfAvailable(self, unit: Unit) -> Union[Path, None]:
"""
Checks if a move is available for the given controller, and if so, returns it
Args:
unit: The given
Returns: The next move if it is available, and None otherwise
"""
moves = self._unitsMoves[unit]
current_move = moves[0] # type: Path
if current_move is None or current_move.finished():
if current_move is not None:
if isinstance(current_move, Path):
self._reactToFinishedMove()
del self._moveDescriptors[current_move]
try:
move = moves[1].get_nowait() # type: Path
self._unitsMoves[unit] = (move, moves[1])
current_move = move
except Empty:
self._unitsMoves[unit] = (None, moves[1])
current_move = None
return current_move
def _checkGameState(self) -> int:
"""
Checks if the game is finished
Returns: 0 = CONTINUE; 2 = END
"""
if self.game.isFinished():
self.winningPlayers = self.game.winningPlayers
return END
return CONTINUE
def _handleEvent(self,
unit: Unit,
event: MoveDescriptor,
player_number: int,
force: bool = False) -> None:
"""
The goal of this method is to handle the given event for the given unit
Args:
unit: The unit that sent the event through its linker
event: The event sent by the controller
"""
try:
move = self.api.createMoveForDescriptor(
unit, event, force=force) # may raise: UnfeasibleMoveException
self._currentTurnTaken = True
self._moveDescriptors[move] = event
self._addMove(unit, move)
except UnfeasibleMoveException:
self._sendEventsToController(player_number, event=WakeEvent())
def _getPipeConnection(self, linker: ControllerWrapper) -> PipeConnection:
"""
Args:
linker: The linker for which we want the pipe connection
Returns: The pipe connection to send and receive game updates
"""
return self.wrappersConnection[linker]
def _getUnitFromControllerWrapper(self, linker: ControllerWrapper) -> Unit:
"""
Args:
linker: The linker for which we want the unit
Returns: The unit for the given linker
"""
return self.wrappers[linker]
def _sendEventsToController(self, player_number: int, event: Event = None):
player_wrapper = self.getWrapperFromPlayerNumber(player_number)
pipe_conn = self._getPipeConnection(player_wrapper)
if event is None:
events = self._eventsToSend[player_wrapper]
if len(events) > 0:
event = MultipleEvents(events)
if event is not None:
pipe_conn.send(event)
self._eventsToSend[player_wrapper] = []
def _informBotOnPerformedMove(self, moved_unit_number: int,
move_descriptor: MoveDescriptor) -> None:
"""
Update the game state of the bot controllers
Args:
moved_unit_number: The number representing the unit that moved and caused the update
move_descriptor: The move that caused the update
"""
for wrapper in self.wrappers:
if issubclass(type(wrapper), BotControllerWrapper):
pipe_conn = self._getPipeConnection(wrapper)
pipe_conn.send(BotEvent(moved_unit_number, move_descriptor))
def _killUnit(self, unit: Unit, linker: ControllerWrapper) -> None:
"""
Kills the given unit and tells its linker
Args:
unit: The unit to kill
linker: The linker, to which tell that the unit is dead
"""
unit.kill()
if not unit.isAlive():
self.wrappersInfoConnection[linker].send(
SpecialEvent(flag=SpecialEvent.UNIT_KILLED))
def _addCollaborationPipes(self, linker: BotControllerWrapper) -> None:
"""
Adds the collaboration pipes between the given linker and its teammate's
Args:
linker: The linker to connect with its teammate
"""
for teammate in self.game.teams[self.game.unitsTeam[
self.wrappers[linker]]]:
if teammate is not self.wrappers[linker]:
teammate_linker = None # type: BotControllerWrapper
for other_linker in self.wrappers:
if self.wrappers[other_linker] is teammate:
teammate_linker = other_linker
break
pipe1, pipe2 = Pipe()
linker.addCollaborationPipe(
teammate_linker.controller.playerNumber, pipe1)
teammate_linker.addCollaborationPipe(
linker.controller.playerNumber, pipe2)
def _prepareLoop(self) -> None:
"""
Launches the processes of the AIs
"""
self.executor = Pool(len(self.wrappers))
try:
self.executor.apipe(lambda: None)
except ValueError:
self.executor.restart()
for wrapper in self.wrappers:
if isinstance(wrapper, BotControllerWrapper):
wrapper.controller.gameState = self.game.copy()
self.executor.apipe(wrapper.run)
for wrapper in self.wrappers:
pipe = self._getPipeConnection(wrapper)
event = pipe.recv(
) # Waiting for the processes to launch correctly
assert (isinstance(event, ReadyEvent))
self._prepared = True
def _addControllerWrapper(self, wrapper: ControllerWrapper,
unit: Unit) -> None:
"""
Adds the linker to the loop, creating the pipe connections
Args:
wrapper: The linker to add
unit: The unit, linked by this linker
"""
self.wrappers[wrapper] = unit
parent_conn, child_conn = Pipe()
parent_info_conn, child_info_conn = Pipe()
self.wrappersConnection[wrapper] = parent_conn
self.wrappersInfoConnection[wrapper] = parent_info_conn
self._eventsToSend[wrapper] = []
wrapper.setMainPipe(child_conn)
wrapper.setGameInfoPipe(child_info_conn)
if isinstance(wrapper, BotControllerWrapper):
self._addCollaborationPipes(wrapper)
@abstractmethod
def _mustSendInitialWakeEvent(self, initial_action: MoveDescriptor,
unit: Unit) -> bool:
pass
@abstractmethod
def _mustRetrieveNextMove(self,
current_wrapper: ControllerWrapper) -> bool:
pass
@abstractmethod
def _getPlayerNumbersToWhichSendEvents(self) -> List[int]:
pass
@abstractmethod
def _reactToFinishedMove(self):
pass
