def test_map_from_seq():
# pylint: disable=missing-docstring
data = PList.from_seq(MSG)
data.invariant()
res = data.map(upper_case).to_seq()
exp = list(upper_case(MSG))
assert res == exp
def test_reduce_sum_non_empty():
# pylint: disable=missing-docstring
slst = SList([1, 2, 3, 4, 5, 6])
plst = PList.from_seq(slst)
exp = 21
res = plst.reduce(operator.add, 0)
assert res == exp
def test_reduce_cons():
# pylint: disable=missing-docstring
slst = SList([1, 2, 3, 4])
plst = PList.from_seq(slst)
res = plst.reduce(operator.add)
exp = 10
assert res == exp
def test_scanl_non_empty():
# pylint: disable=missing-docstring
size = 23
data = PList.init(fun.idt, size)
res = data.scanl(operator.add, 0).to_seq()
exp = SList(range(0, size)).scanl(operator.add, 0)
assert res == exp
def _test_scanl_last(size):
# pylint: disable=missing-docstring
data = PList.init(fun.idt, size)
res_pl, res_scalar = data.scanl_last(operator.add, 0)
res = (res_pl.to_seq(), res_scalar)
exp = SList(range(0, size)).scanl_last(operator.add, 0)
assert res == exp
def _test3():
if SEQ:
data = DSList.init(_vrand, SIZE)
else:
data = DPList.init(_vrand, SIZE)
res1 = test_timing(_vavg, data, "vector average")
res2 = test_timing(_wrapped_vavg, data, "vector average [_run]")
assert res1 == res2
def generate_plist(function, start=0):
"""
Generates a parallel list of random size between 1 and 111, or an empty list,
using ``function`` to initialize the elements of the list. If ``start`` is 1,
a non-empty parallel list is generated.
:param function: callable
:param start: int (0 or 1)
:return: PList
"""
choice = randint(start, 2)
size = randint(1, 111)
if choice == 0:
return PList()
if choice == 1:
return PList.init(function, size)
return PList.from_seq([function(i) for i in range(0, size)])
def get_distribution(plst: PList):
"""
Returns the distribution (as a sequential list) of its argument.
:param plst: PList
:return: list
"""
return plst.get_partition().map(len).to_seq()
def randint(min_, max_):
"""
Returns the same random number on all the processors.
:param min_: int
:param max_: int
:return: int
"""
return PList.from_seq([random.randint(min_, max_)]).to_seq()[0]
def test_map_reduce_cons():
# pylint: disable=missing-docstring
slst = SList([1, 2, 3, 4])
plst = PList.from_seq(slst)
exp = plst.map(fun.incr)
exp = exp.reduce(operator.add)
res = plst.map_reduce(fun.incr, operator.add)
assert res == exp
def test_reduce_sum_empty():
# pylint: disable=missing-docstring
neutral = 0
slst = SList()
plst = PList.from_seq(slst)
exp = neutral
res = plst.reduce(operator.add, neutral)
assert res == exp
def test_map_reduce_nil():
# pylint: disable=missing-docstring
neutral = 0
slst = SList()
plst = PList.from_seq(slst)
res = plst.map_reduce(fun.incr, operator.add, neutral)
exp = neutral
assert res == exp
def _test1():
if SEQ:
input1 = DSList.init(lambda num: random.randint(0, 1000), SIZE)
else:
input1 = DPList.init(lambda num: random.randint(0, 1000), SIZE)
res1 = test_timing(_test_mmr_direct, input1, "map/map/reduce")
res2 = test_timing(_test_mmr_direct, input1, "map/reduce[hc]")
res3 = test_timing(_test_mmr_run, input1, "map/map/reduce[_opt]")
res4 = test_timing(_test_mmr_optimized, input1, "map/map/reduce[_opt/_run]", _opt, _run)
assert res1 == res2 and res1 == res3 and res1 == res4
def _test2():
if SEQ:
input2 = DSList.init(lambda i: random.randint(0, 9) <= 4, SIZE)
else:
input2 = DPList.init(lambda i: random.randint(0, 9) <= 4, SIZE)
res1 = test_timing(_test_bool_direct, input2, "map/reduce bool")
res2 = test_timing(_test_bool_mr, input2, "map_reduce bool")
res3 = test_timing(_test_bool_optimized, input2, "map/reduce bool[_opt]", _opt, _run)
assert res1 == res3
assert res1 == res2
def pssr(input_list: PList) -> PList:
"""
Sort the input list.
Sorts using ``<`` only.
Example::
>>> from pyske.core import PList
>>> pssr(PList.init(lambda i: 10-i, 10)).to_seq()
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
:param input_list: a parallel list.
Pre-condition: the size of the local lists should be at least
equal to the number of processors.
:return: a sorted list that is a permutation of ``input_list``.
"""
nprocs = len(par.procs())
if nprocs == 1:
return input_list.get_partition().map(sorted).flatten()
for local_size in input_list.distribution:
assert local_size >= nprocs
def permutation(index: int):
return int(index / nprocs) + nprocs * (index % nprocs)
def _sample(list_to_sample):
if list_to_sample:
size = len(list_to_sample)
step = int(size / nprocs)
return list_to_sample[step:size:step]
return []
locally_sorted = input_list.get_partition().map(sorted)
first_samples = locally_sorted.map(_sample).gather(0).get_partition()
second_samples = bcast(first_samples.map(_merge).map(_sample), 0)
slices = locally_sorted.map2(_slice, second_samples).flatten()
result = slices.permute(permutation).get_partition().map(_merge).flatten()
return result
def __main():
parallel_list1 = PList.init(lambda i: __MSG[i], len(__MSG))
parallel_list2 = PList.init(lambda x: x, len(__MSG))
parallel_list4 = parallel_list1.map(lambda x: x.capitalize()).zip(
parallel_list2)
parallel_list6 = parallel_list4.map(lambda x: x[0]).mapi(lambda i, x:
(i, x))
parallel_list7 = parallel_list6.map(lambda x: x[1])
parallel_list8 = parallel_list7.map(lambda x: 1)
size = parallel_list8.reduce(lambda x, y: x + y, 0)
parallel_list9 = parallel_list7.get_partition()
parallel_list10 = parallel_list9.map(
lambda l: SList(l).filter(lambda c: c != 'O')).flatten()
parallel_list11 = PList.from_seq(["Hello World!"])
filtered = SList(parallel_list10.get_partition().reduce(concat,
[])).reduce(add)
str1 = SList(parallel_list9.reduce(add)).reduce(add)
str2 = parallel_list11.to_seq()[0]
par.at_root(lambda: print(f'Capitalized: \t{str1}\n'
f'Identity:\t{str2}\n'
f'Length:\t\t{size}\n'
f'Filtered:\t{filtered}'))
def _main():
size, num_iter, _ = util.standard_parse_command_line(data_arg=False)
assert _is_power_of_2(size), "The size should be a power of 2."
assert _is_power_of_2(len(
par.procs())), "The number of processors should be a power of 2."
input_list = PList.init(lambda _: 1.0, size)
timing = Timing()
gc.disable()
for iteration in range(1, 1 + num_iter):
timing.start()
result = fft(input_list)
timing.stop()
gc.collect()
result = result.to_seq()[0]
util.print_experiment(result, timing.get(), par.at_root, iteration)
def test_timing(test_f, data, name,
preprocessing=lambda f, num: lambda: f(num), execute=lambda f: f(),
print_test_f=False):
# pylint: disable=too-many-arguments
"""
Iterate and output timing of the application of a function.
:param test_f: the function to test.
:param data: the input data to the function.
:param name: of the test.
:param preprocessing: a function to apply to the test function.
:param execute: a function to executed the test function.
:return: the result of the application of the test function.
Output the timings on standard output.
"""
assert ITERATIONS > 0
par.at_root(lambda: print(f'Test: {name}'))
skel = preprocessing(test_f, data)
if print_test_f:
par.at_root(lambda: print("Term: ", skel))
gc.collect()
par.barrier()
time: DPList = DPList.init(lambda _: par.wtime())
output = data
for idx in range(0, ITERATIONS):
def printer(val):
return lambda: print(f' Iteration: {val}', end='\r')
par.at_root(printer(idx))
output = execute(skel)
elapsed = time.map(lambda num: par.wtime() - num).map(lambda num: num / ITERATIONS)
par.at_root(lambda: print(30 * ' ', end='\r'))
max_elapsed = elapsed.reduce(max)
avg_elapsed = elapsed.reduce(add) / elapsed.length()
all_elapsed = elapsed.mapi(lambda k, num: "[" + str(k) + "]:" + str(num)).to_seq()
par.at_root(lambda:
print(f'Time (max):\t{max_elapsed}\n'
f'Time (avg):\t{avg_elapsed}\n'
f'Time (all):\t{all_elapsed}'))
return output
def __performance():
size, num_iter = _parse_command_line()
performance.ITERATIONS = num_iter
input_list = PList.init(str, size)
mpi_input_list = None
def mpi_set(value):
nonlocal mpi_input_list
mpi_input_list = value
input_list.get_partition().map(mpi_set)
pyske_input_scatter = \
performance.test_timing(_pyske_gather, input_list, "PySke Gather")
mpi_input_scatter = \
performance.test_timing(_mpi_gather, mpi_input_list, "MPI Gather")
performance.test_timing(_pyske_scatter, pyske_input_scatter,
"PySke Scatter")
performance.test_timing(_mpi_scatter, mpi_input_scatter,
"MPI Scatter (with alltoall)")
performance.test_timing(_mpi_scatter, mpi_input_scatter,
"MPI Scatter (with scatter)")
performance.test_timing(_pyske_bcast, input_list, "PySke Bcast")
performance.test_timing(_mpi_bcast, mpi_input_list, "MPI Bcast")
def __main():
parser = argparse.ArgumentParser()
parser.add_argument("-n",
help="size of the list to generate",
type=int,
default=1000)
parser.add_argument("-a",
help="choice of the algorithm (1-3)",
type=int,
default=1)
parser.add_argument("-s", help="choice of a seed", type=int, default=111)
# Generating a parallel list of the size specified on the command line or 1000
args = parser.parse_args()
size = args.n
algorithm = args.a
seed = args.s
random.seed(seed)
data = PList.init(lambda _: bool(random.getrandbits(1)), size)
par.barrier()
# Solution 1
timer = Timing()
timer.start()
if algorithm == 1:
res = data.map(not_).reduce(and_)
# Solution 2
if algorithm == 2:
res = data.map_reduce(not_, and_, True)
# Solution 3
if algorithm == 3:
res = not data.reduce(or_, False)
timer.stop()
max_t, avg_t, all_t = timer.get()
if algorithm in [1, 2, 3]:
par.at_root(lambda: print(f'Result: \t{res}\n'
f'Time (max):\t{max_t}\n'
f'Time (avg):\t{avg_t}\n'
f'Time (all):\t{all_t}'))
def test_mapi_non_empty_2():
# pylint: disable=missing-docstring
data = PList.init(lambda i: MSG[i], len(MSG))
res = data.mapi(pos_upper).to_seq()
exp = SList(MSG).mapi(pos_upper)
assert res == exp
def test_mapi_empty():
# pylint: disable=missing-docstring
data = PList()
res = data.mapi(pos_upper).to_seq()
exp = []
assert res == exp
def test_zip_empty():
# pylint: disable=missing-docstring
data = PList()
res = data.zip(data).to_seq()
exp = []
assert res == exp
def test_init_to_seq_empty():
# pylint: disable=missing-docstring
plst = PList()
res = plst.to_seq()
exp = []
assert res == exp
def test_init_to_seq_non_empty():
# pylint: disable=missing-docstring
plst = PList.init(alphabet, 17)
res = plst.to_seq()
exp = [alphabet(i) for i in range(0, 17)]
assert res == exp
def is_sorted(input_list: PList) -> bool:
"""Check if the list is sorted."""
return input_list.get_partition().map(_is_sorted_list).reduce(and_, True)
def _filter_even(input_list: PList) -> Distribution:
return input_list.filter(fun.is_even).balance().distribution
def test_map2_empty():
# pylint: disable=missing-docstring
data = PList()
res = data.map2(operator.add, data).to_seq()
exp = []
assert res == exp
def _pyske_gather(input_list: PList):
return input_list.gather(0)
def _pyske_scatter(input_list: PList):
return input_list.scatter(0)