def test_r2b_2():
rn5 = RNode(5)
rn6 = RNode(6)
rn3 = RNode(3, SList([rn5, rn6]))
rn2 = RNode(2)
rn4 = RNode(4)
rn1 = RNode(1, SList([rn2, rn3, rn4]))
res = rn1.r2b()
exp = \
Node(1,
Node(2,
Leaf(None),
Node(3,
Node(5,
Leaf(None),
Node(6,
Leaf(None),
Leaf(None)
)
),
Node(4,
Leaf(None),
Leaf(None)
)
)
),
Leaf(None)
)
assert res == exp
def from_seq(sequence: Sequence[T]) -> 'PList[T]':
p_list = PList()
if _PID == 0:
p_list.__content = SList(sequence)
p_list.__distribution = [
len(sequence) if i == 0 else 0 for i in par.procs()
]
from_root = _COMM.bcast(p_list.__distribution, 0)
p_list.__distribution = Distribution(from_root)
p_list.__local_size = p_list.__distribution[_PID]
p_list.__global_size = p_list.__distribution[0]
p_list.__start_index = SList(p_list.__distribution).scanl(add, 0)[_PID]
return p_list
def flatten(self: 'PList[SList[T]]',
new_distr: Distribution = None) -> 'PList[T]':
p_list = PList()
p_list.__content = self.__content.flatten()
p_list.__local_size = len(p_list.__content)
if new_distr is None:
p_list.__distribution = _COMM.allgather(p_list.__local_size)
else:
p_list.__distribution = new_distr
assert new_distr[_PID] == p_list.__local_size
p_list.__start_index = SList(p_list.__distribution).scanl(add, 0)[_PID]
p_list.__global_size = SList(p_list.__distribution).reduce(add)
return p_list
def reduce(self: 'PList[T]',
binary_op: Callable[[T, T], T],
neutral: Optional[T] = None) -> T:
if neutral is None:
assert self.__global_size >= 1
partial = None if self.__local_size == 0 else SList(
self.__content).reduce(binary_op)
partials = SList(
_COMM.allgather(partial)).filter(lambda x: x is not None)
else:
# assert: (binary_op, neutral) form a monoid
partial = SList(self.__content).reduce(binary_op, neutral)
partials = SList(_COMM.allgather(partial))
return partials.reduce(binary_op, neutral)
def aux(btree):
if btree.is_leaf():
val = btree.get_value()
if val is None:
return SList()
return SList([RNode(val)])
n = btree.get_value()
left = btree.get_left()
right = btree.get_right()
res_l = aux(left)
res_r = aux(right)
res_head = RNode(n, res_l)
res_r.insert(0, res_head)
return res_r
def __init__(self, lt=None):
self.__distribution = SList([])
self.__global_index = SList([])
self.__start_index = 0
self.__nb_segs = 0
self.__content = SList([])
if lt is not None:
(distribution, global_index) = distribute_tree(lt, NPROCS)
self.__distribution = distribution
self.__global_index = global_index
self.__start_index = distribution.scanl(lambda x, y: x + y, 0)[PID]
self.__nb_segs = distribution[PID]
for i_seg in range(self.__start_index,
self.__start_index + self.__nb_segs):
self.__content.extend(lt[i_seg])
def get_full_index(self):
def f(x, y):
(x1, y1) = x
(_, y2) = y
return x1 + y1, y2
return SList(self.__global_index.scanr(f))
def map2(self, f, pt):
"""Map2 skeleton for distributed tree
Precondition
-------------
The distributions of self and pt should be the same
Parameters
----------
pt : :obj:`LTree`
The LTree to zip with the current instance
f : callable
A function to zip values
"""
logger.debug('[START] PID[%s] map2 skeleton', PID)
assert self.__distribution == pt.distribution
content = SList([None] * self.__content.length())
for i in range(
len(self.__global_index[self.__start_index:self.__start_index +
self.__nb_segs])):
(start, offset
) = self.__global_index[self.__start_index:self.__start_index +
self.__nb_segs][i]
logger.debug('[START] PID[%s] map2_local from %s to %s', PID,
start, start + offset)
content[start:start + offset] = Segment(self.__content[start:start + offset]). \
map2(f, Segment(pt.content[start:start + offset]))
logger.debug('[END] PID[%s] map2_local from %s to %s', PID, start,
start + offset)
res = PTree.init(self, content)
logger.debug('[END] PID[%s] map2 skeleton', PID)
return res
def __init__(self: 'PList[T]'):
# pylint: disable=super-init-not-called
self.__content: 'SList[T]' = SList([])
self.__global_size: int = 0
self.__local_size: int = 0
self.__start_index: int = 0
self.__distribution = Distribution([0 for _ in range(0, _NPROCS)])
def racc(self, f, unit_f):
"""Makes a rightward accumulation of the values in the current instance
rAcc (+) (RNode a ts)
= let rs = scanl (+) [root ts[i] | i in [1 .. #ts]]
in RNode unit_(+) [setroot (rAcc (+) ts[i]) r[i] | i in [1 .. #ts]]
Parameters
----------
f : callable
A function to accumulate the value of the current instance with the current accumulator
unit_f
A value such as, forall value, f(value, unit_f) = value
"""
rv = self.get_children().map(lambda x: x.get_value())
rs = rv.scanl(f, unit_f)
ch = SList()
ch0 = self.get_children()
for i in range(0, ch0.length()):
c = ch0[i]
cs = c.racc(f, unit_f)
cs.set_value(rs[i])
ch.append(cs)
return RNode(unit_f, ch)
def test_b2r_node_from_rt():
bt = Node(
1,
Node(
2, Leaf(None),
Node(3, Node(5, Leaf(None), Node(6, Leaf(None), Leaf(None))),
Node(4, Leaf(None), Leaf(None)))), Leaf(None))
rn5 = RNode(5)
rn6 = RNode(6)
rn3 = RNode(3, SList([rn5, rn6]))
rn2 = RNode(2)
rn4 = RNode(4)
exp = RNode(1, SList([rn2, rn3, rn4]))
res = RNode(bt)
assert res == exp
def __init__(self, value, ts=None):
if isinstance(value, BTree):
if value == Leaf(None):
raise ConstructorError(
"A RTree cannot be constructed from a single Leaf that "
"contains None")
# Create a RTree from a BTree
bt = value
rt = RNode.b2r(bt)
self.value = rt.get_value()
self.children = rt.get_children()
else:
self.value = value
if ts is None:
self.children = SList([])
else:
self.children = SList(ts)
def get_partition(self: 'PList[T]') -> 'PList[SList[T]]':
p_list = PList()
p_list.__content = SList([self.__content])
p_list.__global_size = _NPROCS
p_list.__local_size = 1
p_list.__start_index = _PID
p_list.__distribution = [1 for _ in par.procs()]
return p_list
def test_r2b_1():
ch = SList()
ch.append(RNode(2))
ch.append(RNode(3))
rn = RNode(1, ch)
res = rn.r2b()
exp = Node(1, Node(2, Leaf(None), Node(3, Leaf(None), Leaf(None))),
Leaf(None))
assert res == exp
def distribute_tree(lt, n): total_size = lt.map(fun.one, fun.one).reduce(fun.add, fun.idt, fun.add, fun.add, fun.add) avg_elements = int(total_size / n) # Definition of global_index and distribution global_index = SList([]) distribution = SList([]) for i in range(n): distribution.append(0) current_pid = 0 nb_segs = 1 seg = lt[0] seg_length = seg.length() acc_size = seg_length global_index.append((0, seg_length)) for seg_i in range(1, lt.length()): seg = lt[seg_i] curr_seg_length = seg.length() if current_pid == n - 1: # We need to give all the rest to the last processor if seg_i == lt.length() - 1: distribution[current_pid] = (nb_segs + 1) global_index.append((acc_size, curr_seg_length)) else: nb_segs += 1 curr_seg_length = seg.length() global_index.append((acc_size, curr_seg_length)) acc_size = acc_size + curr_seg_length else: if seg_i == lt.length() - 1: distribution[current_pid] = (nb_segs + 1) global_index.append((acc_size, seg.length())) else: curr_seg_length = seg.length() if abs(avg_elements - (acc_size + curr_seg_length)) > abs(avg_elements - acc_size): distribution[current_pid] = nb_segs global_index.append((0, curr_seg_length)) acc_size = curr_seg_length nb_segs = 1 current_pid += 1 else: nb_segs += 1 global_index.append((acc_size, curr_seg_length)) acc_size = acc_size + curr_seg_length return distribution, global_index
def init(value_at: Callable[[int], T], size: int = _NPROCS) -> 'PList[T]':
assert size >= 0
p_list = PList()
p_list.__global_size = size
p_list.__local_size = parimpl.local_size(_PID, size)
distribution_list = [
parimpl.local_size(i, size) for i in range(0, _NPROCS)
]
p_list.__distribution = Distribution(distribution_list)
p_list.__start_index = SList(p_list.__distribution).scanl(
lambda x, y: x + y, 0)[_PID]
p_list.__content = SList([
value_at(i)
for i in range(p_list.__start_index, p_list.__start_index +
p_list.__local_size)
])
p_list.__distribution = [
parimpl.local_size(i, size) for i in range(0, _NPROCS)
]
return p_list
def permute(self: 'PList[T]', bij: Callable[[int], int]) -> 'PList[T]':
p_list = self.__get_shape()
distr = Distribution(self.__distribution)
new_indices = self.mapi(lambda i, x: distr.to_pid(bij(i), x)
).get_partition().map(_group_by)
mapping = new_indices.__content[0]
keys = mapping.keys()
messages = [mapping[pid] if pid in keys else [] for pid in par.procs()]
exchanged = SList(parimpl.COMM.alltoall(messages)).flatten()
exchanged.sort()
p_list.__content = exchanged.map(lambda pair: pair[1])
return p_list
def distribute(self: 'PList[T]', target_distr: Distribution) -> 'PList[T]':
assert Distribution.is_valid(target_distr, self.__global_size)
source_distr = self.__distribution
source_bounds = interval.bounds(source_distr)
target_bounds = interval.bounds(target_distr)
local_interval = source_bounds[_PID]
bounds_to_send = target_bounds.map(
lambda i: interval.intersection(i, local_interval))
msgs = [
interval.to_slice(self.__content,
interval.shift(inter, -self.__start_index))
for inter in bounds_to_send
]
slices = _COMM.alltoall(msgs)
p_list = PList()
p_list.__content = SList(slices).flatten()
p_list.__local_size = target_distr[_PID]
p_list.__global_size = self.__global_size
p_list.__start_index = SList(target_distr).scanl(add, 0)[_PID]
p_list.__distribution = target_distr
return p_list
def map_reduce(self: 'PList[T]',
unary_op: Callable[[T], V],
binary_op: Callable[[V, V], V],
neutral: Optional[V] = None) -> V:
if neutral is None:
assert self.__global_size >= 1
partial = None if self.__local_size == 0 \
else self.__content.map_reduce(unary_op, binary_op)
partials = SList(
_COMM.allgather(partial)).filter(lambda x: x is not None)
return functools.reduce(binary_op, partials)
# assert: (binary_op, neutral) form a monoid
partial = self.__content.map_reduce(unary_op, binary_op, neutral)
partials = _COMM.allgather(partial)
return functools.reduce(binary_op, partials, neutral)
def random_list(frdm, size):
"""
Generates a random list of the given ``size``.
Each element is generated by a call to ``frdm``.
``size`` should be strictly positive.
:param frdm: callable
:param size: int
:return: list
"""
assert size >= 0
res = SList([])
for _ in range(size):
res.append(frdm())
return res
def __local_upwards_accumulation(self, k, phi, psi_l, psi_r):
gt = Segment([None] * self.__nb_segs)
lt2 = SList([None] * self.__nb_segs)
i = 0
for (start, offset) in \
self.__global_index[self.__start_index: self.__start_index + self.__nb_segs]:
logger.debug('[START] PID[%s] uacc_local from %s to %s', PID,
start, start + offset)
(top,
res) = Segment(self.__content[start:start + offset]).uacc_local(
k, phi, psi_l, psi_r)
logger.debug('[END] PID[%s] uacc_local from %s to %s', PID, start,
start + offset)
gt[i] = top
lt2[i] = res
i = i + 1
return i, gt, lt2
def r2b(self):
"""Get a BTree from the current instance
"""
def r2b1(t, ss):
a = t.get_value()
left = r2b2(t.get_children())
right = r2b2(ss)
return Node(a, left, right)
def r2b2(ts):
if not ts:
return Leaf(None)
h = ts[0]
t = ts[1:]
return r2b1(h, t)
return r2b1(self, SList())
def r2b(self):
"""Get a BTree from the current instance
"""
def r2b1(t, ss):
a = t.get_value()
left = r2b2(t.get_children())
right = r2b2(ss)
return Node(a, left, right)
def r2b2(ts):
if ts.empty():
return Leaf(None)
else:
h = ts.head()
t = ts.tail()
return r2b1(h, t)
return r2b1(self, SList())
def init_from_file(filename, parser=int):
"""Instantiate a distributed tree from a file
Parameters
----------
filename : str
The name of the file that contains the PTree to instantiate
parser : callable, optional
A function that transforms a string into a specific type.
By default, string to int
"""
filename = filename + "." + str(PID)
def __parser_couple(s):
s = s.replace("(", "")
s = s.replace(")", "")
ss = s.split(",")
return int(ss[0]), int(ss[1])
p = PTree()
content = SList([])
with open(filename, "r") as f:
count_line = 0
for line in f:
if line.strip()[0] == '#':
continue
# __distribution: PID -> nb of segments
# __global_index: num seg -> (start, offset)
if count_line == 0: # Get the distribution
p.distribution = SList.from_str(line)
p.start_index = p.distribution.scanl(
lambda x, y: x + y, 0)[PID]
p.nb_segs = p.distribution[PID]
elif count_line == 1: # Get the global_index
p.global_index = SList.from_str(line,
parser=__parser_couple)
else: # Get the content
content.extend(Segment.from_str(line, parser=parser))
count_line = count_line + 1
p.content = content
return p
def __local_updates(self, gt, gt2, lt2, k):
content = SList([None] * self.__content.length())
for i in range(
len(self.__global_index[self.__start_index:self.__start_index +
self.__nb_segs])):
(start, offset
) = self.__global_index[self.__start_index:self.__start_index +
self.__nb_segs][i]
logger.debug('[START] PID[%s] uacc_update from %s to %s', PID,
start, start + offset)
if gt[i].is_node():
(lc, rc) = gt2[i].get_value()
val = Segment(self.__content[start:start +
offset]).uacc_update(
lt2[i], k, lc, rc)
else:
val = lt2[i]
logger.debug('[END] PID[%s] uacc_update from %s to %s', PID, start,
start + offset)
content[start:start + offset] = val
return content
def zip(self, rt):
"""Zip the values contained in a second RTree with the ones in the current instance
Precondition
-------------
The lengths of self.children and rt.children should be equal
Parameters
----------
rt : :obj:`RTree`
The RTree to zip with the current instance
"""
ch1 = self.get_children()
ch2 = rt.get_children()
assert ch1.length() == ch2.length(
), "The rose trees cannot be zipped (not the same shape)"
ch = SList([])
for i in range(0, ch1.length()):
ch.append(ch1[i].zip(ch2))
v = (self.get_value(), rt.get_value())
return RNode(v, ch)
def map(self, kl, kn):
"""Map skeleton for distributed tree
Parameters
----------
kl : callable
Function to apply to every leaf value of the current instance
kn : callable
Function to apply to every node value of the current instance
"""
content = SList([None] * self.__content.length())
logger.debug('[START] PID[%s] map skeleton', PID)
for (start, offset) in \
self.__global_index[self.__start_index: self.__start_index + self.__nb_segs]:
logger.debug('[START] PID[%s] map_local from %s to %s', PID, start,
start + offset)
content[start:start + offset] = Segment(
self.__content[start:start + offset]).map_local(kl, kn)
logger.debug('[END] PID[%s] map_local from %s to %s', PID, start,
start + offset)
res = PTree.init(self, content)
logger.debug('[END] PID[%s] map skeleton', PID)
return res
def _lasts(distr):
return SList(scan(distr, add, 0)).tail().map(lambda x: x - 1)
def _firsts(distr):
return SList(distr).scanl(add, 0)
def to_seq(self: 'PList[T]') -> 'SList[T]':
return SList(self.get_partition().reduce(concat, []))
