def racc(self, f, unit_f):
"""Makes a rightward accumulation of the values in the current instance
rAcc (+) (RNode a ts)
= let rs = scanl (+) [root ts[i] | i in [1 .. #ts]]
in RNode unit_(+) [setroot (rAcc (+) ts[i]) r[i] | i in [1 .. #ts]]
Parameters
----------
f : callable
A function to accumulate the value of the current instance with the current accumulator
unit_f
A value such as, forall value, f(value, unit_f) = value
"""
rv = self.get_children().map(lambda x: x.get_value())
rs = rv.scanl(f, unit_f)
ch = SList()
ch0 = self.get_children()
for i in range(0, ch0.length()):
c = ch0[i]
cs = c.racc(f, unit_f)
cs.set_value(rs[i])
ch.append(cs)
return RNode(unit_f, ch)
def test_r2b_2():
rn5 = RNode(5)
rn6 = RNode(6)
rn3 = RNode(3, SList([rn5, rn6]))
rn2 = RNode(2)
rn4 = RNode(4)
rn1 = RNode(1, SList([rn2, rn3, rn4]))
res = rn1.r2b()
exp = \
Node(1,
Node(2,
Leaf(None),
Node(3,
Node(5,
Leaf(None),
Node(6,
Leaf(None),
Leaf(None)
)
),
Node(4,
Leaf(None),
Leaf(None)
)
)
),
Leaf(None)
)
assert res == exp
def test_r2b_1():
ch = SList()
ch.append(RNode(2))
ch.append(RNode(3))
rn = RNode(1, ch)
res = rn.r2b()
exp = Node(1, Node(2, Leaf(None), Node(3, Leaf(None), Leaf(None))),
Leaf(None))
assert res == exp
def permute(self: 'PList[T]', bij: Callable[[int], int]) -> 'PList[T]':
p_list = self.__get_shape()
distr = Distribution(self.__distribution)
new_indices = self.mapi(lambda i, x: distr.to_pid(bij(i), x)
).get_partition().map(_group_by)
mapping = new_indices.__content[0]
keys = mapping.keys()
messages = [mapping[pid] if pid in keys else [] for pid in par.procs()]
exchanged = SList(parimpl.COMM.alltoall(messages)).flatten()
exchanged.sort()
p_list.__content = exchanged.map(lambda pair: pair[1])
return p_list
def from_seq(sequence: Sequence[T]) -> 'PList[T]':
p_list = PList()
if _PID == 0:
p_list.__content = SList(sequence)
p_list.__distribution = [
len(sequence) if i == 0 else 0 for i in par.procs()
]
from_root = _COMM.bcast(p_list.__distribution, 0)
p_list.__distribution = Distribution(from_root)
p_list.__local_size = p_list.__distribution[_PID]
p_list.__global_size = p_list.__distribution[0]
p_list.__start_index = SList(p_list.__distribution).scanl(add, 0)[_PID]
return p_list
def flatten(self: 'PList[SList[T]]',
new_distr: Distribution = None) -> 'PList[T]':
p_list = PList()
p_list.__content = self.__content.flatten()
p_list.__local_size = len(p_list.__content)
if new_distr is None:
p_list.__distribution = _COMM.allgather(p_list.__local_size)
else:
p_list.__distribution = new_distr
assert new_distr[_PID] == p_list.__local_size
p_list.__start_index = SList(p_list.__distribution).scanl(add, 0)[_PID]
p_list.__global_size = SList(p_list.__distribution).reduce(add)
return p_list
def aux(btree):
if btree.is_leaf():
val = btree.get_value()
if val is None:
return SList()
return SList([RNode(val)])
n = btree.get_value()
left = btree.get_left()
right = btree.get_right()
res_l = aux(left)
res_r = aux(right)
res_head = RNode(n, res_l)
res_r.insert(0, res_head)
return res_r
def __init__(self, lt=None):
self.__distribution = SList([])
self.__global_index = SList([])
self.__start_index = 0
self.__nb_segs = 0
self.__content = SList([])
if lt is not None:
(distribution, global_index) = distribute_tree(lt, NPROCS)
self.__distribution = distribution
self.__global_index = global_index
self.__start_index = distribution.scanl(lambda x, y: x + y, 0)[PID]
self.__nb_segs = distribution[PID]
for i_seg in range(self.__start_index,
self.__start_index + self.__nb_segs):
self.__content.extend(lt[i_seg])
def random_list(frdm, size):
"""
Generates a random list of the given ``size``.
Each element is generated by a call to ``frdm``.
``size`` should be strictly positive.
:param frdm: callable
:param size: int
:return: list
"""
assert size >= 0
res = SList([])
for _ in range(size):
res.append(frdm())
return res
def test_b2r_node_from_rt():
bt = Node(
1,
Node(
2, Leaf(None),
Node(3, Node(5, Leaf(None), Node(6, Leaf(None), Leaf(None))),
Node(4, Leaf(None), Leaf(None)))), Leaf(None))
rn5 = RNode(5)
rn6 = RNode(6)
rn3 = RNode(3, SList([rn5, rn6]))
rn2 = RNode(2)
rn4 = RNode(4)
exp = RNode(1, SList([rn2, rn3, rn4]))
res = RNode(bt)
assert res == exp
def get_full_index(self):
def f(x, y):
(x1, y1) = x
(_, y2) = y
return x1 + y1, y2
return SList(self.__global_index.scanr(f))
def __init__(self: 'PList[T]'):
# pylint: disable=super-init-not-called
self.__content: 'SList[T]' = SList([])
self.__global_size: int = 0
self.__local_size: int = 0
self.__start_index: int = 0
self.__distribution = Distribution([0 for _ in range(0, _NPROCS)])
def map2(self, f, pt):
"""Map2 skeleton for distributed tree
Precondition
-------------
The distributions of self and pt should be the same
Parameters
----------
pt : :obj:`LTree`
The LTree to zip with the current instance
f : callable
A function to zip values
"""
logger.debug('[START] PID[%s] map2 skeleton', PID)
assert self.__distribution == pt.distribution
content = SList([None] * self.__content.length())
for i in range(
len(self.__global_index[self.__start_index:self.__start_index +
self.__nb_segs])):
(start, offset
) = self.__global_index[self.__start_index:self.__start_index +
self.__nb_segs][i]
logger.debug('[START] PID[%s] map2_local from %s to %s', PID,
start, start + offset)
content[start:start + offset] = Segment(self.__content[start:start + offset]). \
map2(f, Segment(pt.content[start:start + offset]))
logger.debug('[END] PID[%s] map2_local from %s to %s', PID, start,
start + offset)
res = PTree.init(self, content)
logger.debug('[END] PID[%s] map2 skeleton', PID)
return res
def __init__(self, value, ts=None):
if isinstance(value, BTree):
if value == Leaf(None):
raise ConstructorError(
"A RTree cannot be constructed from a single Leaf that "
"contains None")
# Create a RTree from a BTree
bt = value
rt = RNode.b2r(bt)
self.value = rt.get_value()
self.children = rt.get_children()
else:
self.value = value
if ts is None:
self.children = SList([])
else:
self.children = SList(ts)
def get_partition(self: 'PList[T]') -> 'PList[SList[T]]':
p_list = PList()
p_list.__content = SList([self.__content])
p_list.__global_size = _NPROCS
p_list.__local_size = 1
p_list.__start_index = _PID
p_list.__distribution = [1 for _ in par.procs()]
return p_list
def init(value_at: Callable[[int], T], size: int = _NPROCS) -> 'PList[T]':
assert size >= 0
p_list = PList()
p_list.__global_size = size
p_list.__local_size = parimpl.local_size(_PID, size)
distribution_list = [
parimpl.local_size(i, size) for i in range(0, _NPROCS)
]
p_list.__distribution = Distribution(distribution_list)
p_list.__start_index = SList(p_list.__distribution).scanl(
lambda x, y: x + y, 0)[_PID]
p_list.__content = SList([
value_at(i)
for i in range(p_list.__start_index, p_list.__start_index +
p_list.__local_size)
])
p_list.__distribution = [
parimpl.local_size(i, size) for i in range(0, _NPROCS)
]
return p_list
def distribute(self: 'PList[T]', target_distr: Distribution) -> 'PList[T]':
assert Distribution.is_valid(target_distr, self.__global_size)
source_distr = self.__distribution
source_bounds = interval.bounds(source_distr)
target_bounds = interval.bounds(target_distr)
local_interval = source_bounds[_PID]
bounds_to_send = target_bounds.map(
lambda i: interval.intersection(i, local_interval))
msgs = [
interval.to_slice(self.__content,
interval.shift(inter, -self.__start_index))
for inter in bounds_to_send
]
slices = _COMM.alltoall(msgs)
p_list = PList()
p_list.__content = SList(slices).flatten()
p_list.__local_size = target_distr[_PID]
p_list.__global_size = self.__global_size
p_list.__start_index = SList(target_distr).scanl(add, 0)[_PID]
p_list.__distribution = target_distr
return p_list
def zip(self, rt):
"""Zip the values contained in a second RTree with the ones in the current instance
Precondition
-------------
The lengths of self.children and rt.children should be equal
Parameters
----------
rt : :obj:`RTree`
The RTree to zip with the current instance
"""
ch1 = self.get_children()
ch2 = rt.get_children()
assert ch1.length() == ch2.length(
), "The rose trees cannot be zipped (not the same shape)"
ch = SList([])
for i in range(0, ch1.length()):
ch.append(ch1[i].zip(ch2))
v = (self.get_value(), rt.get_value())
return RNode(v, ch)
def map_reduce(self: 'PList[T]',
unary_op: Callable[[T], V],
binary_op: Callable[[V, V], V],
neutral: Optional[V] = None) -> V:
if neutral is None:
assert self.__global_size >= 1
partial = None if self.__local_size == 0 \
else self.__content.map_reduce(unary_op, binary_op)
partials = SList(
_COMM.allgather(partial)).filter(lambda x: x is not None)
return functools.reduce(binary_op, partials)
# assert: (binary_op, neutral) form a monoid
partial = self.__content.map_reduce(unary_op, binary_op, neutral)
partials = _COMM.allgather(partial)
return functools.reduce(binary_op, partials, neutral)
def reduce(self: 'PList[T]',
binary_op: Callable[[T, T], T],
neutral: Optional[T] = None) -> T:
if neutral is None:
assert self.__global_size >= 1
partial = None if self.__local_size == 0 else SList(
self.__content).reduce(binary_op)
partials = SList(
_COMM.allgather(partial)).filter(lambda x: x is not None)
else:
# assert: (binary_op, neutral) form a monoid
partial = SList(self.__content).reduce(binary_op, neutral)
partials = SList(_COMM.allgather(partial))
return partials.reduce(binary_op, neutral)
def __local_upwards_accumulation(self, k, phi, psi_l, psi_r):
gt = Segment([None] * self.__nb_segs)
lt2 = SList([None] * self.__nb_segs)
i = 0
for (start, offset) in \
self.__global_index[self.__start_index: self.__start_index + self.__nb_segs]:
logger.debug('[START] PID[%s] uacc_local from %s to %s', PID,
start, start + offset)
(top,
res) = Segment(self.__content[start:start + offset]).uacc_local(
k, phi, psi_l, psi_r)
logger.debug('[END] PID[%s] uacc_local from %s to %s', PID, start,
start + offset)
gt[i] = top
lt2[i] = res
i = i + 1
return i, gt, lt2
def r2b(self):
"""Get a BTree from the current instance
"""
def r2b1(t, ss):
a = t.get_value()
left = r2b2(t.get_children())
right = r2b2(ss)
return Node(a, left, right)
def r2b2(ts):
if not ts:
return Leaf(None)
h = ts[0]
t = ts[1:]
return r2b1(h, t)
return r2b1(self, SList())
def r2b(self):
"""Get a BTree from the current instance
"""
def r2b1(t, ss):
a = t.get_value()
left = r2b2(t.get_children())
right = r2b2(ss)
return Node(a, left, right)
def r2b2(ts):
if ts.empty():
return Leaf(None)
else:
h = ts.head()
t = ts.tail()
return r2b1(h, t)
return r2b1(self, SList())
def __local_updates(self, gt, gt2, lt2, k):
content = SList([None] * self.__content.length())
for i in range(
len(self.__global_index[self.__start_index:self.__start_index +
self.__nb_segs])):
(start, offset
) = self.__global_index[self.__start_index:self.__start_index +
self.__nb_segs][i]
logger.debug('[START] PID[%s] uacc_update from %s to %s', PID,
start, start + offset)
if gt[i].is_node():
(lc, rc) = gt2[i].get_value()
val = Segment(self.__content[start:start +
offset]).uacc_update(
lt2[i], k, lc, rc)
else:
val = lt2[i]
logger.debug('[END] PID[%s] uacc_update from %s to %s', PID, start,
start + offset)
content[start:start + offset] = val
return content
def map(self, kl, kn):
"""Map skeleton for distributed tree
Parameters
----------
kl : callable
Function to apply to every leaf value of the current instance
kn : callable
Function to apply to every node value of the current instance
"""
content = SList([None] * self.__content.length())
logger.debug('[START] PID[%s] map skeleton', PID)
for (start, offset) in \
self.__global_index[self.__start_index: self.__start_index + self.__nb_segs]:
logger.debug('[START] PID[%s] map_local from %s to %s', PID, start,
start + offset)
content[start:start + offset] = Segment(
self.__content[start:start + offset]).map_local(kl, kn)
logger.debug('[END] PID[%s] map_local from %s to %s', PID, start,
start + offset)
res = PTree.init(self, content)
logger.debug('[END] PID[%s] map skeleton', PID)
return res
def init_from_file(filename, parser=int):
"""Instantiate a distributed tree from a file
Parameters
----------
filename : str
The name of the file that contains the PTree to instantiate
parser : callable, optional
A function that transforms a string into a specific type.
By default, string to int
"""
filename = filename + "." + str(PID)
def __parser_couple(s):
s = s.replace("(", "")
s = s.replace(")", "")
ss = s.split(",")
return int(ss[0]), int(ss[1])
p = PTree()
content = SList([])
with open(filename, "r") as f:
count_line = 0
for line in f:
if line.strip()[0] == '#':
continue
# __distribution: PID -> nb of segments
# __global_index: num seg -> (start, offset)
if count_line == 0: # Get the distribution
p.distribution = SList.from_str(line)
p.start_index = p.distribution.scanl(
lambda x, y: x + y, 0)[PID]
p.nb_segs = p.distribution[PID]
elif count_line == 1: # Get the global_index
p.global_index = SList.from_str(line,
parser=__parser_couple)
else: # Get the content
content.extend(Segment.from_str(line, parser=parser))
count_line = count_line + 1
p.content = content
return p
def _lasts(distr):
return SList(scan(distr, add, 0)).tail().map(lambda x: x - 1)
def deserialization(self):
"""Get a binary tree from its linear representation
"""
def __graft(bt, lbt, rbt):
val = bt.get_value()
if bt.is_node():
return Node(val, __graft(bt.get_left(), lbt, rbt),
__graft(bt.get_right(), lbt, rbt))
else: # bt.is_leaf()
return Node(val, lbt, rbt) if val.is_critical() else bt
def __remove_annotation(bt):
v = bt.get_value()
return Leaf(v.get_value()) if bt.is_leaf() else Node(
v.get_value(), __remove_annotation(bt.get_left()),
__remove_annotation(bt.get_right()))
def __lv2ibt(segment):
stack = []
has_crit_b = False
if segment.empty():
raise EmptyError(
"An empty Segment cannot be transformed into a BTree")
for i in range(segment.length() - 1, -1, -1):
v = segment[i]
if v.is_leaf():
stack.append(Leaf(v))
elif v.is_critical():
stack.append(Leaf(v))
if has_crit_b:
raise IllFormedError(
"A ill-formed Segment cannot be transformed into a BTree"
)
else:
has_crit_b = True
else:
if len(stack) < 2:
raise IllFormedError(
"A ill-formed Segment cannot be transformed into a BTree"
)
lv = stack.pop()
rv = stack.pop()
stack.append(Node(v, lv, rv))
if len(stack) != 1:
raise IllFormedError(
"A ill-formed Segment cannot be transformed into a BTree")
else:
return has_crit_b, stack[0]
def __rev_segment_to_trees(lb, glob):
stack = []
for i in range(lb.length() - 1, -1, -1):
if glob[i] == TAG_LEAF:
stack.append(lb[i])
else: # gt[i] == VTag_Node
lbt = stack.pop()
rbt = stack.pop()
stack.append(__graft(lb[i], lbt, rbt))
if len(stack) != 1:
raise IllFormedError(
"A ill-formed list of incomplete BTree cannot be transformed into a BTree"
)
else:
return stack[0]
gt = SList()
list_of_btree = SList()
for seg in self:
(has_crit, bt_i) = __lv2ibt(seg)
list_of_btree.append(bt_i)
if has_crit:
gt.append(TAG_NODE)
else:
gt.append(TAG_LEAF)
return __remove_annotation(__rev_segment_to_trees(list_of_btree, gt))
def _firsts(distr):
return SList(distr).scanl(add, 0)
def to_seq(self: 'PList[T]') -> 'SList[T]':
return SList(self.get_partition().reduce(concat, []))
