def racc(self, f, unit_f): """Makes a rightward accumulation of the values in the current instance rAcc (+) (RNode a ts) = let rs = scanl (+) [root ts[i] | i in [1 .. #ts]] in RNode unit_(+) [setroot (rAcc (+) ts[i]) r[i] | i in [1 .. #ts]] Parameters ---------- f : callable A function to accumulate the value of the current instance with the current accumulator unit_f A value such as, forall value, f(value, unit_f) = value """ rv = self.get_children().map(lambda x: x.get_value()) rs = rv.scanl(f, unit_f) ch = SList() ch0 = self.get_children() for i in range(0, ch0.length()): c = ch0[i] cs = c.racc(f, unit_f) cs.set_value(rs[i]) ch.append(cs) return RNode(unit_f, ch)
def test_r2b_2(): rn5 = RNode(5) rn6 = RNode(6) rn3 = RNode(3, SList([rn5, rn6])) rn2 = RNode(2) rn4 = RNode(4) rn1 = RNode(1, SList([rn2, rn3, rn4])) res = rn1.r2b() exp = \ Node(1, Node(2, Leaf(None), Node(3, Node(5, Leaf(None), Node(6, Leaf(None), Leaf(None) ) ), Node(4, Leaf(None), Leaf(None) ) ) ), Leaf(None) ) assert res == exp
def test_r2b_1(): ch = SList() ch.append(RNode(2)) ch.append(RNode(3)) rn = RNode(1, ch) res = rn.r2b() exp = Node(1, Node(2, Leaf(None), Node(3, Leaf(None), Leaf(None))), Leaf(None)) assert res == exp
def permute(self: 'PList[T]', bij: Callable[[int], int]) -> 'PList[T]': p_list = self.__get_shape() distr = Distribution(self.__distribution) new_indices = self.mapi(lambda i, x: distr.to_pid(bij(i), x) ).get_partition().map(_group_by) mapping = new_indices.__content[0] keys = mapping.keys() messages = [mapping[pid] if pid in keys else [] for pid in par.procs()] exchanged = SList(parimpl.COMM.alltoall(messages)).flatten() exchanged.sort() p_list.__content = exchanged.map(lambda pair: pair[1]) return p_list
def from_seq(sequence: Sequence[T]) -> 'PList[T]': p_list = PList() if _PID == 0: p_list.__content = SList(sequence) p_list.__distribution = [ len(sequence) if i == 0 else 0 for i in par.procs() ] from_root = _COMM.bcast(p_list.__distribution, 0) p_list.__distribution = Distribution(from_root) p_list.__local_size = p_list.__distribution[_PID] p_list.__global_size = p_list.__distribution[0] p_list.__start_index = SList(p_list.__distribution).scanl(add, 0)[_PID] return p_list
def flatten(self: 'PList[SList[T]]', new_distr: Distribution = None) -> 'PList[T]': p_list = PList() p_list.__content = self.__content.flatten() p_list.__local_size = len(p_list.__content) if new_distr is None: p_list.__distribution = _COMM.allgather(p_list.__local_size) else: p_list.__distribution = new_distr assert new_distr[_PID] == p_list.__local_size p_list.__start_index = SList(p_list.__distribution).scanl(add, 0)[_PID] p_list.__global_size = SList(p_list.__distribution).reduce(add) return p_list
def aux(btree): if btree.is_leaf(): val = btree.get_value() if val is None: return SList() return SList([RNode(val)]) n = btree.get_value() left = btree.get_left() right = btree.get_right() res_l = aux(left) res_r = aux(right) res_head = RNode(n, res_l) res_r.insert(0, res_head) return res_r
def __init__(self, lt=None): self.__distribution = SList([]) self.__global_index = SList([]) self.__start_index = 0 self.__nb_segs = 0 self.__content = SList([]) if lt is not None: (distribution, global_index) = distribute_tree(lt, NPROCS) self.__distribution = distribution self.__global_index = global_index self.__start_index = distribution.scanl(lambda x, y: x + y, 0)[PID] self.__nb_segs = distribution[PID] for i_seg in range(self.__start_index, self.__start_index + self.__nb_segs): self.__content.extend(lt[i_seg])
def random_list(frdm, size): """ Generates a random list of the given ``size``. Each element is generated by a call to ``frdm``. ``size`` should be strictly positive. :param frdm: callable :param size: int :return: list """ assert size >= 0 res = SList([]) for _ in range(size): res.append(frdm()) return res
def test_b2r_node_from_rt(): bt = Node( 1, Node( 2, Leaf(None), Node(3, Node(5, Leaf(None), Node(6, Leaf(None), Leaf(None))), Node(4, Leaf(None), Leaf(None)))), Leaf(None)) rn5 = RNode(5) rn6 = RNode(6) rn3 = RNode(3, SList([rn5, rn6])) rn2 = RNode(2) rn4 = RNode(4) exp = RNode(1, SList([rn2, rn3, rn4])) res = RNode(bt) assert res == exp
def get_full_index(self): def f(x, y): (x1, y1) = x (_, y2) = y return x1 + y1, y2 return SList(self.__global_index.scanr(f))
def __init__(self: 'PList[T]'): # pylint: disable=super-init-not-called self.__content: 'SList[T]' = SList([]) self.__global_size: int = 0 self.__local_size: int = 0 self.__start_index: int = 0 self.__distribution = Distribution([0 for _ in range(0, _NPROCS)])
def map2(self, f, pt): """Map2 skeleton for distributed tree Precondition ------------- The distributions of self and pt should be the same Parameters ---------- pt : :obj:`LTree` The LTree to zip with the current instance f : callable A function to zip values """ logger.debug('[START] PID[%s] map2 skeleton', PID) assert self.__distribution == pt.distribution content = SList([None] * self.__content.length()) for i in range( len(self.__global_index[self.__start_index:self.__start_index + self.__nb_segs])): (start, offset ) = self.__global_index[self.__start_index:self.__start_index + self.__nb_segs][i] logger.debug('[START] PID[%s] map2_local from %s to %s', PID, start, start + offset) content[start:start + offset] = Segment(self.__content[start:start + offset]). \ map2(f, Segment(pt.content[start:start + offset])) logger.debug('[END] PID[%s] map2_local from %s to %s', PID, start, start + offset) res = PTree.init(self, content) logger.debug('[END] PID[%s] map2 skeleton', PID) return res
def __init__(self, value, ts=None): if isinstance(value, BTree): if value == Leaf(None): raise ConstructorError( "A RTree cannot be constructed from a single Leaf that " "contains None") # Create a RTree from a BTree bt = value rt = RNode.b2r(bt) self.value = rt.get_value() self.children = rt.get_children() else: self.value = value if ts is None: self.children = SList([]) else: self.children = SList(ts)
def get_partition(self: 'PList[T]') -> 'PList[SList[T]]': p_list = PList() p_list.__content = SList([self.__content]) p_list.__global_size = _NPROCS p_list.__local_size = 1 p_list.__start_index = _PID p_list.__distribution = [1 for _ in par.procs()] return p_list
def init(value_at: Callable[[int], T], size: int = _NPROCS) -> 'PList[T]': assert size >= 0 p_list = PList() p_list.__global_size = size p_list.__local_size = parimpl.local_size(_PID, size) distribution_list = [ parimpl.local_size(i, size) for i in range(0, _NPROCS) ] p_list.__distribution = Distribution(distribution_list) p_list.__start_index = SList(p_list.__distribution).scanl( lambda x, y: x + y, 0)[_PID] p_list.__content = SList([ value_at(i) for i in range(p_list.__start_index, p_list.__start_index + p_list.__local_size) ]) p_list.__distribution = [ parimpl.local_size(i, size) for i in range(0, _NPROCS) ] return p_list
def distribute(self: 'PList[T]', target_distr: Distribution) -> 'PList[T]': assert Distribution.is_valid(target_distr, self.__global_size) source_distr = self.__distribution source_bounds = interval.bounds(source_distr) target_bounds = interval.bounds(target_distr) local_interval = source_bounds[_PID] bounds_to_send = target_bounds.map( lambda i: interval.intersection(i, local_interval)) msgs = [ interval.to_slice(self.__content, interval.shift(inter, -self.__start_index)) for inter in bounds_to_send ] slices = _COMM.alltoall(msgs) p_list = PList() p_list.__content = SList(slices).flatten() p_list.__local_size = target_distr[_PID] p_list.__global_size = self.__global_size p_list.__start_index = SList(target_distr).scanl(add, 0)[_PID] p_list.__distribution = target_distr return p_list
def zip(self, rt): """Zip the values contained in a second RTree with the ones in the current instance Precondition ------------- The lengths of self.children and rt.children should be equal Parameters ---------- rt : :obj:`RTree` The RTree to zip with the current instance """ ch1 = self.get_children() ch2 = rt.get_children() assert ch1.length() == ch2.length( ), "The rose trees cannot be zipped (not the same shape)" ch = SList([]) for i in range(0, ch1.length()): ch.append(ch1[i].zip(ch2)) v = (self.get_value(), rt.get_value()) return RNode(v, ch)
def map_reduce(self: 'PList[T]', unary_op: Callable[[T], V], binary_op: Callable[[V, V], V], neutral: Optional[V] = None) -> V: if neutral is None: assert self.__global_size >= 1 partial = None if self.__local_size == 0 \ else self.__content.map_reduce(unary_op, binary_op) partials = SList( _COMM.allgather(partial)).filter(lambda x: x is not None) return functools.reduce(binary_op, partials) # assert: (binary_op, neutral) form a monoid partial = self.__content.map_reduce(unary_op, binary_op, neutral) partials = _COMM.allgather(partial) return functools.reduce(binary_op, partials, neutral)
def reduce(self: 'PList[T]', binary_op: Callable[[T, T], T], neutral: Optional[T] = None) -> T: if neutral is None: assert self.__global_size >= 1 partial = None if self.__local_size == 0 else SList( self.__content).reduce(binary_op) partials = SList( _COMM.allgather(partial)).filter(lambda x: x is not None) else: # assert: (binary_op, neutral) form a monoid partial = SList(self.__content).reduce(binary_op, neutral) partials = SList(_COMM.allgather(partial)) return partials.reduce(binary_op, neutral)
def __local_upwards_accumulation(self, k, phi, psi_l, psi_r): gt = Segment([None] * self.__nb_segs) lt2 = SList([None] * self.__nb_segs) i = 0 for (start, offset) in \ self.__global_index[self.__start_index: self.__start_index + self.__nb_segs]: logger.debug('[START] PID[%s] uacc_local from %s to %s', PID, start, start + offset) (top, res) = Segment(self.__content[start:start + offset]).uacc_local( k, phi, psi_l, psi_r) logger.debug('[END] PID[%s] uacc_local from %s to %s', PID, start, start + offset) gt[i] = top lt2[i] = res i = i + 1 return i, gt, lt2
def r2b(self): """Get a BTree from the current instance """ def r2b1(t, ss): a = t.get_value() left = r2b2(t.get_children()) right = r2b2(ss) return Node(a, left, right) def r2b2(ts): if not ts: return Leaf(None) h = ts[0] t = ts[1:] return r2b1(h, t) return r2b1(self, SList())
def r2b(self): """Get a BTree from the current instance """ def r2b1(t, ss): a = t.get_value() left = r2b2(t.get_children()) right = r2b2(ss) return Node(a, left, right) def r2b2(ts): if ts.empty(): return Leaf(None) else: h = ts.head() t = ts.tail() return r2b1(h, t) return r2b1(self, SList())
def __local_updates(self, gt, gt2, lt2, k): content = SList([None] * self.__content.length()) for i in range( len(self.__global_index[self.__start_index:self.__start_index + self.__nb_segs])): (start, offset ) = self.__global_index[self.__start_index:self.__start_index + self.__nb_segs][i] logger.debug('[START] PID[%s] uacc_update from %s to %s', PID, start, start + offset) if gt[i].is_node(): (lc, rc) = gt2[i].get_value() val = Segment(self.__content[start:start + offset]).uacc_update( lt2[i], k, lc, rc) else: val = lt2[i] logger.debug('[END] PID[%s] uacc_update from %s to %s', PID, start, start + offset) content[start:start + offset] = val return content
def map(self, kl, kn): """Map skeleton for distributed tree Parameters ---------- kl : callable Function to apply to every leaf value of the current instance kn : callable Function to apply to every node value of the current instance """ content = SList([None] * self.__content.length()) logger.debug('[START] PID[%s] map skeleton', PID) for (start, offset) in \ self.__global_index[self.__start_index: self.__start_index + self.__nb_segs]: logger.debug('[START] PID[%s] map_local from %s to %s', PID, start, start + offset) content[start:start + offset] = Segment( self.__content[start:start + offset]).map_local(kl, kn) logger.debug('[END] PID[%s] map_local from %s to %s', PID, start, start + offset) res = PTree.init(self, content) logger.debug('[END] PID[%s] map skeleton', PID) return res
def init_from_file(filename, parser=int): """Instantiate a distributed tree from a file Parameters ---------- filename : str The name of the file that contains the PTree to instantiate parser : callable, optional A function that transforms a string into a specific type. By default, string to int """ filename = filename + "." + str(PID) def __parser_couple(s): s = s.replace("(", "") s = s.replace(")", "") ss = s.split(",") return int(ss[0]), int(ss[1]) p = PTree() content = SList([]) with open(filename, "r") as f: count_line = 0 for line in f: if line.strip()[0] == '#': continue # __distribution: PID -> nb of segments # __global_index: num seg -> (start, offset) if count_line == 0: # Get the distribution p.distribution = SList.from_str(line) p.start_index = p.distribution.scanl( lambda x, y: x + y, 0)[PID] p.nb_segs = p.distribution[PID] elif count_line == 1: # Get the global_index p.global_index = SList.from_str(line, parser=__parser_couple) else: # Get the content content.extend(Segment.from_str(line, parser=parser)) count_line = count_line + 1 p.content = content return p
def _lasts(distr): return SList(scan(distr, add, 0)).tail().map(lambda x: x - 1)
def deserialization(self): """Get a binary tree from its linear representation """ def __graft(bt, lbt, rbt): val = bt.get_value() if bt.is_node(): return Node(val, __graft(bt.get_left(), lbt, rbt), __graft(bt.get_right(), lbt, rbt)) else: # bt.is_leaf() return Node(val, lbt, rbt) if val.is_critical() else bt def __remove_annotation(bt): v = bt.get_value() return Leaf(v.get_value()) if bt.is_leaf() else Node( v.get_value(), __remove_annotation(bt.get_left()), __remove_annotation(bt.get_right())) def __lv2ibt(segment): stack = [] has_crit_b = False if segment.empty(): raise EmptyError( "An empty Segment cannot be transformed into a BTree") for i in range(segment.length() - 1, -1, -1): v = segment[i] if v.is_leaf(): stack.append(Leaf(v)) elif v.is_critical(): stack.append(Leaf(v)) if has_crit_b: raise IllFormedError( "A ill-formed Segment cannot be transformed into a BTree" ) else: has_crit_b = True else: if len(stack) < 2: raise IllFormedError( "A ill-formed Segment cannot be transformed into a BTree" ) lv = stack.pop() rv = stack.pop() stack.append(Node(v, lv, rv)) if len(stack) != 1: raise IllFormedError( "A ill-formed Segment cannot be transformed into a BTree") else: return has_crit_b, stack[0] def __rev_segment_to_trees(lb, glob): stack = [] for i in range(lb.length() - 1, -1, -1): if glob[i] == TAG_LEAF: stack.append(lb[i]) else: # gt[i] == VTag_Node lbt = stack.pop() rbt = stack.pop() stack.append(__graft(lb[i], lbt, rbt)) if len(stack) != 1: raise IllFormedError( "A ill-formed list of incomplete BTree cannot be transformed into a BTree" ) else: return stack[0] gt = SList() list_of_btree = SList() for seg in self: (has_crit, bt_i) = __lv2ibt(seg) list_of_btree.append(bt_i) if has_crit: gt.append(TAG_NODE) else: gt.append(TAG_LEAF) return __remove_annotation(__rev_segment_to_trees(list_of_btree, gt))
def _firsts(distr): return SList(distr).scanl(add, 0)
def to_seq(self: 'PList[T]') -> 'SList[T]': return SList(self.get_partition().reduce(concat, []))