def get_hist_slice(gsum0, hsum0, nodeids, nodecansplit, factor, gcovariate, hcovariate, start, length, slicelen): nodeslices = VectorSlicer(nodeids)(start, length, slicelen) factorslices = factor.slicer(start, length, slicelen) gslices = gcovariate.slicer(start, length, slicelen) hslices = hcovariate.slicer(start, length, slicelen) zipslices = Seq.zip(nodeslices, factorslices, gslices, hslices) return Seq.reduce(f_hist, (gsum0, hsum0, nodecansplit), zipslices)
def __call__(self, start, length, slicelen): dtypes = [f.slicer.dtype for f in self.factors] alluint8 = all([d == np.uint8 for d in dtypes]) dtype = np.uint8 if alluint8 else np.uint16 k = len(self.factors) buf = np.empty((k, slicelen), dtype=dtype) def f(slices): n = len(slices[0]) for i in range(k): if dtypes[i] == dtype: buf[i, :n] = slices[i] else: buf[i, :n] = slices[i].astype(dtype) if n == slicelen: return buf else: return buf[:, :n] return Seq.map( f, Seq.zip(*[f.slicer(start, length, slicelen) for f in self.factors]))
def splitnodeidsslice(nodeids, factors, issplitnode, nodemap, leftpartitions, factorindex, start, length, slicelength): if len(factors) > 0: factorslices = NFactorSlicer(factors)(start, length, slicelength) nodeslices = VectorSlicer(nodeids)(start, length, slicelength) Seq.foreach(lambda x: f_splitnode(x[0], x[1], issplitnode, leftpartitions, factorindex, nodemap), Seq.zip(nodeslices, factorslices))