def dispatch(seq):
vbuf, wbuf = [], []
for (n, v) in seq:
if n == 1:
vbuf.append(v)
elif n == 2:
wbuf.append(v)
return (ResultIterable(vbuf), ResultIterable(wbuf))
def groupByKey(self, numPartitions=None):
"""
Return a new DStream which contains group the values for each key in the
DStream into a single sequence.
Hash-partitions the resulting RDD with into numPartitions partitions in
the DStream.
Note: If you are grouping in order to perform an aggregation (such as a
sum or average) over each key, using reduceByKey will provide much
better performance.
"""
def createCombiner(x):
return [x]
def mergeValue(xs, x):
xs.append(x)
return xs
def mergeCombiners(a, b):
a.extend(b)
return a
return self.combineByKey(
createCombiner, mergeValue, mergeCombiners,
numPartitions).mapValues(lambda x: ResultIterable(x))
def groupByKey(self, numPartitions=None):
"""
Group the values for each key in the RDD into a single sequence.
Hash-partitions the resulting RDD with into numPartitions partitions.
>>> x = sc.parallelize([("a", 1), ("b", 1), ("a", 1)])
>>> map((lambda (x,y): (x, list(y))), sorted(x.groupByKey().collect()))
[('a', [1, 1]), ('b', [1])]
"""
def createCombiner(x):
return [x]
def mergeValue(xs, x):
xs.append(x)
return xs
def mergeCombiners(a, b):
return a + b
return self.combineByKey(createCombiner, mergeValue, mergeCombiners,
numPartitions).mapValues(lambda x: ResultIterable(x))
def dispatch(seq):
bufs = [[] for _ in range(rdd_len)]
for n, v in seq:
bufs[n].append(v)
return tuple(ResultIterable(vs) for vs in bufs)
def test_derivative_should_work(self):
new_values = MonascaDerivativeLDP.derivative(
ResultIterable(self.all_metrics))
new_values = self._values(new_values)
self.assertEqual(new_values,
[0.5, 0.25, -0.5, -0.25, 1.0, 1.5])
def test_derivative_should_remove_duplicate(self):
new_values = MonascaDerivativeLDP.derivative(
ResultIterable(self.all_metrics[0:4]))
new_values = self._values(new_values)
self.assertEqual(new_values,
[0.5, 0.5])
def test_derivative_should_work_on_first_and_last_values(self):
new_values = MonascaDerivativeLDP.derivative(
ResultIterable(self.all_metrics[0:2]))
new_values = self._values(new_values)
self.assertEqual(new_values,
[0.5, 0.5])
def test_derivative_should_do_nothing_with_1_value(self):
self.assertEqual(MonascaDerivativeLDP.derivative(
ResultIterable(self.all_metrics[0:1])),
[])