def test_100_downstream(self):
""" test """
nums = self._pipeline.parallelize([1, 2])
downstreams = []
for i in xrange(100):
downstreams.append(nums.map(lambda x: x))
self.assertEqual(100 * 3, transforms.union(*downstreams).sum().get())
def test_100_downstream(self):
""" test """
nums = self._pipeline.parallelize([1, 2])
downstreams = []
for i in xrange(100):
downstreams.append(nums.map(lambda x: x))
self.assertEqual(100 * 3, transforms.union(*downstreams).sum().get())
def end_serde_test(self):
""" test """
import sys
from bigflow.core import entity
logger.info(str(self._checking_condition))
values = map(lambda condition: condition[1], self._checking_condition)
p_values = self._pipeline.parallelize([values
]) # 避免map结点超过32个（Hadoop的限制）
p_value_list = []
out = []
for (i, (sd, value)) in enumerate(self._checking_condition):
sd1 = serde.of(int)
sd2 = sd
cpp_deserialize_fn = entity.KVDeserializeFn(sd1, sd2)
cpp_serialize_fn = entity.KVSerializeFn(sd1, sd2)
python_deserialize_fn = lambda kv: (sd1.deserialize(kv[0]),
sd2.deserialize(kv[1]))
python_serialize_fn = lambda kv: (sd1.serialize(kv[0]),
sd2.serialize(kv[1]))
serialize_fns = [cpp_serialize_fn, python_serialize_fn]
deserialize_fns = [cpp_deserialize_fn, python_deserialize_fn]
kv_val = (1, value)
def _assert_eq_val(v):
assert v == kv_val
for serialize_fn in serialize_fns:
for deserialize_fn in deserialize_fns:
out.append(
p_values.map(lambda x: (1, x[i])).map(serialize_fn).
map(deserialize_fn).map(_assert_eq_val))
if out:
transforms.union(*out).cache()
else:
print >> sys.stderr, "SKIP a test!!!"
self._pipeline.run()
def test_modify_left_param(self):
""" inner function"""
inp = self._pipeline.parallelize([[1, 2, 3], [6, 5, 4]])
def _sum(x, y):
x[0] += y[0]
x[1] += y[1]
x[2] += y[2]
return x
result = transforms.union(inp.reduce(_sum), inp.reduce(_sum)).get()
self.assertEqual([[7, 7, 7], [7, 7, 7]], result)
def test_modify_left_param(self):
""" inner function"""
inp = self._pipeline.parallelize([[1, 2, 3], [6, 5, 4]])
def _sum(x, y):
x[0] += y[0]
x[1] += y[1]
x[2] += y[2]
return x
result = transforms.union(inp.reduce(_sum), inp.reduce(_sum)).get()
self.assertEqual([[7, 7, 7], [7, 7, 7]], result)
def end_serde_test(self):
""" test """
import sys
from bigflow.core import entity
logger.info(str(self._checking_condition))
values = map(lambda condition: condition[1], self._checking_condition)
p_values = self._pipeline.parallelize([values]) # 避免map结点超过32个（Hadoop的限制）
p_value_list = []
out = []
for (i, (sd, value)) in enumerate(self._checking_condition):
sd1 = serde.of(int)
sd2 = sd
cpp_deserialize_fn = entity.KVDeserializeFn(sd1, sd2)
cpp_serialize_fn = entity.KVSerializeFn(sd1, sd2)
python_deserialize_fn = lambda kv: (sd1.deserialize(kv[0]), sd2.deserialize(kv[1]))
python_serialize_fn = lambda kv: (sd1.serialize(kv[0]), sd2.serialize(kv[1]))
serialize_fns = [cpp_serialize_fn, python_serialize_fn]
deserialize_fns = [cpp_deserialize_fn, python_deserialize_fn]
kv_val = (1, value)
def _assert_eq_val(v):
assert v == kv_val
for serialize_fn in serialize_fns:
for deserialize_fn in deserialize_fns:
out.append(p_values.map(lambda x: (1, x[i]))
.map(serialize_fn)
.map(deserialize_fn)
.map(_assert_eq_val))
if out:
transforms.union(*out).cache()
else:
print >> sys.stderr, "SKIP a test!!!"
self._pipeline.run()
def union(self, other, *others, **option):
"""
将元素与其他PCollection/PObject中的所有元素共同构成PCollection
等同于 ``transforms.union(self, other, *others)``
Args:
other (PCollection or PObject): 其他PCollection/PObject
*others: 其他PCollection/PObject
Returns:
PCollection: 表示结果的PCollection
>>> _p1 = _pipeline.parallelize(1)
>>> _p2 = _pipeline.parallelize([2, 3])
>>> _p1.union(_p2).get()
[1, 2, 3]
"""
return transforms.union(self, other, *others, **option)
def union(self, other, *others, **option):
"""
将元素与其他PCollection/PObject中的所有元素共同构成PCollection
等同于 ``transforms.union(self, other, *others)``
Args:
other (PCollection or PObject): 其他PCollection/PObject
*others: 其他PCollection/PObject
Returns:
PCollection: 表示结果的PCollection
>>> _p1 = _pipeline.parallelize(1)
>>> _p2 = _pipeline.parallelize([2, 3])
>>> _p1.union(_p2).get()
[1, 2, 3]
"""
return transforms.union(self, other, *others, **option)
def union(self, other, *others, **options):
"""
将元素与其他PCollection/PObject中的所有元素共同构成新的PCollection
等同于
:func:`bigflow.transforms.union(self, other, *others)
<bigflow.transforms.union>`
Args:
other (PCollection or PObject): 其他PCollection/PObject
*others: 其他PCollection/PObject
Returns:
PCollection: 表示结果的PCollection
>>> _p1 = _pipeline.parallelize([1, 2, 3, 4])
>>> _p2 = _pipeline.parallelize([5, 6, 7, 8])
>>> _p1.union(_p2).get()
[1, 2, 3, 4, 5, 6, 7, 8]
"""
return transforms.union(self, other, *others, **options)
def union(self, other, *others, **options):
"""
将元素与其他PCollection/PObject中的所有元素共同构成新的PCollection
等同于
:func:`bigflow.transforms.union(self, other, *others)
<bigflow.transforms.union>`
Args:
other (PCollection or PObject): 其他PCollection/PObject
*others: 其他PCollection/PObject
Returns:
PCollection: 表示结果的PCollection
>>> _p1 = _pipeline.parallelize([1, 2, 3, 4])
>>> _p2 = _pipeline.parallelize([5, 6, 7, 8])
>>> _p1.union(_p2).get()
[1, 2, 3, 4, 5, 6, 7, 8]
"""
return transforms.union(self, other, *others, **options)
def _if(cond_val, if_true, if_false):
return transforms.union(if_true.filter(lambda _, v: v, cond_val),
if_false.filter(lambda _, v: not v, cond_val))
def _test_value_fn(a, b, c, d, e):
self.assertEqual(200, e)
return transforms.union(a, b, c, d)
self.passertEqual(expect, analytics)
dict_to_tuple = analytics.apply(schema.dict_to_tuple,
['max_click_num', 'sum_click_num'])
expect = [(2, 5), (1, 1), (3, 7), (3, 6)]
self.passertEqual(expect, dict_to_tuple)
p1 = self._pipeline.parallelize([('a', 1), ('c', 2), ('a', 3),
('b', 2), ('d', 1)])
sp1 = p1.apply(schema.tuple_to_dict, ['websites', 'clicknum'])
p2 = self._pipeline.parallelize([('a', 5), ('c', 6), ('a', 7),
('b', 8), ('d', 9)])
sp2 = p2.apply(schema.tuple_to_dict, ['websites', 'clicknum'])
csp = sp1.apply(schema.cogroup, sp2, fields=['websites'])
gg = csp.apply_values(lambda x, y: transforms.union(x, y)).apply(
schema.flatten)
expect = [{
'clicknum': 8,
'websites': 'b'
}, {
'clicknum': 2,
'websites': 'b'
}, {
'clicknum': 9,
'websites': 'd'
}, {
'clicknum': 1,
'websites': 'd'
}, {
def _if(cond_val, if_true, if_false):
return transforms.union(if_true.filter(lambda _,v : v, cond_val), if_false.filter(lambda _,v:not v, cond_val))