def collapse_partitions_wrapper(input_iterator, key_serdes,
value_serdes, output_writebatch):
f = create_functor(functors[0]._body)
value = f(generator(key_serdes, value_serdes, input_iterator))
if input_iterator.last():
key = input_iterator.key()
output_writebatch.put(key, value_serdes.serialize(value))
def map_partitions_wrapper(input_iterator, key_serdes, value_serdes, output_writebatch):
f = create_functor(functors[0]._body)
value = f(generator(key_serdes, value_serdes, input_iterator))
if isinstance(value, Iterable):
for k1, v1 in value:
if shuffle:
output_writebatch.put((key_serdes.serialize(k1), value_serdes.serialize(v1)))
else:
output_writebatch.put(key_serdes.serialize(k1), value_serdes.serialize(v1))
else:
key = input_iterator.key()
output_writebatch.put((key, value_serdes.serialize(value)))
def map_partitions_wrapper(input_iterator, key_serdes,
value_serdes, shuffle_broker):
f = create_functor(functors[0]._body)
value = f(generator(key_serdes, value_serdes, input_iterator))
if input_iterator.last():
L.info("value of mapPartitions:{}".format(value))
if isinstance(value, Iterable):
for k1, v1 in value:
shuffle_broker.put((key_serdes.serialize(k1),
value_serdes.serialize(v1)))
else:
key = input_iterator.key()
shuffle_broker.put(
(key, value_serdes.serialize(value)))