def collapse_partitions_wrapper(input_iterator, key_serdes, value_serdes, output_writebatch): f = create_functor(functors[0]._body) value = f(generator(key_serdes, value_serdes, input_iterator)) if input_iterator.last(): key = input_iterator.key() output_writebatch.put(key, value_serdes.serialize(value))
def map_partitions_wrapper(input_iterator, key_serdes, value_serdes, output_writebatch): f = create_functor(functors[0]._body) value = f(generator(key_serdes, value_serdes, input_iterator)) if isinstance(value, Iterable): for k1, v1 in value: if shuffle: output_writebatch.put((key_serdes.serialize(k1), value_serdes.serialize(v1))) else: output_writebatch.put(key_serdes.serialize(k1), value_serdes.serialize(v1)) else: key = input_iterator.key() output_writebatch.put((key, value_serdes.serialize(value)))
def map_partitions_wrapper(input_iterator, key_serdes, value_serdes, shuffle_broker): f = create_functor(functors[0]._body) value = f(generator(key_serdes, value_serdes, input_iterator)) if input_iterator.last(): L.info("value of mapPartitions:{}".format(value)) if isinstance(value, Iterable): for k1, v1 in value: shuffle_broker.put((key_serdes.serialize(k1), value_serdes.serialize(v1))) else: key = input_iterator.key() shuffle_broker.put( (key, value_serdes.serialize(value)))