def process_element(self, record: message.KeyRecord):
key = record.key
value = record.value
if key in self.reduce_state:
old_value = self.reduce_state[key]
new_value = self.func.reduce(old_value, value)
self.reduce_state[key] = new_value
self.collect(message.Record(new_value))
else:
self.reduce_state[key] = value
self.collect(record)
def deserialize(self, data):
fields = msgpack.unpackb(data, raw=False)
if fields[0] == RECORD_TYPE_ID:
stream, value = fields[1:]
record = message.Record(value)
record.stream = stream
return record
elif fields[0] == KEY_RECORD_TYPE_ID:
stream, key, value = fields[1:]
key_record = message.KeyRecord(key, value)
key_record.stream = stream
return key_record
else:
raise Exception("Unsupported type id {}, type {}".format(
fields[0], type(fields[0])))
def collect(self, value):
for collector in self._collector_list:
collector.collect(message.Record(value))
def collect(self, value):
for collector in self.collectors:
collector.collect(message.Record(value))
def process_element(self, record):
self.collect(message.Record(self.func.map(record.value)))