def process_element(self, record: message.KeyRecord): key = record.key value = record.value if key in self.reduce_state: old_value = self.reduce_state[key] new_value = self.func.reduce(old_value, value) self.reduce_state[key] = new_value self.collect(message.Record(new_value)) else: self.reduce_state[key] = value self.collect(record)
def deserialize(self, data): fields = msgpack.unpackb(data, raw=False) if fields[0] == RECORD_TYPE_ID: stream, value = fields[1:] record = message.Record(value) record.stream = stream return record elif fields[0] == KEY_RECORD_TYPE_ID: stream, key, value = fields[1:] key_record = message.KeyRecord(key, value) key_record.stream = stream return key_record else: raise Exception("Unsupported type id {}, type {}".format( fields[0], type(fields[0])))
def collect(self, value): for collector in self._collector_list: collector.collect(message.Record(value))
def collect(self, value): for collector in self.collectors: collector.collect(message.Record(value))
def process_element(self, record): self.collect(message.Record(self.func.map(record.value)))