def _is_allowed_event(self, binlogevent, row, log_pos, log_file):
envelopes = self._filter.filter(binlogevent, row, log_pos, log_file)
if len(envelopes) == 0:
return False
for envelope in envelopes:
checkpoint = BinLogMetadata(log_pos, log_file, binlogevent.schema,
binlogevent.table)
self._committer.hold_checkpoint(checkpoint)
self._pending_queue.put(envelope)
return True
def build_envelope(self, binlogevent, row, log_pos, log_file):
row_vals = row['values']
log_metadata = BinLogMetadata(log_pos, log_file, binlogevent.schema,
binlogevent.table, row_vals['id'])
message = MessageEventStore(INSERT_EVENT, binlogevent.schema,
binlogevent.table, binlogevent.primary_key,
row_vals)
envelope = KafkaEnvelope(message, log_metadata, row_vals['topic'],
row_vals['routing_key'])
return envelope
def fetch_lastest_checkpoint(self)->BinLogMetadata:
partition = TopicPartition(self._topic, 0)
self._consumer.assign([partition])
self._consumer.seek_to_end()
offset = self._consumer.position(partition)
if offset == None or offset ==0:
return None
self._consumer.seek(partition, offset-1)
for message in self._consumer:
metadata_attributes = json.loads(message.value.decode())
metadata = BinLogMetadata(log_pos=metadata_attributes['pos'], log_file=metadata_attributes['file'])
return metadata
return None
def _read_stream(self):
log_pos = self._log_metadata.log_pos if self._log_metadata != None else 0
log_file = self._log_metadata.log_file if self._log_metadata != None else ''
for binlogevent in self._stream:
if self._cancellation.is_cancel:
break
self._handle_event(binlogevent)
log_pos = self._stream.log_pos
log_file = self._stream.log_file
self._stream.close()
if log_pos > self._start_position:
self._log_metadata = BinLogMetadata(log_pos=log_pos,
log_file=log_file)
time.sleep(1)
if self._log_metadata != None:
print_log_reader('End of stream, re-connect from {} {}'.format(
log_pos, log_file))
def _commit_not_allowed_event(self, binlogevent, log_pos, log_file):
checkpoint = BinLogMetadata(log_pos, log_file, binlogevent.schema,
binlogevent.table)
self._committer.commit_checkpoint(checkpoint)
def build_envelope(self, binlogevent, row, log_pos, log_file):
log_metadata = BinLogMetadata(log_pos, log_file, binlogevent.schema, binlogevent.table)
message = UpdatedMessage(self._event_type,binlogevent.schema,binlogevent.table,binlogevent.primary_key, self.filter_vals(binlogevent.primary_key, row["before_values"]), self.filter_vals(binlogevent.primary_key, row["after_values"]) )
envelope = KafkaEnvelope(message, log_metadata, self._topic, message.get_routing_key())
return envelope