def binlog_reading(self, server_id, only_tables, only_schemas, log_file,
log_pos):
logger.info('start sync at %s' %
(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
logger.info(f'mysql binlog: {log_file}:{log_pos}')
stream = BinLogStreamReader(connection_settings=dict(
host=self.host,
port=self.port,
user=self.user,
passwd=self.password),
resume_stream=True,
blocking=True,
server_id=int(server_id),
only_tables=only_tables,
only_schemas=only_schemas,
only_events=self.only_events,
log_file=log_file,
log_pos=log_pos,
fail_on_table_metadata_unavailable=True,
slave_heartbeat=10)
for binlog_event in stream:
for row in binlog_event.rows:
event = {}
if isinstance(binlog_event, WriteRowsEvent):
event['action'] = 'insert'
event['values'] = row['values']
event['event_unixtime'] = int(time.time() * 10**6)
event['action_core'] = '2'
elif isinstance(binlog_event, UpdateRowsEvent):
event['action'] = 'insert'
event['values'] = row['after_values']
event['event_unixtime'] = int(time.time() * 10**6)
event['action_core'] = '2'
elif isinstance(binlog_event, DeleteRowsEvent):
event['action'] = 'delete'
event['values'] = row['values']
event['event_unixtime'] = int(time.time() * 10**6)
event['action_core'] = '1'
values = event['values']
for k, v in values.items():
values[k] = complex_decode(v)
event['values'] = values
yield binlog_event.schema, binlog_event.table, event, stream.log_file, stream.log_pos
def convert_values(self, values):
cp_values = deepcopy(values)
for k, v in values.items():
cp_values[k] = complex_decode(v)
return cp_values
def binlog_reading(self, server_id, only_tables, only_schemas, log_file,
log_pos, skip_dmls, skip_update_tables,
skip_delete_tables):
logger.info('start sync at %s' %
(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
logger.info(f'mysql binlog: {log_file}:{log_pos}')
stream = BinLogStreamReader(connection_settings=dict(
host=self.host,
port=self.port,
user=self.user,
passwd=self.password),
resume_stream=True,
blocking=True,
server_id=server_id,
only_tables=only_tables,
only_schemas=only_schemas,
only_events=self.only_events,
log_file=log_file,
log_pos=log_pos,
fail_on_table_metadata_unavailable=True,
slave_heartbeat=10)
for binlog_event in stream:
if isinstance(binlog_event, QueryEvent):
schema = binlog_event.schema.decode()
query = binlog_event.query
if 'alter' not in query:
continue
event = {
'table': None,
'schema': schema,
'action': 'query',
'values': {
'query': parse_mysql_ddl_2_ch(schema, query)
},
'event_unixtime': int(time.time() * 10**6),
'action_core': '0'
}
yield schema, None, event, stream.log_file, stream.log_pos
else:
schema = binlog_event.schema
table = binlog_event.table
skip_dml_table_name = f"{schema}.{table}"
for row in binlog_event.rows:
event = {'table': table, 'schema': schema}
if isinstance(binlog_event, WriteRowsEvent):
event['action'] = 'insert'
event['values'] = row['values']
event['event_unixtime'] = int(time.time() * 10**6)
event['action_core'] = '2'
elif isinstance(binlog_event, UpdateRowsEvent):
if 'update' in skip_dmls or skip_dml_table_name in skip_update_tables:
continue
event['action'] = 'insert'
event['values'] = row['after_values']
event['event_unixtime'] = int(time.time() * 10**6)
event['action_core'] = '2'
elif isinstance(binlog_event, DeleteRowsEvent):
if 'delete' in skip_dmls or skip_dml_table_name in skip_delete_tables:
continue
event['action'] = 'delete'
event['values'] = row['values']
event['event_unixtime'] = int(time.time() * 10**6)
event['action_core'] = '1'
values = event['values']
for k, v in values.items():
values[k] = complex_decode(v)
event['values'] = values
yield binlog_event.schema, binlog_event.table, event, stream.log_file, stream.log_pos