def process_first_event(self, event):
if "{}.{}".format(event.schema,
event.table) not in self.first_rows_passed:
Util.log_row(
event.first_row(), "first row in replication {}.{}".format(
event.schema, event.table))
self.first_rows_passed.append("{}.{}".format(
event.schema, event.table))
def converter_builder(self, which):
if which == CONVERTER_CSV:
return ObjectBuilder(
instance=CSVWriteConverter(
defaults=self.config['converter']['csv']['column_default_value'],
column_skip=self.config['converter']['csv']['column_skip'],
))
elif which == CONVERTER_CH:
if not self.config['converter']['clickhouse']['converter_file'] or not self.config['converter']['clickhouse']['converter_class']:
# default converter
return ObjectBuilder(instance=CHWriteConverter(column_skip=self.config['converter']['clickhouse']['column_skip']))
else:
# explicitly specified converter
_class = Util.class_from_file(
self.config['converter']['clickhouse']['converter_file'],
self.config['converter']['clickhouse']['converter_class']
)
return ObjectBuilder(instance=_class(column_skip=self.config['converter']['clickhouse']['column_skip']))
def __init__(
self,
connection_settings,
server_id,
log_file=None,
log_pos=None,
schemas=None,
tables=None,
tables_prefixes=None,
blocking=None,
resume_stream=None,
nice_pause=None,
binlog_position_file=None,
callbacks={},
):
super().__init__(callbacks=callbacks)
self.connection_settings = connection_settings
self.server_id = server_id
self.log_file = log_file
self.log_pos = log_pos
self.schemas = None if not TableProcessor.extract_dbs(
schemas, Util.join_lists(
tables, tables_prefixes)) else TableProcessor.extract_dbs(
schemas, Util.join_lists(tables, tables_prefixes))
self.tables = None if tables is None else TableProcessor.extract_tables(
tables)
self.tables_prefixes = None if tables_prefixes is None else TableProcessor.extract_tables(
tables_prefixes)
self.blocking = blocking
self.resume_stream = resume_stream
self.nice_pause = nice_pause
self.binlog_position_file = binlog_position_file
self.cache_pool = CachePool()
self.cache_pool.reader = self
logging.info("raw dbs list. len()=%d",
0 if schemas is None else len(schemas))
if schemas is not None:
for schema in schemas:
logging.info(schema)
logging.info("normalised dbs list. len()=%d",
0 if self.schemas is None else len(self.schemas))
if self.schemas is not None:
for schema in self.schemas:
logging.info(schema)
logging.info("raw tables list. len()=%d",
0 if tables is None else len(tables))
if tables is not None:
for table in tables:
logging.info(table)
logging.info("normalised tables list. len()=%d",
0 if self.tables is None else len(self.tables))
if self.tables is not None:
for table in self.tables:
logging.info(table)
logging.info("raw tables-prefixes list. len()=%d",
0 if tables_prefixes is None else len(tables_prefixes))
if tables_prefixes is not None:
for table in tables_prefixes:
logging.info(table)
logging.info(
"normalised tables-prefixes list. len()=%d",
0 if self.tables_prefixes is None else len(self.tables_prefixes))
if self.tables_prefixes is not None:
for table in self.tables_prefixes:
logging.info(table)
if not isinstance(self.server_id, int):
raise Exception(
"Please specify server_id of src server as int. Ex.: --src-server-id=1"
)
self.binlog_stream = BinLogStreamReader(
# MySQL server - data source
connection_settings=self.connection_settings,
server_id=self.server_id,
# we are interested in reading CH-repeatable events only
only_events=[
# Possible events
#BeginLoadQueryEvent,
DeleteRowsEvent,
#ExecuteLoadQueryEvent,
#FormatDescriptionEvent,
#GtidEvent,
#HeartbeatLogEvent,
#IntvarEvent
#NotImplementedEvent,
#QueryEvent,
#RotateEvent,
#StopEvent,
#TableMapEvent,
UpdateRowsEvent,
WriteRowsEvent,
#XidEvent,
],
only_schemas=self.schemas,
# in case we have any prefixes - this means we need to listen to all tables within specified schemas
only_tables=self.tables if not self.tables_prefixes else None,
log_file=self.log_file,
log_pos=self.log_pos,
freeze_schema=
True, # If true do not support ALTER TABLE. It's faster.
blocking=False,
resume_stream=self.resume_stream,
)
logging.debug("mysql connection settings:{}".format(
self.connection_settings))