def process_first_event(self, event): if "{}.{}".format(event.schema, event.table) not in self.first_rows_passed: Util.log_row( event.first_row(), "first row in replication {}.{}".format( event.schema, event.table)) self.first_rows_passed.append("{}.{}".format( event.schema, event.table))
def converter_builder(self, which): if which == CONVERTER_CSV: return ObjectBuilder( instance=CSVWriteConverter( defaults=self.config['converter']['csv']['column_default_value'], column_skip=self.config['converter']['csv']['column_skip'], )) elif which == CONVERTER_CH: if not self.config['converter']['clickhouse']['converter_file'] or not self.config['converter']['clickhouse']['converter_class']: # default converter return ObjectBuilder(instance=CHWriteConverter(column_skip=self.config['converter']['clickhouse']['column_skip'])) else: # explicitly specified converter _class = Util.class_from_file( self.config['converter']['clickhouse']['converter_file'], self.config['converter']['clickhouse']['converter_class'] ) return ObjectBuilder(instance=_class(column_skip=self.config['converter']['clickhouse']['column_skip']))
def __init__( self, connection_settings, server_id, log_file=None, log_pos=None, schemas=None, tables=None, tables_prefixes=None, blocking=None, resume_stream=None, nice_pause=None, binlog_position_file=None, callbacks={}, ): super().__init__(callbacks=callbacks) self.connection_settings = connection_settings self.server_id = server_id self.log_file = log_file self.log_pos = log_pos self.schemas = None if not TableProcessor.extract_dbs( schemas, Util.join_lists( tables, tables_prefixes)) else TableProcessor.extract_dbs( schemas, Util.join_lists(tables, tables_prefixes)) self.tables = None if tables is None else TableProcessor.extract_tables( tables) self.tables_prefixes = None if tables_prefixes is None else TableProcessor.extract_tables( tables_prefixes) self.blocking = blocking self.resume_stream = resume_stream self.nice_pause = nice_pause self.binlog_position_file = binlog_position_file self.cache_pool = CachePool() self.cache_pool.reader = self logging.info("raw dbs list. len()=%d", 0 if schemas is None else len(schemas)) if schemas is not None: for schema in schemas: logging.info(schema) logging.info("normalised dbs list. len()=%d", 0 if self.schemas is None else len(self.schemas)) if self.schemas is not None: for schema in self.schemas: logging.info(schema) logging.info("raw tables list. len()=%d", 0 if tables is None else len(tables)) if tables is not None: for table in tables: logging.info(table) logging.info("normalised tables list. len()=%d", 0 if self.tables is None else len(self.tables)) if self.tables is not None: for table in self.tables: logging.info(table) logging.info("raw tables-prefixes list. len()=%d", 0 if tables_prefixes is None else len(tables_prefixes)) if tables_prefixes is not None: for table in tables_prefixes: logging.info(table) logging.info( "normalised tables-prefixes list. len()=%d", 0 if self.tables_prefixes is None else len(self.tables_prefixes)) if self.tables_prefixes is not None: for table in self.tables_prefixes: logging.info(table) if not isinstance(self.server_id, int): raise Exception( "Please specify server_id of src server as int. Ex.: --src-server-id=1" ) self.binlog_stream = BinLogStreamReader( # MySQL server - data source connection_settings=self.connection_settings, server_id=self.server_id, # we are interested in reading CH-repeatable events only only_events=[ # Possible events #BeginLoadQueryEvent, DeleteRowsEvent, #ExecuteLoadQueryEvent, #FormatDescriptionEvent, #GtidEvent, #HeartbeatLogEvent, #IntvarEvent #NotImplementedEvent, #QueryEvent, #RotateEvent, #StopEvent, #TableMapEvent, UpdateRowsEvent, WriteRowsEvent, #XidEvent, ], only_schemas=self.schemas, # in case we have any prefixes - this means we need to listen to all tables within specified schemas only_tables=self.tables if not self.tables_prefixes else None, log_file=self.log_file, log_pos=self.log_pos, freeze_schema= True, # If true do not support ALTER TABLE. It's faster. blocking=False, resume_stream=self.resume_stream, ) logging.debug("mysql connection settings:{}".format( self.connection_settings))