def _is_unsupported_query_event(self, event):
if (
isinstance(event, QueryEvent) and
not mysql_statement_factory(event.query).is_supported()
):
log.info("Filtered unsupported query event: {} {}".format(
repr(event),
event.query
))
return True
return False
def handle_event(self, event, position):
"""Handles schema change queries. For queries that alter schema,
it also registers the altered schemas with the schematizer.
If the event is blacklisted or the query is skippable or the
query statement is not supported, the method doesn't handle it.
Args:
event: The event containing the query
position: The current position (for saving state)
"""
statement = mysql_statement_factory(event.query)
if self._event_can_be_skipped(event, statement):
return
query = event.query
schema = event.schema
logger.info("Processing supported query {q}".format(q=query))
if self.stats_counter:
self.stats_counter.increment(query)
logger.info("Flushing all messages from producer and saving position")
self.producer.flush()
save_position(
position_data=self.producer.get_checkpoint_position_data(),
state_session=self.db_connections.state_session
)
if not self.mysql_dump_handler.mysql_dump_exists():
# For first time schema event backup
self.mysql_dump_handler.create_schema_dump()
self.mysql_dump_handler.persist_schema_dump()
if self._is_query_alter_and_not_rename_table(statement):
# TODO: DATAPIPE-1963
if schema is None or not schema.strip():
database_name = statement.database_name
else:
database_name = schema
if self.is_blacklisted(event, database_name):
# This blacklist check needs to be called again here, because if
# the statement doesn't have a concrete schema assigned, we
# won't know if it should be executed until this point.
logger.info("Query {e} is blacklisted, skip processing".format(
e=event.query
))
return
table = Table(
cluster_name=self.db_connections.source_cluster_name,
database_name=database_name,
table_name=statement.table
)
self._process_alter_table_event(
query=query,
table=table
)
self._checkpoint(
position=position.to_dict(),
event_type=EventType.SCHEMA_EVENT,
cluster_name=table.cluster_name,
database_name=table.database_name,
table_name=table.table_name
)
else:
if self._does_query_rename_table(statement):
logger.info(
"Rename query {q} detected, clearing schema cache".format(
q=query
)
)
self.schema_wrapper.reset_cache()
database_name = self._get_db_for_statement(statement, schema)
self._execute_query(query=query, database_name=database_name)
self._checkpoint(
position=position.to_dict(),
event_type=EventType.SCHEMA_EVENT,
cluster_name=self.db_connections.source_cluster_name,
database_name=schema,
table_name=None
)
def handle_event(self, event, position):
"""Handles schema change queries. For queries that alter schema,
it also registers the altered schemas with the schematizer.
If the event is blacklisted or the query is skippable or the
query statement is not supported, the method doesn't handle it.
Args:
event: The event containing the query
position: The current position (for saving state)
"""
statement = mysql_statement_factory(event.query)
if self._event_can_be_skipped(event, statement):
return
query = event.query
schema = event.schema
logger.info("Processing supported query {q}".format(q=query))
if self.stats_counter:
self.stats_counter.increment(query)
logger.info("Flushing all messages from producer and saving position")
self.producer.flush()
save_position(
position_data=self.producer.get_checkpoint_position_data(),
state_session=self.db_connections.state_session)
self.mysql_dump_handler.create_and_persist_schema_dump()
if self._is_query_alter_and_not_rename_table(statement):
# TODO: DATAPIPE-1963
if schema is None or not schema.strip():
database_name = statement.database_name
else:
database_name = schema
if self.is_blacklisted(event, database_name):
# This blacklist check needs to be called again here, because if
# the statement doesn't have a concrete schema assigned, we
# won't know if it should be executed until this point.
logger.info("Query {e} is blacklisted, skip processing".format(
e=event.query))
return
table = Table(cluster_name=self.db_connections.source_cluster_name,
database_name=database_name,
table_name=statement.table)
self._process_alter_table_event(query=query, table=table)
self._checkpoint(position=position.to_dict(),
event_type=EventType.SCHEMA_EVENT,
cluster_name=table.cluster_name,
database_name=table.database_name,
table_name=table.table_name)
else:
if self._does_query_rename_table(statement):
logger.info(
"Rename query {q} detected, clearing schema cache".format(
q=query))
self.schema_wrapper.reset_cache()
database_name = self._get_db_for_statement(statement, schema)
self._execute_query(query=query, database_name=database_name)
self._checkpoint(
position=position.to_dict(),
event_type=EventType.SCHEMA_EVENT,
cluster_name=self.db_connections.source_cluster_name,
database_name=schema,
table_name=None)
def statement(self, query):
return mysql_statement_factory(query)
def test_does_rename_table(self, rename_query):
statement = mysql_statement_factory(rename_query)
assert statement.does_rename_table()
def statement(self, query):
return mysql_statement_factory(query)
def test_does_rename_table(self, rename_query):
statement = mysql_statement_factory(rename_query)
assert statement.does_rename_table()