def test_change_log_messages(
containers,
rbrsource,
create_table_query,
schematizer,
namespace,
source,
rbr_source_session,
gtid_enabled
):
if not gtid_enabled:
increment_heartbeat(containers, rbrsource)
execute_query_get_one_row(
containers,
rbrsource,
create_table_query.format(table_name=source)
)
BasicModel = _generate_basic_model(source)
model_1 = BasicModel(id=1, name='insert')
model_2 = BasicModel(id=2, name='insert')
rbr_source_session.add(model_1)
rbr_source_session.add(model_2)
rbr_source_session.commit()
model_1.name = 'update'
rbr_source_session.delete(model_2)
rbr_source_session.commit()
messages = _fetch_messages(
containers,
schematizer,
namespace,
source,
4
)
expected_messages = [
{
'message_type': MessageType.create,
'payload_data': {'id': 1, 'table_name': source, 'table_schema': 'yelp'}
},
{
'message_type': MessageType.create,
'payload_data': {'id': 2, 'table_name': source, 'table_schema': 'yelp'}
},
{
'message_type': MessageType.update,
'payload_data': {'id': 1, 'table_name': source, 'table_schema': 'yelp'},
'previous_payload_data': {'id': 1, 'table_name': source, 'table_schema': 'yelp'}
},
{
'message_type': MessageType.delete,
'payload_data': {'id': 2, 'table_name': source, 'table_schema': 'yelp'}
},
]
_verify_messages(messages, expected_messages)
def test_basic_table(self, containers, replhandler, rbrsource,
create_table_query, namespace, schematizer,
rbr_source_session, gtid_enabled):
if not gtid_enabled:
increment_heartbeat(containers, rbrsource)
source = "{0}_basic_table".format(replhandler)
execute_query_get_one_row(containers, rbrsource,
create_table_query.format(table_name=source))
BasicModel = _generate_basic_model(source)
model_1 = BasicModel(id=1, name='insert')
model_2 = BasicModel(id=2, name='insert')
rbr_source_session.add(model_1)
rbr_source_session.add(model_2)
rbr_source_session.commit()
model_1.name = 'update'
rbr_source_session.delete(model_2)
rbr_source_session.commit()
messages = _fetch_messages(containers, schematizer, namespace, source,
4)
expected_messages = [
{
'message_type': MessageType.create,
'payload_data': {
'id': 1,
'name': 'insert'
}
},
{
'message_type': MessageType.create,
'payload_data': {
'id': 2,
'name': 'insert'
}
},
{
'message_type': MessageType.update,
'payload_data': {
'id': 1,
'name': 'update'
},
'previous_payload_data': {
'id': 1,
'name': 'insert'
}
},
{
'message_type': MessageType.delete,
'payload_data': {
'id': 2,
'name': 'insert'
}
},
]
_verify_messages(messages, expected_messages)
def test_complex_table(self, containers, rbrsource, complex_table_name,
ComplexModel, actual_complex_data,
expected_complex_data, schematizer, namespace,
rbr_source_session, gtid_enabled):
if not gtid_enabled:
increment_heartbeat(containers, rbrsource)
complex_instance = ComplexModel(**actual_complex_data)
rbr_source_session.add(complex_instance)
rbr_source_session.commit()
messages = _fetch_messages(containers, schematizer, namespace,
complex_table_name, 1)
expected_messages = [
{
'message_type': MessageType.create,
'payload_data': expected_complex_data
},
]
_verify_messages(messages, expected_messages)
def test_table_with_contains_pii(self, containers, replhandler, rbrsource,
create_table_query, namespace,
schematizer, rbr_source_session,
gtid_enabled):
with reconfigure(encryption_type='AES_MODE_CBC-1',
key_location='acceptance/configs/data_pipeline/'):
if not gtid_enabled:
increment_heartbeat(containers, rbrsource)
source = "{}_secret_table".format(replhandler)
execute_query_get_one_row(
containers, rbrsource,
create_table_query.format(table_name=source))
BasicModel = _generate_basic_model(source)
model_1 = BasicModel(id=1, name='insert')
model_2 = BasicModel(id=2, name='insert')
rbr_source_session.add(model_1)
rbr_source_session.add(model_2)
rbr_source_session.commit()
messages = _fetch_messages(containers, schematizer, namespace,
source, 2)
expected_messages = [{
'message_type': MessageType.create,
'payload_data': {
'id': 1,
'name': 'insert'
}
}, {
'message_type': MessageType.create,
'payload_data': {
'id': 2,
'name': 'insert'
}
}]
_verify_messages(messages, expected_messages)
def test_processing_table_rename(
self,
containers_without_repl_handler,
rbrsource,
schematizer,
schematracker,
namespace,
start_service,
gtid_enabled
):
"""
This test verifies that the service handles table renames.
"""
table_name_one = "hogwarts_{r}".format(r=self.get_random_string())
table_name_two = "durmstrang_{r}".format(r=self.get_random_string())
create_table_query = """
CREATE TABLE {table_name}
(
`name` VARCHAR(255) NOT NULL PRIMARY KEY,
`house` VARCHAR(255) NOT NULL
)"""
insert_query = """
INSERT INTO {table} VALUES ( "{one}", "{two}" )
"""
change_table_name_query = """
RENAME TABLE {old} TO {new}
"""
if not gtid_enabled:
increment_heartbeat(containers_without_repl_handler, rbrsource)
create_query = create_table_query.format(table_name=table_name_one)
rename_query = change_table_name_query.format(
old=table_name_one,
new=table_name_two
)
execute_query_get_one_row(
containers_without_repl_handler,
rbrsource,
create_query
)
execute_query_get_one_row(
containers_without_repl_handler,
rbrsource,
insert_query.format(
table=table_name_one,
one='Cedric Diggory',
two='Hufflepuff'
)
)
execute_query_get_one_row(
containers_without_repl_handler,
rbrsource,
insert_query.format(
table=table_name_one,
one='Hannah Abbott',
two='Hufflepuff'
)
)
execute_query_get_one_row(
containers_without_repl_handler,
rbrsource,
rename_query
)
execute_query_get_one_row(
containers_without_repl_handler,
rbrsource,
insert_query.format(
table=table_name_two,
one='Viktor Krum',
two='Durmstrang'
)
)
start_service(
resume_stream=True,
num_of_queries_to_process=5,
end_time=30,
is_schema_event_helper_enabled=False,
num_of_schema_events=100,
)
show_query = "SHOW CREATE TABLE {name}"
source_schema = execute_query_get_one_row(
containers_without_repl_handler,
rbrsource,
show_query.format(name=table_name_two)
)
tracker_schema = execute_query_get_one_row(
containers_without_repl_handler,
schematracker,
show_query.format(name=table_name_two)
)
assert source_schema == tracker_schema
messages_two = _fetch_messages(
containers_without_repl_handler,
schematizer,
namespace,
table_name_two,
1
)
messages_one = _fetch_messages(
containers_without_repl_handler,
schematizer,
namespace,
table_name_one,
2
)
expected_messages_one = [
{
'message_type': MessageType.create,
'payload_data': {'name': 'Cedric Diggory',
'house': 'Hufflepuff'}
},
{
'message_type': MessageType.create,
'payload_data': {'name': 'Hannah Abbott', 'house': 'Hufflepuff'}
}
]
_verify_messages(messages_one, expected_messages_one)
expected_messages_two = [
{
'message_type': MessageType.create,
'payload_data': {'name': 'Viktor Krum', 'house': 'Durmstrang'}
}
]
_verify_messages(messages_two, expected_messages_two)
def test_shutdown_processing_events_publishes_events_only_once(
self,
containers_without_repl_handler,
rbrsource,
schematizer,
namespace,
start_service,
gtid_enabled
):
"""Tests the guarantee that the service provides about publishing every
MySQL change only once. A few queries are executed on the source
database and the service is started. The service is asked to halt after
processing a subset of events and then restart and pick up from where it
left off.
"""
table_name = "coursework_{r}".format(r=self.get_random_string())
create_query = """
CREATE TABLE {table}
(name VARCHAR(255) PRIMARY KEY, teacher VARCHAR(255))
"""
insert_query_one = """
INSERT INTO {table} VALUES
("{one}", "{two}")
"""
update_query = """
UPDATE {table}
SET teacher="{one}"
WHERE name="{two}"
"""
if not gtid_enabled:
increment_heartbeat(containers_without_repl_handler, rbrsource)
execute_query_get_one_row(
containers_without_repl_handler,
rbrsource,
create_query.format(table=table_name)
)
execute_query_get_one_row(
containers_without_repl_handler,
rbrsource,
insert_query_one.format(
table=table_name,
one='potions',
two='Severus Snape'
)
)
execute_query_get_one_row(
containers_without_repl_handler,
rbrsource,
insert_query_one.format(
table=table_name,
one='care of magical creatures',
two='Rubeus Hagrid'
)
)
execute_query_get_one_row(
containers_without_repl_handler,
rbrsource,
update_query.format(
table=table_name,
one='Grubbly Plank',
two='care of magical creatures'
)
)
start_service(
resume_stream=False,
num_of_queries_to_process=2,
end_time=30,
is_schema_event_helper_enabled=False,
num_of_schema_events=100,
)
messages_one = _fetch_messages(
containers_without_repl_handler,
schematizer,
namespace,
table_name,
1
)
expected_messages_one = [
{
'message_type': MessageType.create,
'payload_data': {'name': 'potions', 'teacher': 'Severus Snape'}
},
]
_verify_messages(messages_one, expected_messages_one)
start_service(
resume_stream=True,
num_of_queries_to_process=2,
end_time=30,
is_schema_event_helper_enabled=False,
num_of_schema_events=100,
)
messages_two = _fetch_messages(
containers_without_repl_handler,
schematizer,
namespace,
table_name,
3
)
messages_two = messages_two[1:]
expected_messages_two = [
{
'message_type': MessageType.create,
'payload_data': {'name': 'care of magical creatures',
'teacher': 'Rubeus Hagrid'}
},
{
'message_type': MessageType.update,
'payload_data': {'name': 'care of magical creatures',
'teacher': 'Grubbly Plank'},
'previous_payload_data': {'name': 'care of magical creatures',
'teacher': 'Rubeus Hagrid'}
}
]
_verify_messages(messages_two, expected_messages_two)