def test_replication_stream_same_events(
self,
data_event,
patch_config,
position_gtid_1,
position_gtid_2,
patch_restarter,
patch_db_connections,
patch_data_handle_event,
patch_producer,
patch_exit,
patch_save_position,
):
data_event_with_gtid_1 = ReplicationHandlerEvent(
position=position_gtid_1, event=data_event)
data_event_with_gtid_2 = ReplicationHandlerEvent(
position=position_gtid_2, event=data_event)
patch_restarter.return_value.get_stream.return_value.next.side_effect = [
data_event_with_gtid_1, data_event_with_gtid_2
]
self._init_and_run_batch()
assert patch_data_handle_event.call_args_list == [
mock.call(data_event, position_gtid_1),
mock.call(data_event, position_gtid_2)
]
assert patch_data_handle_event.call_count == 2
assert patch_save_position.call_count == 1
def _different_events_builder(self, schema_event, data_event, patch_config,
position_gtid_1, position_gtid_2,
patch_restarter, patch_db_connections,
patch_data_handle_event,
patch_schema_handle_event, patch_producer,
patch_save_position, patch_exit):
schema_event_with_gtid = ReplicationHandlerEvent(
position=position_gtid_1, event=schema_event)
data_event_with_gtid = ReplicationHandlerEvent(
position=position_gtid_2, event=data_event)
patch_restarter.return_value.get_stream.return_value.next.side_effect = [
schema_event_with_gtid,
data_event_with_gtid,
]
def rh_supported_query_event(self):
supported_query_event = mock.Mock(spec=QueryEvent)
supported_query_event.query = 'alter table biz add column name int(11)'
return ReplicationHandlerEvent(
supported_query_event,
LogPosition(log_file='binlog.001', log_pos=50)
)
def rh_unsupported_query_event(self):
unsupported_query_event = mock.Mock(spec=QueryEvent)
unsupported_query_event.query = 'BEGIN'
return ReplicationHandlerEvent(
unsupported_query_event,
LogPosition(log_file='binlog.001', log_pos=10)
)
def test_yield_events_when_gtid_enabled(self, mock_db_connections, patch_stream):
gtid_event_0 = mock.Mock(spec=GtidEvent, gtid="sid:11")
query_event_0 = mock.Mock(spec=QueryEvent)
query_event_1 = mock.Mock(spec=QueryEvent)
gtid_event_1 = mock.Mock(spec=GtidEvent, gtid="sid:12")
data_event_0 = mock.Mock(spec=DataEvent)
data_event_1 = mock.Mock(spec=DataEvent)
data_event_2 = mock.Mock(spec=DataEvent)
event_list = [
gtid_event_0,
query_event_0,
data_event_0,
data_event_1,
data_event_2,
gtid_event_1,
query_event_1,
]
patch_stream.return_value.peek.side_effect = event_list
patch_stream.return_value.pop.side_effect = event_list
# set offset to 1, meaning we want to skip event at offset 0
stream = SimpleBinlogStreamReaderWrapper(
mock_db_connections.source_database_config,
mock_db_connections.tracker_database_config,
GtidPosition(
gtid="sid:10",
offset=1
),
gtid_enabled=True
)
results = [
ReplicationHandlerEvent(
event=data_event_1,
position=GtidPosition(gtid="sid:11", offset=2)
),
ReplicationHandlerEvent(
event=data_event_2,
position=GtidPosition(gtid="sid:11", offset=3)
),
ReplicationHandlerEvent(
event=query_event_1,
position=GtidPosition(gtid="sid:12", offset=0)
)
]
for replication_event, result in zip(stream, results):
assert replication_event.event == result.event
assert replication_event.position.gtid == result.position.gtid
assert replication_event.position.offset == result.position.offset
def _refill_current_events(self):
if not self.current_events:
# If the site goes into readonly mode, there are only heartbeats, we should just
# update the position.
while self._is_position_update(self.stream.peek()):
self._update_upstream_position(self.stream.pop())
event = self.stream.pop()
replication_handler_event = ReplicationHandlerEvent(
position=self._build_position(), event=event)
self._offset += 1
self.current_events.append(replication_handler_event)
def _setup_stream_and_expected_result(
self,
source_database_config,
tracker_database_config,
patch_stream
):
log_pos = 10
log_file = "binlog.001"
row = {"after_values": {
"serial": 123,
# This timestamp is Wed, 21 Oct 2015 12:05:27 GMT
"timestamp": datetime.fromtimestamp(1445429127)
}}
heartbeat_event = mock.Mock(
spec=DataEvent,
schema='yelp_heartbeat',
log_pos=log_pos,
log_file=log_file,
row=row
)
data_event_0 = mock.Mock(spec=DataEvent, table="business", schema="yelp")
data_event_1 = mock.Mock(spec=DataEvent, table="business", schema="yelp")
data_event_2 = mock.Mock(spec=DataEvent, table="business", schema="yelp")
event_list = [
heartbeat_event,
data_event_0,
data_event_1,
data_event_2,
]
patch_stream.return_value.peek.side_effect = event_list
patch_stream.return_value.pop.side_effect = event_list
stream = SimpleBinlogStreamReaderWrapper(
source_database_config,
tracker_database_config,
LogPosition(
log_pos=log_pos,
log_file=log_file,
offset=0
),
gtid_enabled=False,
)
# Since the offset is 0, so the result should start offset 1, and skip
# data_event_0 which is at offset 0.
results = [
ReplicationHandlerEvent(
event=data_event_1,
position=LogPosition(
log_pos=log_pos,
log_file=log_file,
offset=1,
hb_serial=123,
# This is Wed, 21 Oct 2015 12:05:27 GMT
hb_timestamp=1445429127,
)
),
ReplicationHandlerEvent(
event=data_event_2,
position=LogPosition(
log_pos=log_pos,
log_file=log_file,
offset=2,
hb_serial=123,
# This is Wed, 21 Oct 2015 12:05:27 GMT
hb_timestamp=1445429127,
)
)
]
return stream, results
def rh_data_event_before_master_log_pos(self, data_event, position_before_master):
return ReplicationHandlerEvent(
data_event,
position_before_master
)