def test_transaction_id(self, fake_transaction_id_schema_id,
mock_source_cluster_name):
p = GtidPosition(gtid="sid:10", offset=10)
actual_transaction_id = p.get_transaction_id(
fake_transaction_id_schema_id, unicode(mock_source_cluster_name))
expected_transaction_id = get_gtid_meta_attribute(
fake_transaction_id_schema_id, unicode(mock_source_cluster_name),
u"sid:10")
assert actual_transaction_id.schema_id == expected_transaction_id.schema_id
assert actual_transaction_id.payload_data == expected_transaction_id.payload_data
def test_transaction_id(self, fake_transaction_id_schema_id, mock_source_cluster_name):
p = GtidPosition(gtid="sid:10", offset=10)
actual_transaction_id = p.get_transaction_id(
fake_transaction_id_schema_id,
unicode(mock_source_cluster_name)
)
expected_transaction_id = get_gtid_meta_attribute(
fake_transaction_id_schema_id,
unicode(mock_source_cluster_name),
u"sid:10"
)
assert actual_transaction_id.schema_id == expected_transaction_id.schema_id
assert actual_transaction_id.payload_data == expected_transaction_id.payload_data
def _update_upstream_position(self, event):
"""If gtid_enabled and the next event is GtidEvent,
we update the self._upstream_position with GtidPosition, if next event is
not GtidEvent, we keep the current self._upstream_position, if not gtid_enabled,
we update the self.upstream_position with LogPosition.
TODO(cheng|DATAPIPE-172): We may need to skip duplicate heartbeats.
"""
if self.gtid_enabled and isinstance(event, GtidEvent):
self._upstream_position = GtidPosition(gtid=event.gtid)
elif (not self.gtid_enabled
) and event.schema == HEARTBEAT_DB and hasattr(event, 'row'):
# row['after_values']['timestamp'] should be a datetime object without tzinfo.
# we need to give it a local timezone.
timestamp = self._add_tz_info_to_tz_naive_timestamp(
event.row["after_values"]["timestamp"])
if self.sensu_alert_manager and self.meteorite_gauge_manager:
self.sensu_alert_manager.periodic_process(timestamp)
self.meteorite_gauge_manager.periodic_process(timestamp)
self._log_process(timestamp, event.log_file, event.log_pos)
self._upstream_position = LogPosition(
log_pos=event.log_pos,
log_file=event.log_file,
hb_serial=event.row["after_values"]["serial"],
hb_timestamp=calendar.timegm(timestamp.utctimetuple()),
)
self._offset = 0
def test_yield_events_when_gtid_enabled(self, mock_db_connections, patch_stream):
gtid_event_0 = mock.Mock(spec=GtidEvent, gtid="sid:11")
query_event_0 = mock.Mock(spec=QueryEvent)
query_event_1 = mock.Mock(spec=QueryEvent)
gtid_event_1 = mock.Mock(spec=GtidEvent, gtid="sid:12")
data_event_0 = mock.Mock(spec=DataEvent)
data_event_1 = mock.Mock(spec=DataEvent)
data_event_2 = mock.Mock(spec=DataEvent)
event_list = [
gtid_event_0,
query_event_0,
data_event_0,
data_event_1,
data_event_2,
gtid_event_1,
query_event_1,
]
patch_stream.return_value.peek.side_effect = event_list
patch_stream.return_value.pop.side_effect = event_list
# set offset to 1, meaning we want to skip event at offset 0
stream = SimpleBinlogStreamReaderWrapper(
mock_db_connections.source_database_config,
mock_db_connections.tracker_database_config,
GtidPosition(
gtid="sid:10",
offset=1
),
gtid_enabled=True
)
results = [
ReplicationHandlerEvent(
event=data_event_1,
position=GtidPosition(gtid="sid:11", offset=2)
),
ReplicationHandlerEvent(
event=data_event_2,
position=GtidPosition(gtid="sid:11", offset=3)
),
ReplicationHandlerEvent(
event=query_event_1,
position=GtidPosition(gtid="sid:12", offset=0)
)
]
for replication_event, result in zip(stream, results):
assert replication_event.event == result.event
assert replication_event.position.gtid == result.position.gtid
assert replication_event.position.offset == result.position.offset
def _build_position(self):
""" We need to instantiate a new position for each event."""
if self.gtid_enabled:
return GtidPosition(gtid=self._upstream_position.gtid,
offset=self._offset)
else:
return LogPosition(
log_pos=self._upstream_position.log_pos,
log_file=self._upstream_position.log_file,
offset=self._offset,
hb_serial=self._upstream_position.hb_serial,
hb_timestamp=self._upstream_position.hb_timestamp,
)
def test_schema_event(self, mock_db_connections, patch_stream):
gtid_event = mock.Mock(spec=GtidEvent)
schema_event = mock.Mock(spec=QueryEvent)
patch_stream.return_value.fetchone.side_effect = [
gtid_event, schema_event
]
stream = LowLevelBinlogStreamReaderWrapper(
mock_db_connections.source_database_config,
mock_db_connections.tracker_database_config,
GtidPosition(gtid="sid:5"))
assert stream.peek() == gtid_event
assert stream.pop() == gtid_event
assert stream.peek() == schema_event
assert stream.pop() == schema_event
def test_dict_gtid_and_offset(self):
p = GtidPosition(gtid="sid:10", offset=10)
assert p.to_dict() == {"gtid": "sid:10", "offset": 10}
def test_dict_just_gtid(self):
p = GtidPosition(gtid="sid:10")
assert p.to_dict() == {"gtid": "sid:10"}
def test_replication_dict_gtid_and_offset(self):
p = GtidPosition(gtid="sid:10", offset=10)
assert p.to_replication_dict() == {"auto_position": "sid:1-10"}
assert p.offset == 10
def test_replication_dict_just_gtid(self):
p = GtidPosition(gtid="sid:10")
assert p.to_replication_dict() == {"auto_position": "sid:1-11"}
def test_empty_position(self):
p = GtidPosition()
assert p.to_dict() == {}
def test_construct_gtid_position(self):
position_dict = {"gtid": "sid:1", "offset": 10}
position = construct_position(position_dict)
gtid_position = GtidPosition(gtid="sid:1", offset=10)
assert position.gtid == gtid_position.gtid
assert position.offset == gtid_position.offset
def schema_event_position(self):
return GtidPosition(gtid="sid:12")
def test_position(self):
return GtidPosition(gtid="3E11FA47-71CA-11E1-9E33-C80AA9429562:23")
def position(self, gtid_enabled):
if gtid_enabled:
return GtidPosition(gtid="sid:10")
else:
return LogPosition(log_file='binlog', log_pos=100)
def test_empty_position(self):
p = GtidPosition()
assert p.to_dict() == {}
def test_replication_dict_just_gtid(self):
p = GtidPosition(gtid="sid:10")
assert p.to_replication_dict() == {"auto_position": "sid:1-11"}
def test_dict_gtid_and_offset(self):
p = GtidPosition(gtid="sid:10", offset=10)
assert p.to_dict() == {"gtid": "sid:10", "offset": 10}
def test_dict_just_gtid(self):
p = GtidPosition(gtid="sid:10")
assert p.to_dict() == {"gtid": "sid:10"}
def test_replication_dict_gtid_and_offset(self):
p = GtidPosition(gtid="sid:10", offset=10)
assert p.to_replication_dict() == {"auto_position": "sid:1-10"}
assert p.offset == 10
def get_position_to_resume_tailing_from(self):
if self.global_event_state:
return construct_position(self.global_event_state.position)
return GtidPosition() if self.gtid_enabled else LogPosition()
def position_gtid_2(self):
return GtidPosition(gtid="fake_gtid_2")