def runTest(self):
messages = tap_mysql.generate_messages(self.con, {}, self.catalog, {})
record_message = list(filter(lambda m: isinstance(m, singer.RecordMessage), messages))[0]
self.assertTrue(isinstance(record_message, singer.RecordMessage))
self.assertEqual(record_message.record, {'b c': 1})
def test_with_no_state(self):
state = {}
(message_types, versions) = message_types_and_versions(
tap_mysql.generate_messages(self.con, {}, self.catalog, state))
self.assertEqual(['ActivateVersionMessage', 'RecordMessage', 'ActivateVersionMessage'], message_types)
self.assertTrue(isinstance(versions[0], int))
self.assertEqual(versions[0], versions[1])
def runTest(self):
con = get_test_connection()
try:
with con.cursor() as cur:
cur.execute('''
CREATE TABLE tab (
id INTEGER PRIMARY KEY,
a INTEGER,
b INTEGER)
''')
catalog = discover_catalog(con)
catalog.streams[0].stream = 'tab'
catalog.streams[0].schema.selected = True
catalog.streams[0].schema.properties['a'].selected = True
messages = list(
tap_mysql.generate_messages(con, catalog,
tap_mysql.build_state({},
catalog)))
schema_message = list(
filter(lambda m: isinstance(m, singer.SchemaMessage),
messages))[0]
self.assertTrue(isinstance(schema_message, singer.SchemaMessage))
expectedKeys = ['id', 'a']
self.assertEqual(schema_message.schema['properties'].keys(),
set(expectedKeys))
finally:
con.close()
def test_with_state(self):
state = tap_mysql.build_state(
{
'bookmarks': {
'tap_mysql_test-incremental': {
'version': 1,
'replication_key_value': '2017-06-20',
'replication_key': 'updated'
},
'tap_mysql_test-integer_incremental': {
'version': 1,
'replication_key_value': 3,
'replication_key': 'updated'
}
}
}, self.catalog)
(message_types, versions) = message_types_and_versions(
tap_mysql.generate_messages(self.con, self.catalog, state))
self.assertEqual([
'ActivateVersionMessage', 'RecordMessage', 'RecordMessage',
'ActivateVersionMessage', 'RecordMessage'
], message_types)
self.assertTrue(isinstance(versions[0], int))
self.assertEqual(versions[0], versions[1])
self.assertEqual(versions[1], 1)
def test_fail_if_log_file_does_not_exist(self):
log_file = 'chicken'
stream = self.catalog.streams[0]
state = {
'bookmarks': {
stream.tap_stream_id: {
'version': singer.utils.now(),
'log_file': log_file,
'log_pos': 1
}
}
}
failed = False
exception_message = None
expected_exception_message = "Unable to replicate stream({}) with binlog because log file {} does not exist.".format(
stream,
log_file
)
try:
list(tap_mysql.generate_messages(self.con, {}, self.catalog, state))
except Exception as e:
failed = True
exception_message = str(e)
LOGGER.error(exception_message)
def test_initial_full_table(self):
state = {}
expected_log_file, expected_log_pos = binlog.fetch_current_log_file_and_pos(self.con)
messages = list(tap_mysql.generate_messages(self.con, {}, self.catalog, state))
message_types = [type(m) for m in messages]
self.assertEqual(message_types,
[singer.StateMessage,
singer.SchemaMessage,
singer.ActivateVersionMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.StateMessage,
singer.ActivateVersionMessage,
singer.StateMessage,
singer.StateMessage])
activate_version_message = list(filter(lambda m: isinstance(m, singer.ActivateVersionMessage), messages))[0]
record_messages = list(filter(lambda m: isinstance(m, singer.RecordMessage), messages))
self.assertEqual(singer.get_bookmark(state, 'tap_mysql_test-binlog', 'log_file'),
expected_log_file)
self.assertEqual(singer.get_bookmark(state, 'tap_mysql_test-binlog', 'log_pos'),
expected_log_pos)
self.assertEqual(singer.get_bookmark(state, 'tap_mysql_test-binlog', 'version'),
activate_version_message.version)
def runTest(self):
catalog = discover_catalog(self.con)
catalog.streams[0].stream = 'some_stream_name'
catalog.streams[0].schema.selected = True
catalog.streams[0].key_properties = []
catalog.streams[0].schema.properties['b c'].selected = True
messages = tap_mysql.generate_messages(
self.con, catalog, tap_mysql.build_state({}, catalog))
record_message = list(
filter(lambda m: isinstance(m, singer.RecordMessage), messages))[0]
self.assertTrue(isinstance(record_message, singer.RecordMessage))
self.assertEqual(record_message.record, {'b c': 1})
def test_version_not_cleared_from_state_after_incremental_success(self):
state = tap_mysql.build_state(
{'bookmarks': {
'tap_mysql_test-incremental': {
'version': 1,
}
}}, self.catalog)
list(tap_mysql.generate_messages(self.con, self.catalog, state))
self.assertEqual(
state['bookmarks']['tap_mysql_test-incremental']['version'], 1)
def test_version_cleared_from_state_after_full_table_success(self):
state = tap_mysql.build_state(
{'bookmarks': {
'tap_mysql_test-full_table': {
'version': 1,
}
}}, self.catalog)
list(tap_mysql.generate_messages(self.con, self.catalog, state))
self.assertEqual(
state['bookmarks']['tap_mysql_test-full_table']['version'], None)
def test_with_no_version_in_state(self):
state = tap_mysql.build_state(
{'bookmarks': {
'tap_mysql_test-full_table': {
'version': None,
}
}}, self.catalog)
(message_types, versions) = message_types_and_versions(
tap_mysql.generate_messages(self.con, self.catalog, state))
self.assertEqual(['RecordMessage', 'ActivateVersionMessage'],
message_types)
self.assertTrue(isinstance(versions[0], int))
self.assertEqual(versions[0], versions[1])
def test_version_not_cleared_from_state_after_incremental_success(self):
state = {
'bookmarks': {
'tap_mysql_test-incremental': {
'version': 1,
'replication_key_value': '2017-06-20',
'replication_key': 'updated'
}
}
}
list(tap_mysql.generate_messages(self.con, {}, self.catalog, state))
self.assertEqual(state['bookmarks']['tap_mysql_test-incremental']['version'], 1)
def test_version_cleared_from_state_after_full_table_success(self):
state = {
'bookmarks': {
'tap_mysql_test-full_table': {
'version': 1,
'initial_full_table_complete': True
}
}
}
list(tap_mysql.generate_messages(self.con, {}, self.catalog, state))
self.assertFalse('version' in state['bookmarks']['tap_mysql_test-full_table'].keys())
self.assertTrue(state['bookmarks']['tap_mysql_test-full_table']['initial_full_table_complete'])
def test_with_initial_full_table_complete_in_state(self):
common.get_stream_version = lambda a, b: 12345
state = {
'bookmarks': {
'tap_mysql_test-full_table': {
'initial_full_table_complete': True
}
}
}
(message_types, versions) = message_types_and_versions(
tap_mysql.generate_messages(self.con, {}, self.catalog, state))
self.assertEqual(['RecordMessage', 'ActivateVersionMessage'], message_types)
self.assertEqual(versions, [12345, 12345])
def test_fail_on_view(self):
for stream in self.catalog.streams:
stream.is_view = True
state = {}
failed = False
exception_message = None
expected_exception_message = "Unable to replicate stream({}) with binlog because it is a view.".format(self.catalog.streams[0].stream)
try:
list(tap_mysql.generate_messages(self.con, {}, self.catalog, state))
except Exception as e:
failed = True
exception_message = str(e)
LOGGER.error(exception_message)
self.assertTrue(failed)
self.assertEqual(expected_exception_message, exception_message)
def test_binlog_stream(self):
stream = self.catalog.streams[0]
reader = BinLogStreamReader(
connection_settings=get_db_config(),
server_id=binlog.fetch_server_id(self.con),
only_events=[RotateEvent, WriteRowsEvent, UpdateRowsEvent, DeleteRowsEvent],
)
for _ in reader:
expected_log_file = reader.log_file
expected_log_pos = reader.log_pos
messages = list(tap_mysql.generate_messages(self.con, get_db_config(), self.catalog, self.state))
record_messages = list(filter(lambda m: isinstance(m, singer.RecordMessage), messages))
message_types = [type(m) for m in messages]
self.assertEqual(message_types,
[singer.StateMessage,
singer.SchemaMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.StateMessage,
singer.StateMessage])
self.assertEqual([(1, False), (2, False), (3, False), (3, False), (2, True)],
[(m.record['id'], m.record.get(binlog.SDC_DELETED_AT) is not None)
for m in record_messages])
self.assertEqual(singer.get_bookmark(self.state, 'tap_mysql_test-binlog', 'log_file'),
expected_log_file)
self.assertEqual(singer.get_bookmark(self.state, 'tap_mysql_test-binlog', 'log_pos'),
expected_log_pos)
def test_start_at_currently_syncing(self):
state = tap_mysql.build_state(
{'currently_syncing': 'tap_mysql_test-b'}, self.catalog)
messages = list(
tap_mysql.generate_messages(self.con, self.catalog, state))
self.assertRegexpMatches(currently_syncing_seq(messages), '^b+_+')
def test_emit_currently_syncing(self):
state = tap_mysql.build_state({}, self.catalog)
messages = list(
tap_mysql.generate_messages(self.con, self.catalog, state))
self.assertRegexpMatches(currently_syncing_seq(messages), '^a+b+_+')
