def test_sync_messages_are_correct(self):
self.catalog.streams[0] = test_utils.set_replication_method_and_key(self.catalog.streams[0], 'LOG_BASED', None)
self.catalog.streams[0] = test_utils.set_selected(self.catalog.streams[0], True)
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
#inital sync
tap_mysql.do_sync(self.conn, {}, self.catalog, {})
# get schema message to test that it has all the table's columns
schema_message = next(filter(lambda m: isinstance(m, singer.SchemaMessage), SINGER_MESSAGES))
expectedKeys = ['good_pk', 'age']
self.assertEqual(schema_message.schema['properties'].keys(), set(expectedKeys))
# get the records, these are generated by Full table replication
record_messages = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES))
self.assertEqual(len(record_messages), 4)
self.assertListEqual([
{'age': 20, 'good_pk': '61'},
{'age': 30, 'good_pk': '62'},
{'age': 30, 'good_pk': '63'},
{'age': 40, 'good_pk': '64'},
], [rec.record for rec in record_messages])
# get the last state message to be fed to the next sync
state_message = list(filter(lambda m: isinstance(m, singer.StateMessage), SINGER_MESSAGES))[-1]
SINGER_MESSAGES.clear()
# run some queries
with connect_with_backoff(self.conn) as open_conn:
with open_conn.cursor() as cursor:
cursor.execute("UPDATE good_pk_tab set age=age+5")
cursor.execute("INSERT INTO good_pk_tab (good_pk, age) VALUES "
"(BINARY('e'), 16), "
"(BINARY('f'), 5)")
# do a sync and give the state so that binlog replication start from the last synced position
tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state_message.value)
# get the changed/new records
record_messages = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES))
self.assertEqual(len(record_messages), 6)
self.assertListEqual([
{'age': 25, 'good_pk': '61'},
{'age': 35, 'good_pk': '62'},
{'age': 35, 'good_pk': '63'},
{'age': 45, 'good_pk': '64'},
{'age': 16, 'good_pk': '65'},
{'age': 5, 'good_pk': '66'},
], [rec.record for rec in record_messages])
def test_open_connections_with_default_session_sqls(self):
"""Default session parameters should be applied if no custom session SQLs"""
with patch('tap_mysql.connection.MySQLConnection.connect'):
with patch('tap_mysql.connection.run_sql') as run_sql_mock:
run_sql_mock.side_effect = self.run_sql_mock
conn = MySQLConnectionMock(config=test_utils.get_db_config())
connect_with_backoff(conn)
# Test if session variables applied on connection
self.assertEqual(self.executed_queries, tap_mysql.connection.DEFAULT_SESSION_SQLS)
def test_binlog_stream(self):
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
config = test_utils.get_db_config()
config['server_id'] = "100"
tap_mysql.do_sync(self.conn, config, self.catalog, self.state)
record_messages = list(
filter(lambda m: isinstance(m, singer.RecordMessage),
SINGER_MESSAGES))
message_types = [type(m) for m in SINGER_MESSAGES]
self.assertEqual(message_types, [
singer.StateMessage, singer.SchemaMessage, singer.SchemaMessage,
singer.RecordMessage, singer.RecordMessage, singer.RecordMessage,
singer.RecordMessage, singer.RecordMessage, singer.RecordMessage,
singer.RecordMessage, singer.RecordMessage, singer.RecordMessage,
singer.RecordMessage, singer.StateMessage
])
self.assertEqual([
('tap_mysql_test-binlog_1', 1, '2017-06-01T00:00:00+00:00', False),
('tap_mysql_test-binlog_1', 2, '2017-06-20T00:00:00+00:00', False),
('tap_mysql_test-binlog_1', 3, '2017-09-22T00:00:00+00:00', False),
('tap_mysql_test-binlog_2', 1, '2017-10-22T00:00:00+00:00', False),
('tap_mysql_test-binlog_2', 2, '2017-11-10T00:00:00+00:00', False),
('tap_mysql_test-binlog_2', 3, '2017-12-10T00:00:00+00:00', False),
('tap_mysql_test-binlog_1', 3, '2018-06-18T00:00:00+00:00', False),
('tap_mysql_test-binlog_2', 2, '2018-06-18T00:00:00+00:00', False),
('tap_mysql_test-binlog_1', 2, '2017-06-20T00:00:00+00:00', True),
('tap_mysql_test-binlog_2', 1, '2017-10-22T00:00:00+00:00', True)
], [(m.stream, m.record['id'], m.record['updated'],
m.record.get(binlog.SDC_DELETED_AT) is not None)
for m in record_messages])
self.assertIsNotNone(
singer.get_bookmark(self.state, 'tap_mysql_test-binlog_1',
'log_file'))
self.assertIsNotNone(
singer.get_bookmark(self.state, 'tap_mysql_test-binlog_1',
'log_pos'))
self.assertIsNotNone(
singer.get_bookmark(self.state, 'tap_mysql_test-binlog_2',
'log_file'))
self.assertIsNotNone(
singer.get_bookmark(self.state, 'tap_mysql_test-binlog_2',
'log_pos'))
def test_open_connections_with_session_sqls(self):
"""Custom session parameters should be applied if defined"""
session_sqls = [
'SET SESSION max_statement_time=0',
'SET SESSION wait_timeout=28800'
]
with patch('tap_mysql.connection.MySQLConnection.connect'):
with patch('tap_mysql.connection.run_sql') as run_sql_mock:
run_sql_mock.side_effect = self.run_sql_mock
conn = MySQLConnectionMock(config={**test_utils.get_db_config(),
**{'session_sqls': session_sqls}})
connect_with_backoff(conn)
# Test if session variables applied on connection
self.assertEqual(self.executed_queries, session_sqls)
def test_open_connections_with_invalid_session_sqls(self):
"""Invalid SQLs in session_sqls should be ignored"""
session_sqls = [
'SET SESSION max_statement_time=0',
'INVALID-SQL-SHOULD-BE-SILENTLY-IGNORED',
'SET SESSION wait_timeout=28800'
]
with patch('tap_mysql.connection.MySQLConnection.connect'):
with patch('tap_mysql.connection.run_sql') as run_sql_mock:
run_sql_mock.side_effect = self.run_sql_mock
conn = MySQLConnectionMock(config={**test_utils.get_db_config(),
**{'session_sqls': session_sqls}})
connect_with_backoff(conn)
# Test if session variables applied on connection
self.assertEqual(self.executed_queries, ['SET SESSION max_statement_time=0',
'SET SESSION wait_timeout=28800'])
def test_table_3_interrupted(self):
singer.write_message = singer_write_message_no_table_3
state = {}
failed_syncing_table_3 = False
# Do not worry about table_2
for stream in filter(lambda s: s.stream == 'table_2', self.catalog.streams):
md_map = singer.metadata.to_map(stream.metadata)
md_map = singer.metadata.write(md_map,
(),
'selected',
False)
stream.metadata = singer.metadata.to_list(md_map)
config = test_utils.get_db_config()
config['allow_non_auto_increment_pks'] = 'true'
try:
tap_mysql.do_sync(self.conn, config, self.catalog, state)
except Exception as ex:
if str(ex) == 'simulated exception':
failed_syncing_table_3 = True
self.assertTrue(failed_syncing_table_3)
record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
# sorting of the two chicken-* columns is based on lexicographic ordering which in
# this case chicken-11 comes before chicken-2
self.assertEqual(record_messages_1,
[['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 2, 'bar': 'def', 'foo': 200}],
['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}],
['table_3', {'id': "chicken-11", 'bar': 'ghi', 'foo': 300}]])
self.assertEqual(state['currently_syncing'], 'tap_mysql_test-table_3')
table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1']
table_3_bookmark = state['bookmarks']['tap_mysql_test-table_3']
self.assertEqual(table_1_bookmark,
{'initial_full_table_complete': True})
self.assertIsNone(table_3_bookmark.get('initial_full_table_complete'))
table_3_version = table_3_bookmark['version']
self.assertIsNotNone(table_3_version)
self.assertEqual(table_3_bookmark['max_pk_values'], {'id': "turkey"})
self.assertEqual(table_3_bookmark['last_pk_fetched'], {'id': "chicken-11"})
self.assertIsNotNone(table_3_bookmark.get('log_file'))
self.assertIsNotNone(table_3_bookmark.get('log_pos'))
failed_syncing_table_3 = False
singer.write_message = singer_write_message_ok
table_3_RECORD_COUNT = 0
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, config, self.catalog, state)
self.assertFalse(failed_syncing_table_3)
record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(record_messages_2,
[['table_3', {'id': "chicken-2", 'bar': 'def', 'foo': 200}],
['table_3', {'id': "turkey", 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 2, 'bar': 'def', 'foo': 200}],
['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}]])
self.assertIsNone(state['currently_syncing'])
table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1']
table_3_bookmark = state['bookmarks']['tap_mysql_test-table_3']
self.assertEqual(table_1_bookmark,
{'initial_full_table_complete': True})
self.assertIsNone(table_3_bookmark.get('initial_full_table_complete'))
table_3_version = table_3_bookmark['version']
self.assertIsNotNone(table_3_version)
self.assertIsNone(table_3_bookmark.get('max_pk_values'))
self.assertIsNone(table_3_bookmark.get('last_pk_fetched'))
self.assertIsNotNone(table_3_bookmark.get('log_file'))
self.assertIsNotNone(table_3_bookmark.get('log_pos'))
new_table_3_records = [[ "quail-2", 400, 'jkl' ],
[ "quail-100", 500, 'mno' ]]
for record in new_table_3_records:
insert_record_with_specific_id(self.conn, 'table_3', record)
TABLE_3_RECORD_COUNT = 0
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, config, self.catalog, state)
self.assertFalse(failed_syncing_table_3)
record_messages_3 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
# Now that we are performing binlog sync, record output should be based on insert order
self.assertEqual(record_messages_3,
[['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 2, 'bar': 'def', 'foo': 200}],
['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}],
['table_3', {'id': "quail-2" , 'bar': 'jkl', 'foo': 400}],
['table_3', {'id': "quail-100", 'bar': 'mno', 'foo': 500}]])
self.assertIsNone(state['currently_syncing'])
table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1']
table_3_bookmark = state['bookmarks']['tap_mysql_test-table_3']
self.assertEqual(table_1_bookmark,
{'initial_full_table_complete': True})
self.assertIsNone(table_3_bookmark.get('initial_full_table_complete'))
self.assertIsNotNone(table_3_bookmark.get('log_file'))
self.assertIsNotNone(table_3_bookmark.get('log_pos'))
def test_table_2_interrupted(self):
singer.write_message = singer_write_message_no_table_2
state = {}
failed_syncing_table_2 = False
# Do not worry about table_3
for stream in filter(lambda s: s.stream == 'table_3', self.catalog.streams):
md_map = singer.metadata.to_map(stream.metadata)
md_map = singer.metadata.write(md_map,
(),
'selected',
False)
stream.metadata = singer.metadata.to_list(md_map)
try:
tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state)
except Exception as ex:
if str(ex) == 'simulated exception':
failed_syncing_table_2 = True
self.assertTrue(failed_syncing_table_2)
record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(record_messages_1,
[['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 2, 'bar': 'def', 'foo': 200}],
['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}],
['table_2', {'id': 1, 'bar': 'ghi', 'foo': 300}]])
self.assertEqual(state['currently_syncing'], 'tap_mysql_test-table_2')
table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1']
table_2_bookmark = state['bookmarks']['tap_mysql_test-table_2']
self.assertEqual(table_1_bookmark,
{'initial_full_table_complete': True})
self.assertIsNone(table_2_bookmark.get('initial_full_table_complete'))
table_2_version = table_2_bookmark['version']
self.assertIsNotNone(table_2_version)
self.assertEqual(table_2_bookmark['max_pk_values'], {'id': 3})
self.assertEqual(table_2_bookmark['last_pk_fetched'], {'id': 1})
self.assertIsNotNone(table_2_bookmark.get('log_file'))
self.assertIsNotNone(table_2_bookmark.get('log_pos'))
failed_syncing_table_2 = False
singer.write_message = singer_write_message_ok
table_2_RECORD_COUNT = 0
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state)
self.assertFalse(failed_syncing_table_2)
record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(record_messages_2,
[['table_2', {'id': 2, 'bar': 'def', 'foo': 200}],
['table_2', {'id': 3, 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 2, 'bar': 'def', 'foo': 200}],
['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}]])
self.assertIsNone(state['currently_syncing'])
table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1']
table_2_bookmark = state['bookmarks']['tap_mysql_test-table_2']
self.assertEqual(table_1_bookmark,
{'initial_full_table_complete': True})
self.assertIsNone(table_2_bookmark.get('initial_full_table_complete'))
table_2_version = table_2_bookmark['version']
self.assertIsNotNone(table_2_version)
self.assertIsNone(table_2_bookmark.get('max_pk_values'))
self.assertIsNone(table_2_bookmark.get('last_pk_fetched'))
self.assertIsNotNone(table_2_bookmark.get('log_file'))
self.assertIsNotNone(table_2_bookmark.get('log_pos'))
new_table_2_records = [[ 400, 'jkl' ],
[ 500, 'mno' ]]
for record in new_table_2_records:
insert_record(self.conn, 'table_2', record)
TABLE_2_RECORD_COUNT = 0
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state)
self.assertFalse(failed_syncing_table_2)
record_messages_3 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(record_messages_3,
[['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 2, 'bar': 'def', 'foo': 200}],
['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}],
['table_2', {'id': 4, 'bar': 'jkl', 'foo': 400}],
['table_2', {'id': 5, 'bar': 'mno', 'foo': 500}]])
self.assertIsNone(state['currently_syncing'])
table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1']
table_2_bookmark = state['bookmarks']['tap_mysql_test-table_2']
self.assertEqual(table_1_bookmark,
{'initial_full_table_complete': True})
self.assertIsNone(table_2_bookmark.get('initial_full_table_complete'))
self.assertIsNotNone(table_2_bookmark.get('log_file'))
self.assertIsNotNone(table_2_bookmark.get('log_pos'))
def test_table_2_interrupted(self):
singer.write_message = singer_write_message_no_table_2
state = {}
failed_syncing_table_2 = False
try:
tap_mysql.do_sync(self.conn, test_utils.get_db_config(),
self.catalog, state)
except Exception as ex:
if str(ex) == "simulated exception":
failed_syncing_table_2 = True
self.assertTrue(failed_syncing_table_2)
record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(
record_messages_1,
[
["table_1", {
"id": 1,
"bar": "abc",
"foo": 100
}],
["table_1", {
"id": 2,
"bar": "def",
"foo": 200
}],
["table_1", {
"id": 3,
"bar": "ghi",
"foo": 300
}],
["table_2", {
"id": 1,
"bar": "ghi",
"foo": 300
}],
],
)
self.assertEqual(state["currently_syncing"], "tap_mysql_test-table_2")
table_1_bookmark = state["bookmarks"]["tap_mysql_test-table_1"]
table_2_bookmark = state["bookmarks"]["tap_mysql_test-table_2"]
self.assertEqual(table_1_bookmark,
{"initial_full_table_complete": True})
self.assertIsNone(table_2_bookmark.get("initial_full_table_complete"))
table_2_version = table_2_bookmark["version"]
self.assertIsNotNone(table_2_version)
self.assertEqual(table_2_bookmark["max_pk_values"], {"id": 3})
self.assertEqual(table_2_bookmark["last_pk_fetched"], {"id": 1})
self.assertIsNotNone(table_2_bookmark.get("log_file"))
self.assertIsNotNone(table_2_bookmark.get("log_pos"))
failed_syncing_table_2 = False
singer.write_message = singer_write_message_ok
table_2_RECORD_COUNT = 0
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog,
state)
self.assertFalse(failed_syncing_table_2)
record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(
record_messages_2,
[
["table_2", {
"id": 2,
"bar": "def",
"foo": 200
}],
["table_2", {
"id": 3,
"bar": "abc",
"foo": 100
}],
["table_1", {
"id": 1,
"bar": "abc",
"foo": 100
}],
["table_1", {
"id": 2,
"bar": "def",
"foo": 200
}],
["table_1", {
"id": 3,
"bar": "ghi",
"foo": 300
}],
],
)
self.assertIsNone(state["currently_syncing"])
table_1_bookmark = state["bookmarks"]["tap_mysql_test-table_1"]
table_2_bookmark = state["bookmarks"]["tap_mysql_test-table_2"]
self.assertEqual(table_1_bookmark,
{"initial_full_table_complete": True})
self.assertIsNone(table_2_bookmark.get("initial_full_table_complete"))
table_2_version = table_2_bookmark["version"]
self.assertIsNotNone(table_2_version)
self.assertIsNone(table_2_bookmark.get("max_pk_values"))
self.assertIsNone(table_2_bookmark.get("last_pk_fetched"))
self.assertIsNotNone(table_2_bookmark.get("log_file"))
self.assertIsNotNone(table_2_bookmark.get("log_pos"))
new_table_2_records = [[400, "jkl"], [500, "mno"]]
for record in new_table_2_records:
insert_record(self.conn, "table_2", record)
TABLE_2_RECORD_COUNT = 0
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog,
state)
self.assertFalse(failed_syncing_table_2)
record_messages_3 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(
record_messages_3,
[
["table_1", {
"id": 1,
"bar": "abc",
"foo": 100
}],
["table_1", {
"id": 2,
"bar": "def",
"foo": 200
}],
["table_1", {
"id": 3,
"bar": "ghi",
"foo": 300
}],
["table_2", {
"id": 4,
"bar": "jkl",
"foo": 400
}],
["table_2", {
"id": 5,
"bar": "mno",
"foo": 500
}],
],
)
self.assertIsNone(state["currently_syncing"])
table_1_bookmark = state["bookmarks"]["tap_mysql_test-table_1"]
table_2_bookmark = state["bookmarks"]["tap_mysql_test-table_2"]
self.assertEqual(table_1_bookmark,
{"initial_full_table_complete": True})
self.assertIsNone(table_2_bookmark.get("initial_full_table_complete"))
self.assertIsNotNone(table_2_bookmark.get("log_file"))
self.assertIsNotNone(table_2_bookmark.get("log_pos"))
def test_binlog_stream_with_alteration(self):
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
config = test_utils.get_db_config()
config['server_id'] = "100"
tap_mysql.do_sync(self.conn, config, self.catalog, self.state)
with connect_with_backoff(self.conn) as open_conn:
with open_conn.cursor() as cursor:
cursor.execute('ALTER TABLE binlog_1 add column is_cancelled boolean;')
cursor.execute('INSERT INTO binlog_1 (id, updated, is_cancelled) VALUES (2, \'2017-06-20\', true)')
cursor.execute('INSERT INTO binlog_1 (id, updated, is_cancelled) VALUES (3, \'2017-09-21\', false)')
cursor.execute('INSERT INTO binlog_2 (id, updated) VALUES (3, \'2017-12-10\')')
cursor.execute('ALTER TABLE binlog_1 change column updated date_updated datetime;')
cursor.execute('UPDATE binlog_1 set date_updated=\'2018-06-18\' WHERE id = 3')
open_conn.commit()
tap_mysql.do_sync(self.conn, config, self.catalog, self.state)
record_messages = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES))
message_types = [type(m) for m in SINGER_MESSAGES]
self.assertEqual(message_types,
[singer.StateMessage,
singer.SchemaMessage,
singer.SchemaMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.StateMessage, # end of 1st sync
singer.StateMessage, # start of 2nd sync
singer.SchemaMessage,
singer.SchemaMessage,
singer.SchemaMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.StateMessage,
])
self.assertEqual([('tap_mysql_test-binlog_1', 1, '2017-06-01T00:00:00+00:00', False),
('tap_mysql_test-binlog_1', 2, '2017-06-20T00:00:00+00:00', False),
('tap_mysql_test-binlog_1', 3, '2017-09-22T00:00:00+00:00', False),
('tap_mysql_test-binlog_2', 1, '2017-10-22T00:00:00+00:00', False),
('tap_mysql_test-binlog_2', 2, '2017-11-10T00:00:00+00:00', False),
('tap_mysql_test-binlog_2', 3, '2017-12-10T00:00:00+00:00', False),
('tap_mysql_test-binlog_1', 3, '2018-06-18T00:00:00+00:00', False),
('tap_mysql_test-binlog_2', 2, '2018-06-18T00:00:00+00:00', False),
('tap_mysql_test-binlog_1', 2, '2017-06-20T00:00:00+00:00', True),
('tap_mysql_test-binlog_2', 1, '2017-10-22T00:00:00+00:00', True)],
[(m.stream,
m.record['id'],
m.record['updated'],
m.record.get(binlog.SDC_DELETED_AT) is not None)
for m in record_messages[:10]])
self.assertIn('tap_mysql_test-binlog_1', SINGER_MESSAGES[15].stream)
self.assertNotIn('date_updated', SINGER_MESSAGES[15].schema['properties'])
self.assertNotIn('is_cancelled', SINGER_MESSAGES[15].schema['properties'])
self.assertIn('tap_mysql_test-binlog_1', SINGER_MESSAGES[17].stream)
self.assertIn('date_updated', SINGER_MESSAGES[17].schema['properties'])
self.assertIn('is_cancelled', SINGER_MESSAGES[17].schema['properties'])
self.assertIn('tap_mysql_test-binlog_1', SINGER_MESSAGES[18].stream)
self.assertIn('date_updated', SINGER_MESSAGES[18].record)
self.assertIn('is_cancelled', SINGER_MESSAGES[18].record)
self.assertIsNotNone(singer.get_bookmark(self.state, 'tap_mysql_test-binlog_1', 'log_file'))
self.assertIsNotNone(singer.get_bookmark(self.state, 'tap_mysql_test-binlog_1', 'log_pos'))
self.assertIsNotNone(singer.get_bookmark(self.state, 'tap_mysql_test-binlog_2', 'log_file'))
self.assertIsNotNone(singer.get_bookmark(self.state, 'tap_mysql_test-binlog_2', 'log_pos'))
def test_binlog_stream(self):
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
config = test_utils.get_db_config()
config["server_id"] = "100"
tap_mysql.do_sync(self.conn, config, self.catalog, self.state)
record_messages = list(
filter(lambda m: isinstance(m, singer.RecordMessage),
SINGER_MESSAGES))
message_types = [type(m) for m in SINGER_MESSAGES]
self.assertEqual(
message_types,
[
singer.StateMessage,
singer.SchemaMessage,
singer.SchemaMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.RecordMessage,
singer.StateMessage,
],
)
self.assertEqual(
[
("binlog_1", 1, "2017-06-01T00:00:00+00:00", False),
("binlog_1", 2, "2017-06-20T00:00:00+00:00", False),
("binlog_1", 3, "2017-09-22T00:00:00+00:00", False),
("binlog_2", 1, "2017-10-22T00:00:00+00:00", False),
("binlog_2", 2, "2017-11-10T00:00:00+00:00", False),
("binlog_2", 3, "2017-12-10T00:00:00+00:00", False),
("binlog_1", 3, "2018-06-18T00:00:00+00:00", False),
("binlog_2", 2, "2018-06-18T00:00:00+00:00", False),
("binlog_1", 2, "2017-06-20T00:00:00+00:00", True),
("binlog_2", 1, "2017-10-22T00:00:00+00:00", True),
],
[(
m.stream,
m.record["id"],
m.record["updated"],
m.record.get(binlog.SDC_DELETED_AT) is not None,
) for m in record_messages],
)
self.assertIsNotNone(
singer.get_bookmark(self.state, "tap_mysql_test-binlog_1",
"log_file"))
self.assertIsNotNone(
singer.get_bookmark(self.state, "tap_mysql_test-binlog_1",
"log_pos"))
self.assertIsNotNone(
singer.get_bookmark(self.state, "tap_mysql_test-binlog_2",
"log_file"))
self.assertIsNotNone(
singer.get_bookmark(self.state, "tap_mysql_test-binlog_2",
"log_pos"))
