def test_sync_messages_are_correct(self): self.catalog.streams[0] = test_utils.set_replication_method_and_key(self.catalog.streams[0], 'LOG_BASED', None) self.catalog.streams[0] = test_utils.set_selected(self.catalog.streams[0], True) global SINGER_MESSAGES SINGER_MESSAGES.clear() #inital sync tap_mysql.do_sync(self.conn, {}, self.catalog, {}) # get schema message to test that it has all the table's columns schema_message = next(filter(lambda m: isinstance(m, singer.SchemaMessage), SINGER_MESSAGES)) expectedKeys = ['good_pk', 'age'] self.assertEqual(schema_message.schema['properties'].keys(), set(expectedKeys)) # get the records, these are generated by Full table replication record_messages = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES)) self.assertEqual(len(record_messages), 4) self.assertListEqual([ {'age': 20, 'good_pk': '61'}, {'age': 30, 'good_pk': '62'}, {'age': 30, 'good_pk': '63'}, {'age': 40, 'good_pk': '64'}, ], [rec.record for rec in record_messages]) # get the last state message to be fed to the next sync state_message = list(filter(lambda m: isinstance(m, singer.StateMessage), SINGER_MESSAGES))[-1] SINGER_MESSAGES.clear() # run some queries with connect_with_backoff(self.conn) as open_conn: with open_conn.cursor() as cursor: cursor.execute("UPDATE good_pk_tab set age=age+5") cursor.execute("INSERT INTO good_pk_tab (good_pk, age) VALUES " "(BINARY('e'), 16), " "(BINARY('f'), 5)") # do a sync and give the state so that binlog replication start from the last synced position tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state_message.value) # get the changed/new records record_messages = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES)) self.assertEqual(len(record_messages), 6) self.assertListEqual([ {'age': 25, 'good_pk': '61'}, {'age': 35, 'good_pk': '62'}, {'age': 35, 'good_pk': '63'}, {'age': 45, 'good_pk': '64'}, {'age': 16, 'good_pk': '65'}, {'age': 5, 'good_pk': '66'}, ], [rec.record for rec in record_messages])
def test_open_connections_with_default_session_sqls(self): """Default session parameters should be applied if no custom session SQLs""" with patch('tap_mysql.connection.MySQLConnection.connect'): with patch('tap_mysql.connection.run_sql') as run_sql_mock: run_sql_mock.side_effect = self.run_sql_mock conn = MySQLConnectionMock(config=test_utils.get_db_config()) connect_with_backoff(conn) # Test if session variables applied on connection self.assertEqual(self.executed_queries, tap_mysql.connection.DEFAULT_SESSION_SQLS)
def test_binlog_stream(self): global SINGER_MESSAGES SINGER_MESSAGES.clear() config = test_utils.get_db_config() config['server_id'] = "100" tap_mysql.do_sync(self.conn, config, self.catalog, self.state) record_messages = list( filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES)) message_types = [type(m) for m in SINGER_MESSAGES] self.assertEqual(message_types, [ singer.StateMessage, singer.SchemaMessage, singer.SchemaMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.StateMessage ]) self.assertEqual([ ('tap_mysql_test-binlog_1', 1, '2017-06-01T00:00:00+00:00', False), ('tap_mysql_test-binlog_1', 2, '2017-06-20T00:00:00+00:00', False), ('tap_mysql_test-binlog_1', 3, '2017-09-22T00:00:00+00:00', False), ('tap_mysql_test-binlog_2', 1, '2017-10-22T00:00:00+00:00', False), ('tap_mysql_test-binlog_2', 2, '2017-11-10T00:00:00+00:00', False), ('tap_mysql_test-binlog_2', 3, '2017-12-10T00:00:00+00:00', False), ('tap_mysql_test-binlog_1', 3, '2018-06-18T00:00:00+00:00', False), ('tap_mysql_test-binlog_2', 2, '2018-06-18T00:00:00+00:00', False), ('tap_mysql_test-binlog_1', 2, '2017-06-20T00:00:00+00:00', True), ('tap_mysql_test-binlog_2', 1, '2017-10-22T00:00:00+00:00', True) ], [(m.stream, m.record['id'], m.record['updated'], m.record.get(binlog.SDC_DELETED_AT) is not None) for m in record_messages]) self.assertIsNotNone( singer.get_bookmark(self.state, 'tap_mysql_test-binlog_1', 'log_file')) self.assertIsNotNone( singer.get_bookmark(self.state, 'tap_mysql_test-binlog_1', 'log_pos')) self.assertIsNotNone( singer.get_bookmark(self.state, 'tap_mysql_test-binlog_2', 'log_file')) self.assertIsNotNone( singer.get_bookmark(self.state, 'tap_mysql_test-binlog_2', 'log_pos'))
def test_open_connections_with_session_sqls(self): """Custom session parameters should be applied if defined""" session_sqls = [ 'SET SESSION max_statement_time=0', 'SET SESSION wait_timeout=28800' ] with patch('tap_mysql.connection.MySQLConnection.connect'): with patch('tap_mysql.connection.run_sql') as run_sql_mock: run_sql_mock.side_effect = self.run_sql_mock conn = MySQLConnectionMock(config={**test_utils.get_db_config(), **{'session_sqls': session_sqls}}) connect_with_backoff(conn) # Test if session variables applied on connection self.assertEqual(self.executed_queries, session_sqls)
def test_open_connections_with_invalid_session_sqls(self): """Invalid SQLs in session_sqls should be ignored""" session_sqls = [ 'SET SESSION max_statement_time=0', 'INVALID-SQL-SHOULD-BE-SILENTLY-IGNORED', 'SET SESSION wait_timeout=28800' ] with patch('tap_mysql.connection.MySQLConnection.connect'): with patch('tap_mysql.connection.run_sql') as run_sql_mock: run_sql_mock.side_effect = self.run_sql_mock conn = MySQLConnectionMock(config={**test_utils.get_db_config(), **{'session_sqls': session_sqls}}) connect_with_backoff(conn) # Test if session variables applied on connection self.assertEqual(self.executed_queries, ['SET SESSION max_statement_time=0', 'SET SESSION wait_timeout=28800'])
def test_table_3_interrupted(self): singer.write_message = singer_write_message_no_table_3 state = {} failed_syncing_table_3 = False # Do not worry about table_2 for stream in filter(lambda s: s.stream == 'table_2', self.catalog.streams): md_map = singer.metadata.to_map(stream.metadata) md_map = singer.metadata.write(md_map, (), 'selected', False) stream.metadata = singer.metadata.to_list(md_map) config = test_utils.get_db_config() config['allow_non_auto_increment_pks'] = 'true' try: tap_mysql.do_sync(self.conn, config, self.catalog, state) except Exception as ex: if str(ex) == 'simulated exception': failed_syncing_table_3 = True self.assertTrue(failed_syncing_table_3) record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] # sorting of the two chicken-* columns is based on lexicographic ordering which in # this case chicken-11 comes before chicken-2 self.assertEqual(record_messages_1, [['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 2, 'bar': 'def', 'foo': 200}], ['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}], ['table_3', {'id': "chicken-11", 'bar': 'ghi', 'foo': 300}]]) self.assertEqual(state['currently_syncing'], 'tap_mysql_test-table_3') table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1'] table_3_bookmark = state['bookmarks']['tap_mysql_test-table_3'] self.assertEqual(table_1_bookmark, {'initial_full_table_complete': True}) self.assertIsNone(table_3_bookmark.get('initial_full_table_complete')) table_3_version = table_3_bookmark['version'] self.assertIsNotNone(table_3_version) self.assertEqual(table_3_bookmark['max_pk_values'], {'id': "turkey"}) self.assertEqual(table_3_bookmark['last_pk_fetched'], {'id': "chicken-11"}) self.assertIsNotNone(table_3_bookmark.get('log_file')) self.assertIsNotNone(table_3_bookmark.get('log_pos')) failed_syncing_table_3 = False singer.write_message = singer_write_message_ok table_3_RECORD_COUNT = 0 SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, config, self.catalog, state) self.assertFalse(failed_syncing_table_3) record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual(record_messages_2, [['table_3', {'id': "chicken-2", 'bar': 'def', 'foo': 200}], ['table_3', {'id': "turkey", 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 2, 'bar': 'def', 'foo': 200}], ['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}]]) self.assertIsNone(state['currently_syncing']) table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1'] table_3_bookmark = state['bookmarks']['tap_mysql_test-table_3'] self.assertEqual(table_1_bookmark, {'initial_full_table_complete': True}) self.assertIsNone(table_3_bookmark.get('initial_full_table_complete')) table_3_version = table_3_bookmark['version'] self.assertIsNotNone(table_3_version) self.assertIsNone(table_3_bookmark.get('max_pk_values')) self.assertIsNone(table_3_bookmark.get('last_pk_fetched')) self.assertIsNotNone(table_3_bookmark.get('log_file')) self.assertIsNotNone(table_3_bookmark.get('log_pos')) new_table_3_records = [[ "quail-2", 400, 'jkl' ], [ "quail-100", 500, 'mno' ]] for record in new_table_3_records: insert_record_with_specific_id(self.conn, 'table_3', record) TABLE_3_RECORD_COUNT = 0 SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, config, self.catalog, state) self.assertFalse(failed_syncing_table_3) record_messages_3 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] # Now that we are performing binlog sync, record output should be based on insert order self.assertEqual(record_messages_3, [['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 2, 'bar': 'def', 'foo': 200}], ['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}], ['table_3', {'id': "quail-2" , 'bar': 'jkl', 'foo': 400}], ['table_3', {'id': "quail-100", 'bar': 'mno', 'foo': 500}]]) self.assertIsNone(state['currently_syncing']) table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1'] table_3_bookmark = state['bookmarks']['tap_mysql_test-table_3'] self.assertEqual(table_1_bookmark, {'initial_full_table_complete': True}) self.assertIsNone(table_3_bookmark.get('initial_full_table_complete')) self.assertIsNotNone(table_3_bookmark.get('log_file')) self.assertIsNotNone(table_3_bookmark.get('log_pos'))
def test_table_2_interrupted(self): singer.write_message = singer_write_message_no_table_2 state = {} failed_syncing_table_2 = False # Do not worry about table_3 for stream in filter(lambda s: s.stream == 'table_3', self.catalog.streams): md_map = singer.metadata.to_map(stream.metadata) md_map = singer.metadata.write(md_map, (), 'selected', False) stream.metadata = singer.metadata.to_list(md_map) try: tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state) except Exception as ex: if str(ex) == 'simulated exception': failed_syncing_table_2 = True self.assertTrue(failed_syncing_table_2) record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual(record_messages_1, [['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 2, 'bar': 'def', 'foo': 200}], ['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}], ['table_2', {'id': 1, 'bar': 'ghi', 'foo': 300}]]) self.assertEqual(state['currently_syncing'], 'tap_mysql_test-table_2') table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1'] table_2_bookmark = state['bookmarks']['tap_mysql_test-table_2'] self.assertEqual(table_1_bookmark, {'initial_full_table_complete': True}) self.assertIsNone(table_2_bookmark.get('initial_full_table_complete')) table_2_version = table_2_bookmark['version'] self.assertIsNotNone(table_2_version) self.assertEqual(table_2_bookmark['max_pk_values'], {'id': 3}) self.assertEqual(table_2_bookmark['last_pk_fetched'], {'id': 1}) self.assertIsNotNone(table_2_bookmark.get('log_file')) self.assertIsNotNone(table_2_bookmark.get('log_pos')) failed_syncing_table_2 = False singer.write_message = singer_write_message_ok table_2_RECORD_COUNT = 0 SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state) self.assertFalse(failed_syncing_table_2) record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual(record_messages_2, [['table_2', {'id': 2, 'bar': 'def', 'foo': 200}], ['table_2', {'id': 3, 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 2, 'bar': 'def', 'foo': 200}], ['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}]]) self.assertIsNone(state['currently_syncing']) table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1'] table_2_bookmark = state['bookmarks']['tap_mysql_test-table_2'] self.assertEqual(table_1_bookmark, {'initial_full_table_complete': True}) self.assertIsNone(table_2_bookmark.get('initial_full_table_complete')) table_2_version = table_2_bookmark['version'] self.assertIsNotNone(table_2_version) self.assertIsNone(table_2_bookmark.get('max_pk_values')) self.assertIsNone(table_2_bookmark.get('last_pk_fetched')) self.assertIsNotNone(table_2_bookmark.get('log_file')) self.assertIsNotNone(table_2_bookmark.get('log_pos')) new_table_2_records = [[ 400, 'jkl' ], [ 500, 'mno' ]] for record in new_table_2_records: insert_record(self.conn, 'table_2', record) TABLE_2_RECORD_COUNT = 0 SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state) self.assertFalse(failed_syncing_table_2) record_messages_3 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual(record_messages_3, [['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 2, 'bar': 'def', 'foo': 200}], ['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}], ['table_2', {'id': 4, 'bar': 'jkl', 'foo': 400}], ['table_2', {'id': 5, 'bar': 'mno', 'foo': 500}]]) self.assertIsNone(state['currently_syncing']) table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1'] table_2_bookmark = state['bookmarks']['tap_mysql_test-table_2'] self.assertEqual(table_1_bookmark, {'initial_full_table_complete': True}) self.assertIsNone(table_2_bookmark.get('initial_full_table_complete')) self.assertIsNotNone(table_2_bookmark.get('log_file')) self.assertIsNotNone(table_2_bookmark.get('log_pos'))
def test_table_2_interrupted(self): singer.write_message = singer_write_message_no_table_2 state = {} failed_syncing_table_2 = False try: tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state) except Exception as ex: if str(ex) == "simulated exception": failed_syncing_table_2 = True self.assertTrue(failed_syncing_table_2) record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual( record_messages_1, [ ["table_1", { "id": 1, "bar": "abc", "foo": 100 }], ["table_1", { "id": 2, "bar": "def", "foo": 200 }], ["table_1", { "id": 3, "bar": "ghi", "foo": 300 }], ["table_2", { "id": 1, "bar": "ghi", "foo": 300 }], ], ) self.assertEqual(state["currently_syncing"], "tap_mysql_test-table_2") table_1_bookmark = state["bookmarks"]["tap_mysql_test-table_1"] table_2_bookmark = state["bookmarks"]["tap_mysql_test-table_2"] self.assertEqual(table_1_bookmark, {"initial_full_table_complete": True}) self.assertIsNone(table_2_bookmark.get("initial_full_table_complete")) table_2_version = table_2_bookmark["version"] self.assertIsNotNone(table_2_version) self.assertEqual(table_2_bookmark["max_pk_values"], {"id": 3}) self.assertEqual(table_2_bookmark["last_pk_fetched"], {"id": 1}) self.assertIsNotNone(table_2_bookmark.get("log_file")) self.assertIsNotNone(table_2_bookmark.get("log_pos")) failed_syncing_table_2 = False singer.write_message = singer_write_message_ok table_2_RECORD_COUNT = 0 SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state) self.assertFalse(failed_syncing_table_2) record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual( record_messages_2, [ ["table_2", { "id": 2, "bar": "def", "foo": 200 }], ["table_2", { "id": 3, "bar": "abc", "foo": 100 }], ["table_1", { "id": 1, "bar": "abc", "foo": 100 }], ["table_1", { "id": 2, "bar": "def", "foo": 200 }], ["table_1", { "id": 3, "bar": "ghi", "foo": 300 }], ], ) self.assertIsNone(state["currently_syncing"]) table_1_bookmark = state["bookmarks"]["tap_mysql_test-table_1"] table_2_bookmark = state["bookmarks"]["tap_mysql_test-table_2"] self.assertEqual(table_1_bookmark, {"initial_full_table_complete": True}) self.assertIsNone(table_2_bookmark.get("initial_full_table_complete")) table_2_version = table_2_bookmark["version"] self.assertIsNotNone(table_2_version) self.assertIsNone(table_2_bookmark.get("max_pk_values")) self.assertIsNone(table_2_bookmark.get("last_pk_fetched")) self.assertIsNotNone(table_2_bookmark.get("log_file")) self.assertIsNotNone(table_2_bookmark.get("log_pos")) new_table_2_records = [[400, "jkl"], [500, "mno"]] for record in new_table_2_records: insert_record(self.conn, "table_2", record) TABLE_2_RECORD_COUNT = 0 SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state) self.assertFalse(failed_syncing_table_2) record_messages_3 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual( record_messages_3, [ ["table_1", { "id": 1, "bar": "abc", "foo": 100 }], ["table_1", { "id": 2, "bar": "def", "foo": 200 }], ["table_1", { "id": 3, "bar": "ghi", "foo": 300 }], ["table_2", { "id": 4, "bar": "jkl", "foo": 400 }], ["table_2", { "id": 5, "bar": "mno", "foo": 500 }], ], ) self.assertIsNone(state["currently_syncing"]) table_1_bookmark = state["bookmarks"]["tap_mysql_test-table_1"] table_2_bookmark = state["bookmarks"]["tap_mysql_test-table_2"] self.assertEqual(table_1_bookmark, {"initial_full_table_complete": True}) self.assertIsNone(table_2_bookmark.get("initial_full_table_complete")) self.assertIsNotNone(table_2_bookmark.get("log_file")) self.assertIsNotNone(table_2_bookmark.get("log_pos"))
def test_binlog_stream_with_alteration(self): global SINGER_MESSAGES SINGER_MESSAGES.clear() config = test_utils.get_db_config() config['server_id'] = "100" tap_mysql.do_sync(self.conn, config, self.catalog, self.state) with connect_with_backoff(self.conn) as open_conn: with open_conn.cursor() as cursor: cursor.execute('ALTER TABLE binlog_1 add column is_cancelled boolean;') cursor.execute('INSERT INTO binlog_1 (id, updated, is_cancelled) VALUES (2, \'2017-06-20\', true)') cursor.execute('INSERT INTO binlog_1 (id, updated, is_cancelled) VALUES (3, \'2017-09-21\', false)') cursor.execute('INSERT INTO binlog_2 (id, updated) VALUES (3, \'2017-12-10\')') cursor.execute('ALTER TABLE binlog_1 change column updated date_updated datetime;') cursor.execute('UPDATE binlog_1 set date_updated=\'2018-06-18\' WHERE id = 3') open_conn.commit() tap_mysql.do_sync(self.conn, config, self.catalog, self.state) record_messages = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES)) message_types = [type(m) for m in SINGER_MESSAGES] self.assertEqual(message_types, [singer.StateMessage, singer.SchemaMessage, singer.SchemaMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.StateMessage, # end of 1st sync singer.StateMessage, # start of 2nd sync singer.SchemaMessage, singer.SchemaMessage, singer.SchemaMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.StateMessage, ]) self.assertEqual([('tap_mysql_test-binlog_1', 1, '2017-06-01T00:00:00+00:00', False), ('tap_mysql_test-binlog_1', 2, '2017-06-20T00:00:00+00:00', False), ('tap_mysql_test-binlog_1', 3, '2017-09-22T00:00:00+00:00', False), ('tap_mysql_test-binlog_2', 1, '2017-10-22T00:00:00+00:00', False), ('tap_mysql_test-binlog_2', 2, '2017-11-10T00:00:00+00:00', False), ('tap_mysql_test-binlog_2', 3, '2017-12-10T00:00:00+00:00', False), ('tap_mysql_test-binlog_1', 3, '2018-06-18T00:00:00+00:00', False), ('tap_mysql_test-binlog_2', 2, '2018-06-18T00:00:00+00:00', False), ('tap_mysql_test-binlog_1', 2, '2017-06-20T00:00:00+00:00', True), ('tap_mysql_test-binlog_2', 1, '2017-10-22T00:00:00+00:00', True)], [(m.stream, m.record['id'], m.record['updated'], m.record.get(binlog.SDC_DELETED_AT) is not None) for m in record_messages[:10]]) self.assertIn('tap_mysql_test-binlog_1', SINGER_MESSAGES[15].stream) self.assertNotIn('date_updated', SINGER_MESSAGES[15].schema['properties']) self.assertNotIn('is_cancelled', SINGER_MESSAGES[15].schema['properties']) self.assertIn('tap_mysql_test-binlog_1', SINGER_MESSAGES[17].stream) self.assertIn('date_updated', SINGER_MESSAGES[17].schema['properties']) self.assertIn('is_cancelled', SINGER_MESSAGES[17].schema['properties']) self.assertIn('tap_mysql_test-binlog_1', SINGER_MESSAGES[18].stream) self.assertIn('date_updated', SINGER_MESSAGES[18].record) self.assertIn('is_cancelled', SINGER_MESSAGES[18].record) self.assertIsNotNone(singer.get_bookmark(self.state, 'tap_mysql_test-binlog_1', 'log_file')) self.assertIsNotNone(singer.get_bookmark(self.state, 'tap_mysql_test-binlog_1', 'log_pos')) self.assertIsNotNone(singer.get_bookmark(self.state, 'tap_mysql_test-binlog_2', 'log_file')) self.assertIsNotNone(singer.get_bookmark(self.state, 'tap_mysql_test-binlog_2', 'log_pos'))
def test_binlog_stream(self): global SINGER_MESSAGES SINGER_MESSAGES.clear() config = test_utils.get_db_config() config["server_id"] = "100" tap_mysql.do_sync(self.conn, config, self.catalog, self.state) record_messages = list( filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES)) message_types = [type(m) for m in SINGER_MESSAGES] self.assertEqual( message_types, [ singer.StateMessage, singer.SchemaMessage, singer.SchemaMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.StateMessage, ], ) self.assertEqual( [ ("binlog_1", 1, "2017-06-01T00:00:00+00:00", False), ("binlog_1", 2, "2017-06-20T00:00:00+00:00", False), ("binlog_1", 3, "2017-09-22T00:00:00+00:00", False), ("binlog_2", 1, "2017-10-22T00:00:00+00:00", False), ("binlog_2", 2, "2017-11-10T00:00:00+00:00", False), ("binlog_2", 3, "2017-12-10T00:00:00+00:00", False), ("binlog_1", 3, "2018-06-18T00:00:00+00:00", False), ("binlog_2", 2, "2018-06-18T00:00:00+00:00", False), ("binlog_1", 2, "2017-06-20T00:00:00+00:00", True), ("binlog_2", 1, "2017-10-22T00:00:00+00:00", True), ], [( m.stream, m.record["id"], m.record["updated"], m.record.get(binlog.SDC_DELETED_AT) is not None, ) for m in record_messages], ) self.assertIsNotNone( singer.get_bookmark(self.state, "tap_mysql_test-binlog_1", "log_file")) self.assertIsNotNone( singer.get_bookmark(self.state, "tap_mysql_test-binlog_1", "log_pos")) self.assertIsNotNone( singer.get_bookmark(self.state, "tap_mysql_test-binlog_2", "log_file")) self.assertIsNotNone( singer.get_bookmark(self.state, "tap_mysql_test-binlog_2", "log_pos"))