def test_change_replication_key(self): state = { 'bookmarks': { 'tap_mysql_test-incremental': { 'version': 1, 'replication_key_value': '2017-06-20', 'replication_key': 'updated' } } } stream = [x for x in self.catalog.streams if x.stream == 'incremental'][0] stream.metadata = [ {'breadcrumb': (), 'metadata': {'selected': True, 'database-name': 'tap_mysql_test'}}, {'breadcrumb': ('properties', 'val'), 'metadata': {'selected': True}}, {'breadcrumb': ('properties', 'updated'), 'metadata': {'selected': True}} ] test_utils.set_replication_method_and_key(stream, 'INCREMENTAL', 'val') tap_mysql.do_sync(self.conn, {}, self.catalog, state) self.assertEqual(state['bookmarks']['tap_mysql_test-incremental']['replication_key'], 'val') self.assertEqual(state['bookmarks']['tap_mysql_test-incremental']['replication_key_value'], 3) self.assertEqual(state['bookmarks']['tap_mysql_test-incremental']['version'], 1)
def test_with_state(self): state = { 'bookmarks': { 'tap_mysql_test-incremental': { 'version': 1, 'replication_key_value': '2017-06-20', 'replication_key': 'updated' }, 'tap_mysql_test-integer_incremental': { 'version': 1, 'replication_key_value': 3, 'replication_key': 'updated' } } } global SINGER_MESSAGES SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, {}, self.catalog, state) (message_types, versions) = message_types_and_versions(SINGER_MESSAGES) self.assertEqual([ 'ActivateVersionMessage', 'RecordMessage', 'RecordMessage', 'ActivateVersionMessage', 'RecordMessage' ], message_types) self.assertTrue(isinstance(versions[0], int)) self.assertEqual(versions[0], versions[1]) self.assertEqual(versions[1], 1)
def test_version_cleared_from_state_after_full_table_success(self): common.get_stream_version = lambda a, b: 12345 state = { 'bookmarks': { 'tap_mysql_test-full_table': { 'version': 1, 'initial_full_table_complete': True } } } global SINGER_MESSAGES SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, {}, self.catalog, state) (message_types, versions) = message_types_and_versions(SINGER_MESSAGES) self.assertEqual(['RecordMessage', 'ActivateVersionMessage'], message_types) self.assertEqual(versions, [12345, 12345]) self.assertFalse('version' in state['bookmarks'] ['tap_mysql_test-full_table'].keys()) self.assertTrue(state['bookmarks']['tap_mysql_test-full_table'] ['initial_full_table_complete'])
def runTest(self): conn = test_utils.get_test_connection() with connect_with_backoff(conn) as open_conn: with open_conn.cursor() as cur: cur.execute(''' CREATE TABLE tab ( id INTEGER PRIMARY KEY, a INTEGER, b INTEGER) ''') catalog = test_utils.discover_catalog(conn, {}) catalog.streams[0].stream = 'tab' catalog.streams[0].metadata = [ {'breadcrumb': (), 'metadata': {'selected': True, 'database-name': 'tap_mysql_test'}}, {'breadcrumb': ('properties', 'a'), 'metadata': {'selected': True}} ] test_utils.set_replication_method_and_key(catalog.streams[0], 'FULL_TABLE', None) global SINGER_MESSAGES SINGER_MESSAGES.clear() tap_mysql.do_sync(conn, {}, catalog, {}) schema_message = list(filter(lambda m: isinstance(m, singer.SchemaMessage), SINGER_MESSAGES))[0] self.assertTrue(isinstance(schema_message, singer.SchemaMessage)) # tap-mysql selects new fields by default. If a field doesn't appear in the schema, then it should be # selected expectedKeys = ['id', 'a', 'b'] self.assertEqual(schema_message.schema['properties'].keys(), set(expectedKeys))
def test_with_state(self): state = { "bookmarks": { "tap_mysql_test-incremental": { "version": 1, "replication_key_value": "2017-06-20", "replication_key": "updated", }, "tap_mysql_test-integer_incremental": { "version": 1, "replication_key_value": 3, "replication_key": "updated", }, } } global SINGER_MESSAGES SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, {}, self.catalog, state) (message_types, versions) = message_types_and_versions(SINGER_MESSAGES) self.assertEqual( [ "ActivateVersionMessage", "RecordMessage", "RecordMessage", "ActivateVersionMessage", "RecordMessage", ], message_types, ) self.assertTrue(isinstance(versions[0], int)) self.assertEqual(versions[0], versions[1]) self.assertEqual(versions[1], 1)
def test_with_no_state(self): state = {} global SINGER_MESSAGES SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, {}, self.catalog, state) (message_types, versions) = message_types_and_versions(SINGER_MESSAGES) self.assertEqual( [ "ActivateVersionMessage", "RecordMessage", "RecordMessage", "RecordMessage", "ActivateVersionMessage", "RecordMessage", "RecordMessage", "RecordMessage", ], message_types, ) self.assertTrue(isinstance(versions[0], int)) self.assertEqual(versions[0], versions[1])
def test_version_cleared_from_state_after_full_table_success(self): common.get_stream_version = lambda a, b: 12345 state = { "bookmarks": { "tap_mysql_test-full_table": { "version": 1, "initial_full_table_complete": True } } } global SINGER_MESSAGES SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, {}, self.catalog, state) (message_types, versions) = message_types_and_versions(SINGER_MESSAGES) self.assertEqual(["RecordMessage", "ActivateVersionMessage"], message_types) self.assertEqual(versions, [12345, 12345]) self.assertFalse("version" in state["bookmarks"] ["tap_mysql_test-full_table"].keys()) self.assertTrue(state["bookmarks"]["tap_mysql_test-full_table"] ["initial_full_table_complete"])
def runTest(self): global SINGER_MESSAGES SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, {}, self.catalog, {}) record_message = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES))[0] self.assertTrue(isinstance(record_message, singer.RecordMessage)) self.assertEqual(record_message.record, {'val': '{"a": 10, "b": "c"}'})
def test_emit_currently_syncing(self): state = {} global SINGER_MESSAGES SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, {}, self.catalog, state) self.assertRegex(currently_syncing_seq(SINGER_MESSAGES), '^a+b+c+_+')
def test_sync_messages_are_correct(self): self.catalog.streams[0] = test_utils.set_replication_method_and_key(self.catalog.streams[0], 'LOG_BASED', None) self.catalog.streams[0] = test_utils.set_selected(self.catalog.streams[0], True) global SINGER_MESSAGES SINGER_MESSAGES.clear() #inital sync tap_mysql.do_sync(self.conn, {}, self.catalog, {}) # get schema message to test that it has all the table's columns schema_message = next(filter(lambda m: isinstance(m, singer.SchemaMessage), SINGER_MESSAGES)) expectedKeys = ['good_pk', 'age'] self.assertEqual(schema_message.schema['properties'].keys(), set(expectedKeys)) # get the records, these are generated by Full table replication record_messages = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES)) self.assertEqual(len(record_messages), 4) self.assertListEqual([ {'age': 20, 'good_pk': '61'}, {'age': 30, 'good_pk': '62'}, {'age': 30, 'good_pk': '63'}, {'age': 40, 'good_pk': '64'}, ], [rec.record for rec in record_messages]) # get the last state message to be fed to the next sync state_message = list(filter(lambda m: isinstance(m, singer.StateMessage), SINGER_MESSAGES))[-1] SINGER_MESSAGES.clear() # run some queries with connect_with_backoff(self.conn) as open_conn: with open_conn.cursor() as cursor: cursor.execute("UPDATE good_pk_tab set age=age+5") cursor.execute("INSERT INTO good_pk_tab (good_pk, age) VALUES " "(BINARY('e'), 16), " "(BINARY('f'), 5)") # do a sync and give the state so that binlog replication start from the last synced position tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state_message.value) # get the changed/new records record_messages = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES)) self.assertEqual(len(record_messages), 6) self.assertListEqual([ {'age': 25, 'good_pk': '61'}, {'age': 35, 'good_pk': '62'}, {'age': 35, 'good_pk': '63'}, {'age': 45, 'good_pk': '64'}, {'age': 16, 'good_pk': '65'}, {'age': 5, 'good_pk': '66'}, ], [rec.record for rec in record_messages])
def test_initial_full_table(self): state = {} expected_log_file, expected_log_pos = binlog.fetch_current_log_file_and_pos( self.conn) global SINGER_MESSAGES SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, {}, self.catalog, state) message_types = [type(m) for m in SINGER_MESSAGES] self.assertEqual(message_types, [ singer.StateMessage, singer.SchemaMessage, singer.ActivateVersionMessage, singer.RecordMessage, singer.RecordMessage, singer.StateMessage, singer.ActivateVersionMessage, singer.StateMessage, singer.SchemaMessage, singer.ActivateVersionMessage, singer.RecordMessage, singer.RecordMessage, singer.StateMessage, singer.ActivateVersionMessage, singer.StateMessage ]) activate_version_message_1 = list( filter( lambda m: isinstance(m, singer.ActivateVersionMessage) and m. stream == 'binlog_1', SINGER_MESSAGES))[0] activate_version_message_2 = list( filter( lambda m: isinstance(m, singer.ActivateVersionMessage) and m. stream == 'binlog_2', SINGER_MESSAGES))[0] record_messages = list( filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES)) self.assertIsNotNone( singer.get_bookmark(self.state, 'tap_mysql_test-binlog_1', 'log_file')) self.assertIsNotNone( singer.get_bookmark(self.state, 'tap_mysql_test-binlog_1', 'log_pos')) self.assertIsNotNone( singer.get_bookmark(self.state, 'tap_mysql_test-binlog_2', 'log_file')) self.assertIsNotNone( singer.get_bookmark(self.state, 'tap_mysql_test-binlog_2', 'log_pos')) self.assertEqual( singer.get_bookmark(state, 'tap_mysql_test-binlog_1', 'version'), activate_version_message_1.version) self.assertEqual( singer.get_bookmark(state, 'tap_mysql_test-binlog_2', 'version'), activate_version_message_2.version)
def test_table_2_interrupted(self): singer.write_message = singer_write_message_no_table_2 state = {} failed_syncing_table_2 = False try: tap_mysql.do_sync(self.conn, {}, self.catalog, state) except Exception as ex: if str(ex) == 'simulated exception': failed_syncing_table_2 = True self.assertTrue(failed_syncing_table_2) record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual(record_messages_1, [['tap_mysql_test-table_1', {'id': 1, 'bar': 'abc', 'foo': 100}], ['tap_mysql_test-table_1', {'id': 2, 'bar': 'def', 'foo': 200}], ['tap_mysql_test-table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}], ['tap_mysql_test-table_2', {'id': 1, 'bar': 'ghi', 'foo': 300}] ]) failed_syncing_table_2 = False singer.write_message = singer_write_message_ok SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, {}, self.catalog, state) self.assertFalse(failed_syncing_table_2) record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual(record_messages_2, [['tap_mysql_test-table_2', {'id': 2, 'bar': 'def', 'foo': 200}], ['tap_mysql_test-table_2', {'id': 3, 'bar': 'abc', 'foo': 100}], ['tap_mysql_test-table_1', {'id': 1, 'bar': 'abc', 'foo': 100}], ['tap_mysql_test-table_1', {'id': 2, 'bar': 'def', 'foo': 200}], ['tap_mysql_test-table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}]]) expected_state_2 = { 'currently_syncing': None, 'bookmarks': { 'tap_mysql_test-table_1': { 'initial_full_table_complete': True }, 'tap_mysql_test-table_2': { 'initial_full_table_complete': True } } } self.assertEqual(state, expected_state_2)
def test_version_not_cleared_from_state_after_incremental_success(self): state = { 'bookmarks': { 'tap_mysql_test-incremental': { 'version': 1, 'replication_key_value': '2017-06-20', 'replication_key': 'updated' } } } tap_mysql.do_sync(self.conn, {}, self.catalog, state) self.assertEqual(state['bookmarks']['tap_mysql_test-incremental']['version'], 1)
def test_binlog_stream(self): global SINGER_MESSAGES SINGER_MESSAGES.clear() config = test_utils.get_db_config() config['server_id'] = "100" tap_mysql.do_sync(self.conn, config, self.catalog, self.state) record_messages = list( filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES)) message_types = [type(m) for m in SINGER_MESSAGES] self.assertEqual(message_types, [ singer.StateMessage, singer.SchemaMessage, singer.SchemaMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.RecordMessage, singer.StateMessage ]) self.assertEqual([ ('tap_mysql_test-binlog_1', 1, '2017-06-01T00:00:00+00:00', False), ('tap_mysql_test-binlog_1', 2, '2017-06-20T00:00:00+00:00', False), ('tap_mysql_test-binlog_1', 3, '2017-09-22T00:00:00+00:00', False), ('tap_mysql_test-binlog_2', 1, '2017-10-22T00:00:00+00:00', False), ('tap_mysql_test-binlog_2', 2, '2017-11-10T00:00:00+00:00', False), ('tap_mysql_test-binlog_2', 3, '2017-12-10T00:00:00+00:00', False), ('tap_mysql_test-binlog_1', 3, '2018-06-18T00:00:00+00:00', False), ('tap_mysql_test-binlog_2', 2, '2018-06-18T00:00:00+00:00', False), ('tap_mysql_test-binlog_1', 2, '2017-06-20T00:00:00+00:00', True), ('tap_mysql_test-binlog_2', 1, '2017-10-22T00:00:00+00:00', True) ], [(m.stream, m.record['id'], m.record['updated'], m.record.get(binlog.SDC_DELETED_AT) is not None) for m in record_messages]) self.assertIsNotNone( singer.get_bookmark(self.state, 'tap_mysql_test-binlog_1', 'log_file')) self.assertIsNotNone( singer.get_bookmark(self.state, 'tap_mysql_test-binlog_1', 'log_pos')) self.assertIsNotNone( singer.get_bookmark(self.state, 'tap_mysql_test-binlog_2', 'log_file')) self.assertIsNotNone( singer.get_bookmark(self.state, 'tap_mysql_test-binlog_2', 'log_pos'))
def test_change_replication_key(self): state = { "bookmarks": { "tap_mysql_test-incremental": { "version": 1, "replication_key_value": "2017-06-20", "replication_key": "updated", } } } stream = [ x for x in self.catalog.streams if x.stream == "incremental" ][0] stream.metadata = [ { "breadcrumb": (), "metadata": { "selected": True, "database-name": "tap_mysql_test" } }, { "breadcrumb": ("properties", "val"), "metadata": { "selected": True } }, { "breadcrumb": ("properties", "updated"), "metadata": { "selected": True } }, ] test_utils.set_replication_method_and_key(stream, "INCREMENTAL", "val") tap_mysql.do_sync(self.conn, {}, self.catalog, state) self.assertEqual( state["bookmarks"]["tap_mysql_test-incremental"] ["replication_key"], "val") self.assertEqual( state["bookmarks"]["tap_mysql_test-incremental"] ["replication_key_value"], 3) self.assertEqual( state["bookmarks"]["tap_mysql_test-incremental"]["version"], 1)
def test_version_not_cleared_from_state_after_incremental_success(self): state = { "bookmarks": { "tap_mysql_test-incremental": { "version": 1, "replication_key_value": "2017-06-20", "replication_key": "updated", } } } tap_mysql.do_sync(self.conn, {}, self.catalog, state) self.assertEqual( state["bookmarks"]["tap_mysql_test-incremental"]["version"], 1)
def test_fail_on_view(self): for stream in self.catalog.streams: md = singer.metadata.to_map(stream.metadata) singer.metadata.write(md, (), 'is-view', True) state = {} failed = False exception_message = None expected_exception_message = "Unable to replicate stream(tap_mysql_test-{}) with binlog because it is a view.".format( self.catalog.streams[0].stream) with self.assertRaises(Exception) as context: tap_mysql.do_sync(self.conn, {}, self.catalog, state) self.assertEqual(expected_exception_message, str(context.exception))
def runTest(self): conn = test_utils.get_test_connection() with connect_with_backoff(conn) as open_conn: with open_conn.cursor() as cur: cur.execute(""" CREATE TABLE tab ( id INTEGER PRIMARY KEY, a INTEGER, b INTEGER) """) catalog = test_utils.discover_catalog(conn, {}) catalog.streams[0].stream = "tab" catalog.streams[0].metadata = [ { "breadcrumb": (), "metadata": { "selected": True, "database-name": "tap_mysql_test" } }, { "breadcrumb": ("properties", "a"), "metadata": { "selected": True } }, ] test_utils.set_replication_method_and_key(catalog.streams[0], "FULL_TABLE", None) global SINGER_MESSAGES SINGER_MESSAGES.clear() tap_mysql.do_sync(conn, {}, catalog, {}) schema_message = list( filter(lambda m: isinstance(m, singer.SchemaMessage), SINGER_MESSAGES))[0] self.assertTrue(isinstance(schema_message, singer.SchemaMessage)) # tap-mysql selects new fields by default. If a field doesn't appear in the schema, then it should be # selected expectedKeys = ["id", "a", "b"] self.assertEqual(schema_message.schema["properties"].keys(), set(expectedKeys))
def test_start_at_currently_syncing(self): state = { 'currently_syncing': 'tap_mysql_test-b', 'bookmarks': { 'tap_mysql_test-a': { 'version': 123 }, 'tap_mysql_test-b': { 'version': 456 } } } global SINGER_MESSAGES SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, {}, self.catalog, state) self.assertRegex(currently_syncing_seq(SINGER_MESSAGES), '^b+c+a+_+')
def test_start_at_currently_syncing(self): state = { "currently_syncing": "tap_mysql_test-b", "bookmarks": { "tap_mysql_test-a": { "version": 123 }, "tap_mysql_test-b": { "version": 456 }, }, } global SINGER_MESSAGES SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, {}, self.catalog, state) self.assertRegexpMatches(currently_syncing_seq(SINGER_MESSAGES), "^b+c+a+_+")
def test_with_initial_full_table_complete_in_state(self): common.get_stream_version = lambda a, b: 12345 state = { 'bookmarks': { 'tap_mysql_test-full_table': { 'initial_full_table_complete': True } } } global SINGER_MESSAGES SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, {}, self.catalog, state) (message_types, versions) = message_types_and_versions(SINGER_MESSAGES) self.assertEqual(['RecordMessage', 'ActivateVersionMessage'], message_types) self.assertEqual(versions, [12345, 12345])
def test_fail_if_log_file_does_not_exist(self): log_file = 'chicken' stream = self.catalog.streams[0] state = { 'bookmarks': { stream.tap_stream_id: { 'version': singer.utils.now(), 'log_file': log_file, 'log_pos': 1 } } } expected_exception_message = "Unable to replicate stream({}) with binlog because log file {} does not exist.".format( stream, log_file) with self.assertRaises(Exception) as context: tap_mysql.do_sync(self.conn, {}, self.catalog, state) self.assertEqual(expected_exception_message, str(context.exception))
def test_fail_on_view(self): for stream in self.catalog.streams: md = singer.metadata.to_map(stream.metadata) singer.metadata.write(md, (), 'is-view', True) state = {} failed = False exception_message = None expected_exception_message = "Unable to replicate stream({}) with binlog because it is a view.".format( self.catalog.streams[0].stream) try: tap_mysql.do_sync(self.conn, {}, self.catalog, state) except Exception as e: failed = True exception_message = str(e) LOGGER.error(exception_message) self.assertTrue(failed) self.assertEqual(expected_exception_message, exception_message)
def test_initial_full_table(self): state = {} expected_log_file, expected_log_pos = binlog.fetch_current_log_file_and_pos( self.conn) global SINGER_MESSAGES SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, {}, self.catalog, state) message_types = [type(m) for m in SINGER_MESSAGES] self.assertEqual(message_types, [ singer.StateMessage, singer.SchemaMessage, singer.ActivateVersionMessage, singer.RecordMessage, singer.RecordMessage, singer.StateMessage, singer.ActivateVersionMessage, singer.StateMessage ]) record_messages = list( filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES)) # Expected from 0.7.11 expected_records = [{ 'datetime_col': None, 'id': 1, 'timestamp_col': None, 'time_col': '1970-01-01T00:00:00.000000Z', 'date_col': None }, { 'datetime_col': None, 'id': 2, 'timestamp_col': None, 'time_col': None, 'date_col': None }] self.assertEqual(expected_records, [x.asdict()['record'] for x in record_messages])
def test_fail_if_log_file_does_not_exist(self): log_file = 'chicken' stream = self.catalog.streams[0] state = { 'bookmarks': { stream.tap_stream_id: { 'version': singer.utils.now(), 'log_file': log_file, 'log_pos': 1 } } } failed = False exception_message = None expected_exception_message = "Unable to replicate stream({}) with binlog because log file {} does not exist.".format( stream, log_file) try: tap_mysql.do_sync(self.conn, {}, self.catalog, state) except Exception as e: failed = True exception_message = str(e) LOGGER.error(exception_message)
def test_table_2_interrupted(self): singer.write_message = singer_write_message_no_table_2 state = {} failed_syncing_table_2 = False try: tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state) except Exception as ex: if str(ex) == "simulated exception": failed_syncing_table_2 = True self.assertTrue(failed_syncing_table_2) record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual( record_messages_1, [ ["table_1", { "id": 1, "bar": "abc", "foo": 100 }], ["table_1", { "id": 2, "bar": "def", "foo": 200 }], ["table_1", { "id": 3, "bar": "ghi", "foo": 300 }], ["table_2", { "id": 1, "bar": "ghi", "foo": 300 }], ], ) self.assertEqual(state["currently_syncing"], "tap_mysql_test-table_2") table_1_bookmark = state["bookmarks"]["tap_mysql_test-table_1"] table_2_bookmark = state["bookmarks"]["tap_mysql_test-table_2"] self.assertEqual(table_1_bookmark, {"initial_full_table_complete": True}) self.assertIsNone(table_2_bookmark.get("initial_full_table_complete")) table_2_version = table_2_bookmark["version"] self.assertIsNotNone(table_2_version) self.assertEqual(table_2_bookmark["max_pk_values"], {"id": 3}) self.assertEqual(table_2_bookmark["last_pk_fetched"], {"id": 1}) self.assertIsNotNone(table_2_bookmark.get("log_file")) self.assertIsNotNone(table_2_bookmark.get("log_pos")) failed_syncing_table_2 = False singer.write_message = singer_write_message_ok table_2_RECORD_COUNT = 0 SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state) self.assertFalse(failed_syncing_table_2) record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual( record_messages_2, [ ["table_2", { "id": 2, "bar": "def", "foo": 200 }], ["table_2", { "id": 3, "bar": "abc", "foo": 100 }], ["table_1", { "id": 1, "bar": "abc", "foo": 100 }], ["table_1", { "id": 2, "bar": "def", "foo": 200 }], ["table_1", { "id": 3, "bar": "ghi", "foo": 300 }], ], ) self.assertIsNone(state["currently_syncing"]) table_1_bookmark = state["bookmarks"]["tap_mysql_test-table_1"] table_2_bookmark = state["bookmarks"]["tap_mysql_test-table_2"] self.assertEqual(table_1_bookmark, {"initial_full_table_complete": True}) self.assertIsNone(table_2_bookmark.get("initial_full_table_complete")) table_2_version = table_2_bookmark["version"] self.assertIsNotNone(table_2_version) self.assertIsNone(table_2_bookmark.get("max_pk_values")) self.assertIsNone(table_2_bookmark.get("last_pk_fetched")) self.assertIsNotNone(table_2_bookmark.get("log_file")) self.assertIsNotNone(table_2_bookmark.get("log_pos")) new_table_2_records = [[400, "jkl"], [500, "mno"]] for record in new_table_2_records: insert_record(self.conn, "table_2", record) TABLE_2_RECORD_COUNT = 0 SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state) self.assertFalse(failed_syncing_table_2) record_messages_3 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual( record_messages_3, [ ["table_1", { "id": 1, "bar": "abc", "foo": 100 }], ["table_1", { "id": 2, "bar": "def", "foo": 200 }], ["table_1", { "id": 3, "bar": "ghi", "foo": 300 }], ["table_2", { "id": 4, "bar": "jkl", "foo": 400 }], ["table_2", { "id": 5, "bar": "mno", "foo": 500 }], ], ) self.assertIsNone(state["currently_syncing"]) table_1_bookmark = state["bookmarks"]["tap_mysql_test-table_1"] table_2_bookmark = state["bookmarks"]["tap_mysql_test-table_2"] self.assertEqual(table_1_bookmark, {"initial_full_table_complete": True}) self.assertIsNone(table_2_bookmark.get("initial_full_table_complete")) self.assertIsNotNone(table_2_bookmark.get("log_file")) self.assertIsNotNone(table_2_bookmark.get("log_pos"))
def test_table_2_interrupted(self): singer.write_message = singer_write_message_no_table_2 state = {} failed_syncing_table_2 = False try: tap_mysql.do_sync(self.conn, {}, self.catalog, state) except Exception as ex: if str(ex) == "simulated exception": failed_syncing_table_2 = True self.assertTrue(failed_syncing_table_2) record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual( record_messages_1, [ ["table_1", { "id": 1, "bar": "abc", "foo": 100 }], ["table_1", { "id": 2, "bar": "def", "foo": 200 }], ["table_1", { "id": 3, "bar": "ghi", "foo": 300 }], ["table_2", { "id": 1, "bar": "ghi", "foo": 300 }], ], ) expected_state_1 = { "currently_syncing": "tap_mysql_test-table_2", "bookmarks": { "tap_mysql_test-table_2": { "last_pk_fetched": { "id": 1 }, "max_pk_values": { "id": 3 }, }, "tap_mysql_test-table_1": { "initial_full_table_complete": True }, }, } failed_syncing_table_2 = False singer.write_message = singer_write_message_ok TABLE_2_RECORD_COUNT = 0 SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, {}, self.catalog, state) self.assertFalse(failed_syncing_table_2) record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual( record_messages_2, [ ["table_2", { "id": 2, "bar": "def", "foo": 200 }], ["table_2", { "id": 3, "bar": "abc", "foo": 100 }], ["table_1", { "id": 1, "bar": "abc", "foo": 100 }], ["table_1", { "id": 2, "bar": "def", "foo": 200 }], ["table_1", { "id": 3, "bar": "ghi", "foo": 300 }], ], ) expected_state_2 = { "currently_syncing": None, "bookmarks": { "tap_mysql_test-table_1": { "initial_full_table_complete": True }, "tap_mysql_test-table_2": { "initial_full_table_complete": True }, }, } self.assertEqual(state, expected_state_2)
def test_table_2_interrupted(self): singer.write_message = singer_write_message_no_table_2 state = {} failed_syncing_table_2 = False # Do not worry about table_3 for stream in filter(lambda s: s.stream == 'table_3', self.catalog.streams): md_map = singer.metadata.to_map(stream.metadata) md_map = singer.metadata.write(md_map, (), 'selected', False) stream.metadata = singer.metadata.to_list(md_map) try: tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state) except Exception as ex: if str(ex) == 'simulated exception': failed_syncing_table_2 = True self.assertTrue(failed_syncing_table_2) record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual(record_messages_1, [['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 2, 'bar': 'def', 'foo': 200}], ['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}], ['table_2', {'id': 1, 'bar': 'ghi', 'foo': 300}]]) self.assertEqual(state['currently_syncing'], 'tap_mysql_test-table_2') table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1'] table_2_bookmark = state['bookmarks']['tap_mysql_test-table_2'] self.assertEqual(table_1_bookmark, {'initial_full_table_complete': True}) self.assertIsNone(table_2_bookmark.get('initial_full_table_complete')) table_2_version = table_2_bookmark['version'] self.assertIsNotNone(table_2_version) self.assertEqual(table_2_bookmark['max_pk_values'], {'id': 3}) self.assertEqual(table_2_bookmark['last_pk_fetched'], {'id': 1}) self.assertIsNotNone(table_2_bookmark.get('log_file')) self.assertIsNotNone(table_2_bookmark.get('log_pos')) failed_syncing_table_2 = False singer.write_message = singer_write_message_ok table_2_RECORD_COUNT = 0 SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state) self.assertFalse(failed_syncing_table_2) record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual(record_messages_2, [['table_2', {'id': 2, 'bar': 'def', 'foo': 200}], ['table_2', {'id': 3, 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 2, 'bar': 'def', 'foo': 200}], ['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}]]) self.assertIsNone(state['currently_syncing']) table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1'] table_2_bookmark = state['bookmarks']['tap_mysql_test-table_2'] self.assertEqual(table_1_bookmark, {'initial_full_table_complete': True}) self.assertIsNone(table_2_bookmark.get('initial_full_table_complete')) table_2_version = table_2_bookmark['version'] self.assertIsNotNone(table_2_version) self.assertIsNone(table_2_bookmark.get('max_pk_values')) self.assertIsNone(table_2_bookmark.get('last_pk_fetched')) self.assertIsNotNone(table_2_bookmark.get('log_file')) self.assertIsNotNone(table_2_bookmark.get('log_pos')) new_table_2_records = [[ 400, 'jkl' ], [ 500, 'mno' ]] for record in new_table_2_records: insert_record(self.conn, 'table_2', record) TABLE_2_RECORD_COUNT = 0 SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state) self.assertFalse(failed_syncing_table_2) record_messages_3 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual(record_messages_3, [['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 2, 'bar': 'def', 'foo': 200}], ['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}], ['table_2', {'id': 4, 'bar': 'jkl', 'foo': 400}], ['table_2', {'id': 5, 'bar': 'mno', 'foo': 500}]]) self.assertIsNone(state['currently_syncing']) table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1'] table_2_bookmark = state['bookmarks']['tap_mysql_test-table_2'] self.assertEqual(table_1_bookmark, {'initial_full_table_complete': True}) self.assertIsNone(table_2_bookmark.get('initial_full_table_complete')) self.assertIsNotNone(table_2_bookmark.get('log_file')) self.assertIsNotNone(table_2_bookmark.get('log_pos'))
def test_table_3_interrupted(self): singer.write_message = singer_write_message_no_table_3 state = {} failed_syncing_table_3 = False # Do not worry about table_2 for stream in filter(lambda s: s.stream == 'table_2', self.catalog.streams): md_map = singer.metadata.to_map(stream.metadata) md_map = singer.metadata.write(md_map, (), 'selected', False) stream.metadata = singer.metadata.to_list(md_map) config = test_utils.get_db_config() config['allow_non_auto_increment_pks'] = 'true' try: tap_mysql.do_sync(self.conn, config, self.catalog, state) except Exception as ex: if str(ex) == 'simulated exception': failed_syncing_table_3 = True self.assertTrue(failed_syncing_table_3) record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] # sorting of the two chicken-* columns is based on lexicographic ordering which in # this case chicken-11 comes before chicken-2 self.assertEqual(record_messages_1, [['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 2, 'bar': 'def', 'foo': 200}], ['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}], ['table_3', {'id': "chicken-11", 'bar': 'ghi', 'foo': 300}]]) self.assertEqual(state['currently_syncing'], 'tap_mysql_test-table_3') table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1'] table_3_bookmark = state['bookmarks']['tap_mysql_test-table_3'] self.assertEqual(table_1_bookmark, {'initial_full_table_complete': True}) self.assertIsNone(table_3_bookmark.get('initial_full_table_complete')) table_3_version = table_3_bookmark['version'] self.assertIsNotNone(table_3_version) self.assertEqual(table_3_bookmark['max_pk_values'], {'id': "turkey"}) self.assertEqual(table_3_bookmark['last_pk_fetched'], {'id': "chicken-11"}) self.assertIsNotNone(table_3_bookmark.get('log_file')) self.assertIsNotNone(table_3_bookmark.get('log_pos')) failed_syncing_table_3 = False singer.write_message = singer_write_message_ok table_3_RECORD_COUNT = 0 SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, config, self.catalog, state) self.assertFalse(failed_syncing_table_3) record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual(record_messages_2, [['table_3', {'id': "chicken-2", 'bar': 'def', 'foo': 200}], ['table_3', {'id': "turkey", 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 2, 'bar': 'def', 'foo': 200}], ['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}]]) self.assertIsNone(state['currently_syncing']) table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1'] table_3_bookmark = state['bookmarks']['tap_mysql_test-table_3'] self.assertEqual(table_1_bookmark, {'initial_full_table_complete': True}) self.assertIsNone(table_3_bookmark.get('initial_full_table_complete')) table_3_version = table_3_bookmark['version'] self.assertIsNotNone(table_3_version) self.assertIsNone(table_3_bookmark.get('max_pk_values')) self.assertIsNone(table_3_bookmark.get('last_pk_fetched')) self.assertIsNotNone(table_3_bookmark.get('log_file')) self.assertIsNotNone(table_3_bookmark.get('log_pos')) new_table_3_records = [[ "quail-2", 400, 'jkl' ], [ "quail-100", 500, 'mno' ]] for record in new_table_3_records: insert_record_with_specific_id(self.conn, 'table_3', record) TABLE_3_RECORD_COUNT = 0 SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, config, self.catalog, state) self.assertFalse(failed_syncing_table_3) record_messages_3 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] # Now that we are performing binlog sync, record output should be based on insert order self.assertEqual(record_messages_3, [['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 2, 'bar': 'def', 'foo': 200}], ['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}], ['table_3', {'id': "quail-2" , 'bar': 'jkl', 'foo': 400}], ['table_3', {'id': "quail-100", 'bar': 'mno', 'foo': 500}]]) self.assertIsNone(state['currently_syncing']) table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1'] table_3_bookmark = state['bookmarks']['tap_mysql_test-table_3'] self.assertEqual(table_1_bookmark, {'initial_full_table_complete': True}) self.assertIsNone(table_3_bookmark.get('initial_full_table_complete')) self.assertIsNotNone(table_3_bookmark.get('log_file')) self.assertIsNotNone(table_3_bookmark.get('log_pos'))
def test_table_2_interrupted(self): singer.write_message = singer_write_message_no_table_2 state = {} failed_syncing_table_2 = False # Do not worry about table_3 for stream in filter(lambda s: s.stream == 'table_3', self.catalog.streams): md_map = singer.metadata.to_map(stream.metadata) md_map = singer.metadata.write(md_map, (), 'selected', False) stream.metadata = singer.metadata.to_list(md_map) try: tap_mysql.do_sync(self.conn, {}, self.catalog, state) except Exception as ex: if str(ex) == 'simulated exception': failed_syncing_table_2 = True self.assertTrue(failed_syncing_table_2) record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual(record_messages_1, [['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 2, 'bar': 'def', 'foo': 200}], ['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}], ['table_2', {'id': 1, 'bar': 'ghi', 'foo': 300}] ]) expected_state_1 = { 'currently_syncing': 'tap_mysql_test-table_2', 'bookmarks': { 'tap_mysql_test-table_2': { 'last_pk_fetched': {'id': 1}, 'max_pk_values': {'id': 3} }, 'tap_mysql_test-table_1': { 'initial_full_table_complete': True } } } failed_syncing_table_2 = False singer.write_message = singer_write_message_ok TABLE_2_RECORD_COUNT = 0 SINGER_MESSAGES.clear() tap_mysql.do_sync(self.conn, {}, self.catalog, state) self.assertFalse(failed_syncing_table_2) record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)] self.assertEqual(record_messages_2, [['table_2', {'id': 2, 'bar': 'def', 'foo': 200}], ['table_2', {'id': 3, 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}], ['table_1', {'id': 2, 'bar': 'def', 'foo': 200}], ['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}]]) expected_state_2 = { 'currently_syncing': None, 'bookmarks': { 'tap_mysql_test-table_1': { 'initial_full_table_complete': True }, 'tap_mysql_test-table_2': { 'initial_full_table_complete': True }, 'tap_mysql_test-table_4': { 'initial_full_table_complete': True } } } self.assertEqual(state, expected_state_2)