def test_change_replication_key(self):
state = {
'bookmarks': {
'tap_mysql_test-incremental': {
'version': 1,
'replication_key_value': '2017-06-20',
'replication_key': 'updated'
}
}
}
stream = [x for x in self.catalog.streams if x.stream == 'incremental'][0]
stream.metadata = [
{'breadcrumb': (), 'metadata': {'selected': True, 'database-name': 'tap_mysql_test'}},
{'breadcrumb': ('properties', 'val'), 'metadata': {'selected': True}},
{'breadcrumb': ('properties', 'updated'), 'metadata': {'selected': True}}
]
test_utils.set_replication_method_and_key(stream, 'INCREMENTAL', 'val')
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
self.assertEqual(state['bookmarks']['tap_mysql_test-incremental']['replication_key'], 'val')
self.assertEqual(state['bookmarks']['tap_mysql_test-incremental']['replication_key_value'], 3)
self.assertEqual(state['bookmarks']['tap_mysql_test-incremental']['version'], 1)
def test_with_state(self):
state = {
'bookmarks': {
'tap_mysql_test-incremental': {
'version': 1,
'replication_key_value': '2017-06-20',
'replication_key': 'updated'
},
'tap_mysql_test-integer_incremental': {
'version': 1,
'replication_key_value': 3,
'replication_key': 'updated'
}
}
}
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
(message_types, versions) = message_types_and_versions(SINGER_MESSAGES)
self.assertEqual([
'ActivateVersionMessage', 'RecordMessage', 'RecordMessage',
'ActivateVersionMessage', 'RecordMessage'
], message_types)
self.assertTrue(isinstance(versions[0], int))
self.assertEqual(versions[0], versions[1])
self.assertEqual(versions[1], 1)
def test_version_cleared_from_state_after_full_table_success(self):
common.get_stream_version = lambda a, b: 12345
state = {
'bookmarks': {
'tap_mysql_test-full_table': {
'version': 1,
'initial_full_table_complete': True
}
}
}
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
(message_types, versions) = message_types_and_versions(SINGER_MESSAGES)
self.assertEqual(['RecordMessage', 'ActivateVersionMessage'],
message_types)
self.assertEqual(versions, [12345, 12345])
self.assertFalse('version' in state['bookmarks']
['tap_mysql_test-full_table'].keys())
self.assertTrue(state['bookmarks']['tap_mysql_test-full_table']
['initial_full_table_complete'])
def runTest(self):
conn = test_utils.get_test_connection()
with connect_with_backoff(conn) as open_conn:
with open_conn.cursor() as cur:
cur.execute('''
CREATE TABLE tab (
id INTEGER PRIMARY KEY,
a INTEGER,
b INTEGER)
''')
catalog = test_utils.discover_catalog(conn, {})
catalog.streams[0].stream = 'tab'
catalog.streams[0].metadata = [
{'breadcrumb': (), 'metadata': {'selected': True, 'database-name': 'tap_mysql_test'}},
{'breadcrumb': ('properties', 'a'), 'metadata': {'selected': True}}
]
test_utils.set_replication_method_and_key(catalog.streams[0], 'FULL_TABLE', None)
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
tap_mysql.do_sync(conn, {}, catalog, {})
schema_message = list(filter(lambda m: isinstance(m, singer.SchemaMessage), SINGER_MESSAGES))[0]
self.assertTrue(isinstance(schema_message, singer.SchemaMessage))
# tap-mysql selects new fields by default. If a field doesn't appear in the schema, then it should be
# selected
expectedKeys = ['id', 'a', 'b']
self.assertEqual(schema_message.schema['properties'].keys(), set(expectedKeys))
def test_with_state(self):
state = {
"bookmarks": {
"tap_mysql_test-incremental": {
"version": 1,
"replication_key_value": "2017-06-20",
"replication_key": "updated",
},
"tap_mysql_test-integer_incremental": {
"version": 1,
"replication_key_value": 3,
"replication_key": "updated",
},
}
}
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
(message_types, versions) = message_types_and_versions(SINGER_MESSAGES)
self.assertEqual(
[
"ActivateVersionMessage",
"RecordMessage",
"RecordMessage",
"ActivateVersionMessage",
"RecordMessage",
],
message_types,
)
self.assertTrue(isinstance(versions[0], int))
self.assertEqual(versions[0], versions[1])
self.assertEqual(versions[1], 1)
def test_with_no_state(self):
state = {}
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
(message_types, versions) = message_types_and_versions(SINGER_MESSAGES)
self.assertEqual(
[
"ActivateVersionMessage",
"RecordMessage",
"RecordMessage",
"RecordMessage",
"ActivateVersionMessage",
"RecordMessage",
"RecordMessage",
"RecordMessage",
],
message_types,
)
self.assertTrue(isinstance(versions[0], int))
self.assertEqual(versions[0], versions[1])
def test_version_cleared_from_state_after_full_table_success(self):
common.get_stream_version = lambda a, b: 12345
state = {
"bookmarks": {
"tap_mysql_test-full_table": {
"version": 1,
"initial_full_table_complete": True
}
}
}
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
(message_types, versions) = message_types_and_versions(SINGER_MESSAGES)
self.assertEqual(["RecordMessage", "ActivateVersionMessage"],
message_types)
self.assertEqual(versions, [12345, 12345])
self.assertFalse("version" in state["bookmarks"]
["tap_mysql_test-full_table"].keys())
self.assertTrue(state["bookmarks"]["tap_mysql_test-full_table"]
["initial_full_table_complete"])
def runTest(self):
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, {}, self.catalog, {})
record_message = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES))[0]
self.assertTrue(isinstance(record_message, singer.RecordMessage))
self.assertEqual(record_message.record, {'val': '{"a": 10, "b": "c"}'})
def test_emit_currently_syncing(self):
state = {}
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
self.assertRegex(currently_syncing_seq(SINGER_MESSAGES), '^a+b+c+_+')
def test_sync_messages_are_correct(self):
self.catalog.streams[0] = test_utils.set_replication_method_and_key(self.catalog.streams[0], 'LOG_BASED', None)
self.catalog.streams[0] = test_utils.set_selected(self.catalog.streams[0], True)
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
#inital sync
tap_mysql.do_sync(self.conn, {}, self.catalog, {})
# get schema message to test that it has all the table's columns
schema_message = next(filter(lambda m: isinstance(m, singer.SchemaMessage), SINGER_MESSAGES))
expectedKeys = ['good_pk', 'age']
self.assertEqual(schema_message.schema['properties'].keys(), set(expectedKeys))
# get the records, these are generated by Full table replication
record_messages = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES))
self.assertEqual(len(record_messages), 4)
self.assertListEqual([
{'age': 20, 'good_pk': '61'},
{'age': 30, 'good_pk': '62'},
{'age': 30, 'good_pk': '63'},
{'age': 40, 'good_pk': '64'},
], [rec.record for rec in record_messages])
# get the last state message to be fed to the next sync
state_message = list(filter(lambda m: isinstance(m, singer.StateMessage), SINGER_MESSAGES))[-1]
SINGER_MESSAGES.clear()
# run some queries
with connect_with_backoff(self.conn) as open_conn:
with open_conn.cursor() as cursor:
cursor.execute("UPDATE good_pk_tab set age=age+5")
cursor.execute("INSERT INTO good_pk_tab (good_pk, age) VALUES "
"(BINARY('e'), 16), "
"(BINARY('f'), 5)")
# do a sync and give the state so that binlog replication start from the last synced position
tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state_message.value)
# get the changed/new records
record_messages = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES))
self.assertEqual(len(record_messages), 6)
self.assertListEqual([
{'age': 25, 'good_pk': '61'},
{'age': 35, 'good_pk': '62'},
{'age': 35, 'good_pk': '63'},
{'age': 45, 'good_pk': '64'},
{'age': 16, 'good_pk': '65'},
{'age': 5, 'good_pk': '66'},
], [rec.record for rec in record_messages])
def test_initial_full_table(self):
state = {}
expected_log_file, expected_log_pos = binlog.fetch_current_log_file_and_pos(
self.conn)
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
message_types = [type(m) for m in SINGER_MESSAGES]
self.assertEqual(message_types, [
singer.StateMessage, singer.SchemaMessage,
singer.ActivateVersionMessage, singer.RecordMessage,
singer.RecordMessage, singer.StateMessage,
singer.ActivateVersionMessage, singer.StateMessage,
singer.SchemaMessage, singer.ActivateVersionMessage,
singer.RecordMessage, singer.RecordMessage, singer.StateMessage,
singer.ActivateVersionMessage, singer.StateMessage
])
activate_version_message_1 = list(
filter(
lambda m: isinstance(m, singer.ActivateVersionMessage) and m.
stream == 'binlog_1', SINGER_MESSAGES))[0]
activate_version_message_2 = list(
filter(
lambda m: isinstance(m, singer.ActivateVersionMessage) and m.
stream == 'binlog_2', SINGER_MESSAGES))[0]
record_messages = list(
filter(lambda m: isinstance(m, singer.RecordMessage),
SINGER_MESSAGES))
self.assertIsNotNone(
singer.get_bookmark(self.state, 'tap_mysql_test-binlog_1',
'log_file'))
self.assertIsNotNone(
singer.get_bookmark(self.state, 'tap_mysql_test-binlog_1',
'log_pos'))
self.assertIsNotNone(
singer.get_bookmark(self.state, 'tap_mysql_test-binlog_2',
'log_file'))
self.assertIsNotNone(
singer.get_bookmark(self.state, 'tap_mysql_test-binlog_2',
'log_pos'))
self.assertEqual(
singer.get_bookmark(state, 'tap_mysql_test-binlog_1', 'version'),
activate_version_message_1.version)
self.assertEqual(
singer.get_bookmark(state, 'tap_mysql_test-binlog_2', 'version'),
activate_version_message_2.version)
def test_table_2_interrupted(self):
singer.write_message = singer_write_message_no_table_2
state = {}
failed_syncing_table_2 = False
try:
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
except Exception as ex:
if str(ex) == 'simulated exception':
failed_syncing_table_2 = True
self.assertTrue(failed_syncing_table_2)
record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(record_messages_1,
[['tap_mysql_test-table_1', {'id': 1, 'bar': 'abc', 'foo': 100}],
['tap_mysql_test-table_1', {'id': 2, 'bar': 'def', 'foo': 200}],
['tap_mysql_test-table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}],
['tap_mysql_test-table_2', {'id': 1, 'bar': 'ghi', 'foo': 300}]
])
failed_syncing_table_2 = False
singer.write_message = singer_write_message_ok
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
self.assertFalse(failed_syncing_table_2)
record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)]
self.assertEqual(record_messages_2,
[['tap_mysql_test-table_2', {'id': 2, 'bar': 'def', 'foo': 200}],
['tap_mysql_test-table_2', {'id': 3, 'bar': 'abc', 'foo': 100}],
['tap_mysql_test-table_1', {'id': 1, 'bar': 'abc', 'foo': 100}],
['tap_mysql_test-table_1', {'id': 2, 'bar': 'def', 'foo': 200}],
['tap_mysql_test-table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}]])
expected_state_2 = {
'currently_syncing': None,
'bookmarks': {
'tap_mysql_test-table_1': {
'initial_full_table_complete': True
},
'tap_mysql_test-table_2': {
'initial_full_table_complete': True
}
}
}
self.assertEqual(state, expected_state_2)
def test_version_not_cleared_from_state_after_incremental_success(self):
state = {
'bookmarks': {
'tap_mysql_test-incremental': {
'version': 1,
'replication_key_value': '2017-06-20',
'replication_key': 'updated'
}
}
}
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
self.assertEqual(state['bookmarks']['tap_mysql_test-incremental']['version'], 1)
def test_binlog_stream(self):
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
config = test_utils.get_db_config()
config['server_id'] = "100"
tap_mysql.do_sync(self.conn, config, self.catalog, self.state)
record_messages = list(
filter(lambda m: isinstance(m, singer.RecordMessage),
SINGER_MESSAGES))
message_types = [type(m) for m in SINGER_MESSAGES]
self.assertEqual(message_types, [
singer.StateMessage, singer.SchemaMessage, singer.SchemaMessage,
singer.RecordMessage, singer.RecordMessage, singer.RecordMessage,
singer.RecordMessage, singer.RecordMessage, singer.RecordMessage,
singer.RecordMessage, singer.RecordMessage, singer.RecordMessage,
singer.RecordMessage, singer.StateMessage
])
self.assertEqual([
('tap_mysql_test-binlog_1', 1, '2017-06-01T00:00:00+00:00', False),
('tap_mysql_test-binlog_1', 2, '2017-06-20T00:00:00+00:00', False),
('tap_mysql_test-binlog_1', 3, '2017-09-22T00:00:00+00:00', False),
('tap_mysql_test-binlog_2', 1, '2017-10-22T00:00:00+00:00', False),
('tap_mysql_test-binlog_2', 2, '2017-11-10T00:00:00+00:00', False),
('tap_mysql_test-binlog_2', 3, '2017-12-10T00:00:00+00:00', False),
('tap_mysql_test-binlog_1', 3, '2018-06-18T00:00:00+00:00', False),
('tap_mysql_test-binlog_2', 2, '2018-06-18T00:00:00+00:00', False),
('tap_mysql_test-binlog_1', 2, '2017-06-20T00:00:00+00:00', True),
('tap_mysql_test-binlog_2', 1, '2017-10-22T00:00:00+00:00', True)
], [(m.stream, m.record['id'], m.record['updated'],
m.record.get(binlog.SDC_DELETED_AT) is not None)
for m in record_messages])
self.assertIsNotNone(
singer.get_bookmark(self.state, 'tap_mysql_test-binlog_1',
'log_file'))
self.assertIsNotNone(
singer.get_bookmark(self.state, 'tap_mysql_test-binlog_1',
'log_pos'))
self.assertIsNotNone(
singer.get_bookmark(self.state, 'tap_mysql_test-binlog_2',
'log_file'))
self.assertIsNotNone(
singer.get_bookmark(self.state, 'tap_mysql_test-binlog_2',
'log_pos'))
def test_change_replication_key(self):
state = {
"bookmarks": {
"tap_mysql_test-incremental": {
"version": 1,
"replication_key_value": "2017-06-20",
"replication_key": "updated",
}
}
}
stream = [
x for x in self.catalog.streams if x.stream == "incremental"
][0]
stream.metadata = [
{
"breadcrumb": (),
"metadata": {
"selected": True,
"database-name": "tap_mysql_test"
}
},
{
"breadcrumb": ("properties", "val"),
"metadata": {
"selected": True
}
},
{
"breadcrumb": ("properties", "updated"),
"metadata": {
"selected": True
}
},
]
test_utils.set_replication_method_and_key(stream, "INCREMENTAL", "val")
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
self.assertEqual(
state["bookmarks"]["tap_mysql_test-incremental"]
["replication_key"], "val")
self.assertEqual(
state["bookmarks"]["tap_mysql_test-incremental"]
["replication_key_value"], 3)
self.assertEqual(
state["bookmarks"]["tap_mysql_test-incremental"]["version"], 1)
def test_version_not_cleared_from_state_after_incremental_success(self):
state = {
"bookmarks": {
"tap_mysql_test-incremental": {
"version": 1,
"replication_key_value": "2017-06-20",
"replication_key": "updated",
}
}
}
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
self.assertEqual(
state["bookmarks"]["tap_mysql_test-incremental"]["version"], 1)
def test_fail_on_view(self):
for stream in self.catalog.streams:
md = singer.metadata.to_map(stream.metadata)
singer.metadata.write(md, (), 'is-view', True)
state = {}
failed = False
exception_message = None
expected_exception_message = "Unable to replicate stream(tap_mysql_test-{}) with binlog because it is a view.".format(
self.catalog.streams[0].stream)
with self.assertRaises(Exception) as context:
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
self.assertEqual(expected_exception_message, str(context.exception))
def runTest(self):
conn = test_utils.get_test_connection()
with connect_with_backoff(conn) as open_conn:
with open_conn.cursor() as cur:
cur.execute("""
CREATE TABLE tab (
id INTEGER PRIMARY KEY,
a INTEGER,
b INTEGER)
""")
catalog = test_utils.discover_catalog(conn, {})
catalog.streams[0].stream = "tab"
catalog.streams[0].metadata = [
{
"breadcrumb": (),
"metadata": {
"selected": True,
"database-name": "tap_mysql_test"
}
},
{
"breadcrumb": ("properties", "a"),
"metadata": {
"selected": True
}
},
]
test_utils.set_replication_method_and_key(catalog.streams[0],
"FULL_TABLE", None)
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
tap_mysql.do_sync(conn, {}, catalog, {})
schema_message = list(
filter(lambda m: isinstance(m, singer.SchemaMessage),
SINGER_MESSAGES))[0]
self.assertTrue(isinstance(schema_message, singer.SchemaMessage))
# tap-mysql selects new fields by default. If a field doesn't appear in the schema, then it should be
# selected
expectedKeys = ["id", "a", "b"]
self.assertEqual(schema_message.schema["properties"].keys(),
set(expectedKeys))
def test_start_at_currently_syncing(self):
state = {
'currently_syncing': 'tap_mysql_test-b',
'bookmarks': {
'tap_mysql_test-a': {
'version': 123
},
'tap_mysql_test-b': {
'version': 456
}
}
}
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
self.assertRegex(currently_syncing_seq(SINGER_MESSAGES), '^b+c+a+_+')
def test_start_at_currently_syncing(self):
state = {
"currently_syncing": "tap_mysql_test-b",
"bookmarks": {
"tap_mysql_test-a": {
"version": 123
},
"tap_mysql_test-b": {
"version": 456
},
},
}
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
self.assertRegexpMatches(currently_syncing_seq(SINGER_MESSAGES),
"^b+c+a+_+")
def test_with_initial_full_table_complete_in_state(self):
common.get_stream_version = lambda a, b: 12345
state = {
'bookmarks': {
'tap_mysql_test-full_table': {
'initial_full_table_complete': True
}
}
}
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
(message_types, versions) = message_types_and_versions(SINGER_MESSAGES)
self.assertEqual(['RecordMessage', 'ActivateVersionMessage'], message_types)
self.assertEqual(versions, [12345, 12345])
def test_fail_if_log_file_does_not_exist(self):
log_file = 'chicken'
stream = self.catalog.streams[0]
state = {
'bookmarks': {
stream.tap_stream_id: {
'version': singer.utils.now(),
'log_file': log_file,
'log_pos': 1
}
}
}
expected_exception_message = "Unable to replicate stream({}) with binlog because log file {} does not exist.".format(
stream, log_file)
with self.assertRaises(Exception) as context:
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
self.assertEqual(expected_exception_message,
str(context.exception))
def test_fail_on_view(self):
for stream in self.catalog.streams:
md = singer.metadata.to_map(stream.metadata)
singer.metadata.write(md, (), 'is-view', True)
state = {}
failed = False
exception_message = None
expected_exception_message = "Unable to replicate stream({}) with binlog because it is a view.".format(
self.catalog.streams[0].stream)
try:
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
except Exception as e:
failed = True
exception_message = str(e)
LOGGER.error(exception_message)
self.assertTrue(failed)
self.assertEqual(expected_exception_message, exception_message)
def test_initial_full_table(self):
state = {}
expected_log_file, expected_log_pos = binlog.fetch_current_log_file_and_pos(
self.conn)
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
message_types = [type(m) for m in SINGER_MESSAGES]
self.assertEqual(message_types, [
singer.StateMessage, singer.SchemaMessage,
singer.ActivateVersionMessage, singer.RecordMessage,
singer.RecordMessage, singer.StateMessage,
singer.ActivateVersionMessage, singer.StateMessage
])
record_messages = list(
filter(lambda m: isinstance(m, singer.RecordMessage),
SINGER_MESSAGES))
# Expected from 0.7.11
expected_records = [{
'datetime_col': None,
'id': 1,
'timestamp_col': None,
'time_col': '1970-01-01T00:00:00.000000Z',
'date_col': None
}, {
'datetime_col': None,
'id': 2,
'timestamp_col': None,
'time_col': None,
'date_col': None
}]
self.assertEqual(expected_records,
[x.asdict()['record'] for x in record_messages])
def test_fail_if_log_file_does_not_exist(self):
log_file = 'chicken'
stream = self.catalog.streams[0]
state = {
'bookmarks': {
stream.tap_stream_id: {
'version': singer.utils.now(),
'log_file': log_file,
'log_pos': 1
}
}
}
failed = False
exception_message = None
expected_exception_message = "Unable to replicate stream({}) with binlog because log file {} does not exist.".format(
stream, log_file)
try:
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
except Exception as e:
failed = True
exception_message = str(e)
LOGGER.error(exception_message)
def test_table_2_interrupted(self):
singer.write_message = singer_write_message_no_table_2
state = {}
failed_syncing_table_2 = False
try:
tap_mysql.do_sync(self.conn, test_utils.get_db_config(),
self.catalog, state)
except Exception as ex:
if str(ex) == "simulated exception":
failed_syncing_table_2 = True
self.assertTrue(failed_syncing_table_2)
record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(
record_messages_1,
[
["table_1", {
"id": 1,
"bar": "abc",
"foo": 100
}],
["table_1", {
"id": 2,
"bar": "def",
"foo": 200
}],
["table_1", {
"id": 3,
"bar": "ghi",
"foo": 300
}],
["table_2", {
"id": 1,
"bar": "ghi",
"foo": 300
}],
],
)
self.assertEqual(state["currently_syncing"], "tap_mysql_test-table_2")
table_1_bookmark = state["bookmarks"]["tap_mysql_test-table_1"]
table_2_bookmark = state["bookmarks"]["tap_mysql_test-table_2"]
self.assertEqual(table_1_bookmark,
{"initial_full_table_complete": True})
self.assertIsNone(table_2_bookmark.get("initial_full_table_complete"))
table_2_version = table_2_bookmark["version"]
self.assertIsNotNone(table_2_version)
self.assertEqual(table_2_bookmark["max_pk_values"], {"id": 3})
self.assertEqual(table_2_bookmark["last_pk_fetched"], {"id": 1})
self.assertIsNotNone(table_2_bookmark.get("log_file"))
self.assertIsNotNone(table_2_bookmark.get("log_pos"))
failed_syncing_table_2 = False
singer.write_message = singer_write_message_ok
table_2_RECORD_COUNT = 0
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog,
state)
self.assertFalse(failed_syncing_table_2)
record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(
record_messages_2,
[
["table_2", {
"id": 2,
"bar": "def",
"foo": 200
}],
["table_2", {
"id": 3,
"bar": "abc",
"foo": 100
}],
["table_1", {
"id": 1,
"bar": "abc",
"foo": 100
}],
["table_1", {
"id": 2,
"bar": "def",
"foo": 200
}],
["table_1", {
"id": 3,
"bar": "ghi",
"foo": 300
}],
],
)
self.assertIsNone(state["currently_syncing"])
table_1_bookmark = state["bookmarks"]["tap_mysql_test-table_1"]
table_2_bookmark = state["bookmarks"]["tap_mysql_test-table_2"]
self.assertEqual(table_1_bookmark,
{"initial_full_table_complete": True})
self.assertIsNone(table_2_bookmark.get("initial_full_table_complete"))
table_2_version = table_2_bookmark["version"]
self.assertIsNotNone(table_2_version)
self.assertIsNone(table_2_bookmark.get("max_pk_values"))
self.assertIsNone(table_2_bookmark.get("last_pk_fetched"))
self.assertIsNotNone(table_2_bookmark.get("log_file"))
self.assertIsNotNone(table_2_bookmark.get("log_pos"))
new_table_2_records = [[400, "jkl"], [500, "mno"]]
for record in new_table_2_records:
insert_record(self.conn, "table_2", record)
TABLE_2_RECORD_COUNT = 0
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog,
state)
self.assertFalse(failed_syncing_table_2)
record_messages_3 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(
record_messages_3,
[
["table_1", {
"id": 1,
"bar": "abc",
"foo": 100
}],
["table_1", {
"id": 2,
"bar": "def",
"foo": 200
}],
["table_1", {
"id": 3,
"bar": "ghi",
"foo": 300
}],
["table_2", {
"id": 4,
"bar": "jkl",
"foo": 400
}],
["table_2", {
"id": 5,
"bar": "mno",
"foo": 500
}],
],
)
self.assertIsNone(state["currently_syncing"])
table_1_bookmark = state["bookmarks"]["tap_mysql_test-table_1"]
table_2_bookmark = state["bookmarks"]["tap_mysql_test-table_2"]
self.assertEqual(table_1_bookmark,
{"initial_full_table_complete": True})
self.assertIsNone(table_2_bookmark.get("initial_full_table_complete"))
self.assertIsNotNone(table_2_bookmark.get("log_file"))
self.assertIsNotNone(table_2_bookmark.get("log_pos"))
def test_table_2_interrupted(self):
singer.write_message = singer_write_message_no_table_2
state = {}
failed_syncing_table_2 = False
try:
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
except Exception as ex:
if str(ex) == "simulated exception":
failed_syncing_table_2 = True
self.assertTrue(failed_syncing_table_2)
record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(
record_messages_1,
[
["table_1", {
"id": 1,
"bar": "abc",
"foo": 100
}],
["table_1", {
"id": 2,
"bar": "def",
"foo": 200
}],
["table_1", {
"id": 3,
"bar": "ghi",
"foo": 300
}],
["table_2", {
"id": 1,
"bar": "ghi",
"foo": 300
}],
],
)
expected_state_1 = {
"currently_syncing": "tap_mysql_test-table_2",
"bookmarks": {
"tap_mysql_test-table_2": {
"last_pk_fetched": {
"id": 1
},
"max_pk_values": {
"id": 3
},
},
"tap_mysql_test-table_1": {
"initial_full_table_complete": True
},
},
}
failed_syncing_table_2 = False
singer.write_message = singer_write_message_ok
TABLE_2_RECORD_COUNT = 0
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
self.assertFalse(failed_syncing_table_2)
record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(
record_messages_2,
[
["table_2", {
"id": 2,
"bar": "def",
"foo": 200
}],
["table_2", {
"id": 3,
"bar": "abc",
"foo": 100
}],
["table_1", {
"id": 1,
"bar": "abc",
"foo": 100
}],
["table_1", {
"id": 2,
"bar": "def",
"foo": 200
}],
["table_1", {
"id": 3,
"bar": "ghi",
"foo": 300
}],
],
)
expected_state_2 = {
"currently_syncing": None,
"bookmarks": {
"tap_mysql_test-table_1": {
"initial_full_table_complete": True
},
"tap_mysql_test-table_2": {
"initial_full_table_complete": True
},
},
}
self.assertEqual(state, expected_state_2)
def test_table_2_interrupted(self):
singer.write_message = singer_write_message_no_table_2
state = {}
failed_syncing_table_2 = False
# Do not worry about table_3
for stream in filter(lambda s: s.stream == 'table_3', self.catalog.streams):
md_map = singer.metadata.to_map(stream.metadata)
md_map = singer.metadata.write(md_map,
(),
'selected',
False)
stream.metadata = singer.metadata.to_list(md_map)
try:
tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state)
except Exception as ex:
if str(ex) == 'simulated exception':
failed_syncing_table_2 = True
self.assertTrue(failed_syncing_table_2)
record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(record_messages_1,
[['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 2, 'bar': 'def', 'foo': 200}],
['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}],
['table_2', {'id': 1, 'bar': 'ghi', 'foo': 300}]])
self.assertEqual(state['currently_syncing'], 'tap_mysql_test-table_2')
table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1']
table_2_bookmark = state['bookmarks']['tap_mysql_test-table_2']
self.assertEqual(table_1_bookmark,
{'initial_full_table_complete': True})
self.assertIsNone(table_2_bookmark.get('initial_full_table_complete'))
table_2_version = table_2_bookmark['version']
self.assertIsNotNone(table_2_version)
self.assertEqual(table_2_bookmark['max_pk_values'], {'id': 3})
self.assertEqual(table_2_bookmark['last_pk_fetched'], {'id': 1})
self.assertIsNotNone(table_2_bookmark.get('log_file'))
self.assertIsNotNone(table_2_bookmark.get('log_pos'))
failed_syncing_table_2 = False
singer.write_message = singer_write_message_ok
table_2_RECORD_COUNT = 0
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state)
self.assertFalse(failed_syncing_table_2)
record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(record_messages_2,
[['table_2', {'id': 2, 'bar': 'def', 'foo': 200}],
['table_2', {'id': 3, 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 2, 'bar': 'def', 'foo': 200}],
['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}]])
self.assertIsNone(state['currently_syncing'])
table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1']
table_2_bookmark = state['bookmarks']['tap_mysql_test-table_2']
self.assertEqual(table_1_bookmark,
{'initial_full_table_complete': True})
self.assertIsNone(table_2_bookmark.get('initial_full_table_complete'))
table_2_version = table_2_bookmark['version']
self.assertIsNotNone(table_2_version)
self.assertIsNone(table_2_bookmark.get('max_pk_values'))
self.assertIsNone(table_2_bookmark.get('last_pk_fetched'))
self.assertIsNotNone(table_2_bookmark.get('log_file'))
self.assertIsNotNone(table_2_bookmark.get('log_pos'))
new_table_2_records = [[ 400, 'jkl' ],
[ 500, 'mno' ]]
for record in new_table_2_records:
insert_record(self.conn, 'table_2', record)
TABLE_2_RECORD_COUNT = 0
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state)
self.assertFalse(failed_syncing_table_2)
record_messages_3 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(record_messages_3,
[['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 2, 'bar': 'def', 'foo': 200}],
['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}],
['table_2', {'id': 4, 'bar': 'jkl', 'foo': 400}],
['table_2', {'id': 5, 'bar': 'mno', 'foo': 500}]])
self.assertIsNone(state['currently_syncing'])
table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1']
table_2_bookmark = state['bookmarks']['tap_mysql_test-table_2']
self.assertEqual(table_1_bookmark,
{'initial_full_table_complete': True})
self.assertIsNone(table_2_bookmark.get('initial_full_table_complete'))
self.assertIsNotNone(table_2_bookmark.get('log_file'))
self.assertIsNotNone(table_2_bookmark.get('log_pos'))
def test_table_3_interrupted(self):
singer.write_message = singer_write_message_no_table_3
state = {}
failed_syncing_table_3 = False
# Do not worry about table_2
for stream in filter(lambda s: s.stream == 'table_2', self.catalog.streams):
md_map = singer.metadata.to_map(stream.metadata)
md_map = singer.metadata.write(md_map,
(),
'selected',
False)
stream.metadata = singer.metadata.to_list(md_map)
config = test_utils.get_db_config()
config['allow_non_auto_increment_pks'] = 'true'
try:
tap_mysql.do_sync(self.conn, config, self.catalog, state)
except Exception as ex:
if str(ex) == 'simulated exception':
failed_syncing_table_3 = True
self.assertTrue(failed_syncing_table_3)
record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
# sorting of the two chicken-* columns is based on lexicographic ordering which in
# this case chicken-11 comes before chicken-2
self.assertEqual(record_messages_1,
[['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 2, 'bar': 'def', 'foo': 200}],
['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}],
['table_3', {'id': "chicken-11", 'bar': 'ghi', 'foo': 300}]])
self.assertEqual(state['currently_syncing'], 'tap_mysql_test-table_3')
table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1']
table_3_bookmark = state['bookmarks']['tap_mysql_test-table_3']
self.assertEqual(table_1_bookmark,
{'initial_full_table_complete': True})
self.assertIsNone(table_3_bookmark.get('initial_full_table_complete'))
table_3_version = table_3_bookmark['version']
self.assertIsNotNone(table_3_version)
self.assertEqual(table_3_bookmark['max_pk_values'], {'id': "turkey"})
self.assertEqual(table_3_bookmark['last_pk_fetched'], {'id': "chicken-11"})
self.assertIsNotNone(table_3_bookmark.get('log_file'))
self.assertIsNotNone(table_3_bookmark.get('log_pos'))
failed_syncing_table_3 = False
singer.write_message = singer_write_message_ok
table_3_RECORD_COUNT = 0
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, config, self.catalog, state)
self.assertFalse(failed_syncing_table_3)
record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(record_messages_2,
[['table_3', {'id': "chicken-2", 'bar': 'def', 'foo': 200}],
['table_3', {'id': "turkey", 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 2, 'bar': 'def', 'foo': 200}],
['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}]])
self.assertIsNone(state['currently_syncing'])
table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1']
table_3_bookmark = state['bookmarks']['tap_mysql_test-table_3']
self.assertEqual(table_1_bookmark,
{'initial_full_table_complete': True})
self.assertIsNone(table_3_bookmark.get('initial_full_table_complete'))
table_3_version = table_3_bookmark['version']
self.assertIsNotNone(table_3_version)
self.assertIsNone(table_3_bookmark.get('max_pk_values'))
self.assertIsNone(table_3_bookmark.get('last_pk_fetched'))
self.assertIsNotNone(table_3_bookmark.get('log_file'))
self.assertIsNotNone(table_3_bookmark.get('log_pos'))
new_table_3_records = [[ "quail-2", 400, 'jkl' ],
[ "quail-100", 500, 'mno' ]]
for record in new_table_3_records:
insert_record_with_specific_id(self.conn, 'table_3', record)
TABLE_3_RECORD_COUNT = 0
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, config, self.catalog, state)
self.assertFalse(failed_syncing_table_3)
record_messages_3 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
# Now that we are performing binlog sync, record output should be based on insert order
self.assertEqual(record_messages_3,
[['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 2, 'bar': 'def', 'foo': 200}],
['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}],
['table_3', {'id': "quail-2" , 'bar': 'jkl', 'foo': 400}],
['table_3', {'id': "quail-100", 'bar': 'mno', 'foo': 500}]])
self.assertIsNone(state['currently_syncing'])
table_1_bookmark = state['bookmarks']['tap_mysql_test-table_1']
table_3_bookmark = state['bookmarks']['tap_mysql_test-table_3']
self.assertEqual(table_1_bookmark,
{'initial_full_table_complete': True})
self.assertIsNone(table_3_bookmark.get('initial_full_table_complete'))
self.assertIsNotNone(table_3_bookmark.get('log_file'))
self.assertIsNotNone(table_3_bookmark.get('log_pos'))
def test_table_2_interrupted(self):
singer.write_message = singer_write_message_no_table_2
state = {}
failed_syncing_table_2 = False
# Do not worry about table_3
for stream in filter(lambda s: s.stream == 'table_3', self.catalog.streams):
md_map = singer.metadata.to_map(stream.metadata)
md_map = singer.metadata.write(md_map,
(),
'selected',
False)
stream.metadata = singer.metadata.to_list(md_map)
try:
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
except Exception as ex:
if str(ex) == 'simulated exception':
failed_syncing_table_2 = True
self.assertTrue(failed_syncing_table_2)
record_messages_1 = [[m.stream, m.record] for m in SINGER_MESSAGES
if isinstance(m, singer.RecordMessage)]
self.assertEqual(record_messages_1,
[['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 2, 'bar': 'def', 'foo': 200}],
['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}],
['table_2', {'id': 1, 'bar': 'ghi', 'foo': 300}]
])
expected_state_1 = {
'currently_syncing': 'tap_mysql_test-table_2',
'bookmarks': {
'tap_mysql_test-table_2': {
'last_pk_fetched': {'id': 1},
'max_pk_values': {'id': 3}
},
'tap_mysql_test-table_1': {
'initial_full_table_complete': True
}
}
}
failed_syncing_table_2 = False
singer.write_message = singer_write_message_ok
TABLE_2_RECORD_COUNT = 0
SINGER_MESSAGES.clear()
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
self.assertFalse(failed_syncing_table_2)
record_messages_2 = [[m.stream, m.record] for m in SINGER_MESSAGES if isinstance(m, singer.RecordMessage)]
self.assertEqual(record_messages_2,
[['table_2', {'id': 2, 'bar': 'def', 'foo': 200}],
['table_2', {'id': 3, 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 1, 'bar': 'abc', 'foo': 100}],
['table_1', {'id': 2, 'bar': 'def', 'foo': 200}],
['table_1', {'id': 3, 'bar': 'ghi', 'foo': 300}]])
expected_state_2 = {
'currently_syncing': None,
'bookmarks': {
'tap_mysql_test-table_1': {
'initial_full_table_complete': True
},
'tap_mysql_test-table_2': {
'initial_full_table_complete': True
},
'tap_mysql_test-table_4': {
'initial_full_table_complete': True
}
}
}
self.assertEqual(state, expected_state_2)
