def setUp(self):
self.conn = test_utils.get_test_connection()
with connect_with_backoff(self.conn) as open_conn:
with open_conn.cursor() as cursor:
cursor.execute('CREATE TABLE full_table (val int)')
cursor.execute('INSERT INTO full_table (val) VALUES (1)')
self.catalog = test_utils.discover_catalog(self.conn, {})
for stream in self.catalog.streams:
stream.key_properties = []
stream.metadata = [{
'breadcrumb': (),
'metadata': {
'selected': True,
'database-name': 'tap_mysql_test'
}
}, {
'breadcrumb': ('properties', 'val'),
'metadata': {
'selected': True
}
}]
stream.stream = stream.table
test_utils.set_replication_method_and_key(stream, 'FULL_TABLE',
None)
def setUp(self):
self.conn = test_utils.get_test_connection()
with connect_with_backoff(self.conn) as open_conn:
with open_conn.cursor() as cursor:
cursor.execute('CREATE TABLE a (`b c` int)')
cursor.execute('INSERT INTO a (`b c`) VALUES (1)')
self.catalog = test_utils.discover_catalog(self.conn, {})
self.catalog.streams[0].stream = 'some_stream_name'
self.catalog.streams[0].metadata = [{
'breadcrumb': (),
'metadata': {
'selected': True,
'table-key-properties': [],
'database-name': 'tap_mysql_test'
}
}, {
'breadcrumb': ('properties', 'b c'),
'metadata': {
'selected': True
}
}]
test_utils.set_replication_method_and_key(self.catalog.streams[0],
'FULL_TABLE', None)
def test_change_replication_key(self):
state = {
'bookmarks': {
'tap_mysql_test-incremental': {
'version': 1,
'replication_key_value': '2017-06-20',
'replication_key': 'updated'
}
}
}
stream = [x for x in self.catalog.streams if x.stream == 'incremental'][0]
stream.metadata = [
{'breadcrumb': (), 'metadata': {'selected': True, 'database-name': 'tap_mysql_test'}},
{'breadcrumb': ('properties', 'val'), 'metadata': {'selected': True}},
{'breadcrumb': ('properties', 'updated'), 'metadata': {'selected': True}}
]
test_utils.set_replication_method_and_key(stream, 'INCREMENTAL', 'val')
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
self.assertEqual(state['bookmarks']['tap_mysql_test-incremental']['replication_key'], 'val')
self.assertEqual(state['bookmarks']['tap_mysql_test-incremental']['replication_key_value'], 3)
self.assertEqual(state['bookmarks']['tap_mysql_test-incremental']['version'], 1)
def setUp(self):
self.conn = test_utils.get_test_connection()
self.catalog = init_tables(self.conn)
for stream in self.catalog.streams:
stream.metadata = [
{'breadcrumb': (),
'metadata': {'selected': True,
'database-name': 'tap_mysql_test',
'table-key-properties': ['id']}},
{'breadcrumb': ('properties', 'id'), 'metadata': {'selected': True}},
{'breadcrumb': ('properties', 'foo'), 'metadata': {'selected': True}},
{'breadcrumb': ('properties', 'bar'), 'metadata': {'selected': True}},
]
stream.stream = stream.table
if stream.table == 'table_2' or stream.table == 'table_3':
test_utils.set_replication_method_and_key(stream, 'LOG_BASED', None)
else:
test_utils.set_replication_method_and_key(stream, 'FULL_TABLE', None)
global TABLE_2_RECORD_COUNT
TABLE_2_RECORD_COUNT = 0
global TABLE_3_RECORD_COUNT
TABLE_3_RECORD_COUNT = 0
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
def runTest(self):
conn = test_utils.get_test_connection()
with connect_with_backoff(conn) as open_conn:
with open_conn.cursor() as cur:
cur.execute('''
CREATE TABLE tab (
id INTEGER PRIMARY KEY,
a INTEGER,
b INTEGER)
''')
catalog = test_utils.discover_catalog(conn, {})
catalog.streams[0].stream = 'tab'
catalog.streams[0].metadata = [
{'breadcrumb': (), 'metadata': {'selected': True, 'database-name': 'tap_mysql_test'}},
{'breadcrumb': ('properties', 'a'), 'metadata': {'selected': True}}
]
test_utils.set_replication_method_and_key(catalog.streams[0], 'FULL_TABLE', None)
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
tap_mysql.do_sync(conn, {}, catalog, {})
schema_message = list(filter(lambda m: isinstance(m, singer.SchemaMessage), SINGER_MESSAGES))[0]
self.assertTrue(isinstance(schema_message, singer.SchemaMessage))
# tap-mysql selects new fields by default. If a field doesn't appear in the schema, then it should be
# selected
expectedKeys = ['id', 'a', 'b']
self.assertEqual(schema_message.schema['properties'].keys(), set(expectedKeys))
def setUp(self):
self.conn = test_utils.get_test_connection()
with connect_with_backoff(self.conn) as open_conn:
with open_conn.cursor() as cursor:
cursor.execute('CREATE TABLE incremental (val int, updated datetime)')
cursor.execute('INSERT INTO incremental (val, updated) VALUES (1, \'2017-06-01\')')
cursor.execute('INSERT INTO incremental (val, updated) VALUES (2, \'2017-06-20\')')
cursor.execute('INSERT INTO incremental (val, updated) VALUES (3, \'2017-09-22\')')
cursor.execute('CREATE TABLE integer_incremental (val int, updated int)')
cursor.execute('INSERT INTO integer_incremental (val, updated) VALUES (1, 1)')
cursor.execute('INSERT INTO integer_incremental (val, updated) VALUES (2, 2)')
cursor.execute('INSERT INTO integer_incremental (val, updated) VALUES (3, 3)')
self.catalog = test_utils.discover_catalog(self.conn, {})
for stream in self.catalog.streams:
stream.metadata = [
{'breadcrumb': (),
'metadata': {
'selected': True,
'table-key-properties': [],
'database-name': 'tap_mysql_test'
}},
{'breadcrumb': ('properties', 'val'), 'metadata': {'selected': True}}
]
stream.stream = stream.table
test_utils.set_replication_method_and_key(stream, 'INCREMENTAL', 'updated')
def setUp(self):
self.conn = test_utils.get_test_connection()
with connect_with_backoff(self.conn) as open_conn:
with open_conn.cursor() as cursor:
cursor.execute("CREATE TABLE full_table (val int)")
cursor.execute("INSERT INTO full_table (val) VALUES (1)")
self.catalog = test_utils.discover_catalog(self.conn, {})
for stream in self.catalog.streams:
stream.key_properties = []
stream.metadata = [
{
"breadcrumb": (),
"metadata": {
"selected": True,
"database-name": "tap_mysql_test"
},
},
{
"breadcrumb": ("properties", "val"),
"metadata": {
"selected": True
}
},
]
stream.stream = stream.table
test_utils.set_replication_method_and_key(stream, "FULL_TABLE",
None)
def setUp(self):
self.conn = test_utils.get_test_connection()
with connect_with_backoff(self.conn) as open_conn:
with open_conn.cursor() as cursor:
cursor.execute("CREATE TABLE a (`b c` int)")
cursor.execute("INSERT INTO a (`b c`) VALUES (1)")
self.catalog = test_utils.discover_catalog(self.conn, {})
self.catalog.streams[0].stream = "some_stream_name"
self.catalog.streams[0].metadata = [
{
"breadcrumb": (),
"metadata": {
"selected": True,
"table-key-properties": [],
"database-name": "tap_mysql_test",
},
},
{
"breadcrumb": ("properties", "b c"),
"metadata": {
"selected": True
}
},
]
test_utils.set_replication_method_and_key(self.catalog.streams[0],
"FULL_TABLE", None)
def setUp(self):
self.maxDiff = None
self.state = {}
self.conn = test_utils.get_test_connection()
log_file, log_pos = binlog.fetch_current_log_file_and_pos(self.conn)
with connect_with_backoff(self.conn) as open_conn:
with open_conn.cursor() as cursor:
cursor.execute('CREATE TABLE binlog_1 (id int, updated datetime)')
cursor.execute('CREATE TABLE binlog_2 (id int, updated datetime)')
cursor.execute('INSERT INTO binlog_1 (id, updated) VALUES (1, \'2017-06-01\')')
cursor.execute('INSERT INTO binlog_1 (id, updated) VALUES (2, \'2017-06-20\')')
cursor.execute('INSERT INTO binlog_1 (id, updated) VALUES (3, \'2017-09-22\')')
cursor.execute('INSERT INTO binlog_2 (id, updated) VALUES (1, \'2017-10-22\')')
cursor.execute('INSERT INTO binlog_2 (id, updated) VALUES (2, \'2017-11-10\')')
cursor.execute('INSERT INTO binlog_2 (id, updated) VALUES (3, \'2017-12-10\')')
cursor.execute('UPDATE binlog_1 set updated=\'2018-06-18\' WHERE id = 3')
cursor.execute('UPDATE binlog_2 set updated=\'2018-06-18\' WHERE id = 2')
cursor.execute('DELETE FROM binlog_1 WHERE id = 2')
cursor.execute('DELETE FROM binlog_2 WHERE id = 1')
open_conn.commit()
self.catalog = test_utils.discover_catalog(self.conn, {})
for stream in self.catalog.streams:
stream.stream = stream.table
stream.metadata = [
{'breadcrumb': (),
'metadata': {
'selected': True,
'database-name': 'tap_mysql_test',
'table-key-propertes': ['id']
}},
{'breadcrumb': ('properties', 'id'), 'metadata': {'selected': True}},
{'breadcrumb': ('properties', 'updated'), 'metadata': {'selected': True}}
]
test_utils.set_replication_method_and_key(stream, 'LOG_BASED', None)
self.state = singer.write_bookmark(self.state,
stream.tap_stream_id,
'log_file',
log_file)
self.state = singer.write_bookmark(self.state,
stream.tap_stream_id,
'log_pos',
log_pos)
self.state = singer.write_bookmark(self.state,
stream.tap_stream_id,
'version',
singer.utils.now())
def setUp(self):
self.conn = test_utils.get_test_connection()
with connect_with_backoff(self.conn) as open_conn:
with open_conn.cursor() as cursor:
cursor.execute(
"CREATE TABLE incremental (val int, updated datetime)")
cursor.execute(
"INSERT INTO incremental (val, updated) VALUES (1, '2017-06-01')"
)
cursor.execute(
"INSERT INTO incremental (val, updated) VALUES (2, '2017-06-20')"
)
cursor.execute(
"INSERT INTO incremental (val, updated) VALUES (3, '2017-09-22')"
)
cursor.execute(
"CREATE TABLE integer_incremental (val int, updated int)")
cursor.execute(
"INSERT INTO integer_incremental (val, updated) VALUES (1, 1)"
)
cursor.execute(
"INSERT INTO integer_incremental (val, updated) VALUES (2, 2)"
)
cursor.execute(
"INSERT INTO integer_incremental (val, updated) VALUES (3, 3)"
)
self.catalog = test_utils.discover_catalog(self.conn, {})
for stream in self.catalog.streams:
stream.metadata = [
{
"breadcrumb": (),
"metadata": {
"selected": True,
"table-key-properties": [],
"database-name": "tap_mysql_test",
},
},
{
"breadcrumb": ("properties", "val"),
"metadata": {
"selected": True
}
},
]
stream.stream = stream.table
test_utils.set_replication_method_and_key(stream, "INCREMENTAL",
"updated")
def test_change_replication_key(self):
state = {
"bookmarks": {
"tap_mysql_test-incremental": {
"version": 1,
"replication_key_value": "2017-06-20",
"replication_key": "updated",
}
}
}
stream = [
x for x in self.catalog.streams if x.stream == "incremental"
][0]
stream.metadata = [
{
"breadcrumb": (),
"metadata": {
"selected": True,
"database-name": "tap_mysql_test"
}
},
{
"breadcrumb": ("properties", "val"),
"metadata": {
"selected": True
}
},
{
"breadcrumb": ("properties", "updated"),
"metadata": {
"selected": True
}
},
]
test_utils.set_replication_method_and_key(stream, "INCREMENTAL", "val")
tap_mysql.do_sync(self.conn, {}, self.catalog, state)
self.assertEqual(
state["bookmarks"]["tap_mysql_test-incremental"]
["replication_key"], "val")
self.assertEqual(
state["bookmarks"]["tap_mysql_test-incremental"]
["replication_key_value"], 3)
self.assertEqual(
state["bookmarks"]["tap_mysql_test-incremental"]["version"], 1)
def setUp(self):
self.conn = test_utils.get_test_connection()
self.catalog = init_tables(self.conn)
for stream in self.catalog.streams:
stream.metadata = [
{
"breadcrumb": (),
"metadata": {
"selected": True,
"database-name": "tap_mysql_test",
"table-key-properties": ["id"],
},
},
{
"breadcrumb": ("properties", "id"),
"metadata": {
"selected": True
}
},
{
"breadcrumb": ("properties", "foo"),
"metadata": {
"selected": True
}
},
{
"breadcrumb": ("properties", "bar"),
"metadata": {
"selected": True
}
},
]
stream.stream = stream.table
if stream.table == "table_2":
test_utils.set_replication_method_and_key(
stream, "LOG_BASED", None)
else:
test_utils.set_replication_method_and_key(
stream, "FULL_TABLE", None)
global TABLE_2_RECORD_COUNT
TABLE_2_RECORD_COUNT = 0
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
def runTest(self):
conn = test_utils.get_test_connection()
with connect_with_backoff(conn) as open_conn:
with open_conn.cursor() as cur:
cur.execute("""
CREATE TABLE tab (
id INTEGER PRIMARY KEY,
a INTEGER,
b INTEGER)
""")
catalog = test_utils.discover_catalog(conn, {})
catalog.streams[0].stream = "tab"
catalog.streams[0].metadata = [
{
"breadcrumb": (),
"metadata": {
"selected": True,
"database-name": "tap_mysql_test"
}
},
{
"breadcrumb": ("properties", "a"),
"metadata": {
"selected": True
}
},
]
test_utils.set_replication_method_and_key(catalog.streams[0],
"FULL_TABLE", None)
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
tap_mysql.do_sync(conn, {}, catalog, {})
schema_message = list(
filter(lambda m: isinstance(m, singer.SchemaMessage),
SINGER_MESSAGES))[0]
self.assertTrue(isinstance(schema_message, singer.SchemaMessage))
# tap-mysql selects new fields by default. If a field doesn't appear in the schema, then it should be
# selected
expectedKeys = ["id", "a", "b"]
self.assertEqual(schema_message.schema["properties"].keys(),
set(expectedKeys))
def test_sync_messages_are_correct(self):
self.catalog.streams[0] = test_utils.set_replication_method_and_key(self.catalog.streams[0], 'LOG_BASED', None)
self.catalog.streams[0] = test_utils.set_selected(self.catalog.streams[0], True)
global SINGER_MESSAGES
SINGER_MESSAGES.clear()
#inital sync
tap_mysql.do_sync(self.conn, {}, self.catalog, {})
# get schema message to test that it has all the table's columns
schema_message = next(filter(lambda m: isinstance(m, singer.SchemaMessage), SINGER_MESSAGES))
expectedKeys = ['good_pk', 'age']
self.assertEqual(schema_message.schema['properties'].keys(), set(expectedKeys))
# get the records, these are generated by Full table replication
record_messages = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES))
self.assertEqual(len(record_messages), 4)
self.assertListEqual([
{'age': 20, 'good_pk': '61'},
{'age': 30, 'good_pk': '62'},
{'age': 30, 'good_pk': '63'},
{'age': 40, 'good_pk': '64'},
], [rec.record for rec in record_messages])
# get the last state message to be fed to the next sync
state_message = list(filter(lambda m: isinstance(m, singer.StateMessage), SINGER_MESSAGES))[-1]
SINGER_MESSAGES.clear()
# run some queries
with connect_with_backoff(self.conn) as open_conn:
with open_conn.cursor() as cursor:
cursor.execute("UPDATE good_pk_tab set age=age+5")
cursor.execute("INSERT INTO good_pk_tab (good_pk, age) VALUES "
"(BINARY('e'), 16), "
"(BINARY('f'), 5)")
# do a sync and give the state so that binlog replication start from the last synced position
tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state_message.value)
# get the changed/new records
record_messages = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES))
self.assertEqual(len(record_messages), 6)
self.assertListEqual([
{'age': 25, 'good_pk': '61'},
{'age': 35, 'good_pk': '62'},
{'age': 35, 'good_pk': '63'},
{'age': 45, 'good_pk': '64'},
{'age': 16, 'good_pk': '65'},
{'age': 5, 'good_pk': '66'},
], [rec.record for rec in record_messages])
def setUp(self):
self.conn = test_utils.get_test_connection()
self.state = {}
log_file, log_pos = binlog.fetch_current_log_file_and_pos(self.conn)
with connect_with_backoff(self.conn) as open_conn:
with open_conn.cursor() as cursor:
cursor.execute(
'CREATE TABLE datetime_types (id int, datetime_col datetime, timestamp_col timestamp, time_col time, date_col date)'
)
cursor.execute(
'INSERT INTO datetime_types (id, datetime_col, timestamp_col, time_col, date_col) VALUES (1, \'0000-00-00\', \'0000-00-00 00:00:00\', \'00:00:00\', \'0000-00-00\' )'
)
cursor.execute(
'INSERT INTO datetime_types (id, datetime_col, timestamp_col, time_col, date_col) VALUES (2, NULL, NULL, NULL, NULL)'
)
open_conn.commit()
self.catalog = test_utils.discover_catalog(self.conn, {})
for stream in self.catalog.streams:
stream.stream = stream.table
stream.metadata = [{
'breadcrumb': (),
'metadata': {
'selected': True,
'database-name': 'tap_mysql_test',
'table-key-propertes': ['id']
}
}, {
'breadcrumb': ('properties', 'id'),
'metadata': {
'selected': True
}
}, {
'breadcrumb': ('properties', 'datetime_col'),
'metadata': {
'selected': True
}
}, {
'breadcrumb': ('properties', 'timestamp_col'),
'metadata': {
'selected': True
}
}, {
'breadcrumb': ('properties', 'time_col'),
'metadata': {
'selected': True
}
}, {
'breadcrumb': ('properties', 'date_col'),
'metadata': {
'selected': True
}
}]
test_utils.set_replication_method_and_key(stream, 'LOG_BASED',
None)
self.state = singer.write_bookmark(self.state,
stream.tap_stream_id,
'log_file', log_file)
self.state = singer.write_bookmark(self.state,
stream.tap_stream_id, 'log_pos',
log_pos)
self.state = singer.write_bookmark(self.state,
stream.tap_stream_id, 'version',
singer.utils.now())
def setUp(self):
self.maxDiff = None
self.state = {}
self.conn = test_utils.get_test_connection()
log_file, log_pos = binlog.fetch_current_log_file_and_pos(self.conn)
with connect_with_backoff(self.conn) as open_conn:
with open_conn.cursor() as cursor:
cursor.execute(
"CREATE TABLE binlog_1 (id int, updated datetime)")
cursor.execute(
"CREATE TABLE binlog_2 (id int, updated datetime)")
cursor.execute(
"INSERT INTO binlog_1 (id, updated) VALUES (1, '2017-06-01')"
)
cursor.execute(
"INSERT INTO binlog_1 (id, updated) VALUES (2, '2017-06-20')"
)
cursor.execute(
"INSERT INTO binlog_1 (id, updated) VALUES (3, '2017-09-22')"
)
cursor.execute(
"INSERT INTO binlog_2 (id, updated) VALUES (1, '2017-10-22')"
)
cursor.execute(
"INSERT INTO binlog_2 (id, updated) VALUES (2, '2017-11-10')"
)
cursor.execute(
"INSERT INTO binlog_2 (id, updated) VALUES (3, '2017-12-10')"
)
cursor.execute(
"UPDATE binlog_1 set updated='2018-06-18' WHERE id = 3")
cursor.execute(
"UPDATE binlog_2 set updated='2018-06-18' WHERE id = 2")
cursor.execute("DELETE FROM binlog_1 WHERE id = 2")
cursor.execute("DELETE FROM binlog_2 WHERE id = 1")
open_conn.commit()
self.catalog = test_utils.discover_catalog(self.conn, {})
for stream in self.catalog.streams:
stream.stream = stream.table
stream.metadata = [
{
"breadcrumb": (),
"metadata": {
"selected": True,
"database-name": "tap_mysql_test",
"table-key-propertes": ["id"],
},
},
{
"breadcrumb": ("properties", "id"),
"metadata": {
"selected": True
}
},
{
"breadcrumb": ("properties", "updated"),
"metadata": {
"selected": True
}
},
]
test_utils.set_replication_method_and_key(stream, "LOG_BASED",
None)
self.state = singer.write_bookmark(self.state,
stream.tap_stream_id,
"log_file", log_file)
self.state = singer.write_bookmark(self.state,
stream.tap_stream_id, "log_pos",
log_pos)
self.state = singer.write_bookmark(self.state,
stream.tap_stream_id, "version",
singer.utils.now())
