def setUp(self): self.conn = test_utils.get_test_connection() with connect_with_backoff(self.conn) as open_conn: with open_conn.cursor() as cursor: cursor.execute('CREATE TABLE full_table (val int)') cursor.execute('INSERT INTO full_table (val) VALUES (1)') self.catalog = test_utils.discover_catalog(self.conn, {}) for stream in self.catalog.streams: stream.key_properties = [] stream.metadata = [{ 'breadcrumb': (), 'metadata': { 'selected': True, 'database-name': 'tap_mysql_test' } }, { 'breadcrumb': ('properties', 'val'), 'metadata': { 'selected': True } }] stream.stream = stream.table test_utils.set_replication_method_and_key(stream, 'FULL_TABLE', None)
def setUp(self): self.conn = test_utils.get_test_connection() with connect_with_backoff(self.conn) as open_conn: with open_conn.cursor() as cursor: cursor.execute('CREATE TABLE a (`b c` int)') cursor.execute('INSERT INTO a (`b c`) VALUES (1)') self.catalog = test_utils.discover_catalog(self.conn, {}) self.catalog.streams[0].stream = 'some_stream_name' self.catalog.streams[0].metadata = [{ 'breadcrumb': (), 'metadata': { 'selected': True, 'table-key-properties': [], 'database-name': 'tap_mysql_test' } }, { 'breadcrumb': ('properties', 'b c'), 'metadata': { 'selected': True } }] test_utils.set_replication_method_and_key(self.catalog.streams[0], 'FULL_TABLE', None)
def test_change_replication_key(self): state = { 'bookmarks': { 'tap_mysql_test-incremental': { 'version': 1, 'replication_key_value': '2017-06-20', 'replication_key': 'updated' } } } stream = [x for x in self.catalog.streams if x.stream == 'incremental'][0] stream.metadata = [ {'breadcrumb': (), 'metadata': {'selected': True, 'database-name': 'tap_mysql_test'}}, {'breadcrumb': ('properties', 'val'), 'metadata': {'selected': True}}, {'breadcrumb': ('properties', 'updated'), 'metadata': {'selected': True}} ] test_utils.set_replication_method_and_key(stream, 'INCREMENTAL', 'val') tap_mysql.do_sync(self.conn, {}, self.catalog, state) self.assertEqual(state['bookmarks']['tap_mysql_test-incremental']['replication_key'], 'val') self.assertEqual(state['bookmarks']['tap_mysql_test-incremental']['replication_key_value'], 3) self.assertEqual(state['bookmarks']['tap_mysql_test-incremental']['version'], 1)
def setUp(self): self.conn = test_utils.get_test_connection() self.catalog = init_tables(self.conn) for stream in self.catalog.streams: stream.metadata = [ {'breadcrumb': (), 'metadata': {'selected': True, 'database-name': 'tap_mysql_test', 'table-key-properties': ['id']}}, {'breadcrumb': ('properties', 'id'), 'metadata': {'selected': True}}, {'breadcrumb': ('properties', 'foo'), 'metadata': {'selected': True}}, {'breadcrumb': ('properties', 'bar'), 'metadata': {'selected': True}}, ] stream.stream = stream.table if stream.table == 'table_2' or stream.table == 'table_3': test_utils.set_replication_method_and_key(stream, 'LOG_BASED', None) else: test_utils.set_replication_method_and_key(stream, 'FULL_TABLE', None) global TABLE_2_RECORD_COUNT TABLE_2_RECORD_COUNT = 0 global TABLE_3_RECORD_COUNT TABLE_3_RECORD_COUNT = 0 global SINGER_MESSAGES SINGER_MESSAGES.clear()
def runTest(self): conn = test_utils.get_test_connection() with connect_with_backoff(conn) as open_conn: with open_conn.cursor() as cur: cur.execute(''' CREATE TABLE tab ( id INTEGER PRIMARY KEY, a INTEGER, b INTEGER) ''') catalog = test_utils.discover_catalog(conn, {}) catalog.streams[0].stream = 'tab' catalog.streams[0].metadata = [ {'breadcrumb': (), 'metadata': {'selected': True, 'database-name': 'tap_mysql_test'}}, {'breadcrumb': ('properties', 'a'), 'metadata': {'selected': True}} ] test_utils.set_replication_method_and_key(catalog.streams[0], 'FULL_TABLE', None) global SINGER_MESSAGES SINGER_MESSAGES.clear() tap_mysql.do_sync(conn, {}, catalog, {}) schema_message = list(filter(lambda m: isinstance(m, singer.SchemaMessage), SINGER_MESSAGES))[0] self.assertTrue(isinstance(schema_message, singer.SchemaMessage)) # tap-mysql selects new fields by default. If a field doesn't appear in the schema, then it should be # selected expectedKeys = ['id', 'a', 'b'] self.assertEqual(schema_message.schema['properties'].keys(), set(expectedKeys))
def setUp(self): self.conn = test_utils.get_test_connection() with connect_with_backoff(self.conn) as open_conn: with open_conn.cursor() as cursor: cursor.execute('CREATE TABLE incremental (val int, updated datetime)') cursor.execute('INSERT INTO incremental (val, updated) VALUES (1, \'2017-06-01\')') cursor.execute('INSERT INTO incremental (val, updated) VALUES (2, \'2017-06-20\')') cursor.execute('INSERT INTO incremental (val, updated) VALUES (3, \'2017-09-22\')') cursor.execute('CREATE TABLE integer_incremental (val int, updated int)') cursor.execute('INSERT INTO integer_incremental (val, updated) VALUES (1, 1)') cursor.execute('INSERT INTO integer_incremental (val, updated) VALUES (2, 2)') cursor.execute('INSERT INTO integer_incremental (val, updated) VALUES (3, 3)') self.catalog = test_utils.discover_catalog(self.conn, {}) for stream in self.catalog.streams: stream.metadata = [ {'breadcrumb': (), 'metadata': { 'selected': True, 'table-key-properties': [], 'database-name': 'tap_mysql_test' }}, {'breadcrumb': ('properties', 'val'), 'metadata': {'selected': True}} ] stream.stream = stream.table test_utils.set_replication_method_and_key(stream, 'INCREMENTAL', 'updated')
def setUp(self): self.conn = test_utils.get_test_connection() with connect_with_backoff(self.conn) as open_conn: with open_conn.cursor() as cursor: cursor.execute("CREATE TABLE full_table (val int)") cursor.execute("INSERT INTO full_table (val) VALUES (1)") self.catalog = test_utils.discover_catalog(self.conn, {}) for stream in self.catalog.streams: stream.key_properties = [] stream.metadata = [ { "breadcrumb": (), "metadata": { "selected": True, "database-name": "tap_mysql_test" }, }, { "breadcrumb": ("properties", "val"), "metadata": { "selected": True } }, ] stream.stream = stream.table test_utils.set_replication_method_and_key(stream, "FULL_TABLE", None)
def setUp(self): self.conn = test_utils.get_test_connection() with connect_with_backoff(self.conn) as open_conn: with open_conn.cursor() as cursor: cursor.execute("CREATE TABLE a (`b c` int)") cursor.execute("INSERT INTO a (`b c`) VALUES (1)") self.catalog = test_utils.discover_catalog(self.conn, {}) self.catalog.streams[0].stream = "some_stream_name" self.catalog.streams[0].metadata = [ { "breadcrumb": (), "metadata": { "selected": True, "table-key-properties": [], "database-name": "tap_mysql_test", }, }, { "breadcrumb": ("properties", "b c"), "metadata": { "selected": True } }, ] test_utils.set_replication_method_and_key(self.catalog.streams[0], "FULL_TABLE", None)
def setUp(self): self.maxDiff = None self.state = {} self.conn = test_utils.get_test_connection() log_file, log_pos = binlog.fetch_current_log_file_and_pos(self.conn) with connect_with_backoff(self.conn) as open_conn: with open_conn.cursor() as cursor: cursor.execute('CREATE TABLE binlog_1 (id int, updated datetime)') cursor.execute('CREATE TABLE binlog_2 (id int, updated datetime)') cursor.execute('INSERT INTO binlog_1 (id, updated) VALUES (1, \'2017-06-01\')') cursor.execute('INSERT INTO binlog_1 (id, updated) VALUES (2, \'2017-06-20\')') cursor.execute('INSERT INTO binlog_1 (id, updated) VALUES (3, \'2017-09-22\')') cursor.execute('INSERT INTO binlog_2 (id, updated) VALUES (1, \'2017-10-22\')') cursor.execute('INSERT INTO binlog_2 (id, updated) VALUES (2, \'2017-11-10\')') cursor.execute('INSERT INTO binlog_2 (id, updated) VALUES (3, \'2017-12-10\')') cursor.execute('UPDATE binlog_1 set updated=\'2018-06-18\' WHERE id = 3') cursor.execute('UPDATE binlog_2 set updated=\'2018-06-18\' WHERE id = 2') cursor.execute('DELETE FROM binlog_1 WHERE id = 2') cursor.execute('DELETE FROM binlog_2 WHERE id = 1') open_conn.commit() self.catalog = test_utils.discover_catalog(self.conn, {}) for stream in self.catalog.streams: stream.stream = stream.table stream.metadata = [ {'breadcrumb': (), 'metadata': { 'selected': True, 'database-name': 'tap_mysql_test', 'table-key-propertes': ['id'] }}, {'breadcrumb': ('properties', 'id'), 'metadata': {'selected': True}}, {'breadcrumb': ('properties', 'updated'), 'metadata': {'selected': True}} ] test_utils.set_replication_method_and_key(stream, 'LOG_BASED', None) self.state = singer.write_bookmark(self.state, stream.tap_stream_id, 'log_file', log_file) self.state = singer.write_bookmark(self.state, stream.tap_stream_id, 'log_pos', log_pos) self.state = singer.write_bookmark(self.state, stream.tap_stream_id, 'version', singer.utils.now())
def setUp(self): self.conn = test_utils.get_test_connection() with connect_with_backoff(self.conn) as open_conn: with open_conn.cursor() as cursor: cursor.execute( "CREATE TABLE incremental (val int, updated datetime)") cursor.execute( "INSERT INTO incremental (val, updated) VALUES (1, '2017-06-01')" ) cursor.execute( "INSERT INTO incremental (val, updated) VALUES (2, '2017-06-20')" ) cursor.execute( "INSERT INTO incremental (val, updated) VALUES (3, '2017-09-22')" ) cursor.execute( "CREATE TABLE integer_incremental (val int, updated int)") cursor.execute( "INSERT INTO integer_incremental (val, updated) VALUES (1, 1)" ) cursor.execute( "INSERT INTO integer_incremental (val, updated) VALUES (2, 2)" ) cursor.execute( "INSERT INTO integer_incremental (val, updated) VALUES (3, 3)" ) self.catalog = test_utils.discover_catalog(self.conn, {}) for stream in self.catalog.streams: stream.metadata = [ { "breadcrumb": (), "metadata": { "selected": True, "table-key-properties": [], "database-name": "tap_mysql_test", }, }, { "breadcrumb": ("properties", "val"), "metadata": { "selected": True } }, ] stream.stream = stream.table test_utils.set_replication_method_and_key(stream, "INCREMENTAL", "updated")
def test_change_replication_key(self): state = { "bookmarks": { "tap_mysql_test-incremental": { "version": 1, "replication_key_value": "2017-06-20", "replication_key": "updated", } } } stream = [ x for x in self.catalog.streams if x.stream == "incremental" ][0] stream.metadata = [ { "breadcrumb": (), "metadata": { "selected": True, "database-name": "tap_mysql_test" } }, { "breadcrumb": ("properties", "val"), "metadata": { "selected": True } }, { "breadcrumb": ("properties", "updated"), "metadata": { "selected": True } }, ] test_utils.set_replication_method_and_key(stream, "INCREMENTAL", "val") tap_mysql.do_sync(self.conn, {}, self.catalog, state) self.assertEqual( state["bookmarks"]["tap_mysql_test-incremental"] ["replication_key"], "val") self.assertEqual( state["bookmarks"]["tap_mysql_test-incremental"] ["replication_key_value"], 3) self.assertEqual( state["bookmarks"]["tap_mysql_test-incremental"]["version"], 1)
def setUp(self): self.conn = test_utils.get_test_connection() self.catalog = init_tables(self.conn) for stream in self.catalog.streams: stream.metadata = [ { "breadcrumb": (), "metadata": { "selected": True, "database-name": "tap_mysql_test", "table-key-properties": ["id"], }, }, { "breadcrumb": ("properties", "id"), "metadata": { "selected": True } }, { "breadcrumb": ("properties", "foo"), "metadata": { "selected": True } }, { "breadcrumb": ("properties", "bar"), "metadata": { "selected": True } }, ] stream.stream = stream.table if stream.table == "table_2": test_utils.set_replication_method_and_key( stream, "LOG_BASED", None) else: test_utils.set_replication_method_and_key( stream, "FULL_TABLE", None) global TABLE_2_RECORD_COUNT TABLE_2_RECORD_COUNT = 0 global SINGER_MESSAGES SINGER_MESSAGES.clear()
def runTest(self): conn = test_utils.get_test_connection() with connect_with_backoff(conn) as open_conn: with open_conn.cursor() as cur: cur.execute(""" CREATE TABLE tab ( id INTEGER PRIMARY KEY, a INTEGER, b INTEGER) """) catalog = test_utils.discover_catalog(conn, {}) catalog.streams[0].stream = "tab" catalog.streams[0].metadata = [ { "breadcrumb": (), "metadata": { "selected": True, "database-name": "tap_mysql_test" } }, { "breadcrumb": ("properties", "a"), "metadata": { "selected": True } }, ] test_utils.set_replication_method_and_key(catalog.streams[0], "FULL_TABLE", None) global SINGER_MESSAGES SINGER_MESSAGES.clear() tap_mysql.do_sync(conn, {}, catalog, {}) schema_message = list( filter(lambda m: isinstance(m, singer.SchemaMessage), SINGER_MESSAGES))[0] self.assertTrue(isinstance(schema_message, singer.SchemaMessage)) # tap-mysql selects new fields by default. If a field doesn't appear in the schema, then it should be # selected expectedKeys = ["id", "a", "b"] self.assertEqual(schema_message.schema["properties"].keys(), set(expectedKeys))
def test_sync_messages_are_correct(self): self.catalog.streams[0] = test_utils.set_replication_method_and_key(self.catalog.streams[0], 'LOG_BASED', None) self.catalog.streams[0] = test_utils.set_selected(self.catalog.streams[0], True) global SINGER_MESSAGES SINGER_MESSAGES.clear() #inital sync tap_mysql.do_sync(self.conn, {}, self.catalog, {}) # get schema message to test that it has all the table's columns schema_message = next(filter(lambda m: isinstance(m, singer.SchemaMessage), SINGER_MESSAGES)) expectedKeys = ['good_pk', 'age'] self.assertEqual(schema_message.schema['properties'].keys(), set(expectedKeys)) # get the records, these are generated by Full table replication record_messages = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES)) self.assertEqual(len(record_messages), 4) self.assertListEqual([ {'age': 20, 'good_pk': '61'}, {'age': 30, 'good_pk': '62'}, {'age': 30, 'good_pk': '63'}, {'age': 40, 'good_pk': '64'}, ], [rec.record for rec in record_messages]) # get the last state message to be fed to the next sync state_message = list(filter(lambda m: isinstance(m, singer.StateMessage), SINGER_MESSAGES))[-1] SINGER_MESSAGES.clear() # run some queries with connect_with_backoff(self.conn) as open_conn: with open_conn.cursor() as cursor: cursor.execute("UPDATE good_pk_tab set age=age+5") cursor.execute("INSERT INTO good_pk_tab (good_pk, age) VALUES " "(BINARY('e'), 16), " "(BINARY('f'), 5)") # do a sync and give the state so that binlog replication start from the last synced position tap_mysql.do_sync(self.conn, test_utils.get_db_config(), self.catalog, state_message.value) # get the changed/new records record_messages = list(filter(lambda m: isinstance(m, singer.RecordMessage), SINGER_MESSAGES)) self.assertEqual(len(record_messages), 6) self.assertListEqual([ {'age': 25, 'good_pk': '61'}, {'age': 35, 'good_pk': '62'}, {'age': 35, 'good_pk': '63'}, {'age': 45, 'good_pk': '64'}, {'age': 16, 'good_pk': '65'}, {'age': 5, 'good_pk': '66'}, ], [rec.record for rec in record_messages])
def setUp(self): self.conn = test_utils.get_test_connection() self.state = {} log_file, log_pos = binlog.fetch_current_log_file_and_pos(self.conn) with connect_with_backoff(self.conn) as open_conn: with open_conn.cursor() as cursor: cursor.execute( 'CREATE TABLE datetime_types (id int, datetime_col datetime, timestamp_col timestamp, time_col time, date_col date)' ) cursor.execute( 'INSERT INTO datetime_types (id, datetime_col, timestamp_col, time_col, date_col) VALUES (1, \'0000-00-00\', \'0000-00-00 00:00:00\', \'00:00:00\', \'0000-00-00\' )' ) cursor.execute( 'INSERT INTO datetime_types (id, datetime_col, timestamp_col, time_col, date_col) VALUES (2, NULL, NULL, NULL, NULL)' ) open_conn.commit() self.catalog = test_utils.discover_catalog(self.conn, {}) for stream in self.catalog.streams: stream.stream = stream.table stream.metadata = [{ 'breadcrumb': (), 'metadata': { 'selected': True, 'database-name': 'tap_mysql_test', 'table-key-propertes': ['id'] } }, { 'breadcrumb': ('properties', 'id'), 'metadata': { 'selected': True } }, { 'breadcrumb': ('properties', 'datetime_col'), 'metadata': { 'selected': True } }, { 'breadcrumb': ('properties', 'timestamp_col'), 'metadata': { 'selected': True } }, { 'breadcrumb': ('properties', 'time_col'), 'metadata': { 'selected': True } }, { 'breadcrumb': ('properties', 'date_col'), 'metadata': { 'selected': True } }] test_utils.set_replication_method_and_key(stream, 'LOG_BASED', None) self.state = singer.write_bookmark(self.state, stream.tap_stream_id, 'log_file', log_file) self.state = singer.write_bookmark(self.state, stream.tap_stream_id, 'log_pos', log_pos) self.state = singer.write_bookmark(self.state, stream.tap_stream_id, 'version', singer.utils.now())
def setUp(self): self.maxDiff = None self.state = {} self.conn = test_utils.get_test_connection() log_file, log_pos = binlog.fetch_current_log_file_and_pos(self.conn) with connect_with_backoff(self.conn) as open_conn: with open_conn.cursor() as cursor: cursor.execute( "CREATE TABLE binlog_1 (id int, updated datetime)") cursor.execute( "CREATE TABLE binlog_2 (id int, updated datetime)") cursor.execute( "INSERT INTO binlog_1 (id, updated) VALUES (1, '2017-06-01')" ) cursor.execute( "INSERT INTO binlog_1 (id, updated) VALUES (2, '2017-06-20')" ) cursor.execute( "INSERT INTO binlog_1 (id, updated) VALUES (3, '2017-09-22')" ) cursor.execute( "INSERT INTO binlog_2 (id, updated) VALUES (1, '2017-10-22')" ) cursor.execute( "INSERT INTO binlog_2 (id, updated) VALUES (2, '2017-11-10')" ) cursor.execute( "INSERT INTO binlog_2 (id, updated) VALUES (3, '2017-12-10')" ) cursor.execute( "UPDATE binlog_1 set updated='2018-06-18' WHERE id = 3") cursor.execute( "UPDATE binlog_2 set updated='2018-06-18' WHERE id = 2") cursor.execute("DELETE FROM binlog_1 WHERE id = 2") cursor.execute("DELETE FROM binlog_2 WHERE id = 1") open_conn.commit() self.catalog = test_utils.discover_catalog(self.conn, {}) for stream in self.catalog.streams: stream.stream = stream.table stream.metadata = [ { "breadcrumb": (), "metadata": { "selected": True, "database-name": "tap_mysql_test", "table-key-propertes": ["id"], }, }, { "breadcrumb": ("properties", "id"), "metadata": { "selected": True } }, { "breadcrumb": ("properties", "updated"), "metadata": { "selected": True } }, ] test_utils.set_replication_method_and_key(stream, "LOG_BASED", None) self.state = singer.write_bookmark(self.state, stream.tap_stream_id, "log_file", log_file) self.state = singer.write_bookmark(self.state, stream.tap_stream_id, "log_pos", log_pos) self.state = singer.write_bookmark(self.state, stream.tap_stream_id, "version", singer.utils.now())