def test_with_state(self):
state = tap_mysql.build_state(
{
'bookmarks': {
'tap_mysql_test-incremental': {
'version': 1,
'replication_key_value': '2017-06-20',
'replication_key': 'updated'
},
'tap_mysql_test-integer_incremental': {
'version': 1,
'replication_key_value': 3,
'replication_key': 'updated'
}
}
}, self.catalog)
(message_types, versions) = message_types_and_versions(
tap_mysql.generate_messages(self.con, self.catalog, state))
self.assertEqual([
'ActivateVersionMessage', 'RecordMessage', 'RecordMessage',
'ActivateVersionMessage', 'RecordMessage'
], message_types)
self.assertTrue(isinstance(versions[0], int))
self.assertEqual(versions[0], versions[1])
self.assertEqual(versions[1], 1)
def runTest(self):
con = get_test_connection()
try:
with con.cursor() as cur:
cur.execute('''
CREATE TABLE tab (
id INTEGER PRIMARY KEY,
a INTEGER,
b INTEGER)
''')
catalog = discover_catalog(con)
catalog.streams[0].stream = 'tab'
catalog.streams[0].schema.selected = True
catalog.streams[0].schema.properties['a'].selected = True
messages = list(
tap_mysql.generate_messages(con, catalog,
tap_mysql.build_state({},
catalog)))
schema_message = list(
filter(lambda m: isinstance(m, singer.SchemaMessage),
messages))[0]
self.assertTrue(isinstance(schema_message, singer.SchemaMessage))
expectedKeys = ['id', 'a']
self.assertEqual(schema_message.schema['properties'].keys(),
set(expectedKeys))
finally:
con.close()
def test_with_no_state(self):
state = tap_mysql.build_state({}, self.catalog)
(message_types, versions) = message_types_and_versions(
tap_mysql.generate_messages(self.con, self.catalog, state))
self.assertEqual([
'ActivateVersionMessage', 'RecordMessage', 'ActivateVersionMessage'
], message_types)
self.assertTrue(isinstance(versions[0], int))
self.assertEqual(versions[0], versions[1])
def runTest(self):
messages = tap_mysql.generate_messages(
self.con, self.catalog, tap_mysql.build_state({}, self.catalog))
record_message = list(
filter(lambda m: isinstance(m, singer.RecordMessage), messages))[0]
self.assertTrue(isinstance(record_message, singer.RecordMessage))
self.assertEqual(record_message.record, {'b c': 1})
def runTest(self):
catalog = discover_catalog(self.con)
catalog.streams[0].stream = 'some_stream_name'
catalog.streams[0].schema.selected = True
catalog.streams[0].key_properties = []
catalog.streams[0].schema.properties['b c'].selected = True
messages = tap_mysql.generate_messages(
self.con, catalog, tap_mysql.build_state({}, catalog))
record_message = list(
filter(lambda m: isinstance(m, singer.RecordMessage), messages))[0]
self.assertTrue(isinstance(record_message, singer.RecordMessage))
self.assertEqual(record_message.record, {'b c': 1})
def test_version_not_cleared_from_state_after_incremental_success(self):
state = tap_mysql.build_state(
{'bookmarks': {
'tap_mysql_test-incremental': {
'version': 1,
}
}}, self.catalog)
list(tap_mysql.generate_messages(self.con, self.catalog, state))
self.assertEqual(
state['bookmarks']['tap_mysql_test-incremental']['version'], 1)
def test_version_cleared_from_state_after_full_table_success(self):
state = tap_mysql.build_state(
{'bookmarks': {
'tap_mysql_test-full_table': {
'version': 1,
}
}}, self.catalog)
list(tap_mysql.generate_messages(self.con, self.catalog, state))
self.assertEqual(
state['bookmarks']['tap_mysql_test-full_table']['version'], None)
def test_with_no_version_in_state(self):
state = tap_mysql.build_state(
{'bookmarks': {
'tap_mysql_test-full_table': {
'version': None,
}
}}, self.catalog)
(message_types, versions) = message_types_and_versions(
tap_mysql.generate_messages(self.con, self.catalog, state))
self.assertEqual(['RecordMessage', 'ActivateVersionMessage'],
message_types)
self.assertTrue(isinstance(versions[0], int))
self.assertEqual(versions[0], versions[1])
def test_start_at_currently_syncing(self):
state = tap_mysql.build_state(
{'currently_syncing': 'tap_mysql_test-b'}, self.catalog)
messages = list(
tap_mysql.generate_messages(self.con, self.catalog, state))
self.assertRegexpMatches(currently_syncing_seq(messages), '^b+_+')
def test_emit_currently_syncing(self):
state = tap_mysql.build_state({}, self.catalog)
messages = list(
tap_mysql.generate_messages(self.con, self.catalog, state))
self.assertRegexpMatches(currently_syncing_seq(messages), '^a+b+_+')