def test_catalog(self):
singer.write_message = singer_write_message
log_miner.UPDATE_BOOKMARK_PERIOD = 1
with get_test_connection() as conn:
conn.autocommit = True
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
chicken_stream = [s for s in catalog.streams if s.table == 'CHICKEN'][0]
chicken_stream = select_all_of_stream(chicken_stream)
chicken_stream = set_replication_method_for_stream(chicken_stream, 'LOG_BASED')
cur = conn.cursor()
prev_scn = cur.execute("SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
crud_up_log_miner_fixtures(cur, 'CHICKEN',
{
'"name-char-explicit-byte"':'name-char-explicit-byte I',
'"name-char-explicit-char"':'name-char-explicit-char I',
'name_nchar' : 'name-nchar I',
'"name-nvarchar2"' : 'name-nvarchar2 I',
'"name-varchar-explicit-byte"' : 'name-varchar-explicit-byte I',
'"name-varchar-explicit-char"' : 'name-varchar-explicit-char I',
'"name-varchar2-explicit-byte"': 'name-varchar2-explicit-byte I',
'"name-varchar2-explicit-char"': 'name-varchar2-explicit-char I'
}, lambda s: s.replace("I", "II"))
post_scn = cur.execute("SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
LOGGER.info("post SCN: {}".format(post_scn))
state = write_bookmark({}, chicken_stream.tap_stream_id, 'scn', prev_scn)
state = write_bookmark(state, chicken_stream.tap_stream_id, 'version', 1)
tap_oracle.do_sync(get_test_conn_config(), catalog, None, state)
verify_crud_messages(self, CAUGHT_MESSAGES, ['ID'])
#verify message 1 - first insert
insert_rec_1 = CAUGHT_MESSAGES[1].record
self.assertIsNotNone(insert_rec_1.get('scn'))
insert_rec_1.pop('scn')
self.assertEqual(insert_rec_1, {'name-varchar2-explicit-byte': 'name-varchar2-explicit-byte I',
'name-char-explicit-char': 'name-char-explicit-char I ',
'name-nvarchar2': 'name-nvarchar2 I', 'name-varchar-explicit-char': 'name-varchar-explicit-char I',
'name-varchar2-explicit-char': 'name-varchar2-explicit-char I',
'NAME_NCHAR': 'name-nchar I ',
'name-char-explicit-byte': 'name-char-explicit-byte I ',
'_sdc_deleted_at': None, 'name-varchar-explicit-byte': 'name-varchar-explicit-byte I', 'ID': 1})
#verify UPDATE
update_rec = CAUGHT_MESSAGES[5].record
self.assertIsNotNone(update_rec.get('scn'))
update_rec.pop('scn')
self.assertEqual(update_rec, {'name-varchar2-explicit-byte': 'name-varchar2-explicit-byte II',
'name-char-explicit-char': 'name-char-explicit-char II ',
'name-nvarchar2': 'name-nvarchar2 II', 'name-varchar-explicit-char': 'name-varchar-explicit-char II',
'name-varchar2-explicit-char': 'name-varchar2-explicit-char II',
'NAME_NCHAR': 'name-nchar II ',
'name-char-explicit-byte': 'name-char-explicit-byte II ',
'_sdc_deleted_at': None, 'name-varchar-explicit-byte': 'name-varchar-explicit-byte II', 'ID': 1})
#verify first DELETE message
delete_rec = CAUGHT_MESSAGES[9].record
self.assertIsNotNone(delete_rec.get('scn'))
self.assertIsNotNone(delete_rec.get('_sdc_deleted_at'))
delete_rec.pop('scn')
delete_rec.pop('_sdc_deleted_at')
self.assertEqual(delete_rec, {'name-varchar2-explicit-byte': 'name-varchar2-explicit-byte II',
'name-char-explicit-char': 'name-char-explicit-char II ',
'name-nvarchar2': 'name-nvarchar2 II', 'name-varchar-explicit-char': 'name-varchar-explicit-char II',
'name-varchar2-explicit-char': 'name-varchar2-explicit-char II',
'NAME_NCHAR': 'name-nchar II ',
'name-char-explicit-byte': 'name-char-explicit-byte II ',
'name-varchar-explicit-byte': 'name-varchar-explicit-byte II', 'ID': 1})
#verify second DELETE message
delete_rec_2 = CAUGHT_MESSAGES[11].record
self.assertIsNotNone(delete_rec_2.get('scn'))
self.assertIsNotNone(delete_rec_2.get('_sdc_deleted_at'))
delete_rec_2.pop('scn')
delete_rec_2.pop('_sdc_deleted_at')
self.assertEqual(delete_rec_2, {'name-varchar2-explicit-byte': 'name-varchar2-explicit-byte II',
'name-char-explicit-char': 'name-char-explicit-char II ',
'name-nvarchar2': 'name-nvarchar2 II', 'name-varchar-explicit-char': 'name-varchar-explicit-char II',
'name-varchar2-explicit-char': 'name-varchar2-explicit-char II',
'NAME_NCHAR': 'name-nchar II ',
'name-char-explicit-byte': 'name-char-explicit-byte II ',
'name-varchar-explicit-byte': 'name-varchar-explicit-byte II', 'ID': 2})
def test_catalog(self):
singer.write_message = singer_write_message
log_miner.UPDATE_BOOKMARK_PERIOD = 1
with get_test_connection() as conn:
conn.autocommit = True
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
chicken_stream = [
s for s in catalog.streams if s.table == 'CHICKEN'
][0]
chicken_stream = select_all_of_stream(chicken_stream)
chicken_stream = set_replication_method_for_stream(
chicken_stream, 'LOG_BASED')
cur = conn.cursor()
prev_scn = cur.execute(
"SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
our_fake_float = decimal.Decimal('1234567.8901234')
our_real_float = 1234567.8901234
crud_up_log_miner_fixtures(
cur, 'CHICKEN', {
'"our_double_precision"': our_fake_float,
'"our_real"': our_fake_float,
'"our_binary_float"': our_real_float,
'"our_binary_double"': our_real_float,
'"our_nan"': float('nan'),
'"our_+_infinity"': float('+inf'),
'"our_-_infinity"': float('-inf')
}, self.update_add_5)
post_scn = cur.execute(
"SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
LOGGER.info("post SCN: {}".format(post_scn))
state = write_bookmark({}, chicken_stream.tap_stream_id, 'scn',
prev_scn)
state = write_bookmark(state, chicken_stream.tap_stream_id,
'version', 1)
tap_oracle.do_sync(get_test_conn_config(), catalog, None, state)
verify_crud_messages(self, CAUGHT_MESSAGES, ['our_float'])
#verify message 1 - first insert
insert_rec_1 = CAUGHT_MESSAGES[1].record
self.assertIsNotNone(insert_rec_1.get('scn'))
insert_rec_1.pop('scn')
self.assertEqual(float('+inf'), insert_rec_1.get('our_+_infinity'))
insert_rec_1.pop('our_+_infinity')
self.assertEqual(float('-inf'), insert_rec_1.get('our_-_infinity'))
insert_rec_1.pop('our_-_infinity')
self.assertTrue(math.isnan(insert_rec_1.get('our_nan')))
insert_rec_1.pop('our_nan')
self.assertEqual(
insert_rec_1,
{
'our_float': '1',
'our_double_precision': str(our_fake_float),
'our_real': str(our_fake_float),
'our_binary_float': 1234567.88, #weird
'our_binary_double': 1234567.890123,
'_sdc_deleted_at': None
})
#verify UPDATE
update_rec = CAUGHT_MESSAGES[5].record
self.assertIsNotNone(update_rec.get('scn'))
update_rec.pop('scn')
self.assertEqual(float('+inf'), update_rec.get('our_+_infinity'))
update_rec.pop('our_+_infinity')
self.assertEqual(float('-inf'), update_rec.get('our_-_infinity'))
update_rec.pop('our_-_infinity')
self.assertTrue(math.isnan(update_rec.get('our_nan')))
update_rec.pop('our_nan')
self.assertEqual(
update_rec, {
'our_binary_double': 1234572.890123,
'_sdc_deleted_at': None,
'our_binary_float': 1234572.88,
'our_float': '1',
'our_double_precision': '1234572.8901234',
'our_real': '1234572.8901234'
})
#verify first DELETE message
delete_rec = CAUGHT_MESSAGES[9].record
self.assertEqual(float('+inf'), delete_rec.get('our_+_infinity'))
delete_rec.pop('our_+_infinity')
self.assertEqual(float('-inf'), delete_rec.get('our_-_infinity'))
delete_rec.pop('our_-_infinity')
self.assertTrue(math.isnan(delete_rec.get('our_nan')))
delete_rec.pop('our_nan')
self.assertIsNotNone(delete_rec.get('scn'))
self.assertIsNotNone(delete_rec.get('_sdc_deleted_at'))
delete_rec.pop('scn')
delete_rec.pop('_sdc_deleted_at')
self.assertEqual(
delete_rec, {
'our_binary_double': 1234572.890123,
'our_binary_float': 1234572.88,
'our_float': '1',
'our_double_precision': '1234572.8901234',
'our_real': '1234572.8901234'
})
#verify second DELETE message
delete_rec_2 = CAUGHT_MESSAGES[11].record
self.assertEqual(float('+inf'), delete_rec_2.get('our_+_infinity'))
delete_rec_2.pop('our_+_infinity')
self.assertEqual(float('-inf'), delete_rec_2.get('our_-_infinity'))
delete_rec_2.pop('our_-_infinity')
self.assertTrue(math.isnan(delete_rec_2.get('our_nan')))
delete_rec_2.pop('our_nan')
self.assertIsNotNone(delete_rec_2.get('scn'))
self.assertIsNotNone(delete_rec_2.get('_sdc_deleted_at'))
delete_rec_2.pop('scn')
delete_rec_2.pop('_sdc_deleted_at')
self.assertEqual(
delete_rec_2, {
'our_binary_double': 1234572.890123,
'our_binary_float': 1234572.88,
'our_float': '2',
'our_double_precision': '1234572.8901234',
'our_real': '1234572.8901234'
})
def test_catalog(self):
singer.write_message = singer_write_message
log_miner.UPDATE_BOOKMARK_PERIOD = 1
with get_test_connection() as conn:
conn.autocommit = True
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
chicken_stream = [
s for s in catalog.streams if s.table == 'CHICKEN'
][0]
chicken_stream = select_all_of_stream(chicken_stream)
chicken_stream = set_replication_method_for_stream(
chicken_stream, 'LOG_BASED')
cur = conn.cursor()
prev_scn = cur.execute(
"SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
crud_up_log_miner_fixtures(
cur, 'CHICKEN', {
'"our_number_10_2"': decimal.Decimal('100.11'),
'"our_number_38_4"':
decimal.Decimal('99999999999999999.99991')
}, self.update_add_5)
post_scn = cur.execute(
"SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
LOGGER.info("post SCN: {}".format(post_scn))
state = write_bookmark({}, chicken_stream.tap_stream_id, 'scn',
prev_scn)
state = write_bookmark(state, chicken_stream.tap_stream_id,
'version', 1)
tap_oracle.do_sync(get_test_conn_config(), catalog, None, state)
verify_crud_messages(self, CAUGHT_MESSAGES, ['our_number'])
#verify message 1 - first insert
insert_rec_1 = CAUGHT_MESSAGES[1].record
self.assertIsNotNone(insert_rec_1.get('scn'))
insert_rec_1.pop('scn')
self.assertEqual(
insert_rec_1, {
'our_number_38_4':
decimal.Decimal('99999999999999999.9999'),
'our_number': 1,
'our_number_10_2': decimal.Decimal('100.11'),
'_sdc_deleted_at': None
})
#verify UPDATE
update_rec = CAUGHT_MESSAGES[5].record
self.assertIsNotNone(update_rec.get('scn'))
update_rec.pop('scn')
self.assertEqual(
update_rec, {
'our_number_38_4':
decimal.Decimal('100000000000000004.9999'),
'our_number': 1,
'our_number_10_2': decimal.Decimal('105.11'),
'_sdc_deleted_at': None
})
#verify first DELETE message
delete_rec = CAUGHT_MESSAGES[9].record
self.assertIsNotNone(delete_rec.get('scn'))
self.assertIsNotNone(delete_rec.get('_sdc_deleted_at'))
delete_rec.pop('scn')
delete_rec.pop('_sdc_deleted_at')
self.assertEqual(
delete_rec, {
'our_number_38_4':
decimal.Decimal('100000000000000004.9999'),
'our_number': 1,
'our_number_10_2': decimal.Decimal('105.11')
})
#verify second DELETE message
delete_rec_2 = CAUGHT_MESSAGES[11].record
self.assertIsNotNone(delete_rec_2.get('scn'))
self.assertIsNotNone(delete_rec_2.get('_sdc_deleted_at'))
delete_rec_2.pop('scn')
delete_rec_2.pop('_sdc_deleted_at')
self.assertEqual(
delete_rec_2, {
'our_number_38_4':
decimal.Decimal('100000000000000004.9999'),
'our_number': 2,
'our_number_10_2': decimal.Decimal('105.11')
})
def test_catalog(self):
singer.write_message = singer_write_message
log_miner.UPDATE_BOOKMARK_PERIOD = 1
with get_test_connection() as conn:
conn.autocommit = True
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
chicken_stream = [
s for s in catalog.streams if s.table == 'CHICKEN'
][0]
chicken_stream = select_all_of_stream(chicken_stream)
chicken_stream = set_replication_method_for_stream(
chicken_stream, 'LOG_BASED')
cur = conn.cursor()
prev_scn = cur.execute(
"SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
our_date = datetime.date(1996, 6, 6)
our_ts = datetime.datetime(1997, 2, 2, 2, 2, 2, 722184)
nyc_tz = pytz.timezone('America/New_York')
our_ts_tz_edt = nyc_tz.localize(
datetime.datetime(1997, 3, 3, 3, 3, 3, 722184))
our_ts_tz_utc = datetime.datetime(1997, 3, 3, 3, 3, 3, 722184,
pytz.UTC)
auckland_tz = pytz.timezone('Pacific/Auckland')
our_ts_local = auckland_tz.localize(
datetime.datetime(1997, 3, 3, 18, 3, 3, 722184))
crud_up_log_miner_fixtures(
cur, 'CHICKEN', {
'"our_date"': our_date,
'"our_ts"': our_ts,
'"our_ts_tz_edt"': our_ts_tz_edt,
'"our_ts_tz_utc"': our_ts_tz_utc,
'"our_ts_tz_local"': our_ts_local
}, self.update_add_1_day)
post_scn = cur.execute(
"SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
LOGGER.info("post SCN: {}".format(post_scn))
state = write_bookmark({}, chicken_stream.tap_stream_id, 'scn',
prev_scn)
state = write_bookmark(state, chicken_stream.tap_stream_id,
'version', 1)
tap_oracle.do_sync(get_test_conn_config(), catalog, None, state)
verify_crud_messages(self, CAUGHT_MESSAGES, ['ID'])
#verify message 1 - first insert
insert_rec_1 = CAUGHT_MESSAGES[1].record
self.assertIsNotNone(insert_rec_1.get('scn'))
insert_rec_1.pop('scn')
self.assertIsNone(insert_rec_1.get('_sdc_deleted_at'))
self.assertEqual(insert_rec_1.get('ID'), 1)
insert_rec_1.pop('_sdc_deleted_at')
insert_rec_1.pop('ID')
self.assertEqual(
insert_rec_1, {
'our_ts': '1997-02-02T02:02:02.722184+00:00',
'our_ts_tz_edt': '1997-03-03T03:03:03.722184-05:00',
'our_ts_tz_utc': '1997-03-03T03:03:03.722184+00:00',
'our_date': '1996-06-06T00:00:00.00+00:00',
'our_ts_tz_local': '1997-03-03T05:03:03.722184+00:00'
})
for v in insert_rec_1.values():
self.assertTrue(strict_rfc3339.validate_rfc3339(v))
#verify UPDATE
update_rec = CAUGHT_MESSAGES[5].record
self.assertIsNotNone(update_rec.get('scn'))
update_rec.pop('scn')
self.assertEqual(
update_rec, {
'our_ts': '1997-02-03T02:02:02.722184+00:00',
'our_ts_tz_edt': '1997-03-04T03:03:03.722184-05:00',
'our_ts_tz_utc': '1997-03-04T03:03:03.722184+00:00',
'our_date': '1996-06-07T00:00:00.00+00:00',
'_sdc_deleted_at': None,
'our_ts_tz_local': '1997-03-04T05:03:03.722184+00:00',
'ID': 1
})
#verify first DELETE message
delete_rec = CAUGHT_MESSAGES[9].record
self.assertIsNotNone(delete_rec.get('scn'))
self.assertIsNotNone(delete_rec.get('_sdc_deleted_at'))
delete_rec.pop('scn')
delete_rec.pop('_sdc_deleted_at')
self.assertEqual(
delete_rec, {
'our_ts': '1997-02-03T02:02:02.722184+00:00',
'our_ts_tz_edt': '1997-03-04T03:03:03.722184-05:00',
'our_ts_tz_utc': '1997-03-04T03:03:03.722184+00:00',
'our_date': '1996-06-07T00:00:00.00+00:00',
'our_ts_tz_local': '1997-03-04T05:03:03.722184+00:00',
'ID': 1
})
#verify second DELETE message
delete_rec_2 = CAUGHT_MESSAGES[11].record
self.assertIsNotNone(delete_rec_2.get('scn'))
self.assertIsNotNone(delete_rec_2.get('_sdc_deleted_at'))
delete_rec_2.pop('scn')
delete_rec_2.pop('_sdc_deleted_at')
self.assertEqual(
delete_rec_2, {
'our_ts': '1997-02-03T02:02:02.722184+00:00',
'our_ts_tz_edt': '1997-03-04T03:03:03.722184-05:00',
'our_ts_tz_utc': '1997-03-04T03:03:03.722184+00:00',
'our_date': '1996-06-07T00:00:00.00+00:00',
'our_ts_tz_local': '1997-03-04T05:03:03.722184+00:00',
'ID': 2
})