def test_catalog(self): singer.write_message = singer_write_message log_miner.UPDATE_BOOKMARK_PERIOD = 1 with get_test_connection() as conn: conn.autocommit = True catalog = tap_oracle.do_discovery(get_test_conn_config(), []) chicken_stream = [s for s in catalog.streams if s.table == 'CHICKEN'][0] chicken_stream = select_all_of_stream(chicken_stream) chicken_stream = set_replication_method_for_stream(chicken_stream, 'LOG_BASED') cur = conn.cursor() prev_scn = cur.execute("SELECT current_scn FROM V$DATABASE").fetchall()[0][0] crud_up_log_miner_fixtures(cur, 'CHICKEN', { '"name-char-explicit-byte"':'name-char-explicit-byte I', '"name-char-explicit-char"':'name-char-explicit-char I', 'name_nchar' : 'name-nchar I', '"name-nvarchar2"' : 'name-nvarchar2 I', '"name-varchar-explicit-byte"' : 'name-varchar-explicit-byte I', '"name-varchar-explicit-char"' : 'name-varchar-explicit-char I', '"name-varchar2-explicit-byte"': 'name-varchar2-explicit-byte I', '"name-varchar2-explicit-char"': 'name-varchar2-explicit-char I' }, lambda s: s.replace("I", "II")) post_scn = cur.execute("SELECT current_scn FROM V$DATABASE").fetchall()[0][0] LOGGER.info("post SCN: {}".format(post_scn)) state = write_bookmark({}, chicken_stream.tap_stream_id, 'scn', prev_scn) state = write_bookmark(state, chicken_stream.tap_stream_id, 'version', 1) tap_oracle.do_sync(get_test_conn_config(), catalog, None, state) verify_crud_messages(self, CAUGHT_MESSAGES, ['ID']) #verify message 1 - first insert insert_rec_1 = CAUGHT_MESSAGES[1].record self.assertIsNotNone(insert_rec_1.get('scn')) insert_rec_1.pop('scn') self.assertEqual(insert_rec_1, {'name-varchar2-explicit-byte': 'name-varchar2-explicit-byte I', 'name-char-explicit-char': 'name-char-explicit-char I ', 'name-nvarchar2': 'name-nvarchar2 I', 'name-varchar-explicit-char': 'name-varchar-explicit-char I', 'name-varchar2-explicit-char': 'name-varchar2-explicit-char I', 'NAME_NCHAR': 'name-nchar I ', 'name-char-explicit-byte': 'name-char-explicit-byte I ', '_sdc_deleted_at': None, 'name-varchar-explicit-byte': 'name-varchar-explicit-byte I', 'ID': 1}) #verify UPDATE update_rec = CAUGHT_MESSAGES[5].record self.assertIsNotNone(update_rec.get('scn')) update_rec.pop('scn') self.assertEqual(update_rec, {'name-varchar2-explicit-byte': 'name-varchar2-explicit-byte II', 'name-char-explicit-char': 'name-char-explicit-char II ', 'name-nvarchar2': 'name-nvarchar2 II', 'name-varchar-explicit-char': 'name-varchar-explicit-char II', 'name-varchar2-explicit-char': 'name-varchar2-explicit-char II', 'NAME_NCHAR': 'name-nchar II ', 'name-char-explicit-byte': 'name-char-explicit-byte II ', '_sdc_deleted_at': None, 'name-varchar-explicit-byte': 'name-varchar-explicit-byte II', 'ID': 1}) #verify first DELETE message delete_rec = CAUGHT_MESSAGES[9].record self.assertIsNotNone(delete_rec.get('scn')) self.assertIsNotNone(delete_rec.get('_sdc_deleted_at')) delete_rec.pop('scn') delete_rec.pop('_sdc_deleted_at') self.assertEqual(delete_rec, {'name-varchar2-explicit-byte': 'name-varchar2-explicit-byte II', 'name-char-explicit-char': 'name-char-explicit-char II ', 'name-nvarchar2': 'name-nvarchar2 II', 'name-varchar-explicit-char': 'name-varchar-explicit-char II', 'name-varchar2-explicit-char': 'name-varchar2-explicit-char II', 'NAME_NCHAR': 'name-nchar II ', 'name-char-explicit-byte': 'name-char-explicit-byte II ', 'name-varchar-explicit-byte': 'name-varchar-explicit-byte II', 'ID': 1}) #verify second DELETE message delete_rec_2 = CAUGHT_MESSAGES[11].record self.assertIsNotNone(delete_rec_2.get('scn')) self.assertIsNotNone(delete_rec_2.get('_sdc_deleted_at')) delete_rec_2.pop('scn') delete_rec_2.pop('_sdc_deleted_at') self.assertEqual(delete_rec_2, {'name-varchar2-explicit-byte': 'name-varchar2-explicit-byte II', 'name-char-explicit-char': 'name-char-explicit-char II ', 'name-nvarchar2': 'name-nvarchar2 II', 'name-varchar-explicit-char': 'name-varchar-explicit-char II', 'name-varchar2-explicit-char': 'name-varchar2-explicit-char II', 'NAME_NCHAR': 'name-nchar II ', 'name-char-explicit-byte': 'name-char-explicit-byte II ', 'name-varchar-explicit-byte': 'name-varchar-explicit-byte II', 'ID': 2})
def test_catalog(self): singer.write_message = singer_write_message log_miner.UPDATE_BOOKMARK_PERIOD = 1 with get_test_connection() as conn: conn.autocommit = True catalog = tap_oracle.do_discovery(get_test_conn_config(), []) chicken_stream = [ s for s in catalog.streams if s.table == 'CHICKEN' ][0] chicken_stream = select_all_of_stream(chicken_stream) chicken_stream = set_replication_method_for_stream( chicken_stream, 'LOG_BASED') cur = conn.cursor() prev_scn = cur.execute( "SELECT current_scn FROM V$DATABASE").fetchall()[0][0] our_fake_float = decimal.Decimal('1234567.8901234') our_real_float = 1234567.8901234 crud_up_log_miner_fixtures( cur, 'CHICKEN', { '"our_double_precision"': our_fake_float, '"our_real"': our_fake_float, '"our_binary_float"': our_real_float, '"our_binary_double"': our_real_float, '"our_nan"': float('nan'), '"our_+_infinity"': float('+inf'), '"our_-_infinity"': float('-inf') }, self.update_add_5) post_scn = cur.execute( "SELECT current_scn FROM V$DATABASE").fetchall()[0][0] LOGGER.info("post SCN: {}".format(post_scn)) state = write_bookmark({}, chicken_stream.tap_stream_id, 'scn', prev_scn) state = write_bookmark(state, chicken_stream.tap_stream_id, 'version', 1) tap_oracle.do_sync(get_test_conn_config(), catalog, None, state) verify_crud_messages(self, CAUGHT_MESSAGES, ['our_float']) #verify message 1 - first insert insert_rec_1 = CAUGHT_MESSAGES[1].record self.assertIsNotNone(insert_rec_1.get('scn')) insert_rec_1.pop('scn') self.assertEqual(float('+inf'), insert_rec_1.get('our_+_infinity')) insert_rec_1.pop('our_+_infinity') self.assertEqual(float('-inf'), insert_rec_1.get('our_-_infinity')) insert_rec_1.pop('our_-_infinity') self.assertTrue(math.isnan(insert_rec_1.get('our_nan'))) insert_rec_1.pop('our_nan') self.assertEqual( insert_rec_1, { 'our_float': '1', 'our_double_precision': str(our_fake_float), 'our_real': str(our_fake_float), 'our_binary_float': 1234567.88, #weird 'our_binary_double': 1234567.890123, '_sdc_deleted_at': None }) #verify UPDATE update_rec = CAUGHT_MESSAGES[5].record self.assertIsNotNone(update_rec.get('scn')) update_rec.pop('scn') self.assertEqual(float('+inf'), update_rec.get('our_+_infinity')) update_rec.pop('our_+_infinity') self.assertEqual(float('-inf'), update_rec.get('our_-_infinity')) update_rec.pop('our_-_infinity') self.assertTrue(math.isnan(update_rec.get('our_nan'))) update_rec.pop('our_nan') self.assertEqual( update_rec, { 'our_binary_double': 1234572.890123, '_sdc_deleted_at': None, 'our_binary_float': 1234572.88, 'our_float': '1', 'our_double_precision': '1234572.8901234', 'our_real': '1234572.8901234' }) #verify first DELETE message delete_rec = CAUGHT_MESSAGES[9].record self.assertEqual(float('+inf'), delete_rec.get('our_+_infinity')) delete_rec.pop('our_+_infinity') self.assertEqual(float('-inf'), delete_rec.get('our_-_infinity')) delete_rec.pop('our_-_infinity') self.assertTrue(math.isnan(delete_rec.get('our_nan'))) delete_rec.pop('our_nan') self.assertIsNotNone(delete_rec.get('scn')) self.assertIsNotNone(delete_rec.get('_sdc_deleted_at')) delete_rec.pop('scn') delete_rec.pop('_sdc_deleted_at') self.assertEqual( delete_rec, { 'our_binary_double': 1234572.890123, 'our_binary_float': 1234572.88, 'our_float': '1', 'our_double_precision': '1234572.8901234', 'our_real': '1234572.8901234' }) #verify second DELETE message delete_rec_2 = CAUGHT_MESSAGES[11].record self.assertEqual(float('+inf'), delete_rec_2.get('our_+_infinity')) delete_rec_2.pop('our_+_infinity') self.assertEqual(float('-inf'), delete_rec_2.get('our_-_infinity')) delete_rec_2.pop('our_-_infinity') self.assertTrue(math.isnan(delete_rec_2.get('our_nan'))) delete_rec_2.pop('our_nan') self.assertIsNotNone(delete_rec_2.get('scn')) self.assertIsNotNone(delete_rec_2.get('_sdc_deleted_at')) delete_rec_2.pop('scn') delete_rec_2.pop('_sdc_deleted_at') self.assertEqual( delete_rec_2, { 'our_binary_double': 1234572.890123, 'our_binary_float': 1234572.88, 'our_float': '2', 'our_double_precision': '1234572.8901234', 'our_real': '1234572.8901234' })
def test_catalog(self): singer.write_message = singer_write_message log_miner.UPDATE_BOOKMARK_PERIOD = 1 with get_test_connection() as conn: conn.autocommit = True catalog = tap_oracle.do_discovery(get_test_conn_config(), []) chicken_stream = [ s for s in catalog.streams if s.table == 'CHICKEN' ][0] chicken_stream = select_all_of_stream(chicken_stream) chicken_stream = set_replication_method_for_stream( chicken_stream, 'LOG_BASED') cur = conn.cursor() prev_scn = cur.execute( "SELECT current_scn FROM V$DATABASE").fetchall()[0][0] crud_up_log_miner_fixtures( cur, 'CHICKEN', { '"our_number_10_2"': decimal.Decimal('100.11'), '"our_number_38_4"': decimal.Decimal('99999999999999999.99991') }, self.update_add_5) post_scn = cur.execute( "SELECT current_scn FROM V$DATABASE").fetchall()[0][0] LOGGER.info("post SCN: {}".format(post_scn)) state = write_bookmark({}, chicken_stream.tap_stream_id, 'scn', prev_scn) state = write_bookmark(state, chicken_stream.tap_stream_id, 'version', 1) tap_oracle.do_sync(get_test_conn_config(), catalog, None, state) verify_crud_messages(self, CAUGHT_MESSAGES, ['our_number']) #verify message 1 - first insert insert_rec_1 = CAUGHT_MESSAGES[1].record self.assertIsNotNone(insert_rec_1.get('scn')) insert_rec_1.pop('scn') self.assertEqual( insert_rec_1, { 'our_number_38_4': decimal.Decimal('99999999999999999.9999'), 'our_number': 1, 'our_number_10_2': decimal.Decimal('100.11'), '_sdc_deleted_at': None }) #verify UPDATE update_rec = CAUGHT_MESSAGES[5].record self.assertIsNotNone(update_rec.get('scn')) update_rec.pop('scn') self.assertEqual( update_rec, { 'our_number_38_4': decimal.Decimal('100000000000000004.9999'), 'our_number': 1, 'our_number_10_2': decimal.Decimal('105.11'), '_sdc_deleted_at': None }) #verify first DELETE message delete_rec = CAUGHT_MESSAGES[9].record self.assertIsNotNone(delete_rec.get('scn')) self.assertIsNotNone(delete_rec.get('_sdc_deleted_at')) delete_rec.pop('scn') delete_rec.pop('_sdc_deleted_at') self.assertEqual( delete_rec, { 'our_number_38_4': decimal.Decimal('100000000000000004.9999'), 'our_number': 1, 'our_number_10_2': decimal.Decimal('105.11') }) #verify second DELETE message delete_rec_2 = CAUGHT_MESSAGES[11].record self.assertIsNotNone(delete_rec_2.get('scn')) self.assertIsNotNone(delete_rec_2.get('_sdc_deleted_at')) delete_rec_2.pop('scn') delete_rec_2.pop('_sdc_deleted_at') self.assertEqual( delete_rec_2, { 'our_number_38_4': decimal.Decimal('100000000000000004.9999'), 'our_number': 2, 'our_number_10_2': decimal.Decimal('105.11') })
def test_catalog(self): singer.write_message = singer_write_message log_miner.UPDATE_BOOKMARK_PERIOD = 1 with get_test_connection() as conn: conn.autocommit = True catalog = tap_oracle.do_discovery(get_test_conn_config(), []) chicken_stream = [ s for s in catalog.streams if s.table == 'CHICKEN' ][0] chicken_stream = select_all_of_stream(chicken_stream) chicken_stream = set_replication_method_for_stream( chicken_stream, 'LOG_BASED') cur = conn.cursor() prev_scn = cur.execute( "SELECT current_scn FROM V$DATABASE").fetchall()[0][0] our_date = datetime.date(1996, 6, 6) our_ts = datetime.datetime(1997, 2, 2, 2, 2, 2, 722184) nyc_tz = pytz.timezone('America/New_York') our_ts_tz_edt = nyc_tz.localize( datetime.datetime(1997, 3, 3, 3, 3, 3, 722184)) our_ts_tz_utc = datetime.datetime(1997, 3, 3, 3, 3, 3, 722184, pytz.UTC) auckland_tz = pytz.timezone('Pacific/Auckland') our_ts_local = auckland_tz.localize( datetime.datetime(1997, 3, 3, 18, 3, 3, 722184)) crud_up_log_miner_fixtures( cur, 'CHICKEN', { '"our_date"': our_date, '"our_ts"': our_ts, '"our_ts_tz_edt"': our_ts_tz_edt, '"our_ts_tz_utc"': our_ts_tz_utc, '"our_ts_tz_local"': our_ts_local }, self.update_add_1_day) post_scn = cur.execute( "SELECT current_scn FROM V$DATABASE").fetchall()[0][0] LOGGER.info("post SCN: {}".format(post_scn)) state = write_bookmark({}, chicken_stream.tap_stream_id, 'scn', prev_scn) state = write_bookmark(state, chicken_stream.tap_stream_id, 'version', 1) tap_oracle.do_sync(get_test_conn_config(), catalog, None, state) verify_crud_messages(self, CAUGHT_MESSAGES, ['ID']) #verify message 1 - first insert insert_rec_1 = CAUGHT_MESSAGES[1].record self.assertIsNotNone(insert_rec_1.get('scn')) insert_rec_1.pop('scn') self.assertIsNone(insert_rec_1.get('_sdc_deleted_at')) self.assertEqual(insert_rec_1.get('ID'), 1) insert_rec_1.pop('_sdc_deleted_at') insert_rec_1.pop('ID') self.assertEqual( insert_rec_1, { 'our_ts': '1997-02-02T02:02:02.722184+00:00', 'our_ts_tz_edt': '1997-03-03T03:03:03.722184-05:00', 'our_ts_tz_utc': '1997-03-03T03:03:03.722184+00:00', 'our_date': '1996-06-06T00:00:00.00+00:00', 'our_ts_tz_local': '1997-03-03T05:03:03.722184+00:00' }) for v in insert_rec_1.values(): self.assertTrue(strict_rfc3339.validate_rfc3339(v)) #verify UPDATE update_rec = CAUGHT_MESSAGES[5].record self.assertIsNotNone(update_rec.get('scn')) update_rec.pop('scn') self.assertEqual( update_rec, { 'our_ts': '1997-02-03T02:02:02.722184+00:00', 'our_ts_tz_edt': '1997-03-04T03:03:03.722184-05:00', 'our_ts_tz_utc': '1997-03-04T03:03:03.722184+00:00', 'our_date': '1996-06-07T00:00:00.00+00:00', '_sdc_deleted_at': None, 'our_ts_tz_local': '1997-03-04T05:03:03.722184+00:00', 'ID': 1 }) #verify first DELETE message delete_rec = CAUGHT_MESSAGES[9].record self.assertIsNotNone(delete_rec.get('scn')) self.assertIsNotNone(delete_rec.get('_sdc_deleted_at')) delete_rec.pop('scn') delete_rec.pop('_sdc_deleted_at') self.assertEqual( delete_rec, { 'our_ts': '1997-02-03T02:02:02.722184+00:00', 'our_ts_tz_edt': '1997-03-04T03:03:03.722184-05:00', 'our_ts_tz_utc': '1997-03-04T03:03:03.722184+00:00', 'our_date': '1996-06-07T00:00:00.00+00:00', 'our_ts_tz_local': '1997-03-04T05:03:03.722184+00:00', 'ID': 1 }) #verify second DELETE message delete_rec_2 = CAUGHT_MESSAGES[11].record self.assertIsNotNone(delete_rec_2.get('scn')) self.assertIsNotNone(delete_rec_2.get('_sdc_deleted_at')) delete_rec_2.pop('scn') delete_rec_2.pop('_sdc_deleted_at') self.assertEqual( delete_rec_2, { 'our_ts': '1997-02-03T02:02:02.722184+00:00', 'our_ts_tz_edt': '1997-03-04T03:03:03.722184-05:00', 'our_ts_tz_utc': '1997-03-04T03:03:03.722184+00:00', 'our_date': '1996-06-07T00:00:00.00+00:00', 'our_ts_tz_local': '1997-03-04T05:03:03.722184+00:00', 'ID': 2 })