def test_catalog(self):
with get_test_connection() as conn:
catalog = tap_oracle.do_discovery(get_test_conn_config(), ['ROOT'])
discovered_streams = {}
for s in catalog.streams:
schema_name = [
md['metadata']['schema-name'] for md in s.metadata
if md['breadcrumb'] == () and md['metadata']
][0]
if discovered_streams.get(schema_name) is None:
discovered_streams[schema_name] = [s.tap_stream_id]
else:
discovered_streams[schema_name].append(s.tap_stream_id)
self.assertEqual(list(discovered_streams.keys()), ['ROOT'])
def test_catalog(self):
singer.write_message = singer_write_message_no_cow
with get_test_connection() as conn:
conn.autocommit = True
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
cow_stream = [s for s in catalog.streams if s.table == 'COW'][0]
cow_stream = select_all_of_stream(cow_stream)
cow_stream = set_replication_method_for_stream(
cow_stream, 'FULL_TABLE')
chicken_stream = [
s for s in catalog.streams if s.table == 'CHICKEN'
][0]
chicken_stream = select_all_of_stream(chicken_stream)
chicken_stream = set_replication_method_for_stream(
chicken_stream, 'FULL_TABLE')
cur = conn.cursor()
cow_rec = {'NAME': 'betty', 'colour': 'blue'}
insert_record(cur, 'COW', cow_rec)
chicken_rec = {'NAME': 'fred', 'colour': 'red'}
insert_record(cur, 'CHICKEN', chicken_rec)
state = {}
#this will sync the CHICKEN but then blow up on the COW
try:
tap_oracle.do_sync(get_test_conn_config(), catalog, None,
state)
except Exception:
blew_up_on_cow = True
self.assertTrue(blew_up_on_cow)
self.assertEqual(9, len(CAUGHT_MESSAGES))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[0], singer.SchemaMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[1],
singer.StateMessage))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[2], singer.ActivateVersionMessage))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[3], singer.RecordMessage))
self.assertEqual('CHICKEN', CAUGHT_MESSAGES[3].stream)
self.assertTrue(
isinstance(CAUGHT_MESSAGES[4], singer.ActivateVersionMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[5],
singer.StateMessage))
self.assertEqual(
None, singer.get_currently_syncing(CAUGHT_MESSAGES[5].value))
#cow messages
self.assertTrue(
isinstance(CAUGHT_MESSAGES[6], singer.SchemaMessage))
self.assertEqual("COW", CAUGHT_MESSAGES[6].stream)
self.assertTrue(isinstance(CAUGHT_MESSAGES[7],
singer.StateMessage))
old_state = CAUGHT_MESSAGES[7].value
self.assertEqual("ROOT-COW", old_state.get('currently_syncing'))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[8], singer.ActivateVersionMessage))
#run another do_sync which will resume with COW but then also do chicken
singer.write_message = singer_write_message_ok
CAUGHT_MESSAGES.clear()
tap_oracle.do_sync(get_test_conn_config(), catalog, None,
old_state)
#cow messages
self.assertEqual(10, len(CAUGHT_MESSAGES))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[0], singer.SchemaMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[1],
singer.StateMessage))
self.assertEqual(
"ROOT-COW",
singer.get_currently_syncing(CAUGHT_MESSAGES[1].value))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[2], singer.RecordMessage))
self.assertEqual('COW', CAUGHT_MESSAGES[2].stream)
self.assertTrue(
isinstance(CAUGHT_MESSAGES[3], singer.ActivateVersionMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[4],
singer.StateMessage))
self.assertEqual(
None, singer.get_currently_syncing(CAUGHT_MESSAGES[4].value))
#chicken messages
self.assertTrue(
isinstance(CAUGHT_MESSAGES[5], singer.SchemaMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[6],
singer.StateMessage))
self.assertEqual(
"ROOT-CHICKEN",
singer.get_currently_syncing(CAUGHT_MESSAGES[6].value))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[7], singer.RecordMessage))
self.assertEqual('CHICKEN', CAUGHT_MESSAGES[7].stream)
self.assertTrue(
isinstance(CAUGHT_MESSAGES[8], singer.ActivateVersionMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[9],
singer.StateMessage))
self.assertEqual(
None, singer.get_currently_syncing(CAUGHT_MESSAGES[9].value))
def test_catalog(self):
singer.write_message = singer_write_message
with get_test_connection() as conn:
conn.autocommit = True
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
chicken_stream = [
s for s in catalog.streams if s.table == 'CHICKEN'
][0]
mdata = metadata.to_map(chicken_stream.metadata)
self.assertEqual(
mdata, {
('properties', 'AGE'): {
'inclusion': 'available',
'selected-by-default': True,
'sql-datatype': 'NUMBER'
},
(): {
'is-view': False,
'row-count': 0,
'table-key-properties': [],
'schema-name': 'ROOT',
'database-name': os.getenv('TAP_ORACLE_SID')
},
('properties', 'INTERVAL_COLUMN'): {
'inclusion': 'unsupported',
'selected-by-default': False,
'sql-datatype': 'INTERVAL DAY(2) TO SECOND(6)'
}
})
chicken_stream = select_all_of_stream(chicken_stream)
chicken_stream = set_replication_method_for_stream(
chicken_stream, 'FULL_TABLE')
cur = conn.cursor()
cur.execute("""
INSERT INTO CHICKEN (AGE, INTERVAL_COLUMN) values (3,
TIMESTAMP '2001-09-04 17:00:00.000000' - TIMESTAMP '2001-09-03 17:00:00.000000'
)""")
state = {}
tap_oracle.do_sync(get_test_conn_config(), catalog, None, state)
#messages: ActivateVersion, SchemaMessage, Record, Record, State, ActivateVersion
self.assertEqual(6, len(CAUGHT_MESSAGES))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[0], singer.SchemaMessage))
self.assertEqual([], CAUGHT_MESSAGES[0].key_properties)
self.assertTrue(isinstance(CAUGHT_MESSAGES[1],
singer.StateMessage))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[2], singer.ActivateVersionMessage))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[3], singer.RecordMessage))
self.assertEqual({'AGE': 3}, CAUGHT_MESSAGES[3].record)
self.assertTrue(
isinstance(CAUGHT_MESSAGES[4], singer.ActivateVersionMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[5],
singer.StateMessage))
def test_catalog(self):
singer.write_message = singer_write_message
with get_test_connection() as conn:
conn.autocommit = True
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
chicken_stream = [
s for s in catalog.streams if s.table == 'CHICKEN'
][0]
chicken_stream = select_all_of_stream(chicken_stream)
#unselect the NO_SYNC column
chicken_stream = unselect_column(chicken_stream, 'NO_SYNC')
chicken_stream = set_replication_method_for_stream(
chicken_stream, 'FULL_TABLE')
cur = conn.cursor()
our_date = datetime.date(1996, 6, 6)
our_ts = datetime.datetime(1997, 2, 2, 2, 2, 2, 722184)
nyc_tz = pytz.timezone('America/New_York')
our_ts_tz_edt = nyc_tz.localize(
datetime.datetime(1997, 3, 3, 3, 3, 3, 722184))
our_ts_tz_utc = datetime.datetime(1997, 3, 3, 3, 3, 3, 722184,
pytz.UTC)
california_tz = pytz.timezone('America/Los_Angeles')
our_ts_local = california_tz.localize(
datetime.datetime(2018, 4, 1, 10, 0, 0, 000000))
our_float = decimal.Decimal(
'1234567.890123456789012345678901234567890123456789')
our_real = our_float
our_double_precision = our_float
rec_1 = {
'"none_column"': None,
'NO_SYNC': 666, #should not sync this column
'"size_number"': 1E-6,
'"size_number_*"': 100.12345,
'"size_number_4"': 100.12345,
'"size_number_4_0"': 100.12345,
'"size_number_*_0"': 2**128, # 39 Decimal Places in pow(2)
'"size_number_*_38"': 1E-6,
'"size_number_10_-1"': 311.12345,
'"size_number_integer"': 400.12345,
'"size_number_int"': 500.12345,
'"size_number_smallint"': 50000.12345,
'"our_number_10_2"': decimal.Decimal('100.11'),
'"our_number_38_4"':
decimal.Decimal('99999999999999999.99991'),
'"our_double_precision"': our_double_precision,
'"our_real"': our_real,
'"our_float"': our_float,
'"our_binary_float"': 1234567.8901234,
'"our_binary_double"': 1234567.8901234,
'"our_nan"': float('nan'),
'"our_+_infinity"': float('+inf'),
'"our_-_infinity"': float('-inf'),
'"our_date"': our_date,
'"our_ts"': our_ts,
'"our_ts_tz_edt"': our_ts_tz_edt,
'"our_ts_tz_utc"': our_ts_tz_utc,
'"our_ts_tz_local"': our_ts_local,
'"name-char-explicit-byte"': 'name-char-explicit-byte I',
'"name-char-explicit-char"': 'name-char-explicit-char I',
'NAME_NCHAR': 'name-nchar I',
'"name-nvarchar2"': 'name-nvarchar2 I',
'"name-varchar-explicit-byte"': 'name-varchar-explicit-byte I',
'"name-varchar-explicit-char"': 'name-varchar-explicit-char I',
'"name-varchar2-explicit-byte"':
'name-varchar2-explicit-byte I',
'"name-varchar2-explicit-char"':
'name-varchar2-explicit-char I'
}
insert_record(cur, 'CHICKEN', rec_1)
rec_2 = copy.deepcopy(rec_1)
rec_2.update({
'"size_number_4_0"': 101,
'"our_number_10_2"': decimal.Decimal('101.11') + 1,
'"our_double_precision"': our_double_precision + 1,
'"our_date"': our_date + datetime.timedelta(days=1),
'NAME_NCHAR': 'name-nchar II'
})
insert_record(cur, 'CHICKEN', rec_2)
state = {}
tap_oracle.do_sync(get_test_conn_config(), catalog, None, state)
#messages: ActivateVersion, SchemaMessage, Record, Record, State, ActivateVersion
self.assertEqual(7, len(CAUGHT_MESSAGES))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[0], singer.SchemaMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[1],
singer.StateMessage))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[2], singer.ActivateVersionMessage))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[3], singer.RecordMessage))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[4], singer.RecordMessage))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[5], singer.ActivateVersionMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[6],
singer.StateMessage))
state = CAUGHT_MESSAGES[1].value
version = state.get('bookmarks',
{}).get(chicken_stream.tap_stream_id,
{}).get('version')
self.assertIsNotNone(version)
self.assertEqual(CAUGHT_MESSAGES[2].version, version)
self.assertEqual(CAUGHT_MESSAGES[5].version, version)
edt = pytz.timezone('America/New_York')
expected_rec_1 = {
'ID': 1,
'none_column': None,
'size_number': '0.000001',
'size_number_*': '100.12345',
'size_number_4': 100,
'size_number_4_0': 100,
'size_number_*_0': 2**128,
'size_number_*_38': '0.000001',
'size_number_10_-1': 310,
'size_number_integer': 400,
'size_number_int': 500,
'size_number_smallint': 50000,
'our_number_10_2': '100.11',
'our_number_38_4': '99999999999999999.9999',
'our_double_precision':
'1234567.8901234567890123456789012345679',
'our_float': '1234567.8901234567890123456789012345679',
'our_real': '1234567.890123456789',
'our_binary_float': 1234567.875,
'our_binary_double': 1234567.890123,
'our_+_infinity': float('+inf'),
'our_-_infinity': float('-inf'),
'our_date': '1996-06-06T00:00:00.00+00:00',
'our_ts': '1997-02-02T02:02:02.722184+00:00',
'our_ts_tz_edt': '1997-03-03T03:03:03.722184-05:00',
'our_ts_tz_utc': '1997-03-03T03:03:03.722184+00:00',
'our_ts_tz_local': '2018-04-01T17:00:00.000000+00:00',
'name-char-explicit-byte':
'name-char-explicit-byte I ',
'name-char-explicit-char':
'name-char-explicit-char I ',
'NAME_NCHAR':
'name-nchar I ',
'name-nvarchar2': 'name-nvarchar2 I',
'name-varchar-explicit-byte': 'name-varchar-explicit-byte I',
'name-varchar-explicit-char': 'name-varchar-explicit-char I',
'name-varchar2-explicit-byte': 'name-varchar2-explicit-byte I',
'name-varchar2-explicit-char': 'name-varchar2-explicit-char I'
}
self.assertTrue(
math.isnan(CAUGHT_MESSAGES[3].record.get('our_nan')))
CAUGHT_MESSAGES[3].record.pop('our_nan')
self.assertEqual(CAUGHT_MESSAGES[3].record, expected_rec_1)
expected_rec_2 = expected_rec_1
expected_rec_2.update({
'ID':
2,
'size_number_4_0':
101,
'our_number_10_2':
'102.11',
'our_double_precision':
'1234568.890123456789012345679',
'our_date':
'1996-06-07T00:00:00.00+00:00',
'NAME_NCHAR':
'name-nchar II '
})
self.assertTrue(
math.isnan(CAUGHT_MESSAGES[4].record.get('our_nan')))
CAUGHT_MESSAGES[4].record.pop('our_nan')
self.assertEqual(CAUGHT_MESSAGES[4].record, expected_rec_2)
#run another do_sync
CAUGHT_MESSAGES.clear()
tap_oracle.do_sync(get_test_conn_config(), catalog, None, state)
self.assertEqual(6, len(CAUGHT_MESSAGES))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[0], singer.SchemaMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[1],
singer.StateMessage))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[2], singer.RecordMessage))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[3], singer.RecordMessage))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[4], singer.ActivateVersionMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[5],
singer.StateMessage))
nascent_version = CAUGHT_MESSAGES[1].value.get(
'bookmarks', {}).get(chicken_stream.tap_stream_id,
{}).get('version')
self.assertTrue(nascent_version > version)
def test_catalog(self):
singer.write_message = singer_write_message
log_miner.UPDATE_BOOKMARK_PERIOD = 1
with get_test_connection() as conn:
conn.autocommit = True
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
chicken_stream = [s for s in catalog.streams if s.table == 'CHICKEN'][0]
chicken_stream = select_all_of_stream(chicken_stream)
chicken_stream = set_replication_method_for_stream(chicken_stream, 'LOG_BASED')
cur = conn.cursor()
prev_scn = cur.execute("SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
crud_up_log_miner_fixtures(cur, 'CHICKEN',
{
'"name-char-explicit-byte"':'name-char-explicit-byte I',
'"name-char-explicit-char"':'name-char-explicit-char I',
'name_nchar' : 'name-nchar I',
'"name-nvarchar2"' : 'name-nvarchar2 I',
'"name-varchar-explicit-byte"' : 'name-varchar-explicit-byte I',
'"name-varchar-explicit-char"' : 'name-varchar-explicit-char I',
'"name-varchar2-explicit-byte"': 'name-varchar2-explicit-byte I',
'"name-varchar2-explicit-char"': 'name-varchar2-explicit-char I'
}, lambda s: s.replace("I", "II"))
post_scn = cur.execute("SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
LOGGER.info("post SCN: {}".format(post_scn))
state = write_bookmark({}, chicken_stream.tap_stream_id, 'scn', prev_scn)
state = write_bookmark(state, chicken_stream.tap_stream_id, 'version', 1)
tap_oracle.do_sync(get_test_conn_config(), catalog, None, state)
verify_crud_messages(self, CAUGHT_MESSAGES, ['ID'])
#verify message 1 - first insert
insert_rec_1 = CAUGHT_MESSAGES[1].record
self.assertIsNotNone(insert_rec_1.get('scn'))
insert_rec_1.pop('scn')
self.assertEqual(insert_rec_1, {'name-varchar2-explicit-byte': 'name-varchar2-explicit-byte I',
'name-char-explicit-char': 'name-char-explicit-char I ',
'name-nvarchar2': 'name-nvarchar2 I', 'name-varchar-explicit-char': 'name-varchar-explicit-char I',
'name-varchar2-explicit-char': 'name-varchar2-explicit-char I',
'NAME_NCHAR': 'name-nchar I ',
'name-char-explicit-byte': 'name-char-explicit-byte I ',
'_sdc_deleted_at': None, 'name-varchar-explicit-byte': 'name-varchar-explicit-byte I', 'ID': 1})
#verify UPDATE
update_rec = CAUGHT_MESSAGES[5].record
self.assertIsNotNone(update_rec.get('scn'))
update_rec.pop('scn')
self.assertEqual(update_rec, {'name-varchar2-explicit-byte': 'name-varchar2-explicit-byte II',
'name-char-explicit-char': 'name-char-explicit-char II ',
'name-nvarchar2': 'name-nvarchar2 II', 'name-varchar-explicit-char': 'name-varchar-explicit-char II',
'name-varchar2-explicit-char': 'name-varchar2-explicit-char II',
'NAME_NCHAR': 'name-nchar II ',
'name-char-explicit-byte': 'name-char-explicit-byte II ',
'_sdc_deleted_at': None, 'name-varchar-explicit-byte': 'name-varchar-explicit-byte II', 'ID': 1})
#verify first DELETE message
delete_rec = CAUGHT_MESSAGES[9].record
self.assertIsNotNone(delete_rec.get('scn'))
self.assertIsNotNone(delete_rec.get('_sdc_deleted_at'))
delete_rec.pop('scn')
delete_rec.pop('_sdc_deleted_at')
self.assertEqual(delete_rec, {'name-varchar2-explicit-byte': 'name-varchar2-explicit-byte II',
'name-char-explicit-char': 'name-char-explicit-char II ',
'name-nvarchar2': 'name-nvarchar2 II', 'name-varchar-explicit-char': 'name-varchar-explicit-char II',
'name-varchar2-explicit-char': 'name-varchar2-explicit-char II',
'NAME_NCHAR': 'name-nchar II ',
'name-char-explicit-byte': 'name-char-explicit-byte II ',
'name-varchar-explicit-byte': 'name-varchar-explicit-byte II', 'ID': 1})
#verify second DELETE message
delete_rec_2 = CAUGHT_MESSAGES[11].record
self.assertIsNotNone(delete_rec_2.get('scn'))
self.assertIsNotNone(delete_rec_2.get('_sdc_deleted_at'))
delete_rec_2.pop('scn')
delete_rec_2.pop('_sdc_deleted_at')
self.assertEqual(delete_rec_2, {'name-varchar2-explicit-byte': 'name-varchar2-explicit-byte II',
'name-char-explicit-char': 'name-char-explicit-char II ',
'name-nvarchar2': 'name-nvarchar2 II', 'name-varchar-explicit-char': 'name-varchar-explicit-char II',
'name-varchar2-explicit-char': 'name-varchar2-explicit-char II',
'NAME_NCHAR': 'name-nchar II ',
'name-char-explicit-byte': 'name-char-explicit-byte II ',
'name-varchar-explicit-byte': 'name-varchar-explicit-byte II', 'ID': 2})
def test_catalog(self):
with get_test_connection() as conn:
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
chicken_streams = [
s for s in catalog.streams if s.table == 'CHICKEN'
]
self.assertEqual(len(chicken_streams), 1)
stream_dict = chicken_streams[0].to_dict()
stream_dict.get('metadata').sort(key=lambda md: md['breadcrumb'])
self.assertEqual(
{
'schema': {
'properties': {
'our_float': {
'type': ['number'],
'multipleOf': 1e-38
},
'our_double_precision': {
'type': ['null', 'number'],
'multipleOf': 1e-38
},
'our_real': {
'type': ['null', 'number'],
'multipleOf': 1e-18
},
'our_binary_float': {
'type': ['null', 'number']
},
'our_binary_double': {
'type': ['null', 'number']
}
},
'type': 'object'
},
'stream':
'CHICKEN',
'table_name':
'CHICKEN',
'tap_stream_id':
'ROOT-CHICKEN',
'metadata': [{
'breadcrumb': (),
'metadata': {
'table-key-properties': ["our_float"],
'database-name': os.getenv('TAP_ORACLE_SID'),
'schema-name': 'ROOT',
'is-view': False,
'row-count': 0
}
}, {
'breadcrumb': ('properties', 'our_binary_double'),
'metadata': {
'inclusion': 'available',
'sql-datatype': 'BINARY_DOUBLE',
'selected-by-default': True
}
}, {
'breadcrumb': ('properties', 'our_binary_float'),
'metadata': {
'inclusion': 'available',
'sql-datatype': 'BINARY_FLOAT',
'selected-by-default': True
}
}, {
'breadcrumb': ('properties', 'our_double_precision'),
'metadata': {
'inclusion': 'available',
'sql-datatype': 'FLOAT',
'selected-by-default': True
}
}, {
'breadcrumb': ('properties', 'our_float'),
'metadata': {
'inclusion': 'automatic',
'sql-datatype': 'FLOAT',
'selected-by-default': True
}
}, {
'breadcrumb': ('properties', 'our_real'),
'metadata': {
'inclusion': 'available',
'sql-datatype': 'FLOAT',
'selected-by-default': True
}
}]
}, stream_dict)
def test_catalog(self):
with get_test_connection() as conn:
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
chicken_streams = [
s for s in catalog.streams if s.table == 'CHICKEN'
]
self.assertEqual(len(chicken_streams), 1)
stream_dict = chicken_streams[0].to_dict()
self.assertEqual('CHICKEN', stream_dict.get('table_name'))
self.assertEqual('CHICKEN', stream_dict.get('stream'))
self.assertEqual('ROOT-CHICKEN', stream_dict.get('tap_stream_id'))
stream_dict.get('metadata').sort(key=lambda md: md['breadcrumb'])
self.assertEqual(stream_dict.get('metadata'), [{
'metadata': {
'table-key-properties': ['ID'],
'database-name': os.getenv('TAP_ORACLE_SID'),
'schema-name': 'ROOT',
'is-view': False,
'row-count': 0
},
'breadcrumb': ()
}, {
'metadata': {
'inclusion': 'automatic',
'sql-datatype': 'NUMBER',
'selected-by-default': True
},
'breadcrumb': ('properties', 'ID'),
}, {
'metadata': {
'inclusion': 'available',
'sql-datatype': 'CHAR',
'selected-by-default': True
},
'breadcrumb': ('properties', 'name-char-explicit-byte')
}, {
'metadata': {
'inclusion': 'available',
'sql-datatype': 'CHAR',
'selected-by-default': True
},
'breadcrumb': ('properties', 'name-char-explicit-char')
}, {
'metadata': {
'inclusion': 'available',
'sql-datatype': 'NCHAR',
'selected-by-default': True
},
'breadcrumb': ('properties', 'name-nchar')
}, {
'metadata': {
'inclusion': 'available',
'sql-datatype': 'NVARCHAR2',
'selected-by-default': True
},
'breadcrumb': ('properties', 'name-nvarchar2')
}, {
'metadata': {
'inclusion': 'available',
'sql-datatype': 'VARCHAR2',
'selected-by-default': True
},
'breadcrumb': ('properties', 'name-varchar-explicit-byte')
}, {
'metadata': {
'inclusion': 'available',
'sql-datatype': 'VARCHAR2',
'selected-by-default': True
},
'breadcrumb': ('properties', 'name-varchar-explicit-char')
}, {
'metadata': {
'inclusion': 'available',
'sql-datatype': 'VARCHAR2',
'selected-by-default': True
},
'breadcrumb': ('properties', 'name-varchar2-explicit-byte')
}, {
'metadata': {
'inclusion': 'available',
'sql-datatype': 'VARCHAR2',
'selected-by-default': True
},
'breadcrumb': ('properties', 'name-varchar2-explicit-char')
}])
self.assertEqual(
{
'properties': {
'ID': {
'type': ['integer'],
'maximum': 99999999999999999999999999999999999999,
'minimum': -99999999999999999999999999999999999999
},
'name-char-explicit-byte': {
'type': ['null', 'string']
},
'name-char-explicit-char': {
'type': ['null', 'string'],
'maxLength': 250
},
'name-nchar': {
'type': ['null', 'string'],
'maxLength': 123
},
'name-nvarchar2': {
'type': ['null', 'string'],
'maxLength': 234
},
'name-varchar-explicit-byte': {
'type': ['null', 'string']
},
'name-varchar-explicit-char': {
'type': ['null', 'string'],
'maxLength': 251
},
'name-varchar2-explicit-byte': {
'type': ['null', 'string']
},
'name-varchar2-explicit-char': {
'type': ['null', 'string'],
'maxLength': 251
}
},
'type': 'object'
}, stream_dict.get('schema'))
def test_catalog(self):
with get_test_connection() as conn:
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
chicken_streams = [
s for s in catalog.streams if s.table == 'CHICKEN'
]
self.assertEqual(len(chicken_streams), 1)
stream_dict = chicken_streams[0].to_dict()
stream_dict.get('metadata').sort(key=lambda md: md['breadcrumb'])
self.assertEqual(
{
'schema': {
'properties': {
'our_number': {
'maximum':
99999999999999999999999999999999999999,
'minimum':
-99999999999999999999999999999999999999,
'type': ['integer']
},
'our_number_10_2': {
'exclusiveMaximum': True,
'exclusiveMinimum': True,
'maximum': 100000000,
'minimum': -100000000,
'multipleOf': 0.01,
'type': ['null', 'number']
},
'our_number_38_4': {
'exclusiveMaximum': True,
'exclusiveMinimum': True,
'maximum': 10000000000000000000000000000000000,
'minimum':
-10000000000000000000000000000000000,
'multipleOf': 0.0001,
'type': ['null', 'number']
}
},
'type': 'object'
},
'stream':
'CHICKEN',
'table_name':
'CHICKEN',
'tap_stream_id':
'ROOT-CHICKEN',
'metadata': [{
'breadcrumb': (),
'metadata': {
'table-key-properties': ['our_number'],
'database-name': os.getenv('TAP_ORACLE_SID'),
'schema-name': 'ROOT',
'is-view': False,
'row-count': 0
}
}, {
'breadcrumb': ('properties', 'our_number'),
'metadata': {
'inclusion': 'automatic',
'sql-datatype': 'NUMBER',
'selected-by-default': True
}
}, {
'breadcrumb': ('properties', 'our_number_10_2'),
'metadata': {
'inclusion': 'available',
'sql-datatype': 'NUMBER',
'selected-by-default': True
}
}, {
'breadcrumb': ('properties', 'our_number_38_4'),
'metadata': {
'inclusion': 'available',
'sql-datatype': 'NUMBER',
'selected-by-default': True
}
}]
}, stream_dict)
def test_catalog(self):
with get_test_connection() as conn:
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
chicken_streams = [
s for s in catalog.streams if s.table == 'CHICKEN'
]
self.assertEqual(len(chicken_streams), 1)
stream_dict = chicken_streams[0].to_dict()
stream_dict.get('metadata').sort(key=lambda md: md['breadcrumb'])
self.assertEqual(
{
'schema': {
'properties': {
'our_date': {
'type': ['string'],
'format': 'date-time'
},
'our_ts': {
'type': ['null', 'string'],
'format': 'date-time'
},
'our_ts_tz': {
'type': ['null', 'string'],
'format': 'date-time'
},
'our_ts_tz_local': {
'type': ['null', 'string'],
'format': 'date-time'
}
},
'type': 'object'
},
'stream':
'CHICKEN',
'table_name':
'CHICKEN',
'tap_stream_id':
'ROOT-CHICKEN',
'metadata': [{
'breadcrumb': (),
'metadata': {
'table-key-properties': ['our_date'],
'database-name': os.getenv('TAP_ORACLE_SID'),
'schema-name': 'ROOT',
'is-view': 0,
'row-count': 0
}
}, {
'breadcrumb': ('properties', 'our_date'),
'metadata': {
'inclusion': 'automatic',
'sql-datatype': 'DATE',
'selected-by-default': True
}
}, {
'breadcrumb': ('properties', 'our_ts'),
'metadata': {
'inclusion': 'available',
'sql-datatype': 'TIMESTAMP(6)',
'selected-by-default': True
}
}, {
'breadcrumb': ('properties', 'our_ts_tz'),
'metadata': {
'inclusion': 'available',
'sql-datatype': 'TIMESTAMP(6) WITH TIME ZONE',
'selected-by-default': True
}
}, {
'breadcrumb': ('properties', 'our_ts_tz_local'),
'metadata': {
'inclusion': 'available',
'sql-datatype':
'TIMESTAMP(6) WITH LOCAL TIME ZONE',
'selected-by-default': True
}
}]
}, stream_dict)
def test_catalog(self):
singer.write_message = singer_write_message
with get_test_connection() as conn:
conn.autocommit = True
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
chicken_stream = [s for s in catalog.streams if s.table == 'CHICKEN'][0]
chicken_stream = select_all_of_stream(chicken_stream)
#unselect the NO_SYNC column
chicken_stream = unselect_column(chicken_stream, 'NO_SYNC')
#select logminer
chicken_stream = set_replication_method_for_stream(chicken_stream, 'LOG_BASED')
cur = conn.cursor()
rec_1 = {
'"none_column"' : None,
'"our_number_10_2"' : decimal.Decimal('100.11'),
'"our_binary_float"' : 1234567.8901234,
'"our_date"' : datetime.date(1996, 6, 6),
'"name-char-explicit-byte"' :'name-char-explicit-byte I',
}
insert_record(cur, 'CHICKEN', rec_1)
rec_2 = copy.deepcopy(rec_1)
rec_2.update({'"size_number_4_0"' : 101,
'"our_number_10_2"' : decimal.Decimal('101.11') + 1,
'"our_binary_float"' : 1234567.8901234 + 1,
'"our_date"' : datetime.date(1996, 6, 6) + datetime.timedelta(days=1)
})
insert_record(cur, 'CHICKEN', rec_2)
original_state = {}
#initial run should be full_table
tap_oracle.do_sync(get_test_conn_config(), catalog, None, original_state)
#messages for initial full table replication: ActivateVersion, SchemaMessage, Record, Record, State, ActivateVersion
self.assertEqual(7, len(CAUGHT_MESSAGES))
self.assertTrue(isinstance(CAUGHT_MESSAGES[0], singer.SchemaMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[1], singer.StateMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[2], singer.ActivateVersionMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[3], singer.RecordMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[4], singer.RecordMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[5], singer.ActivateVersionMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[6], singer.StateMessage))
state = CAUGHT_MESSAGES[6].value
version = state.get('bookmarks', {}).get(chicken_stream.tap_stream_id, {}).get('version')
scn = state.get('bookmarks', {}).get(chicken_stream.tap_stream_id, {}).get('scn')
self.assertIsNotNone(version)
self.assertIsNotNone(scn)
self.assertEqual(CAUGHT_MESSAGES[2].version, version)
self.assertEqual(CAUGHT_MESSAGES[5].version, version)
#run another do_sync
CAUGHT_MESSAGES.clear()
rec_3 = copy.deepcopy(rec_2)
rec_3.update({'"size_number_4_0"' : 102,
'"our_number_10_2"' : decimal.Decimal('101.11') + 3,
'"our_binary_float"' : 1234567.8901234 + 2,
'"our_date"' : datetime.date(1996, 6, 6) + datetime.timedelta(days=2)
})
insert_record(cur, 'CHICKEN', rec_3)
#this sync should activate logminer because of the scn in state
tap_oracle.do_sync(get_test_conn_config(), catalog, None, state)
#TODO: assert new scn
self.assertEqual(3, len(CAUGHT_MESSAGES))
self.assertTrue(isinstance(CAUGHT_MESSAGES[0], singer.SchemaMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[1], singer.RecordMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[2], singer.StateMessage))
new_scn = CAUGHT_MESSAGES[2].value.get('bookmarks', {}).get(chicken_stream.tap_stream_id, {}).get('scn')
new_version = CAUGHT_MESSAGES[2].value.get('bookmarks', {}).get(chicken_stream.tap_stream_id, {}).get('version')
self.assertTrue(new_scn > scn)
self.assertTrue(version == new_version)
def test_catalog(self):
singer.write_message = singer_write_message
log_miner.UPDATE_BOOKMARK_PERIOD = 1
with get_test_connection() as conn:
conn.autocommit = True
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
chicken_stream = [
s for s in catalog.streams if s.table == 'CHICKEN'
][0]
chicken_stream = select_all_of_stream(chicken_stream)
chicken_stream = set_replication_method_for_stream(
chicken_stream, 'LOG_BASED')
cur = conn.cursor()
prev_scn = cur.execute(
"SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
crud_up_log_miner_fixtures(
cur, 'CHICKEN', {
'"our_number_10_2"': decimal.Decimal('100.11'),
'"our_number_38_4"':
decimal.Decimal('99999999999999999.99991')
}, self.update_add_5)
post_scn = cur.execute(
"SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
LOGGER.info("post SCN: {}".format(post_scn))
state = write_bookmark({}, chicken_stream.tap_stream_id, 'scn',
prev_scn)
state = write_bookmark(state, chicken_stream.tap_stream_id,
'version', 1)
tap_oracle.do_sync(get_test_conn_config(), catalog, None, state)
verify_crud_messages(self, CAUGHT_MESSAGES, ['our_number'])
#verify message 1 - first insert
insert_rec_1 = CAUGHT_MESSAGES[1].record
self.assertIsNotNone(insert_rec_1.get('scn'))
insert_rec_1.pop('scn')
self.assertEqual(
insert_rec_1, {
'our_number_38_4':
decimal.Decimal('99999999999999999.9999'),
'our_number': 1,
'our_number_10_2': decimal.Decimal('100.11'),
'_sdc_deleted_at': None
})
#verify UPDATE
update_rec = CAUGHT_MESSAGES[5].record
self.assertIsNotNone(update_rec.get('scn'))
update_rec.pop('scn')
self.assertEqual(
update_rec, {
'our_number_38_4':
decimal.Decimal('100000000000000004.9999'),
'our_number': 1,
'our_number_10_2': decimal.Decimal('105.11'),
'_sdc_deleted_at': None
})
#verify first DELETE message
delete_rec = CAUGHT_MESSAGES[9].record
self.assertIsNotNone(delete_rec.get('scn'))
self.assertIsNotNone(delete_rec.get('_sdc_deleted_at'))
delete_rec.pop('scn')
delete_rec.pop('_sdc_deleted_at')
self.assertEqual(
delete_rec, {
'our_number_38_4':
decimal.Decimal('100000000000000004.9999'),
'our_number': 1,
'our_number_10_2': decimal.Decimal('105.11')
})
#verify second DELETE message
delete_rec_2 = CAUGHT_MESSAGES[11].record
self.assertIsNotNone(delete_rec_2.get('scn'))
self.assertIsNotNone(delete_rec_2.get('_sdc_deleted_at'))
delete_rec_2.pop('scn')
delete_rec_2.pop('_sdc_deleted_at')
self.assertEqual(
delete_rec_2, {
'our_number_38_4':
decimal.Decimal('100000000000000004.9999'),
'our_number': 2,
'our_number_10_2': decimal.Decimal('105.11')
})
def test_catalog(self):
singer.write_message = singer_write_message_no_cow
conn_config = get_test_conn_config()
catalog = tap_oracle.do_discovery(conn_config, [])
cow_stream = [s for s in catalog.streams if s.table == 'COW'][0]
self.assertIsNotNone(cow_stream)
cow_stream = select_all_of_stream(cow_stream)
cow_stream = set_replication_method_for_stream(cow_stream, 'LOG_BASED')
with get_test_connection() as conn:
conn.autocommit = True
cur = conn.cursor()
cow_rec = {'name': 'betty', 'colour': 'blue'}
insert_record(cur, 'COW', cow_rec)
cow_rec = {'name': 'smelly', 'colour': 'brow'}
insert_record(cur, 'COW', cow_rec)
cow_rec = {'name': 'pooper', 'colour': 'green'}
insert_record(cur, 'COW', cow_rec)
state = {}
#the initial phase of cows logical replication will be a full table.
#it will sync the first record and then blow up on the 2nd record
try:
tap_oracle.do_sync(get_test_conn_config(), catalog, None, state)
except Exception as ex:
blew_up_on_cow = True
self.assertTrue(blew_up_on_cow)
self.assertEqual(7, len(CAUGHT_MESSAGES))
# import pdb
# pdb.set_trace()
self.assertTrue(isinstance(CAUGHT_MESSAGES[0], singer.SchemaMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[1], singer.StateMessage))
self.assertEqual(CAUGHT_MESSAGES[1].value['currently_syncing'],
'ROOT-COW')
self.assertIsNotNone(
CAUGHT_MESSAGES[1].value['bookmarks']['ROOT-COW']['version'])
self.assertEqual(
CAUGHT_MESSAGES[1].value['bookmarks']['ROOT-COW']
['last_replication_method'], 'LOG_BASED')
self.assertIsNone(CAUGHT_MESSAGES[1].value['bookmarks']
['ROOT-COW'].get('ORA_ROWSCN'))
self.assertIsNotNone(
CAUGHT_MESSAGES[1].value['bookmarks']['ROOT-COW'].get('scn'))
end_scn = CAUGHT_MESSAGES[1].value['bookmarks']['ROOT-COW'].get('scn')
first_version = CAUGHT_MESSAGES[1].value['bookmarks']['ROOT-COW'].get(
'version')
self.assertTrue(
isinstance(CAUGHT_MESSAGES[2], singer.ActivateVersionMessage))
self.assertEqual(CAUGHT_MESSAGES[2].version, first_version)
self.assertTrue(isinstance(CAUGHT_MESSAGES[3], singer.RecordMessage))
self.assertEqual(CAUGHT_MESSAGES[3].record, {
'NAME': 'betty',
'ID': 1,
'COLOUR': 'blue'
})
self.assertEqual('ROOT-COW', CAUGHT_MESSAGES[3].stream)
self.assertEqual(first_version, CAUGHT_MESSAGES[3].version)
self.assertTrue(isinstance(CAUGHT_MESSAGES[4], singer.StateMessage))
#ORA_ROWSCN is set while we are processing the full table replication
self.assertIsNotNone(
CAUGHT_MESSAGES[4].value['bookmarks']['ROOT-COW']['ORA_ROWSCN'])
self.assertEqual(
CAUGHT_MESSAGES[4].value['bookmarks']['ROOT-COW']['scn'], end_scn)
self.assertEqual(
first_version,
CAUGHT_MESSAGES[4].value['bookmarks']['ROOT-COW']['version'])
self.assertEqual(CAUGHT_MESSAGES[5].record['NAME'], 'smelly')
self.assertEqual('ROOT-COW', CAUGHT_MESSAGES[5].stream)
self.assertEqual(first_version, CAUGHT_MESSAGES[5].version)
self.assertTrue(isinstance(CAUGHT_MESSAGES[6], singer.StateMessage))
self.assertEqual(
first_version,
CAUGHT_MESSAGES[6].value['bookmarks']['ROOT-COW']['version'])
last_ora_rowscn = CAUGHT_MESSAGES[6].value['bookmarks']['ROOT-COW'][
'ORA_ROWSCN']
old_state = CAUGHT_MESSAGES[6].value
#run another do_sync, should get the remaining record which effectively finishes the initial full_table
#replication portion of the logical replication
singer.write_message = singer_write_message_ok
global COW_RECORD_COUNT
COW_RECORD_COUNT = 0
CAUGHT_MESSAGES.clear()
tap_oracle.do_sync(get_test_conn_config(), catalog, None, old_state)
self.assertEqual(8, len(CAUGHT_MESSAGES))
self.assertTrue(isinstance(CAUGHT_MESSAGES[0], singer.SchemaMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[1], singer.StateMessage))
self.assertEqual(
CAUGHT_MESSAGES[1].value['bookmarks']['ROOT-COW'].get(
'ORA_ROWSCN'), last_ora_rowscn)
self.assertEqual(
CAUGHT_MESSAGES[1].value['bookmarks']['ROOT-COW'].get('scn'),
end_scn)
self.assertEqual(
CAUGHT_MESSAGES[1].value['bookmarks']['ROOT-COW'].get('version'),
first_version)
self.assertTrue(isinstance(CAUGHT_MESSAGES[2], singer.RecordMessage))
self.assertEqual(CAUGHT_MESSAGES[2].record, {
'COLOUR': 'brow',
'ID': 2,
'NAME': 'smelly'
})
self.assertEqual('ROOT-COW', CAUGHT_MESSAGES[2].stream)
self.assertTrue(isinstance(CAUGHT_MESSAGES[3], singer.StateMessage))
self.assertTrue(
CAUGHT_MESSAGES[3].value['bookmarks']['ROOT-COW'].get(
'ORA_ROWSCN'), last_ora_rowscn)
self.assertEqual(
CAUGHT_MESSAGES[3].value['bookmarks']['ROOT-COW'].get('scn'),
end_scn)
self.assertEqual(
CAUGHT_MESSAGES[3].value['bookmarks']['ROOT-COW'].get('version'),
first_version)
self.assertTrue(isinstance(CAUGHT_MESSAGES[4], singer.RecordMessage))
self.assertEqual(CAUGHT_MESSAGES[4].record['NAME'], 'pooper')
self.assertEqual('ROOT-COW', CAUGHT_MESSAGES[4].stream)
self.assertTrue(isinstance(CAUGHT_MESSAGES[5], singer.StateMessage))
self.assertTrue(CAUGHT_MESSAGES[5].value['bookmarks']['ROOT-COW'].get(
'ORA_ROWSCN') > last_ora_rowscn)
self.assertEqual(
CAUGHT_MESSAGES[5].value['bookmarks']['ROOT-COW'].get('scn'),
end_scn)
self.assertEqual(
CAUGHT_MESSAGES[5].value['bookmarks']['ROOT-COW'].get('version'),
first_version)
self.assertTrue(
isinstance(CAUGHT_MESSAGES[6], singer.ActivateVersionMessage))
self.assertEqual(CAUGHT_MESSAGES[6].version, first_version)
self.assertTrue(isinstance(CAUGHT_MESSAGES[7], singer.StateMessage))
self.assertIsNone(CAUGHT_MESSAGES[7].value['bookmarks']
['ROOT-COW'].get('ORA_ROWSCN'))
self.assertEqual(
CAUGHT_MESSAGES[7].value['bookmarks']['ROOT-COW'].get('scn'),
end_scn)
self.assertEqual(
CAUGHT_MESSAGES[7].value['bookmarks']['ROOT-COW'].get('version'),
first_version)
def test_catalog(self):
singer.write_message = singer_write_message_no_cow
conn_config = get_test_conn_config()
catalog = tap_oracle.do_discovery(conn_config, [])
cow_stream = [s for s in catalog.streams if s.table == 'COW'][0]
self.assertIsNotNone(cow_stream)
cow_stream = select_all_of_stream(cow_stream)
cow_stream = set_replication_method_for_stream(cow_stream,
'FULL_TABLE')
chicken_stream = [s for s in catalog.streams
if s.table == 'CHICKEN'][0]
self.assertIsNotNone(chicken_stream)
chicken_stream = select_all_of_stream(chicken_stream)
chicken_stream = set_replication_method_for_stream(
chicken_stream, 'FULL_TABLE')
with get_test_connection() as conn:
conn.autocommit = True
cur = conn.cursor()
cow_rec = {'name': 'betty', 'colour': 'blue'}
insert_record(cur, 'COW', cow_rec)
cow_rec = {'name': 'smelly', 'colour': 'brow'}
insert_record(cur, 'COW', cow_rec)
cow_rec = {'name': 'pooper', 'colour': 'green'}
insert_record(cur, 'COW', cow_rec)
chicken_rec = {'name': 'fred', 'colour': 'red'}
insert_record(cur, 'CHICKEN', chicken_rec)
state = {}
#this will sync the CHICKEN but then blow up on the COW
try:
tap_oracle.do_sync(get_test_conn_config(), catalog, None, state)
except Exception as ex:
# LOGGER.exception(ex)
blew_up_on_cow = True
self.assertTrue(blew_up_on_cow)
self.assertEqual(14, len(CAUGHT_MESSAGES))
self.assertTrue(isinstance(CAUGHT_MESSAGES[0], singer.SchemaMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[1], singer.StateMessage))
self.assertIsNone(CAUGHT_MESSAGES[1].value['bookmarks']
['ROOT-CHICKEN'].get('ORA_ROWSCN'))
self.assertTrue(
isinstance(CAUGHT_MESSAGES[2], singer.ActivateVersionMessage))
new_version = CAUGHT_MESSAGES[2].version
self.assertTrue(isinstance(CAUGHT_MESSAGES[3], singer.RecordMessage))
self.assertEqual('ROOT-CHICKEN', CAUGHT_MESSAGES[3].stream)
self.assertTrue(isinstance(CAUGHT_MESSAGES[4], singer.StateMessage))
#ORA_ROWSCN is set while we are processing the full table replication
self.assertIsNotNone(CAUGHT_MESSAGES[4].value['bookmarks']
['ROOT-CHICKEN']['ORA_ROWSCN'])
self.assertTrue(
isinstance(CAUGHT_MESSAGES[5], singer.ActivateVersionMessage))
self.assertEqual(CAUGHT_MESSAGES[5].version, new_version)
self.assertTrue(isinstance(CAUGHT_MESSAGES[6], singer.StateMessage))
self.assertEqual(
None, singer.get_currently_syncing(CAUGHT_MESSAGES[6].value))
#ORA_ROWSCN is cleared at the end of the full table replication
self.assertIsNone(CAUGHT_MESSAGES[6].value['bookmarks']['ROOT-CHICKEN']
['ORA_ROWSCN'])
#cow messages
self.assertTrue(isinstance(CAUGHT_MESSAGES[7], singer.SchemaMessage))
self.assertEqual("ROOT-COW", CAUGHT_MESSAGES[7].stream)
self.assertTrue(isinstance(CAUGHT_MESSAGES[8], singer.StateMessage))
self.assertIsNone(CAUGHT_MESSAGES[8].value['bookmarks']
['ROOT-COW'].get('ORA_ROWSCN'))
self.assertEqual("ROOT-COW",
CAUGHT_MESSAGES[8].value['currently_syncing'])
self.assertTrue(
isinstance(CAUGHT_MESSAGES[9], singer.ActivateVersionMessage))
cow_version = CAUGHT_MESSAGES[9].version
self.assertTrue(isinstance(CAUGHT_MESSAGES[10], singer.RecordMessage))
self.assertEqual(CAUGHT_MESSAGES[10].record['NAME'], 'betty')
self.assertEqual('ROOT-COW', CAUGHT_MESSAGES[10].stream)
self.assertTrue(isinstance(CAUGHT_MESSAGES[11], singer.StateMessage))
#ORA_ROWSCN is set while we are processing the full table replication
self.assertIsNotNone(
CAUGHT_MESSAGES[11].value['bookmarks']['ROOT-COW']['ORA_ROWSCN'])
betty_ora_row_scn = CAUGHT_MESSAGES[11].value['bookmarks'][
'ROOT-COW'].get('ORA_ROWSCN')
self.assertEqual(CAUGHT_MESSAGES[12].record['NAME'], 'smelly')
self.assertEqual('ROOT-COW', CAUGHT_MESSAGES[12].stream)
old_state = CAUGHT_MESSAGES[13].value
self.assertIsNotNone(
CAUGHT_MESSAGES[13].value['bookmarks']['ROOT-COW']['ORA_ROWSCN'])
smelly_ora_row_scn = CAUGHT_MESSAGES[13].value['bookmarks'][
'ROOT-COW'].get('ORA_ROWSCN')
self.assertGreater(smelly_ora_row_scn, betty_ora_row_scn)
#run another do_sync
singer.write_message = singer_write_message_ok
CAUGHT_MESSAGES.clear()
global COW_RECORD_COUNT
COW_RECORD_COUNT = 0
tap_oracle.do_sync(get_test_conn_config(), catalog, None, old_state)
self.assertTrue(isinstance(CAUGHT_MESSAGES[0], singer.SchemaMessage))
self.assertTrue(isinstance(CAUGHT_MESSAGES[1], singer.StateMessage))
# because we were interrupted, we do not switch versions
self.assertEqual(
CAUGHT_MESSAGES[1].value['bookmarks']['ROOT-COW']['version'],
cow_version)
self.assertIsNotNone(
CAUGHT_MESSAGES[1].value['bookmarks']['ROOT-COW']['ORA_ROWSCN'])
self.assertEqual(
"ROOT-COW", singer.get_currently_syncing(CAUGHT_MESSAGES[1].value))
self.assertTrue(isinstance(CAUGHT_MESSAGES[2], singer.RecordMessage))
self.assertEqual(CAUGHT_MESSAGES[2].record['NAME'], 'smelly')
self.assertEqual('ROOT-COW', CAUGHT_MESSAGES[2].stream)
#after record: activate version, state with no ORA_ROWSCN or currently syncing
self.assertTrue(isinstance(CAUGHT_MESSAGES[3], singer.StateMessage))
#we still have an ORA_ROWSCN for COW because are not yet done with the COW table
self.assertIsNotNone(
CAUGHT_MESSAGES[3].value['bookmarks']['ROOT-COW']['ORA_ROWSCN'])
self.assertEqual(
singer.get_currently_syncing(CAUGHT_MESSAGES[3].value), 'ROOT-COW')
self.assertTrue(isinstance(CAUGHT_MESSAGES[4], singer.RecordMessage))
self.assertEqual(CAUGHT_MESSAGES[4].record['NAME'], 'pooper')
self.assertEqual('ROOT-COW', CAUGHT_MESSAGES[4].stream)
self.assertTrue(isinstance(CAUGHT_MESSAGES[5], singer.StateMessage))
self.assertIsNotNone(
CAUGHT_MESSAGES[5].value['bookmarks']['ROOT-COW']['ORA_ROWSCN'])
self.assertEqual(
singer.get_currently_syncing(CAUGHT_MESSAGES[5].value), 'ROOT-COW')
#ORA_ROWSCN is cleared because we are finished the full table replication
self.assertTrue(
isinstance(CAUGHT_MESSAGES[6], singer.ActivateVersionMessage))
self.assertEqual(CAUGHT_MESSAGES[6].version, cow_version)
self.assertTrue(isinstance(CAUGHT_MESSAGES[7], singer.StateMessage))
self.assertIsNone(
singer.get_currently_syncing(CAUGHT_MESSAGES[7].value))
self.assertIsNone(CAUGHT_MESSAGES[7].value['bookmarks']['ROOT-CHICKEN']
['ORA_ROWSCN'])
self.assertIsNone(
singer.get_currently_syncing(CAUGHT_MESSAGES[7].value))
def test_catalog(self):
singer.write_message = singer_write_message
log_miner.UPDATE_BOOKMARK_PERIOD = 1
with get_test_connection() as conn:
conn.autocommit = True
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
chicken_stream = [
s for s in catalog.streams if s.table == 'CHICKEN'
][0]
chicken_stream = select_all_of_stream(chicken_stream)
chicken_stream = set_replication_method_for_stream(
chicken_stream, 'LOG_BASED')
cur = conn.cursor()
prev_scn = cur.execute(
"SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
our_fake_float = decimal.Decimal('1234567.8901234')
our_real_float = 1234567.8901234
crud_up_log_miner_fixtures(
cur, 'CHICKEN', {
'"our_double_precision"': our_fake_float,
'"our_real"': our_fake_float,
'"our_binary_float"': our_real_float,
'"our_binary_double"': our_real_float,
'"our_nan"': float('nan'),
'"our_+_infinity"': float('+inf'),
'"our_-_infinity"': float('-inf')
}, self.update_add_5)
post_scn = cur.execute(
"SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
LOGGER.info("post SCN: {}".format(post_scn))
state = write_bookmark({}, chicken_stream.tap_stream_id, 'scn',
prev_scn)
state = write_bookmark(state, chicken_stream.tap_stream_id,
'version', 1)
tap_oracle.do_sync(get_test_conn_config(), catalog, None, state)
verify_crud_messages(self, CAUGHT_MESSAGES, ['our_float'])
#verify message 1 - first insert
insert_rec_1 = CAUGHT_MESSAGES[1].record
self.assertIsNotNone(insert_rec_1.get('scn'))
insert_rec_1.pop('scn')
self.assertEqual(float('+inf'), insert_rec_1.get('our_+_infinity'))
insert_rec_1.pop('our_+_infinity')
self.assertEqual(float('-inf'), insert_rec_1.get('our_-_infinity'))
insert_rec_1.pop('our_-_infinity')
self.assertTrue(math.isnan(insert_rec_1.get('our_nan')))
insert_rec_1.pop('our_nan')
self.assertEqual(
insert_rec_1,
{
'our_float': '1',
'our_double_precision': str(our_fake_float),
'our_real': str(our_fake_float),
'our_binary_float': 1234567.88, #weird
'our_binary_double': 1234567.890123,
'_sdc_deleted_at': None
})
#verify UPDATE
update_rec = CAUGHT_MESSAGES[5].record
self.assertIsNotNone(update_rec.get('scn'))
update_rec.pop('scn')
self.assertEqual(float('+inf'), update_rec.get('our_+_infinity'))
update_rec.pop('our_+_infinity')
self.assertEqual(float('-inf'), update_rec.get('our_-_infinity'))
update_rec.pop('our_-_infinity')
self.assertTrue(math.isnan(update_rec.get('our_nan')))
update_rec.pop('our_nan')
self.assertEqual(
update_rec, {
'our_binary_double': 1234572.890123,
'_sdc_deleted_at': None,
'our_binary_float': 1234572.88,
'our_float': '1',
'our_double_precision': '1234572.8901234',
'our_real': '1234572.8901234'
})
#verify first DELETE message
delete_rec = CAUGHT_MESSAGES[9].record
self.assertEqual(float('+inf'), delete_rec.get('our_+_infinity'))
delete_rec.pop('our_+_infinity')
self.assertEqual(float('-inf'), delete_rec.get('our_-_infinity'))
delete_rec.pop('our_-_infinity')
self.assertTrue(math.isnan(delete_rec.get('our_nan')))
delete_rec.pop('our_nan')
self.assertIsNotNone(delete_rec.get('scn'))
self.assertIsNotNone(delete_rec.get('_sdc_deleted_at'))
delete_rec.pop('scn')
delete_rec.pop('_sdc_deleted_at')
self.assertEqual(
delete_rec, {
'our_binary_double': 1234572.890123,
'our_binary_float': 1234572.88,
'our_float': '1',
'our_double_precision': '1234572.8901234',
'our_real': '1234572.8901234'
})
#verify second DELETE message
delete_rec_2 = CAUGHT_MESSAGES[11].record
self.assertEqual(float('+inf'), delete_rec_2.get('our_+_infinity'))
delete_rec_2.pop('our_+_infinity')
self.assertEqual(float('-inf'), delete_rec_2.get('our_-_infinity'))
delete_rec_2.pop('our_-_infinity')
self.assertTrue(math.isnan(delete_rec_2.get('our_nan')))
delete_rec_2.pop('our_nan')
self.assertIsNotNone(delete_rec_2.get('scn'))
self.assertIsNotNone(delete_rec_2.get('_sdc_deleted_at'))
delete_rec_2.pop('scn')
delete_rec_2.pop('_sdc_deleted_at')
self.assertEqual(
delete_rec_2, {
'our_binary_double': 1234572.890123,
'our_binary_float': 1234572.88,
'our_float': '2',
'our_double_precision': '1234572.8901234',
'our_real': '1234572.8901234'
})
def test_catalog(self):
singer.write_message = singer_write_message
log_miner.UPDATE_BOOKMARK_PERIOD = 1
with get_test_connection() as conn:
conn.autocommit = True
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
chicken_stream = [
s for s in catalog.streams if s.table == 'CHICKEN'
][0]
chicken_stream = select_all_of_stream(chicken_stream)
chicken_stream = set_replication_method_for_stream(
chicken_stream, 'LOG_BASED')
cur = conn.cursor()
prev_scn = cur.execute(
"SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
our_date = datetime.date(1996, 6, 6)
our_ts = datetime.datetime(1997, 2, 2, 2, 2, 2, 722184)
nyc_tz = pytz.timezone('America/New_York')
our_ts_tz_edt = nyc_tz.localize(
datetime.datetime(1997, 3, 3, 3, 3, 3, 722184))
our_ts_tz_utc = datetime.datetime(1997, 3, 3, 3, 3, 3, 722184,
pytz.UTC)
auckland_tz = pytz.timezone('Pacific/Auckland')
our_ts_local = auckland_tz.localize(
datetime.datetime(1997, 3, 3, 18, 3, 3, 722184))
crud_up_log_miner_fixtures(
cur, 'CHICKEN', {
'"our_date"': our_date,
'"our_ts"': our_ts,
'"our_ts_tz_edt"': our_ts_tz_edt,
'"our_ts_tz_utc"': our_ts_tz_utc,
'"our_ts_tz_local"': our_ts_local
}, self.update_add_1_day)
post_scn = cur.execute(
"SELECT current_scn FROM V$DATABASE").fetchall()[0][0]
LOGGER.info("post SCN: {}".format(post_scn))
state = write_bookmark({}, chicken_stream.tap_stream_id, 'scn',
prev_scn)
state = write_bookmark(state, chicken_stream.tap_stream_id,
'version', 1)
tap_oracle.do_sync(get_test_conn_config(), catalog, None, state)
verify_crud_messages(self, CAUGHT_MESSAGES, ['ID'])
#verify message 1 - first insert
insert_rec_1 = CAUGHT_MESSAGES[1].record
self.assertIsNotNone(insert_rec_1.get('scn'))
insert_rec_1.pop('scn')
self.assertIsNone(insert_rec_1.get('_sdc_deleted_at'))
self.assertEqual(insert_rec_1.get('ID'), 1)
insert_rec_1.pop('_sdc_deleted_at')
insert_rec_1.pop('ID')
self.assertEqual(
insert_rec_1, {
'our_ts': '1997-02-02T02:02:02.722184+00:00',
'our_ts_tz_edt': '1997-03-03T03:03:03.722184-05:00',
'our_ts_tz_utc': '1997-03-03T03:03:03.722184+00:00',
'our_date': '1996-06-06T00:00:00.00+00:00',
'our_ts_tz_local': '1997-03-03T05:03:03.722184+00:00'
})
for v in insert_rec_1.values():
self.assertTrue(strict_rfc3339.validate_rfc3339(v))
#verify UPDATE
update_rec = CAUGHT_MESSAGES[5].record
self.assertIsNotNone(update_rec.get('scn'))
update_rec.pop('scn')
self.assertEqual(
update_rec, {
'our_ts': '1997-02-03T02:02:02.722184+00:00',
'our_ts_tz_edt': '1997-03-04T03:03:03.722184-05:00',
'our_ts_tz_utc': '1997-03-04T03:03:03.722184+00:00',
'our_date': '1996-06-07T00:00:00.00+00:00',
'_sdc_deleted_at': None,
'our_ts_tz_local': '1997-03-04T05:03:03.722184+00:00',
'ID': 1
})
#verify first DELETE message
delete_rec = CAUGHT_MESSAGES[9].record
self.assertIsNotNone(delete_rec.get('scn'))
self.assertIsNotNone(delete_rec.get('_sdc_deleted_at'))
delete_rec.pop('scn')
delete_rec.pop('_sdc_deleted_at')
self.assertEqual(
delete_rec, {
'our_ts': '1997-02-03T02:02:02.722184+00:00',
'our_ts_tz_edt': '1997-03-04T03:03:03.722184-05:00',
'our_ts_tz_utc': '1997-03-04T03:03:03.722184+00:00',
'our_date': '1996-06-07T00:00:00.00+00:00',
'our_ts_tz_local': '1997-03-04T05:03:03.722184+00:00',
'ID': 1
})
#verify second DELETE message
delete_rec_2 = CAUGHT_MESSAGES[11].record
self.assertIsNotNone(delete_rec_2.get('scn'))
self.assertIsNotNone(delete_rec_2.get('_sdc_deleted_at'))
delete_rec_2.pop('scn')
delete_rec_2.pop('_sdc_deleted_at')
self.assertEqual(
delete_rec_2, {
'our_ts': '1997-02-03T02:02:02.722184+00:00',
'our_ts_tz_edt': '1997-03-04T03:03:03.722184-05:00',
'our_ts_tz_utc': '1997-03-04T03:03:03.722184+00:00',
'our_date': '1996-06-07T00:00:00.00+00:00',
'our_ts_tz_local': '1997-03-04T05:03:03.722184+00:00',
'ID': 2
})
def test_catalog(self):
with get_test_connection() as conn:
catalog = tap_oracle.do_discovery(get_test_conn_config(), [])
chicken_streams = [
s for s in catalog.streams if s.table == 'CHICKEN'
]
self.assertEqual(len(chicken_streams), 1)
stream_dict = chicken_streams[0].to_dict()
stream_dict.get('metadata').sort(key=lambda md: md['breadcrumb'])
self.assertEqual(
{
'schema': {
'properties': {
'size_number_10_-1': {
'type': ['null', 'integer']
},
'size_number_*_0': {
'type': ['null', 'integer']
},
'size_number_integer': {
'type': ['null', 'integer']
},
'size_number_4': {
'type': ['null', 'integer']
},
'size_number_4_0': {
'type': ['null', 'integer']
},
'size_number_int': {
'type': ['null', 'integer']
},
'size_number_smallint': {
'type': ['null', 'integer']
},
'SIZE_PK': {
'type': ['integer']
}
},
'type': 'object'
},
'stream':
'CHICKEN',
'table_name':
'CHICKEN',
'tap_stream_id':
'ROOT-CHICKEN',
'metadata': [{
'metadata': {
'table-key-properties': ['SIZE_PK'],
'database-name': os.getenv('TAP_ORACLE_SID'),
'schema-name': 'ROOT',
'is-view': False,
'row-count': 0
},
'breadcrumb': ()
}, {
'metadata': {
'inclusion': 'automatic',
'sql-datatype': 'NUMBER',
'selected-by-default': True
},
'breadcrumb': ('properties', 'SIZE_PK')
}, {
'metadata': {
'inclusion': 'available',
'sql-datatype': 'NUMBER',
'selected-by-default': True
},
'breadcrumb': ('properties', 'size_number_*_0')
}, {
'metadata': {
'inclusion': 'available',
'sql-datatype': 'NUMBER',
'selected-by-default': True
},
'breadcrumb': ('properties', 'size_number_10_-1')
}, {
'metadata': {
'inclusion': 'available',
'sql-datatype': 'NUMBER',
'selected-by-default': True
},
'breadcrumb': ('properties', 'size_number_4')
}, {
'metadata': {
'inclusion': 'available',
'sql-datatype': 'NUMBER',
'selected-by-default': True
},
'breadcrumb': ('properties', 'size_number_4_0')
}, {
'metadata': {
'inclusion': 'available',
'sql-datatype': 'NUMBER',
'selected-by-default': True
},
'breadcrumb': ('properties', 'size_number_int')
}, {
'metadata': {
'inclusion': 'available',
'sql-datatype': 'NUMBER',
'selected-by-default': True
},
'breadcrumb': ('properties', 'size_number_integer')
}, {
'metadata': {
'inclusion': 'available',
'sql-datatype': 'NUMBER',
'selected-by-default': True
},
'breadcrumb': ('properties', 'size_number_smallint')
}]
}, stream_dict)
