def test_add_record_message__allOf__impossible_schema():
stream_name = 'test'
schema = deepcopy(SIMPLE_ALLOF_SCHEMA)
schema['properties']['allOfKey']['allOf'].append({'type': ['number']})
singer_stream = BufferedSingerStream(stream_name, schema, [])
with pytest.raises(SingerStreamError):
singer_stream.add_record_message({
'type': 'RECORD',
'stream': stream_name,
'record': {
'allOfKey': 'short'
},
'sequence': 0
})
with pytest.raises(SingerStreamError):
singer_stream.add_record_message({
'type': 'RECORD',
'stream': stream_name,
'record': {
'allOfKey': 314159
},
'sequence': 0
})
assert singer_stream.peek_invalid_records()
assert singer_stream.count == 0
assert [] == missing_sdc_properties(singer_stream)
def test_add_record_message__invalid_record():
stream = InvalidCatStream(10)
singer_stream = BufferedSingerStream(CATS_SCHEMA['stream'],
CATS_SCHEMA['schema'],
CATS_SCHEMA['key_properties'])
with pytest.raises(SingerStreamError):
singer_stream.add_record_message(stream.generate_record_message())
assert singer_stream.peek_invalid_records()
assert singer_stream.count == 0
assert [] == missing_sdc_properties(singer_stream)
def test_add_record_message__invalid_record__detection_off():
stream = InvalidCatStream(10)
singer_stream = BufferedSingerStream(CATS_SCHEMA['stream'],
CATS_SCHEMA['schema'],
CATS_SCHEMA['key_properties'],
invalid_records_detect=False)
singer_stream.add_record_message(stream.generate_record_message())
assert singer_stream.peek_invalid_records()
assert singer_stream.count == 0
assert [] == missing_sdc_properties(singer_stream)
def test_add_record_message():
stream = CatStream(10)
singer_stream = BufferedSingerStream(CATS_SCHEMA['stream'],
CATS_SCHEMA['schema'],
CATS_SCHEMA['key_properties'])
assert singer_stream.add_record_message(
stream.generate_record_message()) is None
assert not singer_stream.peek_invalid_records()
assert [] == missing_sdc_properties(singer_stream)
def test_multiple_batches__by_memory():
stream = CatStream(100)
singer_stream = BufferedSingerStream(CATS_SCHEMA['stream'],
CATS_SCHEMA['schema'],
CATS_SCHEMA['key_properties'],
max_buffer_size=1024)
assert len(singer_stream.peek_buffer()) == 0
while not singer_stream.buffer_full:
singer_stream.add_record_message(stream.generate_record_message())
assert len(singer_stream.peek_buffer()) == 1
assert [] == missing_sdc_properties(singer_stream)
singer_stream.flush_buffer()
assert len(singer_stream.peek_buffer()) == 0
def test_add_record_message__allOf__invalid_record():
stream_name = 'test'
singer_stream = BufferedSingerStream(stream_name,
deepcopy(SIMPLE_ALLOF_SCHEMA), [])
with pytest.raises(SingerStreamError):
singer_stream.add_record_message({
'type': 'RECORD',
'stream': stream_name,
'record': {
'allOfKey':
'this is a string which is much too long to be allowed'
},
'sequence': 0
})
assert singer_stream.peek_invalid_records()
assert singer_stream.count == 0
assert [] == missing_sdc_properties(singer_stream)
def test_add_record_message__allOf():
stream_name = 'test'
singer_stream = BufferedSingerStream(stream_name,
deepcopy(SIMPLE_ALLOF_SCHEMA), [])
strs_shorter_than_6 = ['hello', 'I', 'am', 'a set', 'of', 'short', 'strs']
for string in strs_shorter_than_6:
singer_stream.add_record_message({
'type': 'RECORD',
'stream': stream_name,
'record': {
'allOfKey': string
},
'sequence': 0
})
assert not singer_stream.peek_invalid_records()
assert singer_stream.count == len(strs_shorter_than_6)
assert [] == missing_sdc_properties(singer_stream)
def test_add_record_message__multipleOf_invalid_record():
stream_name = 'test'
singer_stream = BufferedSingerStream(
stream_name, deepcopy(SIMPLE_MULTIPLE_OF_INVALID_SCHEMA), [])
multiple_of_values = [1, 2]
for value in multiple_of_values:
with pytest.raises(SingerStreamError):
singer_stream.add_record_message({
'type': 'RECORD',
'stream': stream_name,
'record': {
'multipleOfKey': value
},
'sequence': 0,
RAW_LINE_SIZE: 100
})
assert singer_stream.peek_invalid_records()
assert singer_stream.count == 0
def test_multiple_batches__old_records__by_rows():
stream_oldest = CatStream(100, version=0)
stream_middle_aged = CatStream(100, version=5)
stream_latest = CatStream(100, version=10)
singer_stream = BufferedSingerStream(CATS_SCHEMA['stream'],
CATS_SCHEMA['schema'],
CATS_SCHEMA['key_properties'],
max_rows=20)
assert len(singer_stream.peek_buffer()) == 0
while not singer_stream.buffer_full:
singer_stream.add_record_message(
stream_oldest.generate_record_message())
assert len(singer_stream.peek_buffer()) == 20
singer_stream.flush_buffer()
assert len(singer_stream.peek_buffer()) == 0
singer_stream.add_record_message(stream_latest.generate_record_message())
assert len(singer_stream.peek_buffer()) == 1
reasonable_cutoff = 1000
while not singer_stream.buffer_full and reasonable_cutoff != 0:
singer_stream.add_record_message(
stream_middle_aged.generate_record_message())
reasonable_cutoff -= 1
assert reasonable_cutoff == 0
assert len(singer_stream.peek_buffer()) == 1
assert [] == missing_sdc_properties(singer_stream)
def test_init__empty_key_properties():
singer_stream = BufferedSingerStream(CATS_SCHEMA['stream'],
CATS_SCHEMA['schema'], [])
stream = CatStream(100)
for _ in range(20):
singer_stream.add_record_message(stream.generate_record_message())
assert singer_stream
assert [] == missing_sdc_properties(singer_stream)
assert [SINGER_PK] == singer_stream.key_properties
rows_missing_pk = []
rows_checked = 0
for r in singer_stream.get_batch():
if not r[SINGER_PK]:
rows_missing_pk.append(r)
rows_checked += 1
assert rows_checked > 1
assert [] == rows_missing_pk
def test_add_record_message__multipleOf():
stream_name = 'test'
singer_stream = BufferedSingerStream(
stream_name, deepcopy(SIMPLE_MULTIPLE_OF_VALID_SCHEMA), [])
multiple_of_values = [
'1', '2', '3', '4', '5', '1.1', '2.3', '1.23456789', '20', '100.1'
]
for value in multiple_of_values:
singer_stream.add_record_message({
'type': 'RECORD',
'stream': stream_name,
'record': {
'multipleOfKey': Decimal(value)
},
'sequence': 0,
RAW_LINE_SIZE: 100
})
assert not singer_stream.peek_invalid_records()
assert singer_stream.count == len(multiple_of_values)