def test_add_record_message__invalid_record():
stream = InvalidCatStream(10)
singer_stream = BufferedSingerStream(CATS_SCHEMA['stream'],
CATS_SCHEMA['schema'],
CATS_SCHEMA['key_properties'])
with pytest.raises(SingerStreamError):
singer_stream.add_record_message(stream.generate_record_message())
assert singer_stream.peek_invalid_records()
assert singer_stream.count == 0
assert [] == missing_sdc_properties(singer_stream)
def test_add_record_message__invalid_record__detection_off():
stream = InvalidCatStream(10)
singer_stream = BufferedSingerStream(CATS_SCHEMA['stream'],
CATS_SCHEMA['schema'],
CATS_SCHEMA['key_properties'],
invalid_records_detect=False)
singer_stream.add_record_message(stream.generate_record_message())
assert singer_stream.peek_invalid_records()
assert singer_stream.count == 0
assert [] == missing_sdc_properties(singer_stream)
def test_loading__invalid__records__threshold():
config = deepcopy(CONFIG)
config['invalid_records_threshold'] = 10
target = Target()
with pytest.raises(singer_stream.SingerStreamError, match=r'.*.10*'):
target_tools.stream_to_target(InvalidCatStream(20), target, config=config)
assert len(target.calls['write_batch']) == 0
def test_loading__invalid__records__disable():
config = deepcopy(CONFIG)
config['invalid_records_detect'] = False
target = Target()
target_tools.stream_to_target(InvalidCatStream(100), target, config=config)
## Since all `cat`s records were invalid, we could not persist them, hence, no calls made to `write_batch`
assert len(target.calls['write_batch']) == 1
assert target.calls['write_batch'][0]['records_count'] == 0
def test_loading__invalid__records():
with pytest.raises(singer_stream.SingerStreamError, match=r'.*'):
target_tools.stream_to_target(InvalidCatStream(1), None, config=CONFIG)