def test_pre_parse_s3_debug(s3_mock, log_mock, _): """S3Payload - Pre Parse, Debug On""" # Cache the logger level log_level = LOGGER.getEffectiveLevel() # Increase the logger level to debug LOGGER.setLevel(logging.DEBUG) records = ['_first_line_test_' * 10, '_second_line_test_' * 10] s3_mock.side_effect = [((100, records[0]), (200, records[1]))] raw_record = make_s3_raw_record('unit_bucket_name', 'unit_key_name') s3_payload = load_stream_payload('s3', 'unit_key_name', raw_record) S3Payload.s3_object_size = 350 _ = [_ for _ in s3_payload.pre_parse()] calls = [ call( 'Processed %s S3 records out of an approximate total of %s ' '(average record size: %s bytes, total size: %s bytes)', 100, 350, 1, 350), call( 'Processed %s S3 records out of an approximate total of %s ' '(average record size: %s bytes, total size: %s bytes)', 200, 350, 1, 350) ] log_mock.assert_has_calls(calls) # Reset the logger level and stop the patchers LOGGER.setLevel(log_level)
def test_s3_object_too_large(): """S3Payload - S3ObjectSizeError, Object too Large""" raw_record = make_s3_raw_record('unit_bucket_name', 'unit_key_name') s3_payload = load_stream_payload('s3', 'unit_key_name', raw_record) S3Payload.s3_object_size = (128 * 1024 * 1024) + 10 s3_payload._download_object('region', 'bucket', 'key')
def test_pre_parse_s3_debug(s3_mock, log_mock, _): """S3Payload - Pre Parse, Debug On""" with patch.object(payload, 'LOGGER_DEBUG_ENABLED', True): records = ['_first_line_test_' * 10, '_second_line_test_' * 10] s3_mock.side_effect = [((100, records[0]), (200, records[1]))] raw_record = make_s3_raw_record('unit_bucket_name', 'unit_key_name') s3_payload = load_stream_payload('s3', 'unit_key_name', raw_record) S3Payload.s3_object_size = 350 _ = [_ for _ in s3_payload.pre_parse()] calls = [ call( 'Processed %s S3 records out of an approximate total of %s ' '(average record size: %s bytes, total size: %s bytes)', 100, 350, 1, 350), call( 'Processed %s S3 records out of an approximate total of %s ' '(average record size: %s bytes, total size: %s bytes)', 200, 350, 1, 350) ] log_mock.assert_has_calls(calls)
def test_s3_download_object_zero_size(*_): """S3Payload - Download Object of Zero Size""" raw_record = make_s3_raw_record('unit_bucket_name', 'unit_key_name', 0) s3_payload = load_stream_payload('s3', 'unit_key_name', raw_record) assert_is_none( s3_payload._download_object('us-east-1', 'unit_bucket_name', 'unit_key_name'))
def test_get_object(log_mock, _): """S3Payload - Get S3 Info from Raw Record""" raw_record = make_s3_raw_record('unit_bucket_name', 'unit_key_name') s3_payload = load_stream_payload('s3', 'unit_key_name', raw_record) s3_payload._get_object() log_mock.assert_called_with( 'Pre-parsing record from S3. Bucket: %s, Key: %s, Size: %d', 'unit_bucket_name', 'unit_key_name', 100)
def test_get_object_ioerror(download_object_mock): """S3Payload - IOError Test""" download_object_mock.side_effect = IOError('No space left on device') raw_record = make_s3_raw_record('unit_bucket_name', 'unit_key_name') s3_payload = load_stream_payload('s3', 'unit_key_name', raw_record) result = s3_payload._get_object() assert_equal(result, None)
def test_s3_download_object(log_mock, *_): """S3Payload - Download Object""" raw_record = make_s3_raw_record('unit_bucket_name', 'unit_key_name') s3_payload = load_stream_payload('s3', 'unit_key_name', raw_record) s3_payload._download_object('us-east-1', 'unit_bucket_name', 'unit_key_name') assert_equal(log_mock.call_args_list[1][0][0], 'Completed download in %s seconds')
def test_s3_download_object(*_): """S3Payload - Download Object""" key = 'test/unit/s3-object.gz' raw_record = make_s3_raw_record('unit_bucket_name', key) s3_payload = load_stream_payload('s3', key, raw_record) S3Payload.s3_object_size = (1024 * 1024) downloaded_path = s3_payload._download_object('us-east-1', 'unit_bucket_name', key) assert_true(downloaded_path.endswith('test-unit-s3-object.gz'))
def test_pre_parse_s3(s3_mock, *_): """S3Payload - Pre Parse""" records = ['{"record01": "value01"}', '{"record02": "value02"}'] s3_mock.side_effect = [((0, records[0]), (1, records[1]))] raw_record = make_s3_raw_record('unit_bucket_name', 'unit_key_name') s3_payload = load_stream_payload('s3', 'unit_key_name', raw_record) for index, record in enumerate(s3_payload.pre_parse()): assert_equal(record.pre_parsed_record, records[index])
def test_s3_download_object_mb(log_mock, *_): """S3Payload - Download Object, Size in MB""" raw_record = make_s3_raw_record('unit_bucket_name', 'unit_key_name') s3_payload = load_stream_payload('s3', 'unit_key_name', raw_record) S3Payload.s3_object_size = (127.8 * 1024 * 1024) s3_payload._download_object('us-east-1', 'unit_bucket_name', 'unit_key_name') assert_equal(log_mock.call_args_list[0], call('[S3Payload] Starting download from S3: %s/%s [%s]', 'unit_bucket_name', 'unit_key_name', '127.8MB')) assert_equal(log_mock.call_args_list[1][0][0], 'Completed download in %s seconds')