def test_read_reantrant_with_splitting(self):
file_name = self._write_data()
source = AvroSource(file_name)
splits = [
split for split in source.split(desired_bundle_size=100000)]
assert len(splits) == 1
source_test_utils.assertReentrantReadsSucceed(
(splits[0].source, splits[0].start_position, splits[0].stop_position))
def test_read_reantrant_with_splitting(self):
file_name = self._write_data()
source = AvroSource(file_name)
splits = [split for split in source.split(desired_bundle_size=100000)]
assert len(splits) == 1
source_test_utils.assertReentrantReadsSucceed(
(splits[0].source, splits[0].start_position,
splits[0].stop_position))
def test_read_reentrant_after_splitting(self):
file_name, expected_data = write_data(10)
assert len(expected_data) == 10
source = TextSource(file_name, 0, CompressionTypes.UNCOMPRESSED, True,
coders.StrUtf8Coder())
splits = [split for split in source.split(desired_bundle_size=100000)]
assert len(splits) == 1
source_test_utils.assertReentrantReadsSucceed(
(splits[0].source, splits[0].start_position, splits[0].stop_position))
def test_read_reentrant_after_splitting(self):
file_name, expected_data = write_data(10)
assert len(expected_data) == 10
source = TextSource(file_name, 0, CompressionTypes.UNCOMPRESSED, True,
coders.StrUtf8Coder())
splits = [split for split in source.split(desired_bundle_size=100000)]
assert len(splits) == 1
source_test_utils.assertReentrantReadsSucceed(
(splits[0].source, splits[0].start_position, splits[0].stop_position))
def test_create_source_read_reentrant_with_initial_splits(self):
source = Create._create_source_from_iterable(range(24), self.coder)
for split in source.split(desired_bundle_size=5):
source_test_utils.assertReentrantReadsSucceed((split.source,
split.start_position,
split.stop_position))
def test_create_source_read_reentrant(self): source = Create._create_source_from_iterable(range(9), self.coder) source_test_utils.assertReentrantReadsSucceed((source, None, None))
def test_read_reentrant_without_splitting(self):
file_name, expected_data = write_data(10)
assert len(expected_data) == 10
source = TextSource(file_name, 0, CompressionTypes.UNCOMPRESSED, True,
coders.StrUtf8Coder())
source_test_utils.assertReentrantReadsSucceed((source, None, None))
def test_read_reentrant_without_splitting(self):
file_name = self._write_data()
source = AvroSource(file_name)
source_test_utils.assertReentrantReadsSucceed((source, None, None))
def test_read_reentrant_without_splitting(self): file_name = self._write_data() source = AvroSource(file_name) source_test_utils.assertReentrantReadsSucceed((source, None, None))
def test_read_reentrant_without_splitting(self):
file_name, expected_data = write_data(10)
assert len(expected_data) == 10
source = TextSource(file_name, 0, CompressionTypes.UNCOMPRESSED, True,
coders.StrUtf8Coder())
source_test_utils.assertReentrantReadsSucceed((source, None, None))