def test_read_reantrant_with_splitting(self): file_name = self._write_data() source = _create_avro_source(file_name) splits = [split for split in source.split(desired_bundle_size=100000)] assert len(splits) == 1 source_test_utils.assert_reentrant_reads_succeed( (splits[0].source, splits[0].start_position, splits[0].stop_position))
def test_read_reentrant_without_splitting(self): with TempDir() as tempdir: file_name = self._create_temp_vcf_file( _SAMPLE_HEADER_LINES + _SAMPLE_TEXT_LINES, tempdir) source = VcfSource(file_name) source_test_utils.assert_reentrant_reads_succeed( (source, None, None))
def test_read_reantrant_with_splitting(self): file_name = self._write_data() source = AvroSource(file_name) splits = [ split for split in source.split(desired_bundle_size=100000)] assert len(splits) == 1 source_test_utils.assert_reentrant_reads_succeed( (splits[0].source, splits[0].start_position, splits[0].stop_position))
def test_read_reentrant_after_splitting(self): with TempDir() as tempdir: file_name = self._create_temp_vcf_file(_SAMPLE_HEADER_LINES + _SAMPLE_TEXT_LINES, tempdir) source = VcfSource(file_name) splits = [split for split in source.split(desired_bundle_size=100000)] assert len(splits) == 1 source_test_utils.assert_reentrant_reads_succeed( (splits[0].source, splits[0].start_position, splits[0].stop_position))
def test_read_reentrant_after_splitting(self): file_name, expected_data = write_data(10) assert len(expected_data) == 10 source = TextSource(file_name, 0, CompressionTypes.UNCOMPRESSED, True, coders.StrUtf8Coder()) splits = list(source.split(desired_bundle_size=100000)) assert len(splits) == 1 source_test_utils.assert_reentrant_reads_succeed( (splits[0].source, splits[0].start_position, splits[0].stop_position))
def test_read_reentrant_after_splitting(self): file_name, expected_data = write_data(10) assert len(expected_data) == 10 source = TextSource(file_name, 0, CompressionTypes.UNCOMPRESSED, True, coders.StrUtf8Coder()) splits = list(source.split(desired_bundle_size=100000)) assert len(splits) == 1 source_test_utils.assert_reentrant_reads_succeed( (splits[0].source, splits[0].start_position, splits[0].stop_position))
def test_read_reentrant_after_splitting(self): with TempDir() as tempdir: file_name = self._create_temp_vcf_file( _SAMPLE_HEADER_LINES + _SAMPLE_TEXT_LINES, tempdir) source = VcfSource(file_name) splits = list(split for split in source.split(desired_bundle_size=100000)) assert len(splits) == 1 source_test_utils.assert_reentrant_reads_succeed( (splits[0].source, splits[0].start_position, splits[0].stop_position))
def test_read_reentrant_without_splitting(self): file_name, expected_data = write_data(10) assert len(expected_data) == 10 source = TextSource( file_name, 0, CompressionTypes.UNCOMPRESSED, True, coders.StrUtf8Coder()) source_test_utils.assert_reentrant_reads_succeed((source, None, None))
def test_read_reentrant_without_splitting(self): file_name, expected_data = write_data(10) assert len(expected_data) == 10 source = TextSource(file_name, 0, CompressionTypes.UNCOMPRESSED, True, coders.StrUtf8Coder()) source_test_utils.assert_reentrant_reads_succeed((source, None, None))
def test_create_source_read_reentrant_with_initial_splits(self): source = Create._create_source_from_iterable(range(24), self.coder) for split in source.split(desired_bundle_size=5): source_test_utils.assert_reentrant_reads_succeed((split.source, split.start_position, split.stop_position))
def test_create_source_read_reentrant(self): source = Create._create_source_from_iterable(range(9), self.coder) source_test_utils.assert_reentrant_reads_succeed((source, None, None))
def test_read_reentrant(self): file_name = self._write_data(count=6, row_group_size=3) source = _create_parquet_source(file_name) source_test_utils.assert_reentrant_reads_succeed((source, None, None))
def test_read_reentrant_without_splitting(self): file_name = self._write_data() source = _create_avro_source(file_name, use_fastavro=self.use_fastavro) source_test_utils.assert_reentrant_reads_succeed((source, None, None))
def test_read_reentrant_without_splitting(self): file_name = self._write_data() source = _create_avro_source(file_name, use_fastavro=self.use_fastavro) source_test_utils.assert_reentrant_reads_succeed((source, None, None))
def test_read_reentrant_without_splitting(self): file_name = self._write_data() source = AvroSource(file_name) source_test_utils.assert_reentrant_reads_succeed((source, None, None))
def test_read_reentrant_without_splitting(self): with TempDir() as tempdir: file_name = self._create_temp_vcf_file(_SAMPLE_HEADER_LINES + _SAMPLE_TEXT_LINES, tempdir) source = VcfSource(file_name) source_test_utils.assert_reentrant_reads_succeed((source, None, None))
def test_create_source_read_reentrant_with_initial_splits(self): source = Create._create_source_from_iterable(range(24), self.coder) for split in source.split(desired_bundle_size=5): source_test_utils.assert_reentrant_reads_succeed( (split.source, split.start_position, split.stop_position))
def test_create_source_read_reentrant(self): source = Create._create_source_from_iterable(range(9), self.coder) source_test_utils.assert_reentrant_reads_succeed((source, None, None))
def test_read_reentrant_without_splitting(self): file_name = self._write_data() source = AvroSource(file_name) source_test_utils.assert_reentrant_reads_succeed((source, None, None))