Python _TFRecordSource Examples

Example #1

File: tfrecordio_test.py Project: jasonkuster/incubator-beam

 def test_process_auto(self):
   path = os.path.join(self._new_tempdir(), 'result.gz')
   self._write_file_gzip(path, FOO_BAR_RECORD_BASE64)
   with TestPipeline() as p:
     result = (p
               | beam.Read(
                   _TFRecordSource(
                       path,
                       coder=coders.BytesCoder(),
                       compression_type=fileio.CompressionTypes.AUTO)))
     beam.assert_that(result, beam.equal_to(['foo', 'bar']))

Example #2

 def test_process_auto(self):
     path = os.path.join(self._new_tempdir(), 'result.gz')
     self._write_file_gzip(path, FOO_BAR_RECORD_BASE64)
     with beam.Pipeline(DirectRunner()) as p:
         result = (p
                   | beam.Read(
                       _TFRecordSource(
                           path,
                           coder=coders.BytesCoder(),
                           compression_type=fileio.CompressionTypes.AUTO)))
         beam.assert_that(result, beam.equal_to(['foo', 'bar']))

Example #3

File: tfrecordio_test.py Project: diptikul/incubator-beam

 def test_process_single(self):
     path = os.path.join(self._new_tempdir(), 'result')
     self._write_file(path, FOO_RECORD_BASE64)
     with TestPipeline() as p:
         result = (p
                   | beam.Read(
                       _TFRecordSource(
                           path,
                           coder=coders.BytesCoder(),
                           compression_type=fileio.CompressionTypes.AUTO)))
         beam.assert_that(result, beam.equal_to(['foo']))

Example #4

File: tfrecordio_test.py Project: ssisk/beam

 def test_process_gzip(self):
     path = os.path.join(self._new_tempdir(), 'result')
     self._write_file_gzip(path, FOO_BAR_RECORD_BASE64)
     with TestPipeline() as p:
         result = (p
                   | beam.io.Read(
                       _TFRecordSource(
                           path,
                           coder=coders.BytesCoder(),
                           compression_type=CompressionTypes.GZIP,
                           validate=True)))
         beam.assert_that(result, beam.equal_to(['foo', 'bar']))

Example #5

File: tfrecordio_test.py Project: amarouni/incubator-beam

 def test_process_gzip(self):
   path = os.path.join(self._new_tempdir(), 'result')
   self._write_file_gzip(path, FOO_BAR_RECORD_BASE64)
   with TestPipeline() as p:
     result = (p
               | beam.io.Read(
                   _TFRecordSource(
                       path,
                       coder=coders.BytesCoder(),
                       compression_type=CompressionTypes.GZIP,
                       validate=True)))
     assert_that(result, equal_to(['foo', 'bar']))

Example #6

 def test_process_single(self):
   path = os.path.join(self._new_tempdir(), 'result')
   self._write_file(path, FOO_RECORD_BASE64)
   with TestPipeline() as p:
     result = (p
               | beam.Read(
                   _TFRecordSource(
                       path,
                       coder=coders.BytesCoder(),
                       compression_type=fileio.CompressionTypes.AUTO,
                       validate=True)))
     beam.assert_that(result, beam.equal_to(['foo']))