Example #1
0
    def __init__(self, init_net, filename, schema, num_passes=1, batch_size=1):
        """
        Create op for building a TextFileReader instance in the workspace.

        Args:
            init_net   : Net that will be run only once at startup.
            filename   : Path to file to read from.
            schema     : schema.Struct representing the schema of the data.
                         Currently, only support Struct of strings.
            num_passes : Number of passes over the data.
            batch_size : Number of rows to read at a time.
        """
        assert isinstance(schema, Struct), 'Schema must be a schema.Struct'
        for name, child in schema.get_children():
            assert isinstance(child, Scalar), (
                'Only scalar fields are supported in TextFileReader.')
        field_types = [
            data_type_for_dtype(dtype) for dtype in schema.field_types()]
        Reader.__init__(self, schema)
        self._reader = init_net.CreateTextFileReader(
            [],
            filename=filename,
            num_passes=num_passes,
            field_types=field_types)
        self._batch_size = batch_size
Example #2
0
 def __init__(self, dataset, name, batch_size=1):
     """Don't call this directly. Instead, use dataset.reader()"""
     Reader.__init__(self, dataset.content())
     self.dataset = dataset
     self.name = name or (dataset.name + '_cursor')
     self.batch_size = batch_size
     self.cursor = None
Example #3
0
    def __init__(self, init_net, filename, schema, num_passes=1, batch_size=1):
        """
        Create op for building a HiveReader instance in the workspace.

        Args:
            init_net   : Net that will be run only once at startup.
            filename   : Path to file to read from.
            schema     : schema.Struct representing the schema of the data.
                         Currently, only support Struct of strings.
            num_passes : Number of passes over the data.
            batch_size : Number of rows to read at a time.
        """
        assert isinstance(schema, Struct), 'Schema must be a schema.Struct'
        for name, child in schema.get_children():
            assert isinstance(child, Scalar), (
                'Only scalar fields are supported in TextFileReader.')
        field_types = [
            data_type_for_dtype(dtype) for dtype in schema.field_types()]
        Reader.__init__(self, schema)
        self._reader = init_net.CreateTextFileReader(
            [],
            filename=filename,
            num_passes=num_passes,
            field_types=field_types)
        self._batch_size = batch_size
Example #4
0
 def __init__(self, dataset, name, batch_size=1):
     """Don't call this directly. Instead, use dataset.reader()"""
     Reader.__init__(self, dataset.content())
     self.dataset = dataset
     self.name = name or (dataset.name + '_cursor')
     self.batch_size = batch_size
     self.cursor = None
Example #5
0
 def __init__(self, content, cursor, name, indices, batch_size=1):
     """Don't call this directly. Instead, use dataset.random_reader()"""
     Reader.__init__(self, content)
     self._content = content
     self.cursor = cursor
     self.name = name
     self.indices = indices
     self.batch_size = batch_size
Example #6
0
 def __init__(self, content, cursor, name, batch_size=1):
     """Don't call this directly. Instead, use dataset.reader()"""
     assert isinstance(content, Field)
     Reader.__init__(self, content)
     self._content = content
     self.cursor = cursor
     self.name = name
     self.batch_size = batch_size
Example #7
0
 def __init__(self, content, cursor, name, indices, batch_size=1):
     """Don't call this directly. Instead, use dataset.random_reader()"""
     Reader.__init__(self, content)
     self._content = content
     self.cursor = cursor
     self.name = name
     self.indices = indices
     self.batch_size = batch_size
Example #8
0
 def __init__(self, content, cursor, name, batch_size=1):
     """Don't call this directly. Instead, use dataset.reader()"""
     assert isinstance(content, Field)
     Reader.__init__(self, content)
     self._content = content
     self.cursor = cursor
     self.name = name
     self.batch_size = batch_size
Example #9
0
 def __init__(self, dataset, name, indices, batch_size=1, loop_over=False):
     """Don't call this directly. Instead, use dataset.random_reader()"""
     Reader.__init__(self, dataset.content())
     self.dataset = dataset
     self.cursor = None
     self.name = name or (dataset.name + '_cursor')
     self.indices = indices
     self.batch_size = batch_size
     self.loop_over = loop_over
Example #10
0
 def __init__(self, dataset, name, indices, batch_size=1, loop_over=False,
              enforce_batch_size=False):
     """Don't call this directly. Instead, use dataset.random_reader()"""
     Reader.__init__(self, dataset.content())
     self.dataset = dataset
     self.cursor = None
     self.name = name or (dataset.name + '_cursor')
     self.indices = indices
     self.batch_size = batch_size
     self.loop_over = loop_over
     self.enforce_batch_size = enforce_batch_size
Example #11
0
 def __init__(self, reader, delay):
     Reader.__init__(self, schema=reader._schema)
     self.reader = reader
     self.delay = delay
Example #12
0
 def __init__(self, reader, processor):
     Reader.__init__(self)
     self.reader = reader
     self.processor = make_processor(processor)
Example #13
0
 def __init__(self, reader, delay):
     Reader.__init__(self, schema=reader._schema)
     self.reader = reader
     self.delay = delay