def __init__(self, init_net, filename, schema, num_passes=1, batch_size=1):
"""
Create op for building a TextFileReader instance in the workspace.
Args:
init_net : Net that will be run only once at startup.
filename : Path to file to read from.
schema : schema.Struct representing the schema of the data.
Currently, only support Struct of strings.
num_passes : Number of passes over the data.
batch_size : Number of rows to read at a time.
"""
assert isinstance(schema, Struct), 'Schema must be a schema.Struct'
for name, child in schema.get_children():
assert isinstance(child, Scalar), (
'Only scalar fields are supported in TextFileReader.')
field_types = [
data_type_for_dtype(dtype) for dtype in schema.field_types()]
Reader.__init__(self, schema)
self._reader = init_net.CreateTextFileReader(
[],
filename=filename,
num_passes=num_passes,
field_types=field_types)
self._batch_size = batch_size
def __init__(self, dataset, name, batch_size=1):
"""Don't call this directly. Instead, use dataset.reader()"""
Reader.__init__(self, dataset.content())
self.dataset = dataset
self.name = name or (dataset.name + '_cursor')
self.batch_size = batch_size
self.cursor = None
def __init__(self, init_net, filename, schema, num_passes=1, batch_size=1):
"""
Create op for building a HiveReader instance in the workspace.
Args:
init_net : Net that will be run only once at startup.
filename : Path to file to read from.
schema : schema.Struct representing the schema of the data.
Currently, only support Struct of strings.
num_passes : Number of passes over the data.
batch_size : Number of rows to read at a time.
"""
assert isinstance(schema, Struct), 'Schema must be a schema.Struct'
for name, child in schema.get_children():
assert isinstance(child, Scalar), (
'Only scalar fields are supported in TextFileReader.')
field_types = [
data_type_for_dtype(dtype) for dtype in schema.field_types()]
Reader.__init__(self, schema)
self._reader = init_net.CreateTextFileReader(
[],
filename=filename,
num_passes=num_passes,
field_types=field_types)
self._batch_size = batch_size
def __init__(self, dataset, name, batch_size=1):
"""Don't call this directly. Instead, use dataset.reader()"""
Reader.__init__(self, dataset.content())
self.dataset = dataset
self.name = name or (dataset.name + '_cursor')
self.batch_size = batch_size
self.cursor = None
def __init__(self, content, cursor, name, indices, batch_size=1):
"""Don't call this directly. Instead, use dataset.random_reader()"""
Reader.__init__(self, content)
self._content = content
self.cursor = cursor
self.name = name
self.indices = indices
self.batch_size = batch_size
def __init__(self, content, cursor, name, batch_size=1):
"""Don't call this directly. Instead, use dataset.reader()"""
assert isinstance(content, Field)
Reader.__init__(self, content)
self._content = content
self.cursor = cursor
self.name = name
self.batch_size = batch_size
def __init__(self, content, cursor, name, indices, batch_size=1):
"""Don't call this directly. Instead, use dataset.random_reader()"""
Reader.__init__(self, content)
self._content = content
self.cursor = cursor
self.name = name
self.indices = indices
self.batch_size = batch_size
def __init__(self, content, cursor, name, batch_size=1):
"""Don't call this directly. Instead, use dataset.reader()"""
assert isinstance(content, Field)
Reader.__init__(self, content)
self._content = content
self.cursor = cursor
self.name = name
self.batch_size = batch_size
def __init__(self, dataset, name, indices, batch_size=1, loop_over=False):
"""Don't call this directly. Instead, use dataset.random_reader()"""
Reader.__init__(self, dataset.content())
self.dataset = dataset
self.cursor = None
self.name = name or (dataset.name + '_cursor')
self.indices = indices
self.batch_size = batch_size
self.loop_over = loop_over
def __init__(self, dataset, name, indices, batch_size=1, loop_over=False,
enforce_batch_size=False):
"""Don't call this directly. Instead, use dataset.random_reader()"""
Reader.__init__(self, dataset.content())
self.dataset = dataset
self.cursor = None
self.name = name or (dataset.name + '_cursor')
self.indices = indices
self.batch_size = batch_size
self.loop_over = loop_over
self.enforce_batch_size = enforce_batch_size
def __init__(self, reader, delay):
Reader.__init__(self, schema=reader._schema)
self.reader = reader
self.delay = delay
def __init__(self, reader, processor):
Reader.__init__(self)
self.reader = reader
self.processor = make_processor(processor)
def __init__(self, reader, delay):
Reader.__init__(self, schema=reader._schema)
self.reader = reader
self.delay = delay