def __init__(self, file_path):
if file_path is None:
raise ValueError('A file path is required')
logging.debug('Opening a record reader pointing at %s', file_path)
self._reader = pywrap_tensorflow.PyRecordReader_New(file_path, 0)
# Store it for logging purposes.
self._file_path = file_path
if not self._reader:
raise IOError('Failed to open a record reader pointing to %s' % file_path)
def __init__(self, file_path):
if file_path is None:
raise ValueError("A file path is required")
logging.debug("Opening a record reader pointing at %s", file_path)
self._reader = pywrap_tensorflow.PyRecordReader_New(compat.as_bytes(file_path), 0)
# Store it for logging purposes.
self._file_path = file_path
if not self._reader:
raise IOError("Failed to open a record reader pointing to %s" % file_path)
def Load(self):
# Create a temp file to hold the contents that we haven't seen yet.
with tempfile.NamedTemporaryFile(prefix='tf-gcs-') as temp_file:
name = temp_file.name
logging.debug('Temp file created at %s', name)
gcs.CopyContents(self._gcs_path, self._gcs_offset, temp_file)
reader = pywrap_tensorflow.PyRecordReader_New(compat.as_bytes(name), 0)
while reader.GetNext():
event = event_pb2.Event()
event.ParseFromString(reader.record())
yield event
logging.debug('No more events in %s', name)
self._gcs_offset += reader.offset()
def Load(self):
"""Loads all new values from disk.
Calling Load multiple times in a row will not 'drop' events as long as the
return value is not iterated over.
Yields:
All values that were written to disk that have not been yielded yet.
"""
while self._reader.GetNext():
event = event_pb2.Event()
event.ParseFromString(self._reader.record())
yield event
logging.debug("No more events in %s", self._file_path)
def Load(self):
"""Loads all new values from disk.
Calling Load multiple times in a row will not 'drop' events as long as the
return value is not iterated over.
Yields:
All values that were written to disk that have not been yielded yet.
"""
while self._reader.GetNext():
event = event_pb2.Event()
event.ParseFromString(self._reader.record())
yield event
logging.debug('No more events in %s', self._file_path)
def Load(self):
# Create a temp file to hold the contents that we haven't seen yet.
with tempfile.NamedTemporaryFile(prefix='tf-gcs-') as temp_file:
name = temp_file.name
logging.debug('Temp file created at %s', name)
gcs.CopyContents(self._gcs_path, self._gcs_offset, temp_file)
reader = pywrap_tensorflow.PyRecordReader_New(
compat.as_bytes(name), 0)
while reader.GetNext():
event = event_pb2.Event()
event.ParseFromString(reader.record())
yield event
logging.debug('No more events in %s', name)
self._gcs_offset += reader.offset()