def __init__(self, file_path):
if file_path is None:
raise ValueError('A file path is required')
file_path = platform_util.readahead_file_path(file_path)
logger.debug('Opening a record reader pointing at %s', file_path)
with tf.compat.v1.errors.raise_exception_on_not_ok_status() as status:
self._reader = _pywrap_tensorflow.PyRecordReader_New(
tf.compat.as_bytes(file_path), 0, tf.compat.as_bytes(''), status)
# Store it for logging purposes.
self._file_path = file_path
if not self._reader:
raise IOError('Failed to open a record reader pointing to %s' % file_path)
def __init__(self, file_path, detect_file_replacement=False):
"""Constructs a RawEventFileLoader for the given file path.
Args:
file_path: the event file path to read from
detect_file_replacement: if True, when Load() is called, the loader
will make a stat() call to check the size of the file. If it sees
that the file has grown, it will reopen the file entirely (while
preserving the current offset) before attempting to read from it.
Otherwise, Load() will simply poll at EOF for new data.
"""
if file_path is None:
raise ValueError("A file path is required")
self._file_path = platform_util.readahead_file_path(file_path)
self._detect_file_replacement = detect_file_replacement
self._file_size = None
self._iterator = _make_tf_record_iterator(self._file_path)
if self._detect_file_replacement and not hasattr(
self._iterator, "reopen"):
logger.warning(
"File replacement detection requested, but not enabled because "
"TF record iterator impl does not support reopening. This "
"functionality requires TensorFlow 2.9+")
self._detect_file_replacement = False
def test_readahead_file_path(self):
self.assertEqual('foo/bar',
platform_util.readahead_file_path('foo/bar'))
def __init__(self, file_path):
if file_path is None:
raise ValueError("A file path is required")
self._file_path = platform_util.readahead_file_path(file_path)
self._iterator = _make_tf_record_iterator(self._file_path)
def _open(self):
with tf.errors.raise_exception_on_not_ok_status() as status:
return tf.compat.v1.pywrap_tensorflow.PyRecordReader_New(
platform_util.readahead_file_path(tf.compat.as_bytes(self.path)),
self._offset, tf.compat.as_bytes(''), status)
def test_readahead_file_path(self):
self.assertEqual("foo/bar",
platform_util.readahead_file_path("foo/bar"))
