def get_inspection_units(logdir='', event_file='', tag=''):
"""Returns a list of InspectionUnit objects given either logdir or event_file.
If logdir is given, the number of InspectionUnits should equal the
number of directories or subdirectories that contain event files.
If event_file is given, the number of InspectionUnits should be 1.
Args:
logdir: A log directory that contains event files.
event_file: Or, a particular event file path.
tag: An optional tag name to query for.
Returns:
A list of InspectionUnit objects.
"""
if logdir:
subdirs = event_multiplexer.GetLogdirSubdirectories(logdir)
inspection_units = []
for subdir in subdirs:
generator = itertools.chain(*[
generator_from_event_file(os.path.join(subdir, f))
for f in tf.gfile.ListDirectory(subdir)
if event_accumulator.IsTensorFlowEventsFile(
os.path.join(subdir, f))
])
inspection_units.append(
InspectionUnit(name=subdir,
generator=generator,
field_to_obs=get_field_to_observations_map(
generator, tag)))
if inspection_units:
print('Found event files in:\n{}\n'.format('\n'.join(
[u.name for u in inspection_units])))
elif event_accumulator.IsTensorFlowEventsFile(logdir):
print(
'It seems that {} may be an event file instead of a logdir. If this '
'is the case, use --event_file instead of --logdir to pass '
'it in.'.format(logdir))
else:
print('No event files found within logdir {}'.format(logdir))
return inspection_units
elif event_file:
generator = generator_from_event_file(event_file)
return [
InspectionUnit(name=event_file,
generator=generator,
field_to_obs=get_field_to_observations_map(
generator, tag))
]
return []
def sync_event_files_for_nfs_cache(dir_name: str):
"""Make sure event files are synchronized per the nfs cache"""
for file_name in os.listdir(dir_name):
full_path = os.path.join(dir_name, file_name)
if event_accumulator.IsTensorFlowEventsFile(full_path):
stat = os.stat(full_path)
before = stat.st_mtime
synced_modification_time = scan_log_dirs.stat_nfs_safe_modification_time(
full_path)
if synced_modification_time != before:
etb_logger.info("sync file: %s, b: %f, a: %f - %s",
full_path, before,
synced_modification_time, "CHANGED")
def generators_from_logdir(logdir):
"""Returns a list of event generators for subdirectories with event files.
The number of generators returned should equal the number of directories
within logdir that contain event files. If only logdir contains event files,
returns a list of length one.
Args:
logdir: A log directory that contains event files.
Returns:
List of event generators for each subdirectory with event files.
"""
subdirs = event_multiplexer.GetLogdirSubdirectories(logdir)
generators = [
itertools.chain(*[
generator_from_event_file(os.path.join(subdir, f))
for f in tf.gfile.ListDirectory(subdir)
if event_accumulator.IsTensorFlowEventsFile(os.path.join(subdir, f))
]) for subdir in subdirs
]
return generators