def _GeneratorFromPath(path,
event_file_active_filter=None,
detect_file_replacement=None):
"""Create an event generator for file or directory at given path string."""
if not path:
raise ValueError("path must be a valid string")
if io_wrapper.IsSummaryEventsFile(path):
return event_file_loader.EventFileLoader(path, detect_file_replacement)
elif event_file_active_filter:
loader_factory = (
lambda path: event_file_loader.TimestampedEventFileLoader(
path, detect_file_replacement))
return directory_loader.DirectoryLoader(
path,
loader_factory,
path_filter=io_wrapper.IsSummaryEventsFile,
active_filter=event_file_active_filter,
)
else:
loader_factory = lambda path: event_file_loader.EventFileLoader(
path, detect_file_replacement)
return directory_watcher.DirectoryWatcher(
path,
loader_factory,
io_wrapper.IsSummaryEventsFile,
)
def load_events(base, experiment, series):
p_series = os.path.join(base, experiment, series)
agg = []
for f in os.listdir(p_series):
fpath = os.path.join(p_series, f)
if os.path.isdir(fpath) or 'profile' in f:
continue
#print(f)
efl = event_file_loader.EventFileLoader(fpath)
for i, e in enumerate(efl.Load()):
#if i>=5:break
#print(i, e)
ts = e.wall_time
step = e.step
summary_value = e.summary.value
if len(summary_value) > 0:
# https://developers.google.com/protocol-buffers/docs/pythontutorial
#print(f"Found {type(v)}")
#print(f"Found {summary_value[0].simple_value}")
agg.append({
'ts': ts,
'step': step,
summary_value[0].tag: summary_value[0].simple_value,
})
df = pd.DataFrame(agg) # , columns=['ts', 'step', 'value']
# https://www.science-emergence.com/Articles/How-to-add-metadata-to-a-data-frame-with-pandas-in-python-/
df.base = base
df.experiment = experiment
df.series = series
return df
def get_metrics_from_eval_tfevents(eval_dir):
"""Get metrics from tfevents in eval dir
Args:
eval_dir (str): Root directory where eval summaries are stored
Returns:
list[float], list[int]: average returns, and average episode lengths
"""
event_file = None
for root, dirs, files in os.walk(eval_dir):
for file_name in files:
if "events" in file_name and 'profile' not in file_name:
event_file = os.path.join(root, file_name)
break
assert event_file is not None
logging.info("Parse event file:%s", event_file)
episode_returns = []
episode_lengths = []
for event_str in event_file_loader.EventFileLoader(event_file).Load():
if event_str.summary.value:
for item in event_str.summary.value:
if item.tag == 'Metrics/AverageReturn':
episode_returns.append(item.simple_value)
elif item.tag == 'Metrics/AverageEpisodeLength':
episode_lengths.append(item.simple_value)
assert len(episode_returns) > 0
logging.info("Episode returns, %s, episode lengths: %s", episode_returns,
episode_lengths)
return episode_returns, episode_lengths
def _event_dict_list_from_events_file(event_file_path):
"""Given an event file path, load the event data.
Args:
event_file_path (str): A path to a TensorFlow events.out* file.
"""
loader = event_file_loader.EventFileLoader(event_file_path)
events = []
for event in loader.Load():
events.append(event)
output_events = []
for thing in events:
d = protobuf_to_dict(thing)
output_event = {}
if "wall_time" in d:
output_event["wall_time"] = d["wall_time"]
if "step" in d:
output_event["step"] = d["step"]
if "summary" in d.keys():
values = {}
for value in d["summary"]["value"]:
if "simple_value" in value.keys():
output_event[value["tag"]] = value["simple_value"]
if "loss" in output_event:
output_events.append(output_event)
return output_events
def _GeneratorFromPath(path):
"""Create an event generator for file or directory at given path string."""
if not path:
raise ValueError('path must be a valid string')
if IsTensorFlowEventsFile(path):
return event_file_loader.EventFileLoader(path)
else:
return directory_watcher.DirectoryWatcher(
path, event_file_loader.EventFileLoader, IsTensorFlowEventsFile)
def load_events(filename):
try:
# tensorboard>=1.14.0
from tensorboard.backend.event_processing import event_file_loader
loader = event_file_loader.EventFileLoader(filename)
return list(loader.Load())
except ImportError:
import tensorflow as tf
return list(tf.train.summary_iterator(filename))
def stream_tensorboard_scalars(event_file):
loader = event_file_loader.EventFileLoader(event_file)
for event in loader.Load():
t = event.step
if event.summary is not None and len(event.summary.value):
val = event.summary.value[0]
tag = val.tag
value = val.simple_value or np.array(val.tensor.float_val).reshape(
[d.size for d in val.tensor.tensor_shape.dim])
yield t, tag, value
def test_summary(self):
with tempfile.TemporaryDirectory() as root_dir:
writer = alf.summary.create_summary_writer(root_dir,
flush_secs=10,
max_queue=10)
alf.summary.set_default_writer(writer)
alf.summary.enable_summary()
with alf.summary.scope("root") as scope_name:
self.assertEqual(scope_name, "root/")
alf.summary.scalar("scalar", 2020)
with alf.summary.scope("a") as scope_name:
self.assertEqual(scope_name, "root/a/")
alf.summary.text("text", "sample text")
with alf.summary.record_if(lambda: False):
alf.summary.text("test", "this should not appear")
alf.summary.disable_summary()
alf.summary.text("testa", "this should not appear")
alf.summary.enable_summary()
with alf.summary.scope("b") as scope_name:
self.assertEqual(scope_name, "root/b/")
alf.summary.histogram("histogram",
torch.arange(100).numpy())
writer.close()
event_file = _find_event_file(root_dir)
self.assertIsNotNone(event_file)
tag2val = {
'root/scalar': None,
'root/a/text/text_summary': None,
'root/b/histogram': None
}
for event_str in event_file_loader.EventFileLoader(
event_file).Load():
if event_str.summary.value:
for item in event_str.summary.value:
self.assertTrue(item.tag in tag2val)
if item.HasField('simple_value'):
tag2val[item.tag] = item.simple_value
elif item.HasField('histo'):
tag2val[item.tag] = item.histo
else:
self.assertTrue('tensor')
tag2val[item.tag] = tensor_util.make_ndarray(
item.tensor)
self.assertEqual(tag2val['root/scalar'], 2020)
self.assertEqual(tag2val['root/a/text/text_summary'][0],
b'sample text')
self.assertEqual(tag2val['root/b/histogram'].min, 0)
self.assertEqual(tag2val['root/b/histogram'].max, 99)
self.assertEqual(len(tag2val['root/b/histogram'].bucket_limit), 31)
def _GeneratorFromPath(path):
"""Create an event generator for file or directory at given path string."""
if not path:
raise ValueError("path must be a valid string")
if io_wrapper.IsSummaryEventsFile(path):
return event_file_loader.EventFileLoader(path)
else:
return directory_watcher.DirectoryWatcher(
path,
event_file_loader.EventFileLoader,
io_wrapper.IsSummaryEventsFile,
)
def test_graph_def(self):
# Create a `GraphDef` and write it to disk as an event.
logdir = self.get_temp_dir()
writer = test_util.FileWriter(logdir)
graph_def = graph_pb2.GraphDef()
graph_def.node.add(name="alice", op="Person")
graph_def.node.add(name="bob", op="Person")
graph_def.node.add(name="friendship",
op="Friendship",
input=["alice", "bob"])
writer.add_graph(graph=None, graph_def=graph_def, global_step=123)
writer.flush()
# Read in the `Event` containing the written `graph_def`.
files = os.listdir(logdir)
self.assertLen(files, 1)
event_file = os.path.join(logdir, files[0])
self.assertIn("tfevents", event_file)
loader = event_file_loader.EventFileLoader(event_file)
events = list(loader.Load())
self.assertLen(events, 2)
self.assertEqual(events[0].WhichOneof("what"), "file_version")
self.assertEqual(events[1].WhichOneof("what"), "graph_def")
old_event = events[1]
new_events = self._migrate_event(old_event)
self.assertLen(new_events, 2)
self.assertIs(new_events[0], old_event)
new_event = new_events[1]
self.assertEqual(new_event.WhichOneof("what"), "summary")
self.assertLen(new_event.summary.value, 1)
tensor = tensor_util.make_ndarray(new_event.summary.value[0].tensor)
self.assertEqual(
new_event.summary.value[0].metadata.data_class,
summary_pb2.DATA_CLASS_BLOB_SEQUENCE,
)
self.assertEqual(
new_event.summary.value[0].metadata.plugin_data.plugin_name,
graphs_metadata.PLUGIN_NAME,
)
self.assertEqual(tensor.shape, (1, ))
new_graph_def_bytes = tensor[0]
self.assertIsInstance(new_graph_def_bytes, bytes)
self.assertGreaterEqual(len(new_graph_def_bytes), 16)
new_graph_def = graph_pb2.GraphDef.FromString(new_graph_def_bytes)
self.assertProtoEquals(graph_def, new_graph_def)
def _GeneratorFromPath(path, event_file_active_filter=None):
"""Create an event generator for file or directory at given path string."""
if not path:
raise ValueError('path must be a valid string')
if io_wrapper.IsTensorFlowEventsFile(path):
return event_file_loader.EventFileLoader(path)
elif event_file_active_filter:
return directory_loader.DirectoryLoader(
path,
event_file_loader.TimestampedEventFileLoader,
path_filter=io_wrapper.IsTensorFlowEventsFile,
active_filter=event_file_active_filter)
else:
return directory_watcher.DirectoryWatcher(
path, event_file_loader.EventFileLoader,
io_wrapper.IsTensorFlowEventsFile)
def _latest_eval(eval_dir, eval_metrics):
"""Get the latest global step for which an evaluation result was written."""
if not tf.io.gfile.exists(eval_dir):
return None
expected_tags = set(
[common.join_scope('Metrics', metric.name) for metric in eval_metrics])
# Record which summaries were written for each global step.
events_by_step = collections.defaultdict(set)
for events_file in tf.io.gfile.listdir(eval_dir):
loader = event_file_loader.EventFileLoader(
os.path.join(eval_dir, events_file))
for event in loader.Load():
if event.summary.value:
events_by_step[event.step].add(event.summary.value[0].tag)
# Find the greatest step for which all expected summaries are present.
for step in sorted(list(events_by_step.keys()), key=lambda step: -step):
if events_by_step[step].issuperset(expected_tags):
return step
return None
def get_tensorboard_log_watcher_from_path(path: Text):
"""Create an event generator for file or directory at given path string.
This method creates an event generator using tensorboard directory_watcher.
The generator.load() method will return event logs as they become available.
The generator does not repeat events.
Args:
path: Text representing a directory, file, or Google Cloud Storage
(GCS) for tensorboard logs.
Returns:
A tensorboard directory_watcher event generator.
Raises:
ValueError: if path is not defined.
"""
if not path:
raise ValueError("path must be a valid string")
if io_wrapper.IsSummaryEventsFile(path):
return event_file_loader.EventFileLoader(path)
return directory_watcher.DirectoryWatcher(
path,
event_file_loader.EventFileLoader,
io_wrapper.IsSummaryEventsFile,
)
def _process_health_pill_event(self, node_name_set, mapping, target_step,
file_path):
"""Creates health pills out of data in an event.
Creates health pills out of the event and adds them to the mapping.
Args:
node_name_set: A set of node names that are relevant.
mapping: The mapping from node name to HealthPillEvents.
This object may be destructively modified.
target_step: The target step at which to obtain health pills.
file_path: The path to the file with health pill events.
Returns:
Whether we should stop reading events because future events are no longer
relevant.
"""
events_loader = event_file_loader.EventFileLoader(file_path)
for event in events_loader.Load():
if not event.HasField("summary"):
logger.warning(
"An event in a debugger events file lacks a summary.")
continue
if event.step < target_step:
# This event is not of the relevant step. We perform this check
# first because the majority of events will be eliminated from
# consideration by this check.
continue
if event.step > target_step:
# We have passed the relevant step. No need to read more events.
return True
for value in event.summary.value:
# Obtain the device name from the metadata.
summary_metadata = value.metadata
plugin_data = summary_metadata.plugin_data
if plugin_data.plugin_name == constants.DEBUGGER_PLUGIN_NAME:
try:
content = json.loads(
tf.compat.as_text(
summary_metadata.plugin_data.content))
except ValueError as err:
logger.warning(
"Could not parse the JSON string containing data for "
"the debugger plugin: %r, %r",
content,
err,
)
continue
device_name = content["device"]
output_slot = content["outputSlot"]
else:
logger.error(
"No debugger plugin data found for event with tag %s and node "
"name %s.",
value.tag,
value.node_name,
)
continue
if not value.HasField("tensor"):
logger.warning(
"An event in a debugger events file lacks a tensor value."
)
continue
match = re.match(r"^(.*):(\d+):DebugNumericSummary$",
value.node_name)
if not match:
logger.warning(
("A event with a health pill has an invalid watch, (i.e., an "
"unexpected debug op): %r"),
value.node_name,
)
return None
health_pill = self._process_health_pill_value(
wall_time=event.wall_time,
step=event.step,
device_name=device_name,
output_slot=output_slot,
node_name=match.group(1),
tensor_proto=value.tensor,
node_name_set=node_name_set,
)
if not health_pill:
continue
mapping[health_pill.node_name].append(health_pill)
# Keep reading events.
return False
def generator_from_event_file(event_file): """Returns a generator that yields events from an event file.""" return event_file_loader.EventFileLoader(event_file).Load()
def _LoaderForTestFile(self, filename):
return event_file_loader.EventFileLoader(
os.path.join(self.get_temp_dir(), filename))