def get_param_num(results_dir):
tensorboard_dir = results_dir + "/../"
event_acc = EventAccumulator(tensorboard_dir)
event_acc.Reload()
tensor_event = event_acc.Tensors("model_info")
tensor_np = tf.make_ndarray(tensor_event[0].tensor_proto)
return int(tensor_np[0, 1])
class TensorBoardAnalytics(Analytics):
def __init__(self, log_path):
self.event_acc = EventAccumulator(str(log_path),
size_guidance={'tensors': 1000})
def load(self):
self.event_acc.Reload()
def tensor(self, name) -> List[Event]:
name = name.replace('.', '/')
events = self.event_acc.Tensors(name)
return [Event(e.step, tf.make_ndarray(e.tensor_proto)) for e in events]
def summarize(self, events: List[Event]):
step = np.mean([e.step for e in events])
values = np.sort([e.tensor for e in events])
basis_points = np.percentile(values, BASIS_POINTS)
return np.concatenate(([step], basis_points))
def summarize_compressed_histogram(self, events: List[Event]):
basis_points = [int(b) for b in np.multiply(BASIS_POINTS, 100)]
results = []
for e in events:
buckets = compressor.compress_histogram(e.tensor)
assert (len(buckets) == len(basis_points))
for i, c in enumerate(buckets):
assert (c.basis_point == basis_points[i])
results.append([e.step] + [c.value for c in buckets])
return np.asarray(results)
def parse_summary(self, tfefile):
metric_logs = []
event_accumulator = EventAccumulator(tfefile,
size_guidance={'tensors': 0})
event_accumulator.Reload()
for tag in event_accumulator.Tags()['tensors']:
for m in self.metric_names:
tfefile_parent_dir = os.path.dirname(m) if len(
m.split("/")) >= 2 else os.path.dirname(tfefile)
basedir_name = os.path.dirname(tfefile)
if not tag.startswith(
m.split("/")[-1]) or not basedir_name.endswith(
tfefile_parent_dir):
continue
for wall_time, step, tensor in event_accumulator.Tensors(tag):
ml = api_pb2.MetricLog(time_stamp=rfc3339.rfc3339(
datetime.fromtimestamp(wall_time)),
metric=api_pb2.Metric(
name=m,
value=str(
tf.make_ndarray(tensor))))
metric_logs.append(ml)
return metric_logs
def eval_logs(event_path):
event_acc = EventAccumulator(event_path, size_guidance={'tensors': 100000})
event_acc.Reload()
_, _, vals = zip(*event_acc.Tensors('successful_landing'))
has_landed = [tf.make_ndarray(val) for val in vals]
_, _, vals = zip(*event_acc.Tensors('cr'))
cr = [tf.make_ndarray(val) for val in vals]
_, _, vals = zip(*event_acc.Tensors('cral'))
cral = [tf.make_ndarray(val) for val in vals]
_, _, vals = zip(*event_acc.Tensors('boundary_counter'))
boundary_counter = [tf.make_ndarray(val) for val in vals]
print("Successful Landing:", sum(has_landed) / len(has_landed))
print("Collection ratio:", sum(cr) / len(cr))
print("Collection ratio and landed:", sum(cral) / len(cral))
print("Boundary counter:", sum(boundary_counter) / len(boundary_counter))
def plot_tensorflow_log(path):
# Loading too much data is slow...
tf_size_guidance = {
'compressedHistograms': 10,
'images': 0,
'scalars': 100,
'histograms': 1
}
event_acc = EventAccumulator(path, tf_size_guidance)
event_acc.Reload()
# Show all tags in the log file
print(event_acc.Tags())
timestep_reward = event_acc.Tensors('agent.observe/timestep-reward')
episode_reward = event_acc.Tensors('agent.observe/episode-reward')
#print(type(timestep_reward[0]))
#tensor_event = timestep_reward[0]
#tensor_np = tf.make_ndarray(tensor_event.tensor_proto)
#print(tensor_np)
steps = len(timestep_reward)
print(f"steps: {steps}")
x = np.arange(steps)
y = np.zeros([steps, 2])
for i in range(steps):
y[i, 0] = tf.make_ndarray(timestep_reward[i].tensor_proto)
y[i, 1] = tf.make_ndarray(episode_reward[i].tensor_proto)
plt.plot(x, y[:, 0], label='training accuracy')
plt.plot(x, y[:, 1], label='validation accuracy')
plt.xlabel("Steps")
plt.ylabel("Accuracy")
plt.title("Training Progress")
plt.legend(loc='upper right', frameon=True)
plt.show()
def test_study_name() -> None:
dirname = tempfile.mkdtemp()
metric_name = "target"
study_name = "test_tensorboard_integration"
tbcallback = TensorBoardCallback(dirname, metric_name)
study = optuna.create_study(study_name=study_name)
study.optimize(_objective_func, n_trials=1, callbacks=[tbcallback])
event_acc = EventAccumulator(os.path.join(dirname, "trial-0"))
event_acc.Reload()
try:
assert len(event_acc.Tensors("target")) == 1
except Exception as e:
raise e
finally:
shutil.rmtree(dirname)
class TensorBoardAnalytics(Analytics):
def __init__(self, log_path):
self.event_acc = EventAccumulator(str(log_path),
size_guidance={'tensors': 1000})
try:
import tensorflow as tf
self.tf = tf
except ImportError as e:
raise e
def load(self):
try:
self.event_acc.Reload()
except DirectoryDeletedError:
raise FileNotFoundError()
def tensor(self, name: str, start_step: Optional[int],
end_step: Optional[int]) -> List[Event]:
name = name.replace('.', '/')
events = self.event_acc.Tensors(name)
return [
Event(e.step, self.tf.make_ndarray(e.tensor_proto)) for e in events
if _is_filtered(e, start_step, end_step)
]
def summarize(self, events: List[Event]):
step = np.mean([e.step for e in events])
values = np.sort([e.tensor for e in events])
basis_points = np.percentile(values, BASIS_POINTS)
return np.concatenate(([step], basis_points))
def summarize_compressed_histogram(self, events: List[Event]):
basis_points = [int(b) for b in np.multiply(BASIS_POINTS, 100)]
results = []
for e in events:
buckets = compressor.compress_histogram(e.tensor)
assert (len(buckets) == len(basis_points))
for i, c in enumerate(buckets):
assert (c.basis_point == basis_points[i])
results.append([e.step] + [c.value for c in buckets])
return np.asarray(results)
def tflog2pandas(path: str) -> pd.DataFrame:
"""convert single tensorflow log file to pandas DataFrame
Parameters
----------
path : str
path to tensorflow log file
Returns
-------
pd.DataFrame
converted dataframe
"""
DEFAULT_SIZE_GUIDANCE = {
"compressedHistograms": 1,
"images": 1,
"scalars": 0, # 0 means load all
"tensors": 0, # 0 means load all
"histograms": 1,
}
runlog_data = pd.DataFrame({"metric": [], "value": [], "step": []})
try:
event_acc = EventAccumulator(path, DEFAULT_SIZE_GUIDANCE)
event_acc.Reload()
# tags = event_acc.Tags()["scalars"]
tags = event_acc.Tags()["tensors"]
tags = [t for t in tags if "DetectionBoxes" in t or "Loss" in t]
for tag in tags:
# event_list = event_acc.Scalars(tag)
event_list = event_acc.Tensors(tag)
values = list(
map(lambda x: make_ndarray(x.tensor_proto), event_list))
step = list(map(lambda x: x.step, event_list))
r = {"metric": [tag] * len(step), "value": values, "step": step}
r = pd.DataFrame(r)
runlog_data = pd.concat([runlog_data, r])
# Dirty catch of DataLossError
except Exception:
print("Event file possibly corrupt: {}".format(path))
traceback.print_exc()
return runlog_data
class Analyzer:
"""
# TensorBoard Summary Analyzer
This loads TensorBoard summaries, and provides a set of tools to
create customized charts on Jupyter notebooks.
The data format we use is as follows 👇
```
[index,
0%,
6.68%,
15.87%,
30.85%,
50.00%,
69.15%,
84.13%,
93.32%,
100.00%]
```
Each data point gives a histogram in the the above format.
"""
def __init__(self, lab: Lab, experiment: str):
self.info = ExperimentInfo(lab, experiment)
self.event_acc = EventAccumulator(str(self.info.summary_path))
def load(self):
"""
## Load summaries
"""
self.event_acc.Reload()
def tensor(self, name=None):
"""
## Get a tensor summary
If 'name' is 'None' it returns a list of all available tensors.
"""
if name is None:
return self.event_acc.Tags()['tensors']
return self.event_acc.Tensors(name)
def scalar(self, name=None):
"""
## Get a scalar summary
If 'name' is 'None' it returns a list of all available scalars.
"""
if name is None:
return self.event_acc.Tags()['scalars']
return self.event_acc.Scalars(name)
def histogram(self, name=None):
"""
## Get a histogram summary
If 'name' is 'None' it returns a list of all available histograms.
"""
if name is None:
return self.event_acc.Tags()['histograms']
return self.event_acc.CompressedHistograms(name)
@staticmethod
def summarize(events):
"""
## Merge many data points and get a distribution
"""
step = np.mean([e.step for e in events])
values = np.sort([e.value for e in events])
basis_points = np.percentile(
values,
[0, 6.68, 15.87, 30.85, 50.00, 69.15, 84.13, 93.32, 100.00])
return np.concatenate(([step], basis_points))
def summarize_series(self, events):
"""
### Shrink data points and produce a histogram
"""
# Shrink to 100 histograms
interval = max(1, len(events) // 100)
merged = []
results = []
for i, e in enumerate(events):
if i > 0 and (i + 1) % interval == 0:
results.append(self.summarize(merged))
merged = []
merged.append(e)
if len(merged) > 0:
results.append(self.summarize(merged))
return np.array(results)
@staticmethod
def summarize_compressed_histogram(events):
"""
## Convert a TensorBoard histogram to our format
"""
basis_points = [0, 668, 1587, 3085, 5000, 6915, 8413, 9332, 10000]
results = []
for e in events:
assert (len(e.compressed_histogram_values) == len(basis_points))
for i, c in enumerate(e.compressed_histogram_values):
assert (c.basis_point == basis_points[i])
results.append([e.step] +
[c.value for c in e.compressed_histogram_values])
return np.asarray(results)
@staticmethod
def render_density(ax: Axes,
data,
color,
name,
*,
levels=5,
line_width=1,
alpha=0.6):
"""
## Render a density plot from data
"""
# Mean line
ln = ax.plot(data[:, 0],
data[:, 5],
lw=line_width,
color=color,
alpha=1,
label=name)
# Other percentiles
for i in range(1, levels):
ax.fill_between(data[:, 0],
data[:, 5 - i],
data[:, 5 + i],
color=color,
lw=0,
alpha=alpha**i)
return ln
def render_scalar(self,
name,
ax: Axes,
color,
*,
levels=5,
line_width=1,
alpha=0.6):
"""
## Summarize and render a scalar
"""
data = self.summarize_series(self.scalar(name))
self.render_density(ax,
data,
color,
name,
levels=levels,
line_width=line_width,
alpha=alpha)
def render_histogram(self,
name,
ax: Axes,
color,
*,
levels=5,
line_width=1,
alpha=0.6):
"""
## Summarize and render a histogram
"""
data = self.summarize_compressed_histogram(self.histogram(name))
self.render_density(ax,
data,
color,
name,
levels=levels,
line_width=line_width,
alpha=alpha)
@staticmethod
def render_matrix(matrix, ax, color):
from matplotlib.patches import Rectangle
from matplotlib.collections import PatchCollection
rows, cols = matrix.shape
x_ticks = [matrix[0, i] for i in range(1, cols)]
y_ticks = [matrix[i, 0] for i in range(1, rows)]
max_density = 0
for y in range(rows - 2):
for x in range(cols - 2):
area = (x_ticks[x + 1] - x_ticks[x]) * (y_ticks[y + 1] -
y_ticks[y])
density = matrix[y + 1, x + 1] / area
if max_density < density:
max_density = density
boxes = []
for y in range(rows - 2):
for x in range(cols - 2):
width = x_ticks[x + 1] - x_ticks[x]
height = y_ticks[y + 1] - y_ticks[y]
area = width * height
density = matrix[y + 1, x + 1] / area
density = max(density, 1)
# alpha = np.log(density) / np.log(max_density)
alpha = density / max_density
rect = Rectangle((x_ticks[x], y_ticks[y]),
width,
height,
alpha=alpha,
color=color)
boxes.append(rect)
pc = PatchCollection(boxes, match_original=True)
ax.add_collection(pc)
# Plot something
_ = ax.errorbar([], [], xerr=[], yerr=[], fmt='None', ecolor='k')
return x_ticks[0], x_ticks[-1], y_ticks[0], y_ticks[-1]
def render_tensors(self, tensors: Union[str, List[any]], axes: np.ndarray,
color):
if type(tensors) == str:
tensors = self.tensor(tensors)
assert len(axes.shape) == 2
assert axes.shape[0] * axes.shape[1] == len(tensors)
x_min = x_max = y_min = y_max = 0
for i in range(axes.shape[0]):
for j in range(axes.shape[1]):
idx = i * axes.shape[1] + j
axes[i, j].set_title(f"{tensors[idx].step :,}")
matrix = tf.make_ndarray(tensors[idx].tensor_proto)
x1, x2, y1, y2 = self.render_matrix(matrix, axes[i, j], color)
x_min = min(x_min, x1)
x_max = max(x_max, x2)
y_min = min(y_min, y1)
y_max = max(y_max, y2)
for i in range(axes.shape[0]):
for j in range(axes.shape[1]):
axes[i, j].set_xlim(x_min, x_max)
axes[i, j].set_ylim(y_min, y_max)
import sys
import json
from os import listdir
from os.path import isfile, join
from tensorboard.backend.event_processing.event_accumulator import EventAccumulator
event_acc = EventAccumulator(
'/home/jakemdaly/Documents/courses/cse-291/pa2/data/LOG_RNN_PTB_50_TRIALS/logs/BS=32_LR=0.001_EB=250_RNN_HS=384_L=3_BI=0_LS=48_WD=0.05_ANN=SIGMOID_KLT_TS=2020-May-20-20_58_08'
)
event_acc.Reload()
# Show all tags in the log file
# print(event_acc.Tags())
# E. g. get wall clock, number of steps and value for a scalar 'Accuracy'
w_times, step_nums, vals = zip(*event_acc.Scalars('TRAIN-Epoch/ELBO'))
w_times, step_nums, vals = zip(*event_acc.Tensors('args/text_summary'))
# print(vals[0].string_val[0].decode('utf-8')[9:])
# splits = vals[0].string_val[0].decode('utf-8').split(", ")
# for s in splits:
def main():
data_dir = '/home/jakemdaly/Documents/courses/cse-291/pa2/data/'
loggers = []
for experiment in listdir(data_dir):
# for experiment in ['LOG_GRU_TWITTER_50_TRIALS']:
if 'VAN' in experiment:
print("Skipping VAN")
"""Trying to read tensorboard event files"""
import numpy as np
import tensorflow as tf
from tensorboard.backend.event_processing.event_accumulator import EventAccumulator
import matplotlib.pyplot as plt
x = EventAccumulator(path="massminimize_logs/gof_obs")
x.Reload()
print(x.Tags())
w_times, l_step_nums, l_vals = zip(*x.Tensors('loss[0]'))
#w_times, l_step_nums, l_vals = zip(*x.Tensors('total_loss'))
#w_times, t_step_nums, t_vals = zip(*x.Tensors('theta:0[0]'))
w_times, t_step_nums, t_vals = zip(*x.Tensors('s:0[0]'))
def decode(val):
tensor_bytes = val.tensor_content
tensor_dtype = val.dtype
tensor_shape = [x.size for x in val.tensor_shape.dim]
tensor_array = tf.io.decode_raw(tensor_bytes, tensor_dtype)
tensor_array = tf.reshape(tensor_array, tensor_shape)
return tensor_array
order = np.argsort(l_step_nums)
print(l_step_nums)
l = np.array([decode(l_vals[i]).numpy() for i in order])
print(t_step_nums)
def load_accumulator(path):
"""Extract data from tensorboard summaries"""
event_acc = EventAccumulator(str(path), size_guidance={"tensors": 0})
event_acc.Reload()
w_times, step_nums, vals = zip(*event_acc.Tensors("agent.observe/episode-reward"))
return w_times, step_nums, [float(tf.make_ndarray(a)) for a in vals]
