def get_param_num(results_dir): tensorboard_dir = results_dir + "/../" event_acc = EventAccumulator(tensorboard_dir) event_acc.Reload() tensor_event = event_acc.Tensors("model_info") tensor_np = tf.make_ndarray(tensor_event[0].tensor_proto) return int(tensor_np[0, 1])
class TensorBoardAnalytics(Analytics): def __init__(self, log_path): self.event_acc = EventAccumulator(str(log_path), size_guidance={'tensors': 1000}) def load(self): self.event_acc.Reload() def tensor(self, name) -> List[Event]: name = name.replace('.', '/') events = self.event_acc.Tensors(name) return [Event(e.step, tf.make_ndarray(e.tensor_proto)) for e in events] def summarize(self, events: List[Event]): step = np.mean([e.step for e in events]) values = np.sort([e.tensor for e in events]) basis_points = np.percentile(values, BASIS_POINTS) return np.concatenate(([step], basis_points)) def summarize_compressed_histogram(self, events: List[Event]): basis_points = [int(b) for b in np.multiply(BASIS_POINTS, 100)] results = [] for e in events: buckets = compressor.compress_histogram(e.tensor) assert (len(buckets) == len(basis_points)) for i, c in enumerate(buckets): assert (c.basis_point == basis_points[i]) results.append([e.step] + [c.value for c in buckets]) return np.asarray(results)
def parse_summary(self, tfefile): metric_logs = [] event_accumulator = EventAccumulator(tfefile, size_guidance={'tensors': 0}) event_accumulator.Reload() for tag in event_accumulator.Tags()['tensors']: for m in self.metric_names: tfefile_parent_dir = os.path.dirname(m) if len( m.split("/")) >= 2 else os.path.dirname(tfefile) basedir_name = os.path.dirname(tfefile) if not tag.startswith( m.split("/")[-1]) or not basedir_name.endswith( tfefile_parent_dir): continue for wall_time, step, tensor in event_accumulator.Tensors(tag): ml = api_pb2.MetricLog(time_stamp=rfc3339.rfc3339( datetime.fromtimestamp(wall_time)), metric=api_pb2.Metric( name=m, value=str( tf.make_ndarray(tensor)))) metric_logs.append(ml) return metric_logs
def eval_logs(event_path): event_acc = EventAccumulator(event_path, size_guidance={'tensors': 100000}) event_acc.Reload() _, _, vals = zip(*event_acc.Tensors('successful_landing')) has_landed = [tf.make_ndarray(val) for val in vals] _, _, vals = zip(*event_acc.Tensors('cr')) cr = [tf.make_ndarray(val) for val in vals] _, _, vals = zip(*event_acc.Tensors('cral')) cral = [tf.make_ndarray(val) for val in vals] _, _, vals = zip(*event_acc.Tensors('boundary_counter')) boundary_counter = [tf.make_ndarray(val) for val in vals] print("Successful Landing:", sum(has_landed) / len(has_landed)) print("Collection ratio:", sum(cr) / len(cr)) print("Collection ratio and landed:", sum(cral) / len(cral)) print("Boundary counter:", sum(boundary_counter) / len(boundary_counter))
def plot_tensorflow_log(path): # Loading too much data is slow... tf_size_guidance = { 'compressedHistograms': 10, 'images': 0, 'scalars': 100, 'histograms': 1 } event_acc = EventAccumulator(path, tf_size_guidance) event_acc.Reload() # Show all tags in the log file print(event_acc.Tags()) timestep_reward = event_acc.Tensors('agent.observe/timestep-reward') episode_reward = event_acc.Tensors('agent.observe/episode-reward') #print(type(timestep_reward[0])) #tensor_event = timestep_reward[0] #tensor_np = tf.make_ndarray(tensor_event.tensor_proto) #print(tensor_np) steps = len(timestep_reward) print(f"steps: {steps}") x = np.arange(steps) y = np.zeros([steps, 2]) for i in range(steps): y[i, 0] = tf.make_ndarray(timestep_reward[i].tensor_proto) y[i, 1] = tf.make_ndarray(episode_reward[i].tensor_proto) plt.plot(x, y[:, 0], label='training accuracy') plt.plot(x, y[:, 1], label='validation accuracy') plt.xlabel("Steps") plt.ylabel("Accuracy") plt.title("Training Progress") plt.legend(loc='upper right', frameon=True) plt.show()
def test_study_name() -> None: dirname = tempfile.mkdtemp() metric_name = "target" study_name = "test_tensorboard_integration" tbcallback = TensorBoardCallback(dirname, metric_name) study = optuna.create_study(study_name=study_name) study.optimize(_objective_func, n_trials=1, callbacks=[tbcallback]) event_acc = EventAccumulator(os.path.join(dirname, "trial-0")) event_acc.Reload() try: assert len(event_acc.Tensors("target")) == 1 except Exception as e: raise e finally: shutil.rmtree(dirname)
class TensorBoardAnalytics(Analytics): def __init__(self, log_path): self.event_acc = EventAccumulator(str(log_path), size_guidance={'tensors': 1000}) try: import tensorflow as tf self.tf = tf except ImportError as e: raise e def load(self): try: self.event_acc.Reload() except DirectoryDeletedError: raise FileNotFoundError() def tensor(self, name: str, start_step: Optional[int], end_step: Optional[int]) -> List[Event]: name = name.replace('.', '/') events = self.event_acc.Tensors(name) return [ Event(e.step, self.tf.make_ndarray(e.tensor_proto)) for e in events if _is_filtered(e, start_step, end_step) ] def summarize(self, events: List[Event]): step = np.mean([e.step for e in events]) values = np.sort([e.tensor for e in events]) basis_points = np.percentile(values, BASIS_POINTS) return np.concatenate(([step], basis_points)) def summarize_compressed_histogram(self, events: List[Event]): basis_points = [int(b) for b in np.multiply(BASIS_POINTS, 100)] results = [] for e in events: buckets = compressor.compress_histogram(e.tensor) assert (len(buckets) == len(basis_points)) for i, c in enumerate(buckets): assert (c.basis_point == basis_points[i]) results.append([e.step] + [c.value for c in buckets]) return np.asarray(results)
def tflog2pandas(path: str) -> pd.DataFrame: """convert single tensorflow log file to pandas DataFrame Parameters ---------- path : str path to tensorflow log file Returns ------- pd.DataFrame converted dataframe """ DEFAULT_SIZE_GUIDANCE = { "compressedHistograms": 1, "images": 1, "scalars": 0, # 0 means load all "tensors": 0, # 0 means load all "histograms": 1, } runlog_data = pd.DataFrame({"metric": [], "value": [], "step": []}) try: event_acc = EventAccumulator(path, DEFAULT_SIZE_GUIDANCE) event_acc.Reload() # tags = event_acc.Tags()["scalars"] tags = event_acc.Tags()["tensors"] tags = [t for t in tags if "DetectionBoxes" in t or "Loss" in t] for tag in tags: # event_list = event_acc.Scalars(tag) event_list = event_acc.Tensors(tag) values = list( map(lambda x: make_ndarray(x.tensor_proto), event_list)) step = list(map(lambda x: x.step, event_list)) r = {"metric": [tag] * len(step), "value": values, "step": step} r = pd.DataFrame(r) runlog_data = pd.concat([runlog_data, r]) # Dirty catch of DataLossError except Exception: print("Event file possibly corrupt: {}".format(path)) traceback.print_exc() return runlog_data
class Analyzer: """ # TensorBoard Summary Analyzer This loads TensorBoard summaries, and provides a set of tools to create customized charts on Jupyter notebooks. The data format we use is as follows 👇 ``` [index, 0%, 6.68%, 15.87%, 30.85%, 50.00%, 69.15%, 84.13%, 93.32%, 100.00%] ``` Each data point gives a histogram in the the above format. """ def __init__(self, lab: Lab, experiment: str): self.info = ExperimentInfo(lab, experiment) self.event_acc = EventAccumulator(str(self.info.summary_path)) def load(self): """ ## Load summaries """ self.event_acc.Reload() def tensor(self, name=None): """ ## Get a tensor summary If 'name' is 'None' it returns a list of all available tensors. """ if name is None: return self.event_acc.Tags()['tensors'] return self.event_acc.Tensors(name) def scalar(self, name=None): """ ## Get a scalar summary If 'name' is 'None' it returns a list of all available scalars. """ if name is None: return self.event_acc.Tags()['scalars'] return self.event_acc.Scalars(name) def histogram(self, name=None): """ ## Get a histogram summary If 'name' is 'None' it returns a list of all available histograms. """ if name is None: return self.event_acc.Tags()['histograms'] return self.event_acc.CompressedHistograms(name) @staticmethod def summarize(events): """ ## Merge many data points and get a distribution """ step = np.mean([e.step for e in events]) values = np.sort([e.value for e in events]) basis_points = np.percentile( values, [0, 6.68, 15.87, 30.85, 50.00, 69.15, 84.13, 93.32, 100.00]) return np.concatenate(([step], basis_points)) def summarize_series(self, events): """ ### Shrink data points and produce a histogram """ # Shrink to 100 histograms interval = max(1, len(events) // 100) merged = [] results = [] for i, e in enumerate(events): if i > 0 and (i + 1) % interval == 0: results.append(self.summarize(merged)) merged = [] merged.append(e) if len(merged) > 0: results.append(self.summarize(merged)) return np.array(results) @staticmethod def summarize_compressed_histogram(events): """ ## Convert a TensorBoard histogram to our format """ basis_points = [0, 668, 1587, 3085, 5000, 6915, 8413, 9332, 10000] results = [] for e in events: assert (len(e.compressed_histogram_values) == len(basis_points)) for i, c in enumerate(e.compressed_histogram_values): assert (c.basis_point == basis_points[i]) results.append([e.step] + [c.value for c in e.compressed_histogram_values]) return np.asarray(results) @staticmethod def render_density(ax: Axes, data, color, name, *, levels=5, line_width=1, alpha=0.6): """ ## Render a density plot from data """ # Mean line ln = ax.plot(data[:, 0], data[:, 5], lw=line_width, color=color, alpha=1, label=name) # Other percentiles for i in range(1, levels): ax.fill_between(data[:, 0], data[:, 5 - i], data[:, 5 + i], color=color, lw=0, alpha=alpha**i) return ln def render_scalar(self, name, ax: Axes, color, *, levels=5, line_width=1, alpha=0.6): """ ## Summarize and render a scalar """ data = self.summarize_series(self.scalar(name)) self.render_density(ax, data, color, name, levels=levels, line_width=line_width, alpha=alpha) def render_histogram(self, name, ax: Axes, color, *, levels=5, line_width=1, alpha=0.6): """ ## Summarize and render a histogram """ data = self.summarize_compressed_histogram(self.histogram(name)) self.render_density(ax, data, color, name, levels=levels, line_width=line_width, alpha=alpha) @staticmethod def render_matrix(matrix, ax, color): from matplotlib.patches import Rectangle from matplotlib.collections import PatchCollection rows, cols = matrix.shape x_ticks = [matrix[0, i] for i in range(1, cols)] y_ticks = [matrix[i, 0] for i in range(1, rows)] max_density = 0 for y in range(rows - 2): for x in range(cols - 2): area = (x_ticks[x + 1] - x_ticks[x]) * (y_ticks[y + 1] - y_ticks[y]) density = matrix[y + 1, x + 1] / area if max_density < density: max_density = density boxes = [] for y in range(rows - 2): for x in range(cols - 2): width = x_ticks[x + 1] - x_ticks[x] height = y_ticks[y + 1] - y_ticks[y] area = width * height density = matrix[y + 1, x + 1] / area density = max(density, 1) # alpha = np.log(density) / np.log(max_density) alpha = density / max_density rect = Rectangle((x_ticks[x], y_ticks[y]), width, height, alpha=alpha, color=color) boxes.append(rect) pc = PatchCollection(boxes, match_original=True) ax.add_collection(pc) # Plot something _ = ax.errorbar([], [], xerr=[], yerr=[], fmt='None', ecolor='k') return x_ticks[0], x_ticks[-1], y_ticks[0], y_ticks[-1] def render_tensors(self, tensors: Union[str, List[any]], axes: np.ndarray, color): if type(tensors) == str: tensors = self.tensor(tensors) assert len(axes.shape) == 2 assert axes.shape[0] * axes.shape[1] == len(tensors) x_min = x_max = y_min = y_max = 0 for i in range(axes.shape[0]): for j in range(axes.shape[1]): idx = i * axes.shape[1] + j axes[i, j].set_title(f"{tensors[idx].step :,}") matrix = tf.make_ndarray(tensors[idx].tensor_proto) x1, x2, y1, y2 = self.render_matrix(matrix, axes[i, j], color) x_min = min(x_min, x1) x_max = max(x_max, x2) y_min = min(y_min, y1) y_max = max(y_max, y2) for i in range(axes.shape[0]): for j in range(axes.shape[1]): axes[i, j].set_xlim(x_min, x_max) axes[i, j].set_ylim(y_min, y_max)
import sys import json from os import listdir from os.path import isfile, join from tensorboard.backend.event_processing.event_accumulator import EventAccumulator event_acc = EventAccumulator( '/home/jakemdaly/Documents/courses/cse-291/pa2/data/LOG_RNN_PTB_50_TRIALS/logs/BS=32_LR=0.001_EB=250_RNN_HS=384_L=3_BI=0_LS=48_WD=0.05_ANN=SIGMOID_KLT_TS=2020-May-20-20_58_08' ) event_acc.Reload() # Show all tags in the log file # print(event_acc.Tags()) # E. g. get wall clock, number of steps and value for a scalar 'Accuracy' w_times, step_nums, vals = zip(*event_acc.Scalars('TRAIN-Epoch/ELBO')) w_times, step_nums, vals = zip(*event_acc.Tensors('args/text_summary')) # print(vals[0].string_val[0].decode('utf-8')[9:]) # splits = vals[0].string_val[0].decode('utf-8').split(", ") # for s in splits: def main(): data_dir = '/home/jakemdaly/Documents/courses/cse-291/pa2/data/' loggers = [] for experiment in listdir(data_dir): # for experiment in ['LOG_GRU_TWITTER_50_TRIALS']: if 'VAN' in experiment: print("Skipping VAN")
"""Trying to read tensorboard event files""" import numpy as np import tensorflow as tf from tensorboard.backend.event_processing.event_accumulator import EventAccumulator import matplotlib.pyplot as plt x = EventAccumulator(path="massminimize_logs/gof_obs") x.Reload() print(x.Tags()) w_times, l_step_nums, l_vals = zip(*x.Tensors('loss[0]')) #w_times, l_step_nums, l_vals = zip(*x.Tensors('total_loss')) #w_times, t_step_nums, t_vals = zip(*x.Tensors('theta:0[0]')) w_times, t_step_nums, t_vals = zip(*x.Tensors('s:0[0]')) def decode(val): tensor_bytes = val.tensor_content tensor_dtype = val.dtype tensor_shape = [x.size for x in val.tensor_shape.dim] tensor_array = tf.io.decode_raw(tensor_bytes, tensor_dtype) tensor_array = tf.reshape(tensor_array, tensor_shape) return tensor_array order = np.argsort(l_step_nums) print(l_step_nums) l = np.array([decode(l_vals[i]).numpy() for i in order]) print(t_step_nums)
def load_accumulator(path): """Extract data from tensorboard summaries""" event_acc = EventAccumulator(str(path), size_guidance={"tensors": 0}) event_acc.Reload() w_times, step_nums, vals = zip(*event_acc.Tensors("agent.observe/episode-reward")) return w_times, step_nums, [float(tf.make_ndarray(a)) for a in vals]