def tabulate_events(dir_path): summary_iterators = [ EventAccumulator(os.path.join(dir_path, dname)).Reload() for dname in os.listdir(dir_path) ] tags = summary_iterators[0].Tags()['scalars'] for it in summary_iterators: assert it.Tags()['scalars'] == tags out = defaultdict(list) steps = [] for tag in tags: steps = [e.step for e in summary_iterators[0].Scalars(tag)] wall_times = [e.wall_time for e in summary_iterators[0].Scalars(tag)] for events in zip(*[acc.Scalars(tag) for acc in summary_iterators]): assert len(set(e.step for e in events)) == 1 out[tag].append([e.value for e in events]) return out, steps, wall_times
def tabulate_events(inpath, outpath, tags): summary_iterators = [ EventAccumulator(os.path.join(inpath, dname)).Reload() for dname in os.listdir(inpath) ] # tags = summary_iterators[0].Tags()['scalars'] # # for it in summary_iterators: # print(it.Tags()) # assert it.Tags()['scalars'] == tags for tag in tags: for it in summary_iterators: try: run_name = ntpath.basename(it.path) csv_path = os.path.join( outpath, run_name + '_' + tag.replace('/', '_') + '.csv') pd.DataFrame(it.Scalars(tag)).to_csv(csv_path) print('Success: {} - {}'.format(tag, it.path)) except KeyError: print('Error: {} - {}'.format(tag, it.path)) pass
def extract_data(summary_path: os.PathLike, logs_path: os.PathLike) -> Tuple[List[float], Dict, str]: config = Config.load(summary_path).config events_acc = EventAccumulator(logs_path) events_acc.Reload() _, _, val_acc_diffs = zip(*events_acc.Scalars('diff/val/acc')) hp = config.hp.to_dict() hp['n_conv_layers'] = len(config.hp.conv_model_config.conv_sizes) if 'Minimum_test' in events_acc.images.Keys(): image_test = events_acc.Images('Minimum_test')[0].encoded_image_string image_train = events_acc.Images( 'Minimum_train')[0].encoded_image_string else: image_test = None image_train = None return val_acc_diffs, hp, image_test, image_train
def tensorboard_loaddata(dir, ppo=PPO): if ppo: keys = ['batch/reward_mean', 'eval/reward_sum'] # ppo else: keys = ['reward/batch', 'eval/reward_sum'] # td3 x = EventAccumulator(path=dir) x.Reload() x.FirstEventTimestamp() steps = [] wall_time = [] index = [] count = [] data = [] for k in keys: steps.append([e.step for e in x.Scalars(k)]) wall_time.append([e.wall_time for e in x.Scalars(k)]) # index.append([e.index for e in x.Scalars(k)]) # count.append([e.count for e in x.Scalars(k)]) data.append([e.value for e in x.Scalars(k)]) return wall_time, steps, data
def tflog2pandas(path: str) -> pd.DataFrame: """convert single tensorflow log file to pandas DataFrame Parameters ---------- path : str path to tensorflow log file Returns ------- pd.DataFrame converted dataframe """ DEFAULT_SIZE_GUIDANCE = { "compressedHistograms": 1, "images": 1, "scalars": 0, # 0 means load all "histograms": 1, } try: event_acc = EventAccumulator(path, DEFAULT_SIZE_GUIDANCE) event_acc.Reload() tags = event_acc.Tags()["scalars"] tag_train = [t for t in tags if 'test' not in t] tag_test = [t for t in tags if 'test' in t] d_train = {t: [] for t in tag_train} d_test = {t: [] for t in tag_test} for tag in tag_train: event_list = event_acc.Scalars(tag) values = list(map(lambda x: x.value, event_list)) d_train[tag] = values for tag in tag_test: event_list = event_acc.Scalars(tag) values = list(map(lambda x: x.value, event_list)) d_test[tag] = values d_train = pd.DataFrame(d_train) d_test = pd.DataFrame(d_test) # Dirty catch of DataLossError except Exception: print("Event file possibly corrupt: {}".format(path)) traceback.print_exc() return {'train': d_train, 'test': d_test}
def plot_tensorflow_log(path): # Loading too much data is slow... tf_size_guidance = { 'compressedHistograms': 10, 'images': 0, 'scalars': 100, 'histograms': 1 } event_acc = EventAccumulator(path, tf_size_guidance) event_acc.Reload() # Show all tags in the log file print(event_acc.Tags()) timestep_reward = event_acc.Tensors('agent.observe/timestep-reward') episode_reward = event_acc.Tensors('agent.observe/episode-reward') #print(type(timestep_reward[0])) #tensor_event = timestep_reward[0] #tensor_np = tf.make_ndarray(tensor_event.tensor_proto) #print(tensor_np) steps = len(timestep_reward) print(f"steps: {steps}") x = np.arange(steps) y = np.zeros([steps, 2]) for i in range(steps): y[i, 0] = tf.make_ndarray(timestep_reward[i].tensor_proto) y[i, 1] = tf.make_ndarray(episode_reward[i].tensor_proto) plt.plot(x, y[:, 0], label='training accuracy') plt.plot(x, y[:, 1], label='validation accuracy') plt.xlabel("Steps") plt.ylabel("Accuracy") plt.title("Training Progress") plt.legend(loc='upper right', frameon=True) plt.show()
def pick_up_summary(self, summary_path): event_acc = EventAccumulator(summary_path) event_acc.Reload() # E. g. get wall clock, number of steps and value for a scalar 'Accuracy' _, steps_loss, vals_loss = zip(*event_acc.Scalars('Loss')) _, steps_accuracy, vals_accuracy = zip(*event_acc.Scalars('Accuracy')) _, steps_validation_loss, vals_validation_loss = zip( *event_acc.Scalars('Validation_Loss')) _, steps_validation_accuracy, vals_validation_accuracy = zip( *event_acc.Scalars('Validation_Accuracy')) training_history = dict() training_history['steps_loss'] = steps_loss training_history['steps_accuracy'] = steps_accuracy training_history['steps_validation_loss'] = steps_validation_loss training_history[ 'steps_validation_accuracy'] = steps_validation_accuracy training_history['values_loss'] = vals_loss training_history['values_accuracy'] = vals_accuracy training_history['values_validation_loss'] = vals_validation_loss training_history[ 'values_validation_accuracy'] = vals_validation_accuracy self.test_result['training_history'] = training_history
def logCSV(log_dir, seed, csv_path, tag_name_lst=None): data = EventAccumulator(log_dir) data.Reload() for tag in data.Tags().get('scalars'): print(tag) if tag_name_lst is not None and len(tag_name_lst) > 0: if tag in tag_name_lst: w_times, step_nums, vals = zip(*data.Scalars(tag)) in_data = {'w_times': w_times, 'step': step_nums, 'value': vals} in_data = pd.DataFrame(in_data) in_data.to_csv("{}/{}_seed_{}.csv".format(csv_path, str(tag), str(seed)), index=False) else: w_times, step_nums, vals = zip(*data.Scalars(tag)) in_data = {'wall_time': w_times, 'step': step_nums, 'value': vals} in_data = pd.DataFrame(in_data) in_data.to_csv("{}/{}_seed_{}.csv".format(csv_path, str(tag), str(seed)), index=False)
def read_tensorboard_log(path): "Reads the .tfevents file and returns the scalars." # First let's read the config file config = json.load(open(os.path.join(path, "config.json"))) event_acc = EventAccumulator(path) event_acc.Reload() # print(event_acc.Tags()) # shows all tags in the log file wall_times, step_nums, loss_train = zip(*event_acc.Scalars("Loss/train")) # We assume that all scalars are recorded at each step _, _, loss_valid = zip(*event_acc.Scalars("Loss/valid")) _, _, acc_train = zip(*event_acc.Scalars("Accuracy/train")) _, _, acc_valid = zip(*event_acc.Scalars("Accuracy/valid")) info = { "loss_train": loss_train, "loss_valid": loss_valid, "acc_train": acc_train, "acc_valid": acc_valid, "step_nums": step_nums, "wall_times": wall_times, "config": config, } return info
def plot_tensorflow_log(path): # Loading too much data is slow... tf_size_guidance = { 'compressedHistograms': 10, 'images': 0, 'scalars': 100, 'histograms': 1 } event_acc = EventAccumulator(path, tf_size_guidance) event_acc.Reload() # Show all tags in the log file print(event_acc.Tags()) ipdb.set_trace() training_accuracies = event_acc.Scalars('eval/vqa/accuracy') validation_accuracies = event_acc.Scalars('loss/vqa') print(training_accuracies) print(validation_accuracies) steps = 10 x = np.arange(steps) y = np.zeros([steps, 2]) for i in range(steps): y[i, 0] = training_accuracies[i][2] # value y[i, 1] = validation_accuracies[i][2] plt.plot(x, y[:,0], label='training accuracy') plt.plot(x, y[:,1], label='loss/vqa') plt.xlabel("Steps") plt.ylabel("Accuracy") plt.title("Training Progress") plt.legend(loc='upper right', frameon=True) plt.show()
def plot(params): log_path = params['logdir'] acc = EventAccumulator(log_path) acc.Reload() # only support scalar now scalar_list = acc.Tags()['scalars'] tag_groups = [ ('Loss', ['Loss/train', 'Loss/validation']), ('Linear Accuracy', ['Accuracy/train/linear', 'Accuracy/validation/linear']), ('Angular Accuracy', ['Accuracy/train/angular', 'Accuracy/validation/angular']), ] # for tags in scalar_list: for title, tags in tag_groups: for tag in tags: x = [int(s.step) for s in acc.Scalars(tag)] y = [s.value for s in acc.Scalars(tag)] color = BLUE if 'train' in tag else ORANGE plt.plot(x, y, color=colors.to_rgba(color, alpha=0.9)) plt.legend( title='', loc='upper left', labels=['Traning', 'Validation'], ) plt.title(title) plt.xlabel('Epoch') if 'accuracy' in tag.lower(): plt.ylabel('Accuracy %') plt.savefig( f"{params['prefix']}_{title.replace(' ', '_').lower()}.png") plt.clf()
from tensorboard.backend.event_processing.event_accumulator import EventAccumulator from matplotlib import pyplot as plt pixelcnn = EventAccumulator('logs/pixelcnn') noncausal = EventAccumulator('logs/noncausal') denoising = EventAccumulator('logs/denoising') pixelcnn.Reload() noncausal.Reload() denoising.Reload() _, step_nums, vals = zip(*pixelcnn.Scalars('train_loss')) pixelcnn_train = (step_nums, vals) _, step_nums, vals = zip(*pixelcnn.Scalars('test_loss')) pixelcnn_test = (step_nums, vals) _, step_nums, vals = zip(*noncausal.Scalars('train_loss')) noncausal_train = (step_nums, vals) _, step_nums, vals = zip(*noncausal.Scalars('test_loss')) noncausal_test = (step_nums, vals) _, step_nums, vals = zip(*denoising.Scalars('train_loss')) denoising_train = (step_nums, vals) _, step_nums, vals = zip(*denoising.Scalars('test_loss')) denoising_test = (step_nums, vals) plt.plot(*pixelcnn_train, label='PixelCNN Train') plt.plot(*pixelcnn_test, label='PixelCNN Test') plt.xlabel('Iteration')
def saveFig(tb, t1, t2, t3, pltn): fb, fnb = tb f1, fn1 = t1 f2, fn2 = t2 f3, fn3 =t3 CoLA_baseline = EventAccumulator(fb) CoLA_on_MRPC = EventAccumulator(f1) CoLA_on_RTE = EventAccumulator(f2) CoLA_on_WNLI = EventAccumulator(f3) CoLA_baseline.Reload() CoLA_on_MRPC.Reload() CoLA_on_RTE.Reload() CoLA_on_WNLI.Reload() def getLoss(event): iterations = [] losses = [] for wall_time, iteration, loss in event.Scalars("loss"): iterations.append(iteration) losses.append(loss) return iterations, losses baseline = getLoss(CoLA_baseline) MRPC = getLoss(CoLA_on_MRPC) RTE = getLoss(CoLA_on_RTE) WNLI = getLoss(CoLA_on_WNLI) plt.clf() plt.plot(baseline[0], baseline[1], label=fnb) plt.plot(MRPC[0], MRPC[1], label=fn1) plt.plot(RTE[0], RTE[1], label=fn2) plt.plot(WNLI[0], WNLI[1], label=fn3) plt.legend(loc="upper right") plt.savefig(pltn + ".jpg")
# Evaluate logs fix, ax = plt.subplots(figsize=(15, 10)) for i, config in enumerate(log_configs): print('Parsing %d/%d (%s)' % (i + 1, len(log_configs), config['label'])) folder = os.path.join(rootdir, config['folder']) eventfiles = glob.glob(path.join(folder, 'events.out.tfevents.*')) if len(eventfiles) == 0: logging.warning('No event file found in %s. Skipping config.' % config['folder']) continue elif len(eventfiles) > 1: logging.warning('Multiple event files found in %s.' % config['folder']) event_acc = EventAccumulator(folder) event_acc.Reload() time_out = pd.DataFrame( event_acc.Scalars('inception_score/Wall_clock_time')) inception_out = pd.DataFrame(event_acc.Scalars('inception_score/mean')) df = pd.merge(time_out[['step', 'value']], inception_out[['step', 'value']], on='step') df.columns = ['step', 'time', 'inception score'] df['inception score'] = df['inception score'].rolling(window=mean_window, center=False).mean() ax.plot(df[xaxis], df['inception score'], label=config['label'])
from matplotlib.ticker import MultipleLocator testcase = sys.argv[1] # K80_vgg19_32 print(testcase) base_dir = '/scratch/li.baol/tsrbrd_log/pwr_meas/round1/' result_base_dir = '/scratch/li.baol/GPU_time_meas/tensorflow/round1/' log_dir = base_dir + testcase + '_*/' dirs = glob.glob(log_dir) dirs.sort() time_all = [] # execution time for all 10 reps for tc in dirs: model = tc.split('/')[5+1] iterator = EventAccumulator(tc).Reload() tag = iterator.Tags()['scalars'][2] # this is tag for loss wall_time = [t.wall_time for t in iterator.Scalars(tag)] relative_time = [(time - wall_time[0])/3600 for time in wall_time] num_epoch = len(relative_time) time_all.append(relative_time[num_epoch - 1] / (num_epoch - 1)) df = pd.DataFrame(time_all, columns=["time(h)"]) df.to_csv(result_base_dir + 'csv/' + testcase + '.csv', index=False) fig, axs = plt.subplots(1, 1, gridspec_kw={'hspace': 0, 'wspace': 0}, figsize=(12,5)) fig.suptitle(testcase + " GPU time (h) to train 50 epochs") x = np.arange(len(time_all)) axs.bar(x, time_all)
def plot_tensorflow_log(path, epochs): valdir = os.path.join(path, 'validation') traindir = os.path.join(path, 'train') metricdir = os.path.join(path, 'metrics') val_events = os.path.join(valdir, os.listdir(valdir)[0]) train_events = os.path.join(traindir, os.listdir(traindir)[0]) metric_events = os.path.join(metricdir, os.listdir(metricdir)[0]) #for summary in summary_iterator(metric_events): # print(summary) val_acc = EventAccumulator(val_events) val_acc.Reload() train_acc = EventAccumulator(train_events) train_acc.Reload() metric_acc = EventAccumulator(metric_events) metric_acc.Reload() # Show all tags in the log file #print(val_acc.Tags()) #print(train_acc.Tags()) #print(metric_acc.Tags()) val_epoch_loss = val_acc.Scalars('epoch_loss') val_epoch_accuracy = val_acc.Scalars('epoch_accuracy') #print(val_epoch_accuracy) train_epoch_loss = train_acc.Scalars('epoch_loss') train_epoch_accuracy = train_acc.Scalars('epoch_accuracy') #training_accuracies = event_acc.Scalars('training-accuracy') #validation_accuracies = event_acc.Scalars('validation_accuracy') steps = len(train_epoch_accuracy) x = np.arange(steps) y = np.zeros([steps, 2]) for i in range(steps): y[i, 0] = train_epoch_accuracy[i][2] # value y[i, 1] = val_epoch_accuracy[i][2] print('Final train accuracy:', train_epoch_accuracy[-1]) print('Final val accuracy:', val_epoch_accuracy[-1]) plt.plot(x, y[:, 0], label='training accuracy') plt.plot(x, y[:, 1], label='validation accuracy') plt.xlim(0, epochs) plt.ylim(0, 1) plt.xlabel("Epochs") plt.ylabel("Accuracy") plt.title("Training Progress") plt.grid(True, axis='both') plt.legend(loc='lower right', frameon=True) plt.show()
def train_eval_graphs(path): train = {} eval = {} if not os.path.isdir(path): return {} train_events = [ os.path.join(path, f) for f in os.listdir(path) if f.startswith('events.out.tfevents') ] if len(train_events) == 0: return {} train_events.sort(key=lambda x: os.path.getmtime(x)) train_summary = train_events[0] summary_iterator = EventAccumulator(train_summary).Reload() tags = [ m for m in summary_iterator.Tags()['scalars'] if m.split('_1')[0] in ['accuracy', 'r_squared', 'loss'] ] if len(tags) == 0: return {} train['steps'] = [e.step for e in summary_iterator.Scalars(tags[0])] for tag in tags: train[tag.split('_1')[0]] = [] for e in summary_iterator.Scalars(tag): train[tag.split('_1')[0]].append(e.value) eval_events = [] if os.path.isdir(os.path.join(path, 'eval')): eval_events = [ os.path.join(path, 'eval', f) for f in os.listdir(os.path.join(path, 'eval')) if f.startswith('events.out.tfevents') ] if len(eval_events) == 0: return {'train': train} eval_events.sort(key=lambda x: os.path.getmtime(x)) eval_summary = eval_events[0] summary_iterator = EventAccumulator(eval_summary).Reload() tags = [ m for m in summary_iterator.Tags()['scalars'] if m.split('_1')[0] in ['accuracy', 'r_squared', 'loss'] ] if len(tags) == 0: return {'train': train} eval['steps'] = [e.step for e in summary_iterator.Scalars(tags[0])] for tag in tags: eval[tag.split('_1')[0]] = [] for e in summary_iterator.Scalars(tag): eval[tag.split('_1')[0]].append(e.value) return {'train': train, 'eval': eval}
def main(_): search_path = os.path.join(os.getcwd(), 'models') + '/**/eval' all_dirs = glob.glob(search_path, recursive=True) df_eval = pd.DataFrame() all_models = [] for d in all_dirs: # Grab all of the accuracy results for each model and put into Pandas dataframe event_acc = EventAccumulator(d) event_acc.Reload() # Show all tags in the log file print(event_acc.Tags()) try: s = event_acc.Scalars('PASCAL/Precision/[email protected]') df = pd.DataFrame(s) if not df.empty: dir_name = d.split('eval')[0] model_name = dir_name.split('/')[-2] a = meta.ModelMetadata(model_name) all_models.append(a) time_start = df.wall_time[0] # convert wall time and value to rounded values df['wall_time'] = df['wall_time'].apply(wallToGPUTime, args=(time_start, )) df['value'] = df['value'].apply(valueTomAP) # rename columns df.columns = ['GPU Time', 'step', 'Overall mAP'] df['model'] = np.full(len(df), a.name) df_eval = df_eval.append(df) except Exception as ex: print(ex) # drop the step column as it's no longer needed df_eval = df_eval.drop(['step'], axis=1) df_final = df_eval[df_eval['GPU Time'] < 200] df_mean = df_eval[(df_eval['GPU Time'] < 200) & (df_eval['GPU Time'] > 50)] print(df_mean.groupby(['model']).mean().sort_values('Overall mAP')) all_model_index = df_final.set_index(['model', 'GPU Time']).sort_index() with plt.style.context('ggplot'): # start a new figure - size is in inches fig = plt.figure(figsize=(6, 4), dpi=400) ax1 = plt.subplot(aspect='equal') ax1.set_xlim(0, 300) ax1.set_ylim(0, 100) for model in all_models: model_plot(all_model_index, model, ax1) ax1.set_ylim([0, 100]) ax1.set_ylabel('mAP') ax1.set_xlabel('GPU Time (minutes)') ax1.set_title('Mean Average Precision') markers = [] names = [] for name, marker in arch_markers.items(): s = plt.Line2D((0, 1), (0, 0), color='grey', marker=marker, linestyle='') names.append(name) markers.append(s) ax1.legend(markers, names, loc=0) inc = 40 ax1.text(180, 45, r'Resolution', fontsize=8) for size, color in sz_colors.items(): ax1.text(190, inc - 2, r'{0}'.format(size), fontsize=8) c = mpatches.Circle((180, inc), 2, edgecolor='black', facecolor=color) ax1.add_patch(c) inc -= 10 inc = 40 ax1.text(240, 45, r'Box Proposals', fontsize=8) for size, color in sorted(sz_proposals.items()): ax1.text(250, inc - 2, r'{0}'.format(size), fontsize=8) c = mpatches.Circle((240, inc), 2, edgecolor='black', facecolor=color) ax1.add_patch(c) inc -= 10 plt.savefig('mAP.png', format='png', bbox_inches='tight') plt.show() print('Done creating mAP.png')
def extract(dpath, dname): import tensorflow.compat.v1 as tf from tensorflow.core.util.event_pb2 import Event scalar_accumulators = [ EventAccumulator(os.path.join(dpath, dname)).Reload().scalars ] # Filter non event files scalar_accumulators = [ scalar_accumulator for scalar_accumulator in scalar_accumulators if scalar_accumulator.Keys() ] # Get and validate all scalar keys all_keys = [ tuple(scalar_accumulator.Keys()) for scalar_accumulator in scalar_accumulators ] assert ( len(set(all_keys)) == 1 ), "All runs need to have the same scalar keys. There are mismatches in {}".format( all_keys) keys = all_keys[0] all_scalar_events_per_key = [[ scalar_accumulator.Items(key) for scalar_accumulator in scalar_accumulators ] for key in keys] # Get and validate all steps per key all_steps_per_key = [[ tuple(scalar_event.step for scalar_event in scalar_events) for scalar_events in all_scalar_events ] for all_scalar_events in all_scalar_events_per_key] for i, all_steps in enumerate(all_steps_per_key): assert ( len(set(all_steps)) == 1 ), "For scalar {} the step numbering or count doesn't match. Step count for all runs: {}".format( keys[i], [len(steps) for steps in all_steps]) steps_per_key = [all_steps[0] for all_steps in all_steps_per_key] # Get and average wall times per step per key wall_times_per_key = [ np.mean( [ tuple(scalar_event.wall_time for scalar_event in scalar_events) for scalar_events in all_scalar_events ], axis=0, ) for all_scalar_events in all_scalar_events_per_key ] # Get values per step per key values_per_key = [[[scalar_event.value for scalar_event in scalar_events] for scalar_events in all_scalar_events] for all_scalar_events in all_scalar_events_per_key] all_per_key = dict( zip(keys, zip(steps_per_key, wall_times_per_key, values_per_key))) return all_per_key
def merge_runs(dir_path, result_name, new_dir_name='tf_merged', tensorboard=False): diff_run_types = list( set([ get_rid_of_num(list(name)) for name in os.listdir(dir_path) if name != 'merge_runs.py' ])) summary_iterators = [] for name in diff_run_types: summary_iterators.append([ EventAccumulator(os.path.join(dir_path, dname)).Reload() for dname in os.listdir(dir_path) if name in dname ]) tags = [iterator[0].Tags()['scalars'] for iterator in summary_iterators] for idx, sum_it in enumerate(summary_iterators): for it in sum_it: assert it.Tags()['scalars'] == tags[idx] to_merge = ['episode_reward'] for tag in to_merge: fig, ax = plt.subplots(1) ax.set_title(tag) ax.set_xlabel('steps') ax.set_ylabel('episode reward') ax.grid() colors = ['red', 'green', 'blue', 'yellow'] fig.tight_layout() for idx, sum_it in enumerate(summary_iterators): summaries_events = [summary.Scalars(tag) for summary in sum_it] end_point = min([events[-1].step for events in summaries_events]) start_point = max([events[0].step for events in summaries_events]) steps = [step for step in range(start_point, end_point + 1)] interpolated_data = [] for events in summaries_events: event_steps = [event.step for event in events] event_data = [event.value for event in events] interpolated_data.append(interp1d(event_steps, event_data)) matrix_form = [] for step in steps: matrix_form.append( [data(step).item(0) for data in interpolated_data]) matrix_form = np.asarray(matrix_form) max_values = np.amax(matrix_form, axis=1) min_values = np.amin(matrix_form, axis=1) mean = matrix_form.mean(axis=1) sigma = matrix_form.std(axis=1) #fig, ax = plt.subplots(1) ax.plot(steps, mean, lw=1, label=diff_run_types[idx], color=colors[idx % len(colors)]) ax.fill_between(steps, mean + sigma, mean - sigma, facecolor=colors[idx % len(colors)], alpha=0.5) if tensorboard: merged_data_ = tf.placeholder(tf.float32) summary_op = tf.summary.histogram(tag + '_merged', merged_data_) with tf.Session() as sess: writer = tf.summary.FileWriter('./log/' + new_dir_name) for step in steps: merged_summary = sess.run( summary_op, feed_dict={ merged_data_: [ data(step).item(0) for data in interpolated_data ] }) writer.add_summary(merged_summary, step) lgd = ax.legend(loc='upper left') plt.savefig(result_name, bbox_extra_artists=(lgd, ), bbox_inches='tight') plt.close()
def extract(dpath, subpath, args): scalar_accumulators = [ EventAccumulator(str(dpath / dname / subpath)).Reload().scalars for dname in os.listdir(dpath) if dname != FOLDER_NAME ] # Filter non event files scalar_accumulators = [ scalar_accumulator for scalar_accumulator in scalar_accumulators if scalar_accumulator.Keys() ] # Get and validate all scalar keys all_keys = [ tuple(scalar_accumulator.Keys()) for scalar_accumulator in scalar_accumulators ] all_keys_set = [ set(scalar_accumulator.Keys()) for scalar_accumulator in scalar_accumulators ] #assert len(set(all_keys_set)) == 1, "All runs need to have the same scalar keys. There are mismatches in {}".format(all_keys_set) keys = all_keys[0] if args.allowed_keys: allowed_keys = args.allowed_keys else: allowed_keys = [ 'evaluate', 'standard_evaluate', 'forgetting_metric', 'weight_stat' ] found_keys = [] for key in all_keys_set[0]: # Check if current key occurs starts with any of allowed keys. log_key = (len(list(filter(lambda x: key.startswith(x), allowed_keys))) > 0) if log_key: present_in_all = True for av_set in all_keys_set: if not key in av_set: present_in_all = False if present_in_all: found_keys.append(key) keys = found_keys keys_list = all_keys[0] all_scalar_events_per_key = [[ scalar_accumulator.Items(key) for scalar_accumulator in scalar_accumulators ] for key in keys] # Get and validate all steps per key all_steps_per_key = [[ tuple(scalar_event.step for scalar_event in scalar_events) for scalar_events in all_scalar_events ] for all_scalar_events in all_scalar_events_per_key] for i, all_steps in enumerate(all_steps_per_key): print(i, all_steps) assert len( set(all_steps) ) == 1, "For scalar {} the step numbering or count doesn't match. Step count for all runs: {}".format( keys[i], [len(steps) for steps in all_steps]) #del keys_list[i] all_scalar_events_per_key = [[ scalar_accumulator.Items(key) for scalar_accumulator in scalar_accumulators ] for key in keys] print(all_scalar_events_per_key) # Get and validate all steps per key all_steps_per_key = [[ tuple(scalar_event.step for scalar_event in scalar_events) for scalar_events in all_scalar_events ] for all_scalar_events in all_scalar_events_per_key] steps_per_key = [all_steps[0] for all_steps in all_steps_per_key] # Get and average wall times per step per key wall_times_per_key = [ np.mean([ tuple(scalar_event.wall_time for scalar_event in scalar_events) for scalar_events in all_scalar_events ], axis=0) for all_scalar_events in all_scalar_events_per_key ] # Get values per step per key values_per_key = [[[scalar_event.value for scalar_event in scalar_events] for scalar_events in all_scalar_events] for all_scalar_events in all_scalar_events_per_key] all_per_key = dict( zip(keys, zip(steps_per_key, wall_times_per_key, values_per_key))) return all_per_key
'wspace': 0 }, figsize=(12, 5)) fig.suptitle("accuracy curve during training") for key, value in testcases.items(): log_dir = base_dir + value + '_*/' dirs = glob.glob(log_dir) dirs.sort() time_all = [] # time for all 10 reps accuracy_all = [] for tc in dirs: model = tc.split('/')[5 + 1] iterator = EventAccumulator(tc).Reload() tag = iterator.Tags()['scalars'][3] # this is tag for accuracy accuracy = [item.value for item in iterator.Scalars(tag)] wall_time = [t.wall_time for t in iterator.Scalars(tag)] relative_time = [(time - wall_time[0]) / 3600 for time in wall_time] time_all.append(relative_time) accuracy_all.append(accuracy) time_avg = [0] * len(time_all[0]) accuracy_avg = [0] * len(time_all[0]) for j in range(len(time_all)): # 10 time_avg = np.add(time_all[j], time_avg) accuracy_avg = np.add(accuracy_all[j], accuracy_avg)
def extract(experiments): scalar_accumulators = [ EventAccumulator(experiment_dir).Reload().scalars for experiment_dir in experiments ] # Filter non event files scalar_accumulators = [ scalar_accumulator for scalar_accumulator in scalar_accumulators if scalar_accumulator.Keys() ] # Get and validate all scalar keys all_keys = [ tuple(sorted(scalar_accumulator.Keys())) for scalar_accumulator in scalar_accumulators ] assert len(set(all_keys)) == 1, \ "All runs need to have the same scalar keys. There are mismatches in {}".format(all_keys) keys = all_keys[0] all_scalar_events_per_key = [[ scalar_accumulator.Items(key) for scalar_accumulator in scalar_accumulators ] for key in keys] # Get and validate all steps per key x_per_key = [[ tuple(scalar_event.step for scalar_event in sorted(scalar_events)) for scalar_events in sorted(all_scalar_events) ] for all_scalar_events in all_scalar_events_per_key] plot_step = PLOT_STEP all_steps_per_key = [[ tuple(int(step_id) for step_id in range(0, TOTAL_STEP, plot_step)) for _ in sorted(all_scalar_events) ] for all_scalar_events in all_scalar_events_per_key] for i, all_steps in enumerate(all_steps_per_key): assert len( set(all_steps) ) == 1, "For scalar {} the step numbering or count doesn't match. Step count for all runs: {}".format( keys[i], [len(steps) for steps in all_steps]) steps_per_key = [all_steps[0] for all_steps in all_steps_per_key] # Get values per step per key values_per_key = [[[scalar_event.value for scalar_event in scalar_events] for scalar_events in all_scalar_events] for all_scalar_events in all_scalar_events_per_key] interpolated_keys = dict() for tmp_id in range(len(PLOTS)): key_idx = keys.index(PLOTS[tmp_id]['key']) values = values_per_key[key_idx] x = steps_per_key[key_idx] x_steps = x_per_key[key_idx] interpolated_y = [[] for _ in values] for i in range(len(values)): idx = 0 tmp_min_step = min(len(x_steps[i]), len(values[i])) values[i] = values[i][2:tmp_min_step] x_steps[i] = x_steps[i][2:tmp_min_step] assert len(x_steps[i]) == len(values[i]) for x_idx in x: while idx < len(x_steps[i]) - 1 and x_steps[i][idx] < x_idx: idx += 1 if x_idx == 0: interpolated_value = values[i][idx] elif idx < len(values[i]) - 1: interpolated_value = (values[i][idx] + values[i][idx + 1]) / 2 else: interpolated_value = values[i][idx] interpolated_y[i].append(interpolated_value) assert len(interpolated_y[i]) == len(x) print(interpolated_y[0][:30]) interpolated_keys[PLOTS[tmp_id]['key']] = (x, interpolated_y) return interpolated_keys
'scalars': 1000, } # check - if losses in folder, use that plot_values = {} plot_steps = {} for variant, variant_runs in variant_paths.items(): min_steps = 0 for i, run in enumerate(variant_runs): accum_path = os.path.join(run, plot_key_folder) loss_data = {} if osp.exists(osp.join(accum_path, 'losses')): for task in tasks: loss_path = osp.join(accum_path, 'losses', task) event_acc = EventAccumulator(loss_path, tf_size_guidance) event_acc.Reload() print(event_acc.Tags()) scalars = event_acc.Scalars('losses') steps_and_values = np.stack([ np.asarray([scalar.step, scalar.value]) for scalar in scalars ]) # # unload steps to make sure the number of frames is equal across runs steps = steps_and_values[:, 0] values = steps_and_values[:, 1] plot_steps[task] = [steps] plot_values[task] = [values] else: print("Nope")
from tensorboard.backend.event_processing.event_accumulator import EventAccumulator import numpy as np import os path = 'C:\\tmp\\train_stats\\W_NORM_SW_test_c9_lr_0.0010000,kp_1.00,bs_300,nh_128,rl_20' path_out = 'D:\\ardetector\\data\\analytics' event_acc = EventAccumulator(path) event_acc.Reload() # Save train loss path_out2 = str.join('\\', [path_out, 'train_loss.txt']) out_file = open(path_out2, 'w') np.savetxt(out_file, event_acc.Scalars('train_loss'), delimiter=',', fmt='%.5f') out_file.close() # Save test loss path_out2 = str.join('\\', [path_out, 'test_loss.txt']) out_file = open(path_out2, 'w') np.savetxt(out_file, event_acc.Scalars('Test_cost'), delimiter=',', fmt='%.5f') out_file.close() # Save test f1 ar path_out2 = str.join('\\', [path_out, 'test_f1ar.txt']) out_file = open(path_out2, 'w') np.savetxt(out_file, event_acc.Scalars('Test_sample_F1_score'), delimiter=',', fmt='%.5f')
summary_home = r'D:\tmp\fujian\temp_move2' metric = namedtuple('metric', ['wall_time', 'step', 'acc', 'miou']) res = [] for each_dir in os.listdir(summary_home): if not os.path.isdir(join(summary_home, each_dir)): continue print(each_dir) model, dataset, _, epochs, batch_size, init_lr, end_lr, iterations, crop_size, bn_scale, ignore_label, structure_model, extra_message = each_dir.split( '#') # front_end, back_end = model.split('_') # front_end, stride = front_end.split('@') cur_metrics = set() for each_summary_file in os.listdir(join(summary_home, each_dir, 'eval')): event_acc = EventAccumulator( join(summary_home, each_dir, 'eval', each_summary_file)) event_acc.Reload() acc = event_acc.Scalars('accuracy') miou = event_acc.Scalars('mean_iou') for each_acc, each_miou in zip(acc, miou): cur_metrics.add( metric(each_acc.wall_time, each_acc.step, each_acc.value, each_miou.value)) # print(cur_metrics) # exit(1) cur_metrics = sorted(cur_metrics, key=attrgetter('step')) run_time = cur_metrics[-1].wall_time - cur_metrics[0].wall_time best_acc = max(cur_metrics, key=attrgetter('acc')) best_miou = max(cur_metrics, key=attrgetter('miou'))
def main(dirs_dataset_filter="mnist", more_discard_dirs=[], filtering_criteria_models=[ "resnet_50", "resnet_34", "resnet_18", "bagnet_tem_9", "bagnet_tem_17" ], filtering_criteria_frames=[], filtering_criteria_others=[], filtering_criteria_annotation_path=[], filtering_criteria_sampling=["center"]): res_dirs = [f for f in os.listdir("results/") if dirs_dataset_filter in f] discard_dirs = ["motion", "blackframes", "val_1tstride" ] + more_discard_dirs res_dirs = [f for f in res_dirs if all([d not in f for d in discard_dirs])] for r in res_dirs: # if os.path.exists("results/"+r+"/opts.json"): # with open("results/"+r+"/opts.json", "r") as f: # opts = json.load(f) # print(opts["annotation_path"]) if [f for f in os.listdir("results/"+r) if "events.out" in f] and \ any([c in r for c in filtering_criteria_models]) and \ any([c in r for c in filtering_criteria_frames]) and \ any([c in r for c in filtering_criteria_others]) and \ os.path.exists("results/"+r+"/opts.json"): print(r) event_acc = EventAccumulator("results/" + r) event_acc.Reload() with open("results/" + r + "/opts.json", "r") as f: opts = json.load(f) print(opts['annotation_path']) if any([c in opts['annotation_path'] for c in filtering_criteria_annotation_path]) \ and any ([c in opts['train_t_crop'] for c in filtering_criteria_sampling]): if event_acc.scalars.Keys() != []: train_losses, train_epochs, train_accs = zip( *event_acc.Scalars('train/acc')) val_losses, val_epochs, val_accs = zip( *event_acc.Scalars('val/acc')) if len(val_losses) < 10: os.system("rm -r results/" + r) if len(val_losses) >= 10 or train_accs[-1] > 0.95: # print(len(val_losses)) print(r) print("train", round(np.max(train_accs)*100, 2), np.argmax(train_accs), \ "val", round(np.max(val_accs)*100, 2), np.argmax(val_accs)) if os.path.exists("results/" + r + "/checkpoints_test_results.json"): with open( "results/" + r + "/checkpoints_test_results.json", "r") as f: test = json.load(f) print(test)
def output(tensorboard_dir, output_dir, metrics_keys, steps, output_file_base="metrics"): """Output csv and markdown file which accumulated tensorflow event by step and metrics_keys.""" subdirs = GetLogdirSubdirectories(tensorboard_dir) event_accumulators = [] for subdir in subdirs: event_accumulator = EventAccumulator(subdir) # init event accumulator event_accumulator.Reload() event_accumulators.append(event_accumulator) if not metrics_keys: metrics_keys = { metrics_key for event_accumulator in event_accumulators for metrics_key in _get_metrics_keys(event_accumulator) } columns = [ _column_name(event_accumulator, metrics_key) for event_accumulator, metrics_key in itertools.product( event_accumulators, metrics_keys) ] columns.sort() df = pd.DataFrame([], columns=columns) for event_accumulator in event_accumulators: for metrics_key in metrics_keys: value_step_list = _value_step_list(event_accumulator, metrics_key) for value, step in value_step_list: column_name = _column_name(event_accumulator, metrics_key) df.loc[step, column_name] = value if steps: df = df[steps, :] df = df.sort_index(ascending=False) # index to column. and re-order column. df["step"] = df.index df = df[["step"] + columns] output_csv = os.path.join(output_dir, "{}.csv".format(output_file_base)) df.to_csv(output_csv, index=False) output_md = os.path.join(output_dir, "{}.md".format(output_file_base)) writer = pytablewriter.MarkdownTableWriter() writer.char_left_side_row = "|" # fix for github writer.from_dataframe(df) with open(output_md, "w") as file_stream: writer.stream = file_stream writer.write_table() message = """ output success output csv: {} output md: {} """.format(output_csv, output_md) print(message)
def get_miou_list_class_all(log_dir, class_num): # add the tensorboard in the tf1x version tf1x_dir = os.path.join(os.path.dirname(os.path.dirname(tf1x_python)), 'lib', 'python3.7', 'site-packages') sys.path.insert(0, tf1x_dir) from tensorboard.backend.event_processing.event_accumulator import EventAccumulator # Loading too much data is slow... # tf_size_guidance on how much data the EventAccumulator should # | store in memory. The DEFAULT_SIZE_GUIDANCE tries not to store too much # | so as to avoid OOMing the client. The size_guidance should be a map # | from a `tagType` string to an integer representing the number of # | items to keep per tag for items of that `tagType`. If the size is 0, # | all events are stored. tf_size_guidance = { 'compressedHistograms': 10, 'images': 0, 'scalars': 0, # set a 0, to load all scalars 'histograms': 1 } miou_dic = {'step': [0]} # step 0, need some where events_files = io_function.get_file_list_by_pattern(log_dir, 'events*') if len(events_files) < 1: print('warning, No events file in %s' % log_dir) return miou_dic event_acc = EventAccumulator(log_dir, tf_size_guidance) event_acc.Reload() # Show all tags in the log file tag_dict = event_acc.Tags() # io_function.save_dict_to_txt_json('event_acc.txt',tag_dict) scalar_tags = tag_dict['scalars'] # print(scalar_tags) for class_id in range(class_num): name = 'class_%d' % class_id tag = 'eval/miou_1.0_' + name if tag in scalar_tags: miou_class_event = event_acc.Scalars(tag) miou_class_list = [ item[2] for item in miou_class_event ] # item[0] is wall_time, item[1] is step, item [2] is the value # step_list = [item[1] for item in miou_class_event] # print(step_list) miou_dic[name] = miou_class_list tag = 'eval/miou_1.0_overall' if tag in scalar_tags: miou_class_overall = event_acc.Scalars('eval/miou_1.0_overall') miou_class_list = [item[2] for item in miou_class_overall] step_list = [item[1] for item in miou_class_overall] wall_time_list = [item[0] for item in miou_class_overall] # print(step_list) miou_dic['overall'] = miou_class_list miou_dic['step'] = step_list miou_dic[ 'wall_time'] = wall_time_list # we can use datetime.fromtimestamp() to convert datetime io_function.save_dict_to_txt_json(os.path.join(log_dir, 'miou.txt'), miou_dic) return miou_dic
def plot_func(ax, method_dirs, color, label, marker, smooth_index=10, alpha=0.4, linewidth=1.0, scatter_space=500): ''' input: method_dirs : ['algo1/seed1','~algo1/seed4'] ''' ##### extrat data from all seeds ##### y_seeds = [] y_length = [] seed_count = len(method_dirs) for dir in method_dirs: event = EventAccumulator(dir) event.Reload() if (label == 'SAC'): y = event.scalars.Items('episode_threshold') # threshold_value else: y = event.scalars.Items('episode_reward') y_len = len(y) # threshold_len y_length.append(y_len) ###### smoothing ######### smooth_array = np.zeros(y_len) for i in range(y_len): smooth_array[i] = np.array( [j.value for j in y[i:i + smooth_index]]).mean() y_seeds.append(smooth_array) ###### reshape y_data into [num_seeds, min_y_length] ##### min_y_length = min(y_length) y_seeds_new = np.zeros((seed_count, min_y_length)) for i in range(seed_count): y_seeds_new[i] = y_seeds[i][0:min_y_length] y_seeds = y_seeds_new # [4, min_y_len] print(y_seeds.shape) ###### plot y_seeds with color_mean, scatter_marker, std_shadow ############# y_mean = y_seeds.mean(axis=0) x_mean = [i for i in range(min_y_length)] ax.plot(x_mean, y_mean, color=color, linewidth=linewidth) x_scatter = np.linspace(0, min_y_length - 1, int(min_y_length / scatter_space), dtype=np.int) y_scatter = y_mean[x_scatter] ax.scatter(x_scatter, y_scatter, color=color, label=label, marker=marker, s=100) y_std = y_seeds.std(axis=0) upper_bound = y_mean + y_std lower_bound = y_mean - y_std ax.fill_between(x_mean, upper_bound, lower_bound, where=upper_bound > lower_bound, facecolor=color, interpolate=True, alpha=alpha)