def tabulate_events(dir_path):
summary_iterators = [
EventAccumulator(os.path.join(dir_path, dname)).Reload()
for dname in os.listdir(dir_path)
]
tags = summary_iterators[0].Tags()['scalars']
for it in summary_iterators:
assert it.Tags()['scalars'] == tags
out = defaultdict(list)
steps = []
for tag in tags:
steps = [e.step for e in summary_iterators[0].Scalars(tag)]
wall_times = [e.wall_time for e in summary_iterators[0].Scalars(tag)]
for events in zip(*[acc.Scalars(tag) for acc in summary_iterators]):
assert len(set(e.step for e in events)) == 1
out[tag].append([e.value for e in events])
return out, steps, wall_times
def tabulate_events(inpath, outpath, tags):
summary_iterators = [
EventAccumulator(os.path.join(inpath, dname)).Reload()
for dname in os.listdir(inpath)
]
# tags = summary_iterators[0].Tags()['scalars']
#
# for it in summary_iterators:
# print(it.Tags())
# assert it.Tags()['scalars'] == tags
for tag in tags:
for it in summary_iterators:
try:
run_name = ntpath.basename(it.path)
csv_path = os.path.join(
outpath, run_name + '_' + tag.replace('/', '_') + '.csv')
pd.DataFrame(it.Scalars(tag)).to_csv(csv_path)
print('Success: {} - {}'.format(tag, it.path))
except KeyError:
print('Error: {} - {}'.format(tag, it.path))
pass
def extract_data(summary_path: os.PathLike,
logs_path: os.PathLike) -> Tuple[List[float], Dict, str]:
config = Config.load(summary_path).config
events_acc = EventAccumulator(logs_path)
events_acc.Reload()
_, _, val_acc_diffs = zip(*events_acc.Scalars('diff/val/acc'))
hp = config.hp.to_dict()
hp['n_conv_layers'] = len(config.hp.conv_model_config.conv_sizes)
if 'Minimum_test' in events_acc.images.Keys():
image_test = events_acc.Images('Minimum_test')[0].encoded_image_string
image_train = events_acc.Images(
'Minimum_train')[0].encoded_image_string
else:
image_test = None
image_train = None
return val_acc_diffs, hp, image_test, image_train
def tensorboard_loaddata(dir, ppo=PPO):
if ppo:
keys = ['batch/reward_mean', 'eval/reward_sum'] # ppo
else:
keys = ['reward/batch', 'eval/reward_sum'] # td3
x = EventAccumulator(path=dir)
x.Reload()
x.FirstEventTimestamp()
steps = []
wall_time = []
index = []
count = []
data = []
for k in keys:
steps.append([e.step for e in x.Scalars(k)])
wall_time.append([e.wall_time for e in x.Scalars(k)])
# index.append([e.index for e in x.Scalars(k)])
# count.append([e.count for e in x.Scalars(k)])
data.append([e.value for e in x.Scalars(k)])
return wall_time, steps, data
def tflog2pandas(path: str) -> pd.DataFrame:
"""convert single tensorflow log file to pandas DataFrame
Parameters
----------
path : str
path to tensorflow log file
Returns
-------
pd.DataFrame
converted dataframe
"""
DEFAULT_SIZE_GUIDANCE = {
"compressedHistograms": 1,
"images": 1,
"scalars": 0, # 0 means load all
"histograms": 1,
}
try:
event_acc = EventAccumulator(path, DEFAULT_SIZE_GUIDANCE)
event_acc.Reload()
tags = event_acc.Tags()["scalars"]
tag_train = [t for t in tags if 'test' not in t]
tag_test = [t for t in tags if 'test' in t]
d_train = {t: [] for t in tag_train}
d_test = {t: [] for t in tag_test}
for tag in tag_train:
event_list = event_acc.Scalars(tag)
values = list(map(lambda x: x.value, event_list))
d_train[tag] = values
for tag in tag_test:
event_list = event_acc.Scalars(tag)
values = list(map(lambda x: x.value, event_list))
d_test[tag] = values
d_train = pd.DataFrame(d_train)
d_test = pd.DataFrame(d_test)
# Dirty catch of DataLossError
except Exception:
print("Event file possibly corrupt: {}".format(path))
traceback.print_exc()
return {'train': d_train, 'test': d_test}
def plot_tensorflow_log(path):
# Loading too much data is slow...
tf_size_guidance = {
'compressedHistograms': 10,
'images': 0,
'scalars': 100,
'histograms': 1
}
event_acc = EventAccumulator(path, tf_size_guidance)
event_acc.Reload()
# Show all tags in the log file
print(event_acc.Tags())
timestep_reward = event_acc.Tensors('agent.observe/timestep-reward')
episode_reward = event_acc.Tensors('agent.observe/episode-reward')
#print(type(timestep_reward[0]))
#tensor_event = timestep_reward[0]
#tensor_np = tf.make_ndarray(tensor_event.tensor_proto)
#print(tensor_np)
steps = len(timestep_reward)
print(f"steps: {steps}")
x = np.arange(steps)
y = np.zeros([steps, 2])
for i in range(steps):
y[i, 0] = tf.make_ndarray(timestep_reward[i].tensor_proto)
y[i, 1] = tf.make_ndarray(episode_reward[i].tensor_proto)
plt.plot(x, y[:, 0], label='training accuracy')
plt.plot(x, y[:, 1], label='validation accuracy')
plt.xlabel("Steps")
plt.ylabel("Accuracy")
plt.title("Training Progress")
plt.legend(loc='upper right', frameon=True)
plt.show()
def pick_up_summary(self, summary_path):
event_acc = EventAccumulator(summary_path)
event_acc.Reload()
# E. g. get wall clock, number of steps and value for a scalar 'Accuracy'
_, steps_loss, vals_loss = zip(*event_acc.Scalars('Loss'))
_, steps_accuracy, vals_accuracy = zip(*event_acc.Scalars('Accuracy'))
_, steps_validation_loss, vals_validation_loss = zip(
*event_acc.Scalars('Validation_Loss'))
_, steps_validation_accuracy, vals_validation_accuracy = zip(
*event_acc.Scalars('Validation_Accuracy'))
training_history = dict()
training_history['steps_loss'] = steps_loss
training_history['steps_accuracy'] = steps_accuracy
training_history['steps_validation_loss'] = steps_validation_loss
training_history[
'steps_validation_accuracy'] = steps_validation_accuracy
training_history['values_loss'] = vals_loss
training_history['values_accuracy'] = vals_accuracy
training_history['values_validation_loss'] = vals_validation_loss
training_history[
'values_validation_accuracy'] = vals_validation_accuracy
self.test_result['training_history'] = training_history
def logCSV(log_dir, seed, csv_path, tag_name_lst=None):
data = EventAccumulator(log_dir)
data.Reload()
for tag in data.Tags().get('scalars'):
print(tag)
if tag_name_lst is not None and len(tag_name_lst) > 0:
if tag in tag_name_lst:
w_times, step_nums, vals = zip(*data.Scalars(tag))
in_data = {'w_times': w_times,
'step': step_nums,
'value': vals}
in_data = pd.DataFrame(in_data)
in_data.to_csv("{}/{}_seed_{}.csv".format(csv_path, str(tag), str(seed)), index=False)
else:
w_times, step_nums, vals = zip(*data.Scalars(tag))
in_data = {'wall_time': w_times,
'step': step_nums,
'value': vals}
in_data = pd.DataFrame(in_data)
in_data.to_csv("{}/{}_seed_{}.csv".format(csv_path, str(tag), str(seed)), index=False)
def read_tensorboard_log(path):
"Reads the .tfevents file and returns the scalars."
# First let's read the config file
config = json.load(open(os.path.join(path, "config.json")))
event_acc = EventAccumulator(path)
event_acc.Reload()
# print(event_acc.Tags()) # shows all tags in the log file
wall_times, step_nums, loss_train = zip(*event_acc.Scalars("Loss/train"))
# We assume that all scalars are recorded at each step
_, _, loss_valid = zip(*event_acc.Scalars("Loss/valid"))
_, _, acc_train = zip(*event_acc.Scalars("Accuracy/train"))
_, _, acc_valid = zip(*event_acc.Scalars("Accuracy/valid"))
info = {
"loss_train": loss_train,
"loss_valid": loss_valid,
"acc_train": acc_train,
"acc_valid": acc_valid,
"step_nums": step_nums,
"wall_times": wall_times,
"config": config,
}
return info
def plot_tensorflow_log(path):
# Loading too much data is slow...
tf_size_guidance = {
'compressedHistograms': 10,
'images': 0,
'scalars': 100,
'histograms': 1
}
event_acc = EventAccumulator(path, tf_size_guidance)
event_acc.Reload()
# Show all tags in the log file
print(event_acc.Tags())
ipdb.set_trace()
training_accuracies = event_acc.Scalars('eval/vqa/accuracy')
validation_accuracies = event_acc.Scalars('loss/vqa')
print(training_accuracies)
print(validation_accuracies)
steps = 10
x = np.arange(steps)
y = np.zeros([steps, 2])
for i in range(steps):
y[i, 0] = training_accuracies[i][2] # value
y[i, 1] = validation_accuracies[i][2]
plt.plot(x, y[:,0], label='training accuracy')
plt.plot(x, y[:,1], label='loss/vqa')
plt.xlabel("Steps")
plt.ylabel("Accuracy")
plt.title("Training Progress")
plt.legend(loc='upper right', frameon=True)
plt.show()
def plot(params):
log_path = params['logdir']
acc = EventAccumulator(log_path)
acc.Reload()
# only support scalar now
scalar_list = acc.Tags()['scalars']
tag_groups = [
('Loss', ['Loss/train', 'Loss/validation']),
('Linear Accuracy',
['Accuracy/train/linear', 'Accuracy/validation/linear']),
('Angular Accuracy',
['Accuracy/train/angular', 'Accuracy/validation/angular']),
]
# for tags in scalar_list:
for title, tags in tag_groups:
for tag in tags:
x = [int(s.step) for s in acc.Scalars(tag)]
y = [s.value for s in acc.Scalars(tag)]
color = BLUE if 'train' in tag else ORANGE
plt.plot(x, y, color=colors.to_rgba(color, alpha=0.9))
plt.legend(
title='',
loc='upper left',
labels=['Traning', 'Validation'],
)
plt.title(title)
plt.xlabel('Epoch')
if 'accuracy' in tag.lower():
plt.ylabel('Accuracy %')
plt.savefig(
f"{params['prefix']}_{title.replace(' ', '_').lower()}.png")
plt.clf()
from tensorboard.backend.event_processing.event_accumulator import EventAccumulator
from matplotlib import pyplot as plt
pixelcnn = EventAccumulator('logs/pixelcnn')
noncausal = EventAccumulator('logs/noncausal')
denoising = EventAccumulator('logs/denoising')
pixelcnn.Reload()
noncausal.Reload()
denoising.Reload()
_, step_nums, vals = zip(*pixelcnn.Scalars('train_loss'))
pixelcnn_train = (step_nums, vals)
_, step_nums, vals = zip(*pixelcnn.Scalars('test_loss'))
pixelcnn_test = (step_nums, vals)
_, step_nums, vals = zip(*noncausal.Scalars('train_loss'))
noncausal_train = (step_nums, vals)
_, step_nums, vals = zip(*noncausal.Scalars('test_loss'))
noncausal_test = (step_nums, vals)
_, step_nums, vals = zip(*denoising.Scalars('train_loss'))
denoising_train = (step_nums, vals)
_, step_nums, vals = zip(*denoising.Scalars('test_loss'))
denoising_test = (step_nums, vals)
plt.plot(*pixelcnn_train, label='PixelCNN Train')
plt.plot(*pixelcnn_test, label='PixelCNN Test')
plt.xlabel('Iteration')
def saveFig(tb, t1, t2, t3, pltn):
fb, fnb = tb
f1, fn1 = t1
f2, fn2 = t2
f3, fn3 =t3
CoLA_baseline = EventAccumulator(fb)
CoLA_on_MRPC = EventAccumulator(f1)
CoLA_on_RTE = EventAccumulator(f2)
CoLA_on_WNLI = EventAccumulator(f3)
CoLA_baseline.Reload()
CoLA_on_MRPC.Reload()
CoLA_on_RTE.Reload()
CoLA_on_WNLI.Reload()
def getLoss(event):
iterations = []
losses = []
for wall_time, iteration, loss in event.Scalars("loss"):
iterations.append(iteration)
losses.append(loss)
return iterations, losses
baseline = getLoss(CoLA_baseline)
MRPC = getLoss(CoLA_on_MRPC)
RTE = getLoss(CoLA_on_RTE)
WNLI = getLoss(CoLA_on_WNLI)
plt.clf()
plt.plot(baseline[0], baseline[1], label=fnb)
plt.plot(MRPC[0], MRPC[1], label=fn1)
plt.plot(RTE[0], RTE[1], label=fn2)
plt.plot(WNLI[0], WNLI[1], label=fn3)
plt.legend(loc="upper right")
plt.savefig(pltn + ".jpg")
# Evaluate logs
fix, ax = plt.subplots(figsize=(15, 10))
for i, config in enumerate(log_configs):
print('Parsing %d/%d (%s)' % (i + 1, len(log_configs), config['label']))
folder = os.path.join(rootdir, config['folder'])
eventfiles = glob.glob(path.join(folder, 'events.out.tfevents.*'))
if len(eventfiles) == 0:
logging.warning('No event file found in %s. Skipping config.' %
config['folder'])
continue
elif len(eventfiles) > 1:
logging.warning('Multiple event files found in %s.' % config['folder'])
event_acc = EventAccumulator(folder)
event_acc.Reload()
time_out = pd.DataFrame(
event_acc.Scalars('inception_score/Wall_clock_time'))
inception_out = pd.DataFrame(event_acc.Scalars('inception_score/mean'))
df = pd.merge(time_out[['step', 'value']],
inception_out[['step', 'value']],
on='step')
df.columns = ['step', 'time', 'inception score']
df['inception score'] = df['inception score'].rolling(window=mean_window,
center=False).mean()
ax.plot(df[xaxis], df['inception score'], label=config['label'])
from matplotlib.ticker import MultipleLocator
testcase = sys.argv[1] # K80_vgg19_32
print(testcase)
base_dir = '/scratch/li.baol/tsrbrd_log/pwr_meas/round1/'
result_base_dir = '/scratch/li.baol/GPU_time_meas/tensorflow/round1/'
log_dir = base_dir + testcase + '_*/'
dirs = glob.glob(log_dir)
dirs.sort()
time_all = [] # execution time for all 10 reps
for tc in dirs:
model = tc.split('/')[5+1]
iterator = EventAccumulator(tc).Reload()
tag = iterator.Tags()['scalars'][2] # this is tag for loss
wall_time = [t.wall_time for t in iterator.Scalars(tag)]
relative_time = [(time - wall_time[0])/3600 for time in wall_time]
num_epoch = len(relative_time)
time_all.append(relative_time[num_epoch - 1] / (num_epoch - 1))
df = pd.DataFrame(time_all, columns=["time(h)"])
df.to_csv(result_base_dir + 'csv/' + testcase + '.csv', index=False)
fig, axs = plt.subplots(1, 1, gridspec_kw={'hspace': 0, 'wspace': 0}, figsize=(12,5))
fig.suptitle(testcase + " GPU time (h) to train 50 epochs")
x = np.arange(len(time_all))
axs.bar(x, time_all)
def plot_tensorflow_log(path, epochs):
valdir = os.path.join(path, 'validation')
traindir = os.path.join(path, 'train')
metricdir = os.path.join(path, 'metrics')
val_events = os.path.join(valdir, os.listdir(valdir)[0])
train_events = os.path.join(traindir, os.listdir(traindir)[0])
metric_events = os.path.join(metricdir, os.listdir(metricdir)[0])
#for summary in summary_iterator(metric_events):
# print(summary)
val_acc = EventAccumulator(val_events)
val_acc.Reload()
train_acc = EventAccumulator(train_events)
train_acc.Reload()
metric_acc = EventAccumulator(metric_events)
metric_acc.Reload()
# Show all tags in the log file
#print(val_acc.Tags())
#print(train_acc.Tags())
#print(metric_acc.Tags())
val_epoch_loss = val_acc.Scalars('epoch_loss')
val_epoch_accuracy = val_acc.Scalars('epoch_accuracy')
#print(val_epoch_accuracy)
train_epoch_loss = train_acc.Scalars('epoch_loss')
train_epoch_accuracy = train_acc.Scalars('epoch_accuracy')
#training_accuracies = event_acc.Scalars('training-accuracy')
#validation_accuracies = event_acc.Scalars('validation_accuracy')
steps = len(train_epoch_accuracy)
x = np.arange(steps)
y = np.zeros([steps, 2])
for i in range(steps):
y[i, 0] = train_epoch_accuracy[i][2] # value
y[i, 1] = val_epoch_accuracy[i][2]
print('Final train accuracy:', train_epoch_accuracy[-1])
print('Final val accuracy:', val_epoch_accuracy[-1])
plt.plot(x, y[:, 0], label='training accuracy')
plt.plot(x, y[:, 1], label='validation accuracy')
plt.xlim(0, epochs)
plt.ylim(0, 1)
plt.xlabel("Epochs")
plt.ylabel("Accuracy")
plt.title("Training Progress")
plt.grid(True, axis='both')
plt.legend(loc='lower right', frameon=True)
plt.show()
def train_eval_graphs(path):
train = {}
eval = {}
if not os.path.isdir(path):
return {}
train_events = [
os.path.join(path, f) for f in os.listdir(path)
if f.startswith('events.out.tfevents')
]
if len(train_events) == 0:
return {}
train_events.sort(key=lambda x: os.path.getmtime(x))
train_summary = train_events[0]
summary_iterator = EventAccumulator(train_summary).Reload()
tags = [
m for m in summary_iterator.Tags()['scalars']
if m.split('_1')[0] in ['accuracy', 'r_squared', 'loss']
]
if len(tags) == 0:
return {}
train['steps'] = [e.step for e in summary_iterator.Scalars(tags[0])]
for tag in tags:
train[tag.split('_1')[0]] = []
for e in summary_iterator.Scalars(tag):
train[tag.split('_1')[0]].append(e.value)
eval_events = []
if os.path.isdir(os.path.join(path, 'eval')):
eval_events = [
os.path.join(path, 'eval', f)
for f in os.listdir(os.path.join(path, 'eval'))
if f.startswith('events.out.tfevents')
]
if len(eval_events) == 0:
return {'train': train}
eval_events.sort(key=lambda x: os.path.getmtime(x))
eval_summary = eval_events[0]
summary_iterator = EventAccumulator(eval_summary).Reload()
tags = [
m for m in summary_iterator.Tags()['scalars']
if m.split('_1')[0] in ['accuracy', 'r_squared', 'loss']
]
if len(tags) == 0:
return {'train': train}
eval['steps'] = [e.step for e in summary_iterator.Scalars(tags[0])]
for tag in tags:
eval[tag.split('_1')[0]] = []
for e in summary_iterator.Scalars(tag):
eval[tag.split('_1')[0]].append(e.value)
return {'train': train, 'eval': eval}
def main(_):
search_path = os.path.join(os.getcwd(), 'models') + '/**/eval'
all_dirs = glob.glob(search_path, recursive=True)
df_eval = pd.DataFrame()
all_models = []
for d in all_dirs:
# Grab all of the accuracy results for each model and put into Pandas dataframe
event_acc = EventAccumulator(d)
event_acc.Reload()
# Show all tags in the log file
print(event_acc.Tags())
try:
s = event_acc.Scalars('PASCAL/Precision/[email protected]')
df = pd.DataFrame(s)
if not df.empty:
dir_name = d.split('eval')[0]
model_name = dir_name.split('/')[-2]
a = meta.ModelMetadata(model_name)
all_models.append(a)
time_start = df.wall_time[0]
# convert wall time and value to rounded values
df['wall_time'] = df['wall_time'].apply(wallToGPUTime,
args=(time_start, ))
df['value'] = df['value'].apply(valueTomAP)
# rename columns
df.columns = ['GPU Time', 'step', 'Overall mAP']
df['model'] = np.full(len(df), a.name)
df_eval = df_eval.append(df)
except Exception as ex:
print(ex)
# drop the step column as it's no longer needed
df_eval = df_eval.drop(['step'], axis=1)
df_final = df_eval[df_eval['GPU Time'] < 200]
df_mean = df_eval[(df_eval['GPU Time'] < 200) & (df_eval['GPU Time'] > 50)]
print(df_mean.groupby(['model']).mean().sort_values('Overall mAP'))
all_model_index = df_final.set_index(['model', 'GPU Time']).sort_index()
with plt.style.context('ggplot'):
# start a new figure - size is in inches
fig = plt.figure(figsize=(6, 4), dpi=400)
ax1 = plt.subplot(aspect='equal')
ax1.set_xlim(0, 300)
ax1.set_ylim(0, 100)
for model in all_models:
model_plot(all_model_index, model, ax1)
ax1.set_ylim([0, 100])
ax1.set_ylabel('mAP')
ax1.set_xlabel('GPU Time (minutes)')
ax1.set_title('Mean Average Precision')
markers = []
names = []
for name, marker in arch_markers.items():
s = plt.Line2D((0, 1), (0, 0),
color='grey',
marker=marker,
linestyle='')
names.append(name)
markers.append(s)
ax1.legend(markers, names, loc=0)
inc = 40
ax1.text(180, 45, r'Resolution', fontsize=8)
for size, color in sz_colors.items():
ax1.text(190, inc - 2, r'{0}'.format(size), fontsize=8)
c = mpatches.Circle((180, inc),
2,
edgecolor='black',
facecolor=color)
ax1.add_patch(c)
inc -= 10
inc = 40
ax1.text(240, 45, r'Box Proposals', fontsize=8)
for size, color in sorted(sz_proposals.items()):
ax1.text(250, inc - 2, r'{0}'.format(size), fontsize=8)
c = mpatches.Circle((240, inc),
2,
edgecolor='black',
facecolor=color)
ax1.add_patch(c)
inc -= 10
plt.savefig('mAP.png', format='png', bbox_inches='tight')
plt.show()
print('Done creating mAP.png')
def extract(dpath, dname):
import tensorflow.compat.v1 as tf
from tensorflow.core.util.event_pb2 import Event
scalar_accumulators = [
EventAccumulator(os.path.join(dpath, dname)).Reload().scalars
]
# Filter non event files
scalar_accumulators = [
scalar_accumulator for scalar_accumulator in scalar_accumulators
if scalar_accumulator.Keys()
]
# Get and validate all scalar keys
all_keys = [
tuple(scalar_accumulator.Keys())
for scalar_accumulator in scalar_accumulators
]
assert (
len(set(all_keys)) == 1
), "All runs need to have the same scalar keys. There are mismatches in {}".format(
all_keys)
keys = all_keys[0]
all_scalar_events_per_key = [[
scalar_accumulator.Items(key)
for scalar_accumulator in scalar_accumulators
] for key in keys]
# Get and validate all steps per key
all_steps_per_key = [[
tuple(scalar_event.step for scalar_event in scalar_events)
for scalar_events in all_scalar_events
] for all_scalar_events in all_scalar_events_per_key]
for i, all_steps in enumerate(all_steps_per_key):
assert (
len(set(all_steps)) == 1
), "For scalar {} the step numbering or count doesn't match. Step count for all runs: {}".format(
keys[i], [len(steps) for steps in all_steps])
steps_per_key = [all_steps[0] for all_steps in all_steps_per_key]
# Get and average wall times per step per key
wall_times_per_key = [
np.mean(
[
tuple(scalar_event.wall_time for scalar_event in scalar_events)
for scalar_events in all_scalar_events
],
axis=0,
) for all_scalar_events in all_scalar_events_per_key
]
# Get values per step per key
values_per_key = [[[scalar_event.value for scalar_event in scalar_events]
for scalar_events in all_scalar_events]
for all_scalar_events in all_scalar_events_per_key]
all_per_key = dict(
zip(keys, zip(steps_per_key, wall_times_per_key, values_per_key)))
return all_per_key
def merge_runs(dir_path,
result_name,
new_dir_name='tf_merged',
tensorboard=False):
diff_run_types = list(
set([
get_rid_of_num(list(name)) for name in os.listdir(dir_path)
if name != 'merge_runs.py'
]))
summary_iterators = []
for name in diff_run_types:
summary_iterators.append([
EventAccumulator(os.path.join(dir_path, dname)).Reload()
for dname in os.listdir(dir_path) if name in dname
])
tags = [iterator[0].Tags()['scalars'] for iterator in summary_iterators]
for idx, sum_it in enumerate(summary_iterators):
for it in sum_it:
assert it.Tags()['scalars'] == tags[idx]
to_merge = ['episode_reward']
for tag in to_merge:
fig, ax = plt.subplots(1)
ax.set_title(tag)
ax.set_xlabel('steps')
ax.set_ylabel('episode reward')
ax.grid()
colors = ['red', 'green', 'blue', 'yellow']
fig.tight_layout()
for idx, sum_it in enumerate(summary_iterators):
summaries_events = [summary.Scalars(tag) for summary in sum_it]
end_point = min([events[-1].step for events in summaries_events])
start_point = max([events[0].step for events in summaries_events])
steps = [step for step in range(start_point, end_point + 1)]
interpolated_data = []
for events in summaries_events:
event_steps = [event.step for event in events]
event_data = [event.value for event in events]
interpolated_data.append(interp1d(event_steps, event_data))
matrix_form = []
for step in steps:
matrix_form.append(
[data(step).item(0) for data in interpolated_data])
matrix_form = np.asarray(matrix_form)
max_values = np.amax(matrix_form, axis=1)
min_values = np.amin(matrix_form, axis=1)
mean = matrix_form.mean(axis=1)
sigma = matrix_form.std(axis=1)
#fig, ax = plt.subplots(1)
ax.plot(steps,
mean,
lw=1,
label=diff_run_types[idx],
color=colors[idx % len(colors)])
ax.fill_between(steps,
mean + sigma,
mean - sigma,
facecolor=colors[idx % len(colors)],
alpha=0.5)
if tensorboard:
merged_data_ = tf.placeholder(tf.float32)
summary_op = tf.summary.histogram(tag + '_merged',
merged_data_)
with tf.Session() as sess:
writer = tf.summary.FileWriter('./log/' + new_dir_name)
for step in steps:
merged_summary = sess.run(
summary_op,
feed_dict={
merged_data_: [
data(step).item(0)
for data in interpolated_data
]
})
writer.add_summary(merged_summary, step)
lgd = ax.legend(loc='upper left')
plt.savefig(result_name,
bbox_extra_artists=(lgd, ),
bbox_inches='tight')
plt.close()
def extract(dpath, subpath, args):
scalar_accumulators = [
EventAccumulator(str(dpath / dname / subpath)).Reload().scalars
for dname in os.listdir(dpath) if dname != FOLDER_NAME
]
# Filter non event files
scalar_accumulators = [
scalar_accumulator for scalar_accumulator in scalar_accumulators
if scalar_accumulator.Keys()
]
# Get and validate all scalar keys
all_keys = [
tuple(scalar_accumulator.Keys())
for scalar_accumulator in scalar_accumulators
]
all_keys_set = [
set(scalar_accumulator.Keys())
for scalar_accumulator in scalar_accumulators
]
#assert len(set(all_keys_set)) == 1, "All runs need to have the same scalar keys. There are mismatches in {}".format(all_keys_set)
keys = all_keys[0]
if args.allowed_keys:
allowed_keys = args.allowed_keys
else:
allowed_keys = [
'evaluate', 'standard_evaluate', 'forgetting_metric', 'weight_stat'
]
found_keys = []
for key in all_keys_set[0]:
# Check if current key occurs starts with any of allowed keys.
log_key = (len(list(filter(lambda x: key.startswith(x), allowed_keys)))
> 0)
if log_key:
present_in_all = True
for av_set in all_keys_set:
if not key in av_set:
present_in_all = False
if present_in_all:
found_keys.append(key)
keys = found_keys
keys_list = all_keys[0]
all_scalar_events_per_key = [[
scalar_accumulator.Items(key)
for scalar_accumulator in scalar_accumulators
] for key in keys]
# Get and validate all steps per key
all_steps_per_key = [[
tuple(scalar_event.step for scalar_event in scalar_events)
for scalar_events in all_scalar_events
] for all_scalar_events in all_scalar_events_per_key]
for i, all_steps in enumerate(all_steps_per_key):
print(i, all_steps)
assert len(
set(all_steps)
) == 1, "For scalar {} the step numbering or count doesn't match. Step count for all runs: {}".format(
keys[i], [len(steps) for steps in all_steps])
#del keys_list[i]
all_scalar_events_per_key = [[
scalar_accumulator.Items(key)
for scalar_accumulator in scalar_accumulators
] for key in keys]
print(all_scalar_events_per_key)
# Get and validate all steps per key
all_steps_per_key = [[
tuple(scalar_event.step for scalar_event in scalar_events)
for scalar_events in all_scalar_events
] for all_scalar_events in all_scalar_events_per_key]
steps_per_key = [all_steps[0] for all_steps in all_steps_per_key]
# Get and average wall times per step per key
wall_times_per_key = [
np.mean([
tuple(scalar_event.wall_time for scalar_event in scalar_events)
for scalar_events in all_scalar_events
],
axis=0) for all_scalar_events in all_scalar_events_per_key
]
# Get values per step per key
values_per_key = [[[scalar_event.value for scalar_event in scalar_events]
for scalar_events in all_scalar_events]
for all_scalar_events in all_scalar_events_per_key]
all_per_key = dict(
zip(keys, zip(steps_per_key, wall_times_per_key, values_per_key)))
return all_per_key
'wspace': 0
},
figsize=(12, 5))
fig.suptitle("accuracy curve during training")
for key, value in testcases.items():
log_dir = base_dir + value + '_*/'
dirs = glob.glob(log_dir)
dirs.sort()
time_all = [] # time for all 10 reps
accuracy_all = []
for tc in dirs:
model = tc.split('/')[5 + 1]
iterator = EventAccumulator(tc).Reload()
tag = iterator.Tags()['scalars'][3] # this is tag for accuracy
accuracy = [item.value for item in iterator.Scalars(tag)]
wall_time = [t.wall_time for t in iterator.Scalars(tag)]
relative_time = [(time - wall_time[0]) / 3600 for time in wall_time]
time_all.append(relative_time)
accuracy_all.append(accuracy)
time_avg = [0] * len(time_all[0])
accuracy_avg = [0] * len(time_all[0])
for j in range(len(time_all)): # 10
time_avg = np.add(time_all[j], time_avg)
accuracy_avg = np.add(accuracy_all[j], accuracy_avg)
def extract(experiments):
scalar_accumulators = [
EventAccumulator(experiment_dir).Reload().scalars
for experiment_dir in experiments
]
# Filter non event files
scalar_accumulators = [
scalar_accumulator for scalar_accumulator in scalar_accumulators
if scalar_accumulator.Keys()
]
# Get and validate all scalar keys
all_keys = [
tuple(sorted(scalar_accumulator.Keys()))
for scalar_accumulator in scalar_accumulators
]
assert len(set(all_keys)) == 1, \
"All runs need to have the same scalar keys. There are mismatches in {}".format(all_keys)
keys = all_keys[0]
all_scalar_events_per_key = [[
scalar_accumulator.Items(key)
for scalar_accumulator in scalar_accumulators
] for key in keys]
# Get and validate all steps per key
x_per_key = [[
tuple(scalar_event.step for scalar_event in sorted(scalar_events))
for scalar_events in sorted(all_scalar_events)
] for all_scalar_events in all_scalar_events_per_key]
plot_step = PLOT_STEP
all_steps_per_key = [[
tuple(int(step_id) for step_id in range(0, TOTAL_STEP, plot_step))
for _ in sorted(all_scalar_events)
] for all_scalar_events in all_scalar_events_per_key]
for i, all_steps in enumerate(all_steps_per_key):
assert len(
set(all_steps)
) == 1, "For scalar {} the step numbering or count doesn't match. Step count for all runs: {}".format(
keys[i], [len(steps) for steps in all_steps])
steps_per_key = [all_steps[0] for all_steps in all_steps_per_key]
# Get values per step per key
values_per_key = [[[scalar_event.value for scalar_event in scalar_events]
for scalar_events in all_scalar_events]
for all_scalar_events in all_scalar_events_per_key]
interpolated_keys = dict()
for tmp_id in range(len(PLOTS)):
key_idx = keys.index(PLOTS[tmp_id]['key'])
values = values_per_key[key_idx]
x = steps_per_key[key_idx]
x_steps = x_per_key[key_idx]
interpolated_y = [[] for _ in values]
for i in range(len(values)):
idx = 0
tmp_min_step = min(len(x_steps[i]), len(values[i]))
values[i] = values[i][2:tmp_min_step]
x_steps[i] = x_steps[i][2:tmp_min_step]
assert len(x_steps[i]) == len(values[i])
for x_idx in x:
while idx < len(x_steps[i]) - 1 and x_steps[i][idx] < x_idx:
idx += 1
if x_idx == 0:
interpolated_value = values[i][idx]
elif idx < len(values[i]) - 1:
interpolated_value = (values[i][idx] +
values[i][idx + 1]) / 2
else:
interpolated_value = values[i][idx]
interpolated_y[i].append(interpolated_value)
assert len(interpolated_y[i]) == len(x)
print(interpolated_y[0][:30])
interpolated_keys[PLOTS[tmp_id]['key']] = (x, interpolated_y)
return interpolated_keys
'scalars': 1000,
}
# check - if losses in folder, use that
plot_values = {}
plot_steps = {}
for variant, variant_runs in variant_paths.items():
min_steps = 0
for i, run in enumerate(variant_runs):
accum_path = os.path.join(run, plot_key_folder)
loss_data = {}
if osp.exists(osp.join(accum_path, 'losses')):
for task in tasks:
loss_path = osp.join(accum_path, 'losses', task)
event_acc = EventAccumulator(loss_path, tf_size_guidance)
event_acc.Reload()
print(event_acc.Tags())
scalars = event_acc.Scalars('losses')
steps_and_values = np.stack([
np.asarray([scalar.step, scalar.value])
for scalar in scalars
])
# # unload steps to make sure the number of frames is equal across runs
steps = steps_and_values[:, 0]
values = steps_and_values[:, 1]
plot_steps[task] = [steps]
plot_values[task] = [values]
else:
print("Nope")
from tensorboard.backend.event_processing.event_accumulator import EventAccumulator
import numpy as np
import os
path = 'C:\\tmp\\train_stats\\W_NORM_SW_test_c9_lr_0.0010000,kp_1.00,bs_300,nh_128,rl_20'
path_out = 'D:\\ardetector\\data\\analytics'
event_acc = EventAccumulator(path)
event_acc.Reload()
# Save train loss
path_out2 = str.join('\\', [path_out, 'train_loss.txt'])
out_file = open(path_out2, 'w')
np.savetxt(out_file,
event_acc.Scalars('train_loss'),
delimiter=',',
fmt='%.5f')
out_file.close()
# Save test loss
path_out2 = str.join('\\', [path_out, 'test_loss.txt'])
out_file = open(path_out2, 'w')
np.savetxt(out_file, event_acc.Scalars('Test_cost'), delimiter=',', fmt='%.5f')
out_file.close()
# Save test f1 ar
path_out2 = str.join('\\', [path_out, 'test_f1ar.txt'])
out_file = open(path_out2, 'w')
np.savetxt(out_file,
event_acc.Scalars('Test_sample_F1_score'),
delimiter=',',
fmt='%.5f')
summary_home = r'D:\tmp\fujian\temp_move2'
metric = namedtuple('metric', ['wall_time', 'step', 'acc', 'miou'])
res = []
for each_dir in os.listdir(summary_home):
if not os.path.isdir(join(summary_home, each_dir)):
continue
print(each_dir)
model, dataset, _, epochs, batch_size, init_lr, end_lr, iterations, crop_size, bn_scale, ignore_label, structure_model, extra_message = each_dir.split(
'#')
# front_end, back_end = model.split('_')
# front_end, stride = front_end.split('@')
cur_metrics = set()
for each_summary_file in os.listdir(join(summary_home, each_dir, 'eval')):
event_acc = EventAccumulator(
join(summary_home, each_dir, 'eval', each_summary_file))
event_acc.Reload()
acc = event_acc.Scalars('accuracy')
miou = event_acc.Scalars('mean_iou')
for each_acc, each_miou in zip(acc, miou):
cur_metrics.add(
metric(each_acc.wall_time, each_acc.step, each_acc.value,
each_miou.value))
# print(cur_metrics)
# exit(1)
cur_metrics = sorted(cur_metrics, key=attrgetter('step'))
run_time = cur_metrics[-1].wall_time - cur_metrics[0].wall_time
best_acc = max(cur_metrics, key=attrgetter('acc'))
best_miou = max(cur_metrics, key=attrgetter('miou'))
def main(dirs_dataset_filter="mnist",
more_discard_dirs=[],
filtering_criteria_models=[
"resnet_50", "resnet_34", "resnet_18", "bagnet_tem_9",
"bagnet_tem_17"
],
filtering_criteria_frames=[],
filtering_criteria_others=[],
filtering_criteria_annotation_path=[],
filtering_criteria_sampling=["center"]):
res_dirs = [f for f in os.listdir("results/") if dirs_dataset_filter in f]
discard_dirs = ["motion", "blackframes", "val_1tstride"
] + more_discard_dirs
res_dirs = [f for f in res_dirs if all([d not in f for d in discard_dirs])]
for r in res_dirs:
# if os.path.exists("results/"+r+"/opts.json"):
# with open("results/"+r+"/opts.json", "r") as f:
# opts = json.load(f)
# print(opts["annotation_path"])
if [f for f in os.listdir("results/"+r) if "events.out" in f] and \
any([c in r for c in filtering_criteria_models]) and \
any([c in r for c in filtering_criteria_frames]) and \
any([c in r for c in filtering_criteria_others]) and \
os.path.exists("results/"+r+"/opts.json"):
print(r)
event_acc = EventAccumulator("results/" + r)
event_acc.Reload()
with open("results/" + r + "/opts.json", "r") as f:
opts = json.load(f)
print(opts['annotation_path'])
if any([c in opts['annotation_path'] for c in filtering_criteria_annotation_path]) \
and any ([c in opts['train_t_crop'] for c in filtering_criteria_sampling]):
if event_acc.scalars.Keys() != []:
train_losses, train_epochs, train_accs = zip(
*event_acc.Scalars('train/acc'))
val_losses, val_epochs, val_accs = zip(
*event_acc.Scalars('val/acc'))
if len(val_losses) < 10:
os.system("rm -r results/" + r)
if len(val_losses) >= 10 or train_accs[-1] > 0.95:
# print(len(val_losses))
print(r)
print("train", round(np.max(train_accs)*100, 2), np.argmax(train_accs), \
"val", round(np.max(val_accs)*100, 2), np.argmax(val_accs))
if os.path.exists("results/" + r +
"/checkpoints_test_results.json"):
with open(
"results/" + r +
"/checkpoints_test_results.json",
"r") as f:
test = json.load(f)
print(test)
def output(tensorboard_dir,
output_dir,
metrics_keys,
steps,
output_file_base="metrics"):
"""Output csv and markdown file which accumulated tensorflow event by step and metrics_keys."""
subdirs = GetLogdirSubdirectories(tensorboard_dir)
event_accumulators = []
for subdir in subdirs:
event_accumulator = EventAccumulator(subdir)
# init event accumulator
event_accumulator.Reload()
event_accumulators.append(event_accumulator)
if not metrics_keys:
metrics_keys = {
metrics_key
for event_accumulator in event_accumulators
for metrics_key in _get_metrics_keys(event_accumulator)
}
columns = [
_column_name(event_accumulator, metrics_key)
for event_accumulator, metrics_key in itertools.product(
event_accumulators, metrics_keys)
]
columns.sort()
df = pd.DataFrame([], columns=columns)
for event_accumulator in event_accumulators:
for metrics_key in metrics_keys:
value_step_list = _value_step_list(event_accumulator, metrics_key)
for value, step in value_step_list:
column_name = _column_name(event_accumulator, metrics_key)
df.loc[step, column_name] = value
if steps:
df = df[steps, :]
df = df.sort_index(ascending=False)
# index to column. and re-order column.
df["step"] = df.index
df = df[["step"] + columns]
output_csv = os.path.join(output_dir, "{}.csv".format(output_file_base))
df.to_csv(output_csv, index=False)
output_md = os.path.join(output_dir, "{}.md".format(output_file_base))
writer = pytablewriter.MarkdownTableWriter()
writer.char_left_side_row = "|" # fix for github
writer.from_dataframe(df)
with open(output_md, "w") as file_stream:
writer.stream = file_stream
writer.write_table()
message = """
output success
output csv: {}
output md: {}
""".format(output_csv, output_md)
print(message)
def get_miou_list_class_all(log_dir, class_num):
# add the tensorboard in the tf1x version
tf1x_dir = os.path.join(os.path.dirname(os.path.dirname(tf1x_python)),
'lib', 'python3.7', 'site-packages')
sys.path.insert(0, tf1x_dir)
from tensorboard.backend.event_processing.event_accumulator import EventAccumulator
# Loading too much data is slow...
# tf_size_guidance on how much data the EventAccumulator should
# | store in memory. The DEFAULT_SIZE_GUIDANCE tries not to store too much
# | so as to avoid OOMing the client. The size_guidance should be a map
# | from a `tagType` string to an integer representing the number of
# | items to keep per tag for items of that `tagType`. If the size is 0,
# | all events are stored.
tf_size_guidance = {
'compressedHistograms': 10,
'images': 0,
'scalars': 0, # set a 0, to load all scalars
'histograms': 1
}
miou_dic = {'step': [0]} # step 0, need some where
events_files = io_function.get_file_list_by_pattern(log_dir, 'events*')
if len(events_files) < 1:
print('warning, No events file in %s' % log_dir)
return miou_dic
event_acc = EventAccumulator(log_dir, tf_size_guidance)
event_acc.Reload()
# Show all tags in the log file
tag_dict = event_acc.Tags()
# io_function.save_dict_to_txt_json('event_acc.txt',tag_dict)
scalar_tags = tag_dict['scalars']
# print(scalar_tags)
for class_id in range(class_num):
name = 'class_%d' % class_id
tag = 'eval/miou_1.0_' + name
if tag in scalar_tags:
miou_class_event = event_acc.Scalars(tag)
miou_class_list = [
item[2] for item in miou_class_event
] # item[0] is wall_time, item[1] is step, item [2] is the value
# step_list = [item[1] for item in miou_class_event]
# print(step_list)
miou_dic[name] = miou_class_list
tag = 'eval/miou_1.0_overall'
if tag in scalar_tags:
miou_class_overall = event_acc.Scalars('eval/miou_1.0_overall')
miou_class_list = [item[2] for item in miou_class_overall]
step_list = [item[1] for item in miou_class_overall]
wall_time_list = [item[0] for item in miou_class_overall]
# print(step_list)
miou_dic['overall'] = miou_class_list
miou_dic['step'] = step_list
miou_dic[
'wall_time'] = wall_time_list # we can use datetime.fromtimestamp() to convert datetime
io_function.save_dict_to_txt_json(os.path.join(log_dir, 'miou.txt'),
miou_dic)
return miou_dic
def plot_func(ax,
method_dirs,
color,
label,
marker,
smooth_index=10,
alpha=0.4,
linewidth=1.0,
scatter_space=500):
'''
input: method_dirs : ['algo1/seed1','~algo1/seed4']
'''
##### extrat data from all seeds #####
y_seeds = []
y_length = []
seed_count = len(method_dirs)
for dir in method_dirs:
event = EventAccumulator(dir)
event.Reload()
if (label == 'SAC'):
y = event.scalars.Items('episode_threshold') # threshold_value
else:
y = event.scalars.Items('episode_reward')
y_len = len(y) # threshold_len
y_length.append(y_len)
###### smoothing #########
smooth_array = np.zeros(y_len)
for i in range(y_len):
smooth_array[i] = np.array(
[j.value for j in y[i:i + smooth_index]]).mean()
y_seeds.append(smooth_array)
###### reshape y_data into [num_seeds, min_y_length] #####
min_y_length = min(y_length)
y_seeds_new = np.zeros((seed_count, min_y_length))
for i in range(seed_count):
y_seeds_new[i] = y_seeds[i][0:min_y_length]
y_seeds = y_seeds_new # [4, min_y_len]
print(y_seeds.shape)
###### plot y_seeds with color_mean, scatter_marker, std_shadow #############
y_mean = y_seeds.mean(axis=0)
x_mean = [i for i in range(min_y_length)]
ax.plot(x_mean, y_mean, color=color, linewidth=linewidth)
x_scatter = np.linspace(0,
min_y_length - 1,
int(min_y_length / scatter_space),
dtype=np.int)
y_scatter = y_mean[x_scatter]
ax.scatter(x_scatter,
y_scatter,
color=color,
label=label,
marker=marker,
s=100)
y_std = y_seeds.std(axis=0)
upper_bound = y_mean + y_std
lower_bound = y_mean - y_std
ax.fill_between(x_mean,
upper_bound,
lower_bound,
where=upper_bound > lower_bound,
facecolor=color,
interpolate=True,
alpha=alpha)
