def stream_tfevents(path, file_api, run, step=0, namespace=""):
"""Parses and streams a tfevents file to the server"""
last_step = 0
row = {}
buffer = []
last_row = {}
global_step_key = namespaced_tag("global_step", namespace)
try:
for summary in summary_iterator(path):
parsed = tf_summary_to_dict(summary, namespace=namespace)
if parsed is None:
continue
if last_step != parsed[global_step_key]:
last_step = parsed[global_step_key]
if len(row) > 3: # Must have more than _timestamp, _step, and global_step
step += 1
row["_step"] = step
last_row = history_dict_to_json(run, deepcopy(row))
file_api.push("wandb-history.jsonl", util.json_dumps_safer_history(last_row))
row = {}
row.update(parsed)
except tf.errors.DataLossError:
wandb.termwarn("Found a truncated record in tfevents file, stopping parse")
step += 1
row["_step"] = step
last_row = history_dict_to_json(run, deepcopy(row))
file_api.push("wandb-history.jsonl", util.json_dumps_safer_history(last_row))
return last_row
def max_val_acc(self, event_file_name):
max_val_acc = 0
for e in summary_iterator(event_file_name):
for v in e.summary.value:
if v.tag == 'val_acc':
if v.simple_value > max_val_acc:
max_val_acc = v.simple_value
return max_val_acc
def get_actual_step(log_file):
"""
returna actual step
"""
actual_step = 0
for e in summary_iterator(log_file):
if hasattr(e, 'step'):
actual_step = e.step
return actual_step + 1
def __init__(self,
filepath,
score=None,
tensorboard_logfile=None,
monitor='val_loss',
verbose=0,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1):
"""
:param filepath: Keras' filepath argument. "filepath: string, path to save the model file."
:param score: float, the score of the existing model you wish to use (either score or
tensorboard_logfile must be provided).
:param tensorboard_logfile: string, path to the tensorboard log file you wish to pull
score metrics from (either score or tensorboard_logfile must be provided).
:param monitor: Keras' monitor argument. "monitor: quantity to monitor."
:param verbose: Keras' verbose argument. "verbose: verbosity mode, 0 or 1."
:param save_best_only: Keras' save_best_only argument, NOTE: This is defaulted to True: "save_best_only:
if `save_best_only=True`, the latest best model according to the quantity monitored will not be
overwritten".
:param save_weights_only: Keras' save_weights_only argument: "save_weights_only: if True,
then only the model's weights will be saved (`model.save_weights(filepath)`), else the full model
is saved (`model.save(filepath)`)."
:param mode: Keras' mode argument: "mode: one of {auto, min, max}. If `save_best_only=True`, the decision
to overwrite the current save file is made based on either the maximization or the
minimization of the monitored quantity. For `val_acc`, this should be `max`, for `val_loss` this should
be `min`, etc. In `auto` mode, the direction is automatically inferred from the name of the monitored
quantity."
:param period: Keras' period argument: "Interval (number of epochs) between checkpoints".
"""
super(MemoryModelCheckpoint,
self).__init__(filepath, monitor, verbose, save_best_only,
save_weights_only, mode, period)
if (score is None and tensorboard_logfile is None) or (
score is not None and tensorboard_logfile is not None):
raise Exception("Either (XOR) score or logfile must be provided.")
elif score is not None:
self.best = score
elif tensorboard_logfile is not None:
scores = []
for x in summary_iterator(tensorboard_logfile):
for val in x.summary.value:
if val.tag == 'loss':
scores.append(val.simple_value)
if self.monitor_op == np.greater:
self.best = np.max(scores)
else:
self.best = np.min(scores)
def load_log(log_file, tags=['train/acc', 'val/acc', 'train/iou', 'val/iou']):
"""
return best metrics
"""
best_metrics = {}
for e in summary_iterator(log_file):
for v in e.summary.value:
if v.tag in tags:
if v.tag not in best_metrics.keys():
best_metrics[v.tag] = v.simple_value
elif v.tag.find('loss') >= 0:
best_metrics[v.tag] = min(best_metrics[v.tag],
v.simple_value)
else:
best_metrics[v.tag] = max(best_metrics[v.tag],
v.simple_value)
# print(best_metrics)
return best_metrics
def get_rewards(paths, smoothing=None, extend=True):
all_steps = None
all_rewards = None
for path in paths:
if isdir(path):
files = [f for f in listdir(path) if isfile(join(path, f))]
for f in sorted(files):
if "events.out.tfevents" in f:
path = join(path, f)
# print("path:", path)
break
steps = []
rewards = []
for summary in summary_iterator(path):
step = None
reward = None
for v in summary.summary.value:
if v.tag == "ray/tune/train-steps":
step = v.simple_value
if v.tag == "ray/tune/evaluation/return-average":
reward = v.simple_value
if step is not None and reward is not None:
if len(steps) > 0 and step < steps[-1]:
print("detected backwards time")
break
steps.append(step)
rewards.append(reward)
if all_steps is None:
all_steps = steps
all_rewards = [rewards]
else:
all_rewards.append(rewards)
if len(all_steps) < len(steps):
all_steps = steps
print(path, len(rewards))
if not extend:
min_length = int(min([len(r) for r in all_rewards]))
for i in range(len(all_rewards)):
all_rewards[i] = all_rewards[i][:min_length]
if extend:
max_length = int(max([len(r) for r in all_rewards]))
print("max_length:", max_length)
for i in range(len(all_rewards)):
print("len before:", len(all_rewards[i]))
all_rewards[i].extend([all_rewards[i][-1]] *
(max_length - len(all_rewards[i])))
print("len after:", len(all_rewards[i]))
min_length = int(min([len(r) for r in all_rewards]))
if smoothing is not None:
all_rewards = [moving_average(r, n=smoothing) for r in all_rewards]
if smoothing is not None:
all_steps = moving_average(all_steps, n=smoothing)
all_steps = np.array(all_steps[:min_length])
all_rewards = np.array(all_rewards)
print("all_steps", all_steps.shape)
print("all_rewards:", all_rewards.shape)
mean_rewards = np.mean(all_rewards, axis=0)
std_dev = np.std(all_rewards, axis=0)
std_error = std_dev / np.sqrt(all_rewards.shape[0])
print("mean_rewards:", mean_rewards.shape)
print("std_error:", std_error.shape)
return all_steps, mean_rewards, std_error