def log_trial_result(self, iteration: int, trial: "Trial", result: Dict):
"""
Log the current result of a Trial upon each iteration.
"""
if trial not in self._trial_experiments:
self.log_trial_start(trial)
experiment = self._trial_experiments[trial]
step = result["training_iteration"]
config_update = result.pop("config", {}).copy()
config_update.pop("callbacks", None) # Remove callbacks
for k, v in config_update.items():
if isinstance(v, dict):
experiment.log_parameters(flatten_dict({k: v}, "/"), step=step)
else:
experiment.log_parameter(k, v, step=step)
other_logs = {}
metric_logs = {}
system_logs = {}
episode_logs = {}
flat_result = flatten_dict(result, delimiter="/")
for k, v in flat_result.items():
if any(self._check_key_name(k, item) for item in self._to_exclude):
continue
if any(self._check_key_name(k, item) for item in self._to_other):
other_logs[k] = v
elif any(
self._check_key_name(k, item) for item in self._to_system):
system_logs[k] = v
elif any(
self._check_key_name(k, item)
for item in self._to_episodes):
episode_logs[k] = v
else:
metric_logs[k] = v
experiment.log_others(other_logs)
experiment.log_metrics(metric_logs, step=step)
for k, v in system_logs.items():
experiment.log_system_info(k, v)
for k, v in episode_logs.items():
experiment.log_curve(k, x=range(len(v)), y=v, step=step)
def _get_trial_info(trial, parameters, metrics):
"""Returns the following information about a trial:
name | status | loc | params... | metrics...
Args:
trial (Trial): Trial to get information for.
parameters (list[str]): Names of trial parameters to include.
metrics (list[str]): Names of metrics to include.
"""
result = flatten_dict(trial.last_result)
config = flatten_dict(trial.config)
trial_info = [str(trial), trial.status, str(trial.location)]
trial_info += [config.get(param) for param in parameters]
trial_info += [result.get(metric) for metric in metrics]
return trial_info
def convert_search_space(spec: Dict) -> Dict:
spec = flatten_dict(spec, prevent_delimiter=True)
resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
if grid_vars:
raise ValueError(
"Grid search parameters cannot be automatically converted "
"to a BayesOpt search space.")
def resolve_value(domain: Domain) -> Tuple[float, float]:
sampler = domain.get_sampler()
if isinstance(sampler, Quantized):
logger.warning(
"BayesOpt search does not support quantization. "
"Dropped quantization.")
sampler = sampler.get_sampler()
if isinstance(domain, Float):
if domain.sampler is not None:
logger.warning(
"BayesOpt does not support specific sampling methods. "
"The {} sampler will be dropped.".format(sampler))
return (domain.lower, domain.upper)
raise ValueError("BayesOpt does not support parameters of type "
"`{}`".format(type(domain).__name__))
# Parameter name is e.g. "a/b/c" for nested dicts
bounds = {
"/".join(path): resolve_value(domain)
for path, domain in domain_vars
}
return bounds
def on_result(self, result):
step = result.get(TIMESTEPS_TOTAL) or result[TRAINING_ITERATION]
tmp = result.copy()
for k in [
"config", "pid", "timestamp", TIME_TOTAL_S, TRAINING_ITERATION
]:
if k in tmp:
del tmp[k] # not useful to log these
flat_result = flatten_dict(tmp, delimiter="/")
path = ["ray", "tune"]
valid_result = {}
for attr, value in flat_result.items():
full_attr = "/".join(path + [attr])
if type(value) in VALID_SUMMARY_TYPES:
valid_result[full_attr] = value
self._file_writer.add_scalar(
full_attr, value, global_step=step)
elif type(value) is list and len(value) > 0:
valid_result[full_attr] = value
self._file_writer.add_histogram(
full_attr, value, global_step=step)
self.last_result = valid_result
self._file_writer.flush()
def update_last_result(self, result, terminate=False):
result.update(trial_id=self.trial_id, done=terminate)
if self.experiment_tag:
result.update(experiment_tag=self.experiment_tag)
if self.verbose and (terminate or time.time() - self.last_debug >
DEBUG_PRINT_INTERVAL):
print("Result for {}:".format(self))
print(" {}".format(pretty_print(result).replace("\n", "\n ")))
self.last_debug = time.time()
self.set_location(Location(result.get("node_ip"), result.get("pid")))
self.last_result = result
self.last_update_time = time.time()
self.result_logger.on_result(self.last_result)
for metric, value in flatten_dict(result).items():
if isinstance(value, Number):
if metric not in self.metric_analysis:
self.metric_analysis[metric] = {
"max": value,
"min": value,
"avg": value,
"last": value
}
else:
step = result["training_iteration"] or 1
self.metric_analysis[metric]["max"] = max(
value, self.metric_analysis[metric]["max"])
self.metric_analysis[metric]["min"] = min(
value, self.metric_analysis[metric]["min"])
self.metric_analysis[metric]["avg"] = 1 / step * (
value +
(step - 1) * self.metric_analysis[metric]["avg"])
self.metric_analysis[metric]["last"] = value
def on_result(self, result):
experiment_tag = result.get('experiment_tag', 'no_experiment_tag')
experiment_id = result.get('experiment_id', 'no_experiment_id')
if experiment_tag not in self.metrics_queue_dict:
print("=" * 50)
print("Setting up new w&b logger")
print("Experiment tag:", experiment_tag)
print("Experiment id:", experiment_id)
config = result.get("config")
queue = multiprocessing.Queue()
p = multiprocessing.Process(target=wandb_process,
args=(
queue,
config,
))
p.start()
self.metrics_queue_dict[experiment_tag] = queue
print("=" * 50)
queue = self.metrics_queue_dict[experiment_tag]
tmp = result.copy()
for k in ["done", "config", "pid", "timestamp"]:
if k in tmp:
del tmp[k]
metrics = {}
for key, value in flatten_dict(tmp, delimiter="/").items():
if not isinstance(value, numbers.Number):
continue
metrics[key] = value
queue.put(metrics)
def _parse_configs(cfg_path):
try:
with open(cfg_path) as f:
cfg_dict = flatten_dict(json.load(f))
except Exception:
logger.exception("Config parsing failed.")
return cfg_dict
def _log_hparams(self):
if hasattr(self, 'trial'):
if self.trial and self.trial.evaluated_params:
ep = flatten_dict(self.trial.evaluated_params, '/')
ep = {format_keys(p): v for p, v in ep.items()}
for key, value in ep.items():
self.client.log_param(self._run_id, key, value)
def create_trial_if_possible(self, experiment_spec: Dict,
output_path: str) -> Optional[Trial]:
logger.debug("creating trial")
trial_id = Trial.generate_id()
suggested_config = self.searcher.suggest(trial_id)
if suggested_config == Searcher.FINISHED:
self._finished = True
logger.debug("Searcher has finished.")
return
if suggested_config is None:
return
spec = copy.deepcopy(experiment_spec)
spec["config"] = merge_dicts(spec["config"],
copy.deepcopy(suggested_config))
# Create a new trial_id if duplicate trial is created
flattened_config = resolve_nested_dict(spec["config"])
self._counter += 1
tag = "{0}_{1}".format(str(self._counter),
format_vars(flattened_config))
trial = create_trial_from_spec(
spec,
output_path,
self._parser,
evaluated_params=flatten_dict(suggested_config),
experiment_tag=tag,
trial_id=trial_id)
return trial
def _generate_trials(self, experiment_spec, output_path=""):
"""Generates trials with configurations from `_suggest`.
Creates a trial_id that is passed into `_suggest`.
Yields:
Trial objects constructed according to `spec`
"""
if "run" not in experiment_spec:
raise TuneError("Must specify `run` in {}".format(experiment_spec))
for _ in range(experiment_spec.get("num_samples", 1)):
trial_id = Trial.generate_id()
while True:
suggested_config = self._suggest(trial_id)
if suggested_config is None:
yield None
else:
break
spec = copy.deepcopy(experiment_spec)
spec["config"] = merge_dicts(spec["config"],
copy.deepcopy(suggested_config))
flattened_config = resolve_nested_dict(spec["config"])
self._counter += 1
tag = "{0}_{1}".format(str(self._counter),
format_vars(flattened_config))
yield create_trial_from_spec(
spec,
output_path,
self._parser,
evaluated_params=flatten_dict(suggested_config),
experiment_tag=tag,
trial_id=trial_id)
def on_result(self, result):
step = result.get(TIMESTEPS_TOTAL) or result[TRAINING_ITERATION]
tmp = result.copy()
for k in [
"config", "pid", "timestamp", TIME_TOTAL_S, TRAINING_ITERATION
]:
if k in tmp:
del tmp[k] # not useful to log these
# log system usage
perf = result.get("perf", None)
if FilteredTBXLogger.log_sys_usage and perf is not None:
self.log_system_usage(step, perf)
flat_result = flatten_dict(tmp, delimiter="/")
path = ["scalars"]
valid_result = {
"/".join(path + [attr]): value
for attr, value in flat_result.items()
if type(value) in VALID_SUMMARY_TYPES
and attr in FilteredTBXLogger.keep_fields
}
# log scalars
for attr, value in valid_result.items():
self._file_writer.add_scalar(attr, value, global_step=step)
# log videos
videos = result.get("eval_frames", [])
if FilteredTBXLogger.log_videos and len(videos) > 0:
self.log_videos(step, videos, "rollout_frames")
self.last_result = valid_result
self._file_writer.flush()
def get_all_configs(self, prefix: bool = False) -> Dict[str, Dict]:
"""Returns a list of all configurations.
Args:
prefix: If True, flattens the config dict
and prepends `config/`.
Returns:
Dict[str, Dict]: Dict of all configurations of trials, indexed by
their trial dir.
"""
fail_count = 0
for path in self._get_trial_paths():
try:
with open(os.path.join(path, EXPR_PARAM_FILE)) as f:
config = json.load(f)
if prefix:
self._configs[path] = flatten_dict({CONFIG_PREFIX: config})
else:
self._configs[path] = config
except Exception:
fail_count += 1
if fail_count:
logger.warning(
"Couldn't read config from {} paths".format(fail_count))
return self._configs
def on_result(self, result):
step = result.get(TIMESTEPS_TOTAL) or result[TRAINING_ITERATION]
# Log scalars
logged_results = ['episode_reward_max', 'episode_reward_mean', 'episode_reward_min', 'episode_len_mean',
'custom_metrics', 'sampler_perf', 'info', 'perf']
result_copy = result.copy()
for key, val in result.items():
if key not in logged_results:
del result_copy[key]
flat_result = flatten_dict(result_copy, delimiter="/")
self.wandb_run.log(flat_result, step=step, sync=False)
# Log histograms
for key, val in result['hist_stats'].items():
try:
if key != '_robot_coordinates':
self.wandb_run.log({"Histograms/"+key: wandb.Histogram(val)}, step=step, sync=False)
except ValueError:
logger.warning("Unable to log histogram for {}".format(key))
# Log trajectories
traj_fig = plot_trajectories(result['hist_stats']['_robot_coordinates'])
traj_fig.savefig("Trajectory.png")
self.wandb_run.log({'Episode Trajectories': wandb.Image(traj_fig)}, step=step, sync=False)
plt.close(traj_fig)
def _try_log_hparams(self, trial: "Trial", result: Dict):
# TBX currently errors if the hparams value is None.
flat_params = flatten_dict(trial.evaluated_params)
scrubbed_params = {
k: v
for k, v in flat_params.items()
if isinstance(v, self.VALID_HPARAMS)
}
removed = {
k: v
for k, v in flat_params.items()
if not isinstance(v, self.VALID_HPARAMS)
}
if removed:
logger.info(
"Removed the following hyperparameter values when "
"logging to tensorboard: %s", str(removed))
from tensorboardX.summary import hparams
try:
experiment_tag, session_start_tag, session_end_tag = hparams(
hparam_dict=scrubbed_params, metric_dict=result)
self._trial_writer[trial].file_writer.add_summary(experiment_tag)
self._trial_writer[trial].file_writer.add_summary(
session_start_tag)
self._trial_writer[trial].file_writer.add_summary(session_end_tag)
except Exception:
logger.exception("TensorboardX failed to log hparams. "
"This may be due to an unsupported type "
"in the hyperparameter values.")
def _dict_hash(config, precision):
flatconfig = flatten_dict(config)
for param, value in flatconfig.items():
if isinstance(value, float):
flatconfig[param] = "{:.{digits}f}".format(value, digits=precision)
hashed = json.dumps(flatconfig, sort_keys=True, default=str)
return hashed
def to_tf_values(result, path):
flat_result = flatten_dict(result, delimiter="/")
values = [
tf.Summary.Value(tag="/".join(path + [attr]), simple_value=value)
for attr, value in flat_result.items()
if type(value) in VALID_SUMMARY_TYPES
]
return values
def get_result(root, friction_list, num_episodes=20, select_key=None):
results = []
root = os.path.abspath(root)
a_count = 0
for p in os.listdir(root):
trial = osp.join(root, p)
if not osp.isdir(trial):
continue
if select_key is not None:
if select_key not in trial:
print(
"We filter out the trial: {} since it does not contain keyword: {}."
.format(trial, select_key))
continue
assert p.startswith("PPO")
a_count += 1
exps = [pp for pp in os.listdir(trial) if pp.startswith("checkpoint")]
exps.sort(key=lambda v: eval(v.split("_")[1]))
if not exps:
print("Empty!")
assert exps
# Largest checkpoint index
ckpt = osp.join(trial, exps[-1], exps[-1].replace("_", "-"))
config_file = osp.join(trial, "params.json")
with open(config_file, "r") as f:
config = json.load(f)
config = flatten_dict(config)
for friction in friction_list:
trainer = get_trainer(friction, ckpt)
print("\n===== Start Evaluating {} agent =====".format(a_count))
print("friction: {}\nCheckpoint: {}\n".format(friction, ckpt))
# evaluate
ret = evaluate(trainer, num_episodes)
results.append(
dict(path=ckpt,
trial=trial,
friction=friction,
**ret,
**config))
trainer.cleanup()
del trainer
print("Finish {} agents.".format(a_count))
ret = pd.DataFrame(results)
return ret
def results_df(self) -> DataFrame:
if not pd:
raise ValueError("`best_result_df` requires pandas. Install with "
"`pip install pandas`.")
return pd.DataFrame.from_records([
flatten_dict(trial.last_result, delimiter=".")
for trial in self.trials
],
index="trial_id")
def convert_search_space(spec: Dict, join: bool = False) -> Dict:
resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
if grid_vars:
raise ValueError(
"Grid search parameters cannot be automatically converted "
"to a SkOpt search space.")
# Flatten and resolve again after checking for grid search.
spec = flatten_dict(spec, prevent_delimiter=True)
resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
def resolve_value(domain: Domain) -> Union[Tuple, List]:
sampler = domain.get_sampler()
if isinstance(sampler, Quantized):
logger.warning("SkOpt search does not support quantization. "
"Dropped quantization.")
sampler = sampler.get_sampler()
if isinstance(domain, Float):
if isinstance(domain.sampler, LogUniform):
return sko.space.Real(domain.lower,
domain.upper,
prior="log-uniform")
return sko.space.Real(domain.lower,
domain.upper,
prior="uniform")
elif isinstance(domain, Integer):
if isinstance(domain.sampler, LogUniform):
return sko.space.Integer(domain.lower,
domain.upper - 1,
prior="log-uniform")
return sko.space.Integer(domain.lower,
domain.upper - 1,
prior="uniform")
elif isinstance(domain, Categorical):
return sko.space.Categorical(domain.categories)
raise ValueError("SkOpt does not support parameters of type "
"`{}` with samplers of type `{}`".format(
type(domain).__name__,
type(domain.sampler).__name__))
# Parameter name is e.g. "a/b/c" for nested dicts
space = {
"/".join(path): resolve_value(domain)
for path, domain in domain_vars
}
if join:
spec.update(space)
space = spec
return space
def on_result(self, result):
"""
The following is copied from the parent class; however, non-serializable
config values are saved as the repr's so that they are all yaml
serializable. See for details:
- https://github.com/wandb/client/issues/586
"""
config = deepcopy(result.get("config"))
if config and self._config is None:
for k in config.keys():
if wandb.config.get(k) is None:
s = repr(config[k])
try:
ast.literal_eval(s)
wandb.config[k] = config[k]
except (ValueError, SyntaxError):
# Non-serializable
wandb.config[k] = s
self._config = config
tmp = result.copy()
for k in ["done", "config", "pid", "timestamp"]:
if k in tmp:
del tmp[k]
if self.result_to_time_series_fn is not None:
assert self._config is not None
time_series_dict = self.result_to_time_series_fn(tmp, self._config)
for t, d in sorted(time_series_dict.items(), key=lambda x: x[0]):
metrics = {}
for key, value in flatten_dict(d, delimiter="/").items():
if not isinstance(value, self.accepted_types):
continue
metrics[key] = value
wandb.log(metrics, step=t)
else:
metrics = {}
for key, value in flatten_dict(tmp, delimiter="/").items():
if not isinstance(value, self.accepted_types):
continue
metrics[key] = value
wandb.log(metrics)
def results_df(self) -> DataFrame:
"""Get all the last results as a pandas dataframe."""
if not pd:
raise ValueError("`results_df` requires pandas. Install with "
"`pip install pandas`.")
return pd.DataFrame.from_records([
flatten_dict(trial.last_result, delimiter=self._delimiter())
for trial in self.trials
],
index="trial_id")
def _parse_results(res_path):
res_dict = {}
try:
with open(res_path) as f:
# Get last line in file
for line in f:
pass
res_dict = flatten_dict(json.loads(line.strip()))
except Exception:
logger.exception("Importing %s failed...Perhaps empty?" % res_path)
return res_dict
def close(self):
if self._file_writer is not None:
if self.trial and self.trial.evaluated_params and self.last_result:
flat_result = flatten_dict(self.last_result, delimiter="/")
scrubbed_result = {
k: value
for k, value in flat_result.items()
if isinstance(value, tuple(VALID_SUMMARY_TYPES))
}
self._try_log_hparams(scrubbed_result)
self._file_writer.close()
def log_trial_result(self, iteration: int, trial: "Trial", result: Dict):
if trial not in self._trial_writer:
self.log_trial_start(trial)
step = result.get(TIMESTEPS_TOTAL) or result[TRAINING_ITERATION]
tmp = result.copy()
for k in [
"config", "pid", "timestamp", TIME_TOTAL_S, TRAINING_ITERATION
]:
if k in tmp:
del tmp[k] # not useful to log these
flat_result = flatten_dict(tmp, delimiter="/")
path = ["ray", "tune"]
valid_result = {}
for attr, value in flat_result.items():
full_attr = "/".join(path + [attr])
if isinstance(value,
tuple(VALID_SUMMARY_TYPES)) and not np.isnan(value):
valid_result[full_attr] = value
self._trial_writer[trial].add_scalar(full_attr,
value,
global_step=step)
elif (isinstance(value, list)
and len(value) > 0) or (isinstance(value, np.ndarray)
and value.size > 0):
valid_result[full_attr] = value
# Must be video
if isinstance(value, np.ndarray) and value.ndim == 5:
self._trial_writer[trial].add_video(full_attr,
value,
global_step=step,
fps=20)
continue
try:
self._trial_writer[trial].add_histogram(full_attr,
value,
global_step=step)
# In case TensorboardX still doesn't think it's a valid value
# (e.g. `[[]]`), warn and move on.
except (ValueError, TypeError):
if log_once("invalid_tbx_value"):
logger.warning(
"You are trying to log an invalid value ({}={}) "
"via {}!".format(full_attr, value,
type(self).__name__))
self._trial_result[trial] = valid_result
self._trial_writer[trial].flush()
def log_trial_end(self, trial: "Trial", failed: bool = False):
if trial in self._trial_writer:
if trial and trial.evaluated_params and self._trial_result[trial]:
flat_result = flatten_dict(self._trial_result[trial], delimiter="/")
scrubbed_result = {
k: value
for k, value in flat_result.items()
if isinstance(value, tuple(VALID_SUMMARY_TYPES))
}
self._try_log_hparams(trial, scrubbed_result)
self._trial_writer[trial].close()
del self._trial_writer[trial]
del self._trial_result[trial]
def on_result(self, result: Dict):
tmp = result.copy()
if "config" in tmp:
del tmp["config"]
result = flatten_dict(tmp, delimiter="/")
if self._csv_out is None:
self._csv_out = csv.DictWriter(self._file, result.keys())
if not self._continuing:
self._csv_out.writeheader()
self._csv_out.writerow(
{k: v
for k, v in result.items() if k in self._csv_out.fieldnames})
self._file.flush()
def log_result(result: dict):
# Avoid logging the config every iteration
# Only log Jsonable objects
tmp = result.copy()
for k in ["done", "config", "pid", "timestamp"]:
if k in tmp:
del tmp[k]
metrics = {}
for key, value in flatten_dict(tmp, delimiter="/").items():
if not isinstance(value, numbers.Number):
continue
metrics[key] = value
wandb.log(metrics)
def _try_log_hparams(self, result):
# TBX currently errors if the hparams value is None.
flat_params = flatten_dict(self.trial.evaluated_params)
scrubbed_params = {
k: v
for k, v in flat_params.items() if v is not None
}
from tensorboardX.summary import hparams
experiment_tag, session_start_tag, session_end_tag = hparams(
hparam_dict=scrubbed_params, metric_dict=result)
self._file_writer.file_writer.add_summary(experiment_tag)
self._file_writer.file_writer.add_summary(session_start_tag)
self._file_writer.file_writer.add_summary(session_end_tag)
def __init__(self, config, logdir, trial):
super(WeightsAndBiasesLogger, self).__init__(config, logdir, trial)
# logger.warning("WeightsAndBiasesLogger.__init__() called! Trial.experiment_tag: {}".format(trial.experiment_tag))
self.trial = trial
self.experiment_tag = trial.experiment_tag
self.wandb_run = wandb.init(project=weights_and_biases_project,
name=config['env_config']['experiment_name'] + '_' + trial.experiment_tag,
reinit=True)
valid_config = config.copy()
del valid_config['callbacks']
valid_config = flatten_dict(valid_config, delimiter="/")
self.wandb_run.config.update(valid_config, allow_val_change=True)
def _get_report_dict(self, evals_log: Dict[str, Dict[str, list]]) -> dict:
result_dict = flatten_dict(evals_log, delimiter="-")
if not self._metrics:
report_dict = result_dict
else:
report_dict = {}
for key in self._metrics:
if isinstance(self._metrics, dict):
metric = self._metrics[key]
else:
metric = key
report_dict[key] = result_dict[metric]
return report_dict
