v = int(v)
if k in config:
print("Overwriting {:20} {:30} -> {:}".format(k, config[k], v))
config[k] = v
except Exception as e:
print(e)
print("Ignoring argument", u)
for o in dir(opts):
if not o.startswith("_"):
if o in config:
print("Overwriting {:20} {:30} -> {:}".format(
o, config[k], getattr(opts, o)))
config[o] = getattr(opts, o)
comet_exp.log_asset(opts.config)
max_iter = config["max_iter"]
display_size = config["display_size"]
config["vgg_model_path"] = opts.output_path
comet_exp.log_parameters(config)
print("Using model", opts.trainer)
# Setup model and data loader
if opts.trainer == "MUNIT":
trainer = MUNIT_Trainer(config, comet_exp)
elif opts.trainer == "UNIT":
trainer = UNIT_Trainer(config)
elif opts.trainer == "DoubleMUNIT":
trainer = DoubleMUNIT_Trainer(config, comet_exp)
else:
class CometLogger(Logger):
def __init__(
self,
batch_size: int,
snapshot_dir: Optional[str] = None,
snapshot_mode: str = "last",
snapshot_gap: int = 1,
exp_set: Optional[str] = None,
use_print_exp: bool = False,
saved_exp: Optional[str] = None,
**kwargs,
):
"""
:param kwargs: passed to comet's Experiment at init.
"""
if use_print_exp:
self.experiment = PrintExperiment()
else:
from comet_ml import Experiment, ExistingExperiment, OfflineExperiment
if saved_exp:
self.experiment = ExistingExperiment(
previous_experiment=saved_exp, **kwargs
)
else:
try:
self.experiment = Experiment(**kwargs)
except ValueError: # no API key
log_dir = Path.home() / "logs"
log_dir.mkdir(exist_ok=True)
self.experiment = OfflineExperiment(offline_directory=str(log_dir))
self.experiment.log_parameter("complete", False)
if exp_set:
self.experiment.log_parameter("exp_set", exp_set)
if snapshot_dir:
snapshot_dir = Path(snapshot_dir) / self.experiment.get_key()
# log_traj_window (int): How many trajectories to hold in deque for computing performance statistics.
self.log_traj_window = 100
self._cum_metrics = {
"n_unsafe_actions": 0,
"constraint_used": 0,
"cum_completed_trajs": 0,
"logging_time": 0,
}
self._new_completed_trajs = 0
self._last_step = 0
self._start_time = self._last_time = time()
self._last_snapshot_upload = 0
self._snaphot_upload_time = 30 * 60
super().__init__(batch_size, snapshot_dir, snapshot_mode, snapshot_gap)
def log_fast(
self,
step: int,
traj_infos: Sequence[Dict[str, float]],
opt_info: Optional[Tuple[Sequence[float], ...]] = None,
test: bool = False,
) -> None:
if not traj_infos:
return
start = time()
self._new_completed_trajs += len(traj_infos)
self._cum_metrics["cum_completed_trajs"] += len(traj_infos)
# TODO: do we need to support sum(t[k]) if key in k?
# without that, this doesn't include anything from extra eval samplers
for key in self._cum_metrics:
if key == "cum_completed_trajs":
continue
self._cum_metrics[key] += sum(t.get(key, 0) for t in traj_infos)
self._cum_metrics["logging_time"] += time() - start
def log(
self,
step: int,
traj_infos: Sequence[Dict[str, float]],
opt_info: Optional[Tuple[Sequence[float], ...]] = None,
test: bool = False,
):
self.log_fast(step, traj_infos, opt_info, test)
start = time()
with (self.experiment.test() if test else nullcontext()):
step *= self.batch_size
if opt_info is not None:
# grad norm is left on the GPU for some reason
# https://github.com/astooke/rlpyt/issues/163
self.experiment.log_metrics(
{
k: np.mean(v)
for k, v in opt_info._asdict().items()
if k != "gradNorm"
},
step=step,
)
if traj_infos:
agg_vals = {}
for key in traj_infos[0].keys():
if key in self._cum_metrics:
continue
agg_vals[key] = sum(t[key] for t in traj_infos) / len(traj_infos)
self.experiment.log_metrics(agg_vals, step=step)
if not test:
now = time()
self.experiment.log_metrics(
{
"new_completed_trajs": self._new_completed_trajs,
"steps_per_second": (step - self._last_step)
/ (now - self._last_time),
},
step=step,
)
self._last_time = now
self._last_step = step
self._new_completed_trajs = 0
self.experiment.log_metrics(self._cum_metrics, step=step)
self._cum_metrics["logging_time"] += time() - start
def log_metric(self, name, val):
self.experiment.log_metric(name, val)
def log_parameters(self, parameters):
self.experiment.log_parameters(parameters)
def log_config(self, config):
self.experiment.log_parameter("config", json.dumps(convert_dict(config)))
def upload_snapshot(self):
if self.snapshot_dir:
self.experiment.log_asset(self._previous_snapshot_fname)
def save_itr_params(
self, step: int, params: Dict[str, Any], metric: Optional[float] = None
) -> None:
super().save_itr_params(step, params, metric)
now = time()
if now - self._last_snapshot_upload > self._snaphot_upload_time:
self._last_snapshot_upload = now
self.upload_snapshot()
def shutdown(self, error: bool = False) -> None:
if not error:
self.upload_snapshot()
self.experiment.log_parameter("complete", True)
self.experiment.end()
