class CometLogger(): def __init__(self, enabled, is_existing=False, prev_exp_key=None): """ Handles logging of experiment to comet and also persistence to local file system. Supports resumption of stopped experiments. """ disabled = not enabled if not is_existing: self.experiment = Experiment(api_key=COMET_API_KEY, workspace=COMET_WORKSPACE, project_name=PROJECT_NAME, disabled=disabled) else: if prev_exp_key is None: raise ValueError( "Requested existing experiment, but no key provided") print("Continuing existing experiment with key: ", prev_exp_key) self.experiment = ExistingExperiment( api_key=COMET_API_KEY, workspace=COMET_WORKSPACE, project_name=PROJECT_NAME, disabled=disabled, previous_experiment=prev_exp_key) self.disabled = disabled def get_experiment_key(self): return self.experiment.get_key()[:9] def add_tag(self, tag): self.experiment.add_tag(tag) def log_metric(self, name, value, step=None): self.experiment.log_metric(name, value, step=step) def log_metrics(self, metrics_dict, prefix, step=None): self.experiment.log_metrics(metrics_dict, prefix=prefix, step=step) def log_params(self, params_dict): self.experiment.log_parameters(params_dict) def set_name(self, name_str): self.experiment.set_name(name_str) def log_dataset(self, dataset: SpeakerVerificationDataset): if self.disabled: return dataset_string = "" dataset_string += "<b>Speakers</b>: %s\n" % len(dataset.speakers) dataset_string += "\n" + dataset.get_logs() dataset_string = dataset_string.replace("\n", "<br>") self.vis.text(dataset_string, opts={"title": "Dataset"}) def log_implementation(self, params): if self.disabled: return implementation_string = "" for param, value in params.items(): implementation_string += "<b>%s</b>: %s\n" % (param, value) implementation_string = implementation_string.replace("\n", "<br>") self.implementation_string = implementation_string self.implementation_win = self.vis.text( implementation_string, opts={"title": "Training implementation"}) def draw_projections(self, embeds, utterances_per_speaker, step, out_fpath=None, max_speakers=16): if self.disabled: return max_speakers = min(max_speakers, len(colormap)) embeds = embeds[:max_speakers * utterances_per_speaker] n_speakers = len(embeds) // utterances_per_speaker ground_truth = np.repeat(np.arange(n_speakers), utterances_per_speaker) colors = [colormap[i] for i in ground_truth] reducer = umap.UMAP() projected = reducer.fit_transform(embeds) plt.scatter(projected[:, 0], projected[:, 1], c=colors) plt.gca().set_aspect("equal", "datalim") plt.title("UMAP projection (step %d)" % step) if out_fpath is not None: plt.savefig(out_fpath) plt.clf() self.experiment.log_image(out_fpath, step=step)
1, 1, args.data_h, args.data_w).repeat(1, 3, 1, 1)) valid_images.append(255.0 * dec_[0].resize_( 1, 1, args.data_h, args.data_w).repeat(1, 3, 1, 1)) print( "validloss: {:.6f}, epoch : {:02d}".format( loss_val / len(CloudCasttestloader), epoch), end="\r", flush=True, ) dict2 = { "epoch": epoch, "learningRate": get_lr(optimizer), "trainloss": loss_train / len(CloudCasttestloader), "valloss": loss_val / len(CloudCasttestloader), } comet_exp.log_metric("epoch", dict2["epoch"], epoch=epoch) comet_exp.log_metric("learningRate", dict2["learningRate"], epoch=epoch) comet_exp.log_metric("trainloss", dict2["trainloss"], epoch=epoch) comet_exp.log_metric("valloss", dict2["valloss"], epoch=epoch) upload_images( valid_images, epoch, exp=comet_exp, im_per_row=2, rows_per_log=int(len(valid_images) / 2), ) if (epoch % args.save_every) == 0: dict1 = {
itr = int(trainIndex + epoch * (len(trainloader))) # writer.add_scalars( # "Loss", # {"trainLoss": iLoss / args.progress_iter, "validationLoss": vLoss}, # itr, # ) # writer.add_scalar("PSNR", psnr, itr) # writer.add_image("Validation", valImg, itr) comet_exp.log_metrics( {"trainLoss": iLoss / args.progress_iter, "validationLoss": vLoss}, step=itr, epoch=epoch, ) comet_exp.log_metric("PSNR", psnr, step=itr, epoch=epoch) comet_exp.log_metric("SSIM", ssim_val, step=itr, epoch=epoch) # valImage = torch.movedim(valImg, 0, -1) # print(type(valImage)) # print(valImage.shape) # print(valImage.max()) # print(valImage.min()) # comet_exp.log_image( # valImage, # name="iter: " + str(iter) + ";epoch: " + str(epoch), # image_format="jpg", # step=itr, # ) #####
def main(): args = get_args() results_filename = f"logs/{args.env_name}-seed-{args.seed}-num-steps-{args.num_steps}-num-env-steps-{args.num_env_steps}-results.csv" save_path = os.path.join(args.save_dir, args.algo, str(args.seed)) torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) if args.cuda and torch.cuda.is_available() and args.cuda_deterministic: torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True log_dir = os.path.join(save_path, args.env_name) eval_log_dir = log_dir + "_eval" utils.cleanup_log_dir(log_dir) utils.cleanup_log_dir(eval_log_dir) torch.set_num_threads(1) device = torch.device("cuda:0" if args.cuda else "cpu") envs = make_vec_envs(args.env_name, args.seed, args.num_processes, args.gamma, log_dir, device, False, args.custom_gym) if "Train" in args.env_name: test_envs = make_vec_envs(args.env_name.replace("Train", "Test"), args.seed, 1, args.gamma, log_dir, device, False, args.custom_gym) base = NaviBaseTemp obs_shape = envs.observation_space.shape save_j = 0 try: os.makedirs(save_path) except FileExistsError: pass # Recover from job pre-emption try: actor_critic, ob_rms = \ torch.load(os.path.join(save_path, args.env_name + ".pt"), map_location='cpu') j = json.load( open(os.path.join(save_path, args.env_name + "-state.json"), 'r')) save_j = j['save_j'] episode_total = j['episode_total'] test_episode_total = j['test_episode_total'] rollouts = pickle.load( open(os.path.join(save_path, args.env_name + "-rollout.pkl"), 'rb')) rollouts.to(device) obs = envs.reset() rollouts.obs[0].copy_(obs) rollouts.to(device) test_rollouts = pickle.load( open(os.path.join(save_path, args.env_name + "-test-rollout.pkl"), 'rb')) test_rollouts.to(device) test_obs = test_envs.reset() test_rollouts.obs[0].copy_(test_obs) test_rollouts.to(device) optimizer_state_dict = pickle.load( open( os.path.join(save_path, args.env_name + "-optim-state-dict.pkl"), 'rb')) episode_rewards = pickle.load( open( os.path.join(save_path, args.env_name + "-episode_rewards.pkl"), 'rb')) episode_length = pickle.load( open( os.path.join(save_path, args.env_name + "-episode_length.pkl"), 'rb')) episode_success_rate = pickle.load( open( os.path.join(save_path, args.env_name + "-episode_success_rate.pkl"), 'rb')) test_episode_rewards = pickle.load( open( os.path.join(save_path, args.env_name + "-test_episode_rewards.pkl"), 'rb')) test_episode_length = pickle.load( open( os.path.join(save_path, args.env_name + "-test_episode_length.pkl"), 'rb')) test_episode_success_rate = pickle.load( open( os.path.join(save_path, args.env_name + "-test_episode_success_rate.pkl"), 'rb')) if comet_loaded and len(args.comet) > 0: comet_credentials = args.comet.split("/") experiment = ExistingExperiment(api_key=comet_credentials[2], previous_experiment=j['comet_id']) for key, value in vars(args).items(): experiment.log_parameter(key, value) else: experiment = None with open(results_filename, "a") as f: for key, value in vars(args).items(): f.write(f"{key}, {value}\n") f.close() except Exception: # create a new model actor_critic = Policy( obs_shape, envs.action_space, base_kwargs={'recurrent': args.recurrent_policy}, base=base, ) rollouts = RolloutStorage(args.num_steps, args.num_processes, envs.observation_space.shape, envs.action_space, actor_critic.recurrent_hidden_state_size) obs = envs.reset() rollouts.obs[0].copy_(obs) rollouts.to(device) test_rollouts = RolloutStorage( args.num_steps, 1, envs.observation_space.shape, envs.action_space, actor_critic.recurrent_hidden_state_size) if "Train" in args.env_name: test_obs = test_envs.reset() test_rollouts.obs[0].copy_(test_obs) test_rollouts.to(device) episode_rewards = deque(maxlen=10) episode_length = deque(maxlen=10) episode_success_rate = deque(maxlen=100) episode_total = 0 test_episode_rewards = deque(maxlen=10) test_episode_length = deque(maxlen=10) test_episode_success_rate = deque(maxlen=100) test_episode_total = 0 if comet_loaded and len(args.comet) > 0: comet_credentials = args.comet.split("/") experiment = Experiment(api_key=comet_credentials[2], project_name=comet_credentials[1], workspace=comet_credentials[0]) for key, value in vars(args).items(): experiment.log_parameter(key, value) else: experiment = None with open(results_filename, "w+") as f: for key, value in vars(args).items(): f.write(f"{key}, {value}\n") f.close() actor_critic.to(device) if args.algo == 'ppo': agent = algo.PPO(actor_critic, args.clip_param, args.ppo_epoch, args.num_mini_batch, args.value_loss_coef, args.entropy_coef, lr=args.lr, eps=args.eps, max_grad_norm=args.max_grad_norm) elif args.algo == 'random': agent = algo.RANDOM_AGENT(actor_critic, args.value_loss_coef, args.entropy_coef) actor_critic = RandomPolicy( obs_shape, envs.action_space, base_kwargs={'recurrent': args.recurrent_policy}, base=base, ) try: agent.optimizer.load_state_dict(optimizer_state_dict) except Exception: pass start = time.time() num_updates = int( args.num_env_steps) // args.num_steps // args.num_processes for j in range(num_updates - save_j): j = j + save_j if args.use_linear_lr_decay: # decrease learning rate linearly utils.update_linear_schedule(agent.optimizer, j, num_updates, args.lr) print("args.num_steps: " + str(args.num_steps)) for step in range(args.num_steps): # Sample actions with torch.no_grad(): value, action, action_log_prob, recurrent_hidden_states = actor_critic.act( rollouts.obs[step], rollouts.recurrent_hidden_states[step], rollouts.masks[step]) # Observe reward and next obs obs, reward, done, infos = envs.step(action) for idx, info in enumerate(infos): if 'episode' in info.keys(): episode_rewards.append(info['episode']['r']) episode_length.append(info['episode']['l']) if "Explorer" not in args.env_name: episode_success_rate.append( info['was_successful_trajectory']) episode_total += 1 # If done then clean the history of observations. masks = torch.FloatTensor([[0.0] if done_ else [1.0] for done_ in done]) bad_masks = torch.FloatTensor( [[0.0] if 'bad_transition' in info.keys() else [1.0] for info in infos]) rollouts.insert(obs, recurrent_hidden_states, action, action_log_prob, value, reward, masks, bad_masks) with torch.no_grad(): next_value = actor_critic.get_value( rollouts.obs[-1], rollouts.recurrent_hidden_states[-1], rollouts.masks[-1]).detach() rollouts.compute_returns(next_value, args.use_gae, args.gamma, args.gae_lambda, args.use_proper_time_limits) value_loss, action_loss, dist_entropy = agent.update(rollouts) rollouts.after_update() # Run on test if "Train" in args.env_name: for step in range(args.num_steps): # Sample actions with torch.no_grad(): value, action, action_log_prob, recurrent_hidden_states = actor_critic.act( test_rollouts.obs[step], test_rollouts.recurrent_hidden_states[step], test_rollouts.masks[step]) # Observe reward and next obs obs, reward, done, infos = test_envs.step(action) for idx, info in enumerate(infos): if 'episode' in info.keys(): test_episode_rewards.append(info['episode']['r']) test_episode_length.append(info['episode']['l']) test_episode_success_rate.append( info['was_successful_trajectory']) test_episode_total += 1 # save for every interval-th episode or for the last epoch if (j % args.save_interval == 0 or j == num_updates - 1) and args.save_dir != "" and j > args.save_after: if args.save_multiple: torch.save([ actor_critic, getattr(utils.get_vec_normalize(envs), 'ob_rms', None) ], os.path.join(save_path, str(j) + "-" + args.env_name + ".pt")) else: torch.save([ actor_critic, getattr(utils.get_vec_normalize(envs), 'ob_rms', None) ], os.path.join(save_path, args.env_name + ".pt")) json.dump( { 'save_j': j, 'episode_total': episode_total, 'test_episode_total': test_episode_total, 'comet_id': experiment.id }, open( os.path.join(save_path, args.env_name + "-state.json"), 'w+')) pickle.dump( agent.optimizer.state_dict(), open( os.path.join(save_path, args.env_name + "-optim-state-dict.pkl"), 'wb+')) pickle.dump( rollouts, open( os.path.join(save_path, args.env_name + "-rollout.pkl"), 'wb+')) pickle.dump( test_rollouts, open( os.path.join(save_path, args.env_name + "-test-rollout.pkl"), 'wb+')) pickle.dump( episode_rewards, open( os.path.join(save_path, args.env_name + "-episode_rewards.pkl"), 'wb+')) pickle.dump( episode_length, open( os.path.join(save_path, args.env_name + "-episode_length.pkl"), 'wb+')) pickle.dump( episode_success_rate, open( os.path.join( save_path, args.env_name + "-episode_success_rate.pkl"), 'wb+')) pickle.dump( test_episode_rewards, open( os.path.join( save_path, args.env_name + "-test_episode_rewards.pkl"), 'wb+')) pickle.dump( test_episode_length, open( os.path.join( save_path, args.env_name + "-test_episode_length.pkl"), 'wb+')) pickle.dump( test_episode_success_rate, open( os.path.join( save_path, args.env_name + "-test_episode_success_rate.pkl"), 'wb+')) if j % args.log_interval == 0 and len(episode_rewards) > 1: total_num_steps = (j + 1) * args.num_processes * args.num_steps end = time.time() if experiment is not None: experiment.log_metric("Reward Mean", np.mean(episode_rewards), step=total_num_steps) experiment.log_metric("Reward Min", np.min(episode_rewards), step=total_num_steps) experiment.log_metric("Reward Max", np.max(episode_rewards), step=total_num_steps) experiment.log_metric("Episode Length Mean ", np.mean(episode_length), step=total_num_steps) experiment.log_metric("Episode Length Min", np.min(episode_length), step=total_num_steps) experiment.log_metric("Episode Length Max", np.max(episode_length), step=total_num_steps) experiment.log_metric("# Trajectories (Total)", j, step=total_num_steps) if "Explorer" not in args.env_name: experiment.log_metric("Episodic Success Rate", np.mean(episode_success_rate), step=total_num_steps) else: with open(results_filename, "a") as f: f.write( f"Reward Mean, {np.mean(episode_rewards)}, {total_num_steps}\n" ) f.write( f"Reward Min, {np.min(episode_rewards)}, {total_num_steps}\n" ) f.write( f"Reward Max, {np.max(episode_rewards)}, {total_num_steps}\n" ) f.write( f"Episode Length Mean, {np.mean(episode_rewards)}, {total_num_steps}\n" ) f.write( f"Episode Length Min, {np.min(episode_rewards)}, {total_num_steps}\n" ) f.write( f"Episode Length Max, {np.max(episode_rewards)}, {total_num_steps}\n" ) f.write( f"# Trajectories (Total), {j}, {total_num_steps}\n") if "Explorer" not in args.env_name: f.write( f"Episodic Success Rate, {np.mean(episode_success_rate)}, {total_num_steps}\n" ) f.close() print( "Updates {}, num timesteps {}, FPS {} \n Last {} training episodes: mean/median reward {:.1f}/{:.1f}, min/max reward {:.1f}/{:.1f}\n" .format(j, total_num_steps, int(total_num_steps / (end - start)), len(episode_rewards), np.mean(episode_rewards), np.median(episode_rewards), np.min(episode_rewards), np.max(episode_rewards), dist_entropy, value_loss, action_loss)) # Test Generalization if "Train" in args.env_name and j % args.log_interval == 0 and len( test_episode_rewards) > 1: masks = torch.FloatTensor([[0.0] if done_ else [1.0] for done_ in done]) bad_masks = torch.FloatTensor( [[0.0] if 'bad_transition' in info.keys() else [1.0] for info in infos]) test_rollouts.insert(obs, recurrent_hidden_states, action, action_log_prob, value, reward, masks, bad_masks) with torch.no_grad(): next_value = actor_critic.get_value( test_rollouts.obs[-1], test_rollouts.recurrent_hidden_states[-1], test_rollouts.masks[-1]).detach() test_rollouts.after_update() print( f"Test Episode Total: {test_episode_total}, Mean Test rewards: {np.mean(test_episode_rewards)}, Test Episode Length: {np.mean(test_episode_length)}, Test Episode Success Rate: {np.mean(test_episode_success_rate)}" ) test_total_num_steps = (j + 1) * args.num_steps experiment.log_metric("Test Reward Mean", np.mean(test_episode_rewards), step=test_total_num_steps) experiment.log_metric("Test Reward Min", np.min(test_episode_rewards), step=test_total_num_steps) experiment.log_metric("Test Reward Max", np.max(test_episode_rewards), step=test_total_num_steps) experiment.log_metric("Test Episode Length Mean ", np.mean(test_episode_length), step=test_total_num_steps) experiment.log_metric("Test Episode Length Min", np.min(test_episode_length), step=test_total_num_steps) experiment.log_metric("Test Episode Length Max", np.max(test_episode_length), step=test_total_num_steps) experiment.log_metric("# Test Trajectories (Total)", j) experiment.log_metric("Test Episodic Success Rate", np.mean(test_episode_success_rate), step=test_total_num_steps) if (args.eval_interval is not None and len(episode_rewards) > 1 and j % args.eval_interval == 0): ob_rms = utils.get_vec_normalize(envs).ob_rms evaluate(actor_critic, ob_rms, args.env_name, args.seed, args.num_processes, eval_log_dir, device)
torch.nn.utils.clip_grad_norm_(model.parameters(), 1) for p in model.parameters(): vx[p].mul_(args.momentum).add_(p.grad.data) p.grad.data = vx[p] optimizer.step() running_loss += loss.item() * labels.size(0) predicted = torch.round(torch.sigmoid(outputs.data)) correct += (predicted == labels).sum().item() total += labels.size(0) loss = running_loss / total acc = 100 * float(correct) / total if experiment: with experiment.train(): experiment.log_metric('loss', loss, step=epoch) experiment.log_metric('acc', acc, step=epoch) print('TRAIN - Epoch: %d, Loss: %.3f, Accuracy: %.3f' % ( epoch, loss, acc)) running_loss = 0.0 correct = 0 total = 0 with torch.no_grad(): for data in validation_dataloader: inputs, labels = data inputs, labels = inputs.cuda(), labels.cuda() outputs = model(inputs) loss = criterion(outputs, labels)
optimizer.step() train["loss"] += loss.item() train["bce"] += bce.item() train["kld"] += kld.item() to_print = "Epoch[{}/{}] Loss: {:.3f} {:.3f} {:.3f}".format( epoch, args.epochs, train["loss"] / (batch_size * len(CloudCasttrainloader)), train["bce"] / (batch_size * len(CloudCasttrainloader)), train["kld"] / (batch_size * len(CloudCasttrainloader)), ) print(to_print) comet_exp.log_metric( "trainloss", train["loss"] / (batch_size * len(CloudCasttrainloader)), epoch=epoch, ) comet_exp.log_metric("bce", train["bce"] / (batch_size * len(CloudCasttrainloader)), epoch=epoch) comet_exp.log_metric("kld", train["kld"] / (batch_size * len(CloudCasttrainloader)), epoch=epoch) # validate valid_image_idxs = [] valid_images = [] with torch.no_grad(): for i, d in enumerate(CloudCasttrainloader, 0):
class CometLogger(): def __init__(self, disabled, is_existing=False, prev_exp_key=None): """ Handles logging of experiment to comet and also persistence to local file system. Supports resumption of stopped experiments. """ if not is_existing: self.experiment = Experiment(api_key=COMET_API_KEY, workspace=COMET_WORKSPACE, project_name=PROJECT_NAME, disabled=disabled) else: if prev_exp_key is None: raise ValueError("Requested existing experiment, but no key provided") print("Continuing existing experiment with key: ", prev_exp_key) self.experiment = ExistingExperiment(api_key=COMET_API_KEY, workspace=COMET_WORKSPACE, project_name=PROJECT_NAME, disabled=disabled, previous_experiment=prev_exp_key) self.disabled = disabled self.name = None def get_experiment_key(self): return self.experiment.get_key()[:9] def add_tag(self, tag): self.experiment.add_tag(tag) def log_metric(self, name, value, step=None): self.experiment.log_metric(name, value, step=step) def log_metrics(self, metrics_dict, prefix, step=None): self.experiment.log_metrics(metrics_dict, prefix=prefix, step=step) def log_params(self, params_dict): self.experiment.log_parameters(params_dict) def set_name(self, name_str): self.experiment.set_name(name_str) self.name = name_str def save_act_grads(self, log_dict): """Save a dictionary of activation/gradients records to disk""" assert isinstance(log_dict, dict) if self.name is None: warnings.warn("Experiment name not set, not saving") return # Save the log dictionary file_name = f"./.{self.name}.record" with open(file_name, 'wb') as f: pickle.dump(log_dict, f) # TODO: need to rewrite before can be used for MNIST. def draw_projections(self, embeds, utterances_per_speaker, step, out_fpath=None, max_speakers=16): import umap if self.disabled: return max_speakers = min(max_speakers, len(colormap)) embeds = embeds[:max_speakers * utterances_per_speaker] n_speakers = len(embeds) // utterances_per_speaker ground_truth = np.repeat(np.arange(n_speakers), utterances_per_speaker) colors = [colormap[i] for i in ground_truth] reducer = umap.UMAP() projected = reducer.fit_transform(embeds) plt.scatter(projected[:, 0], projected[:, 1], c=colors) plt.gca().set_aspect("equal", "datalim") plt.title("UMAP projection (step %d)" % step) if out_fpath is not None: plt.savefig(out_fpath) plt.clf() self.experiment.log_image(out_fpath, step=step)
# set epsilon to 0.05 for evaluation current_state = list(queue) action = select_action(policy_net, current_state, 0.05) next_observation, reward, done = take_step(env, action) queue.appendleft(next_observation) score += reward.item() if done: break scores.append(score) mean_score = np.mean(scores) last_evaluation = num_steps if args.api_key: with experiment.validate(): experiment.log_metric('score', mean_score, step=num_steps) is_best = mean_score > best_score best_score = max(mean_score, best_score) checkpoint = { 'num_steps': num_steps, 'best_score': best_score, 'epsilon': epsilon, 'last_evaluation': last_evaluation, 'state_dict': policy_net.state_dict(), 'optimizer': optimizer.state_dict(), } if args.api_key: checkpoint['experiment_key'] = experiment.get_key() save_checkpoint(checkpoint, is_best)