class CometLogger():
def __init__(self, enabled, is_existing=False, prev_exp_key=None):
"""
Handles logging of experiment to comet and also persistence to local file system.
Supports resumption of stopped experiments.
"""
disabled = not enabled
if not is_existing:
self.experiment = Experiment(api_key=COMET_API_KEY,
workspace=COMET_WORKSPACE,
project_name=PROJECT_NAME,
disabled=disabled)
else:
if prev_exp_key is None:
raise ValueError(
"Requested existing experiment, but no key provided")
print("Continuing existing experiment with key: ", prev_exp_key)
self.experiment = ExistingExperiment(
api_key=COMET_API_KEY,
workspace=COMET_WORKSPACE,
project_name=PROJECT_NAME,
disabled=disabled,
previous_experiment=prev_exp_key)
self.disabled = disabled
def get_experiment_key(self):
return self.experiment.get_key()[:9]
def add_tag(self, tag):
self.experiment.add_tag(tag)
def log_metric(self, name, value, step=None):
self.experiment.log_metric(name, value, step=step)
def log_metrics(self, metrics_dict, prefix, step=None):
self.experiment.log_metrics(metrics_dict, prefix=prefix, step=step)
def log_params(self, params_dict):
self.experiment.log_parameters(params_dict)
def set_name(self, name_str):
self.experiment.set_name(name_str)
def log_dataset(self, dataset: SpeakerVerificationDataset):
if self.disabled:
return
dataset_string = ""
dataset_string += "<b>Speakers</b>: %s\n" % len(dataset.speakers)
dataset_string += "\n" + dataset.get_logs()
dataset_string = dataset_string.replace("\n", "<br>")
self.vis.text(dataset_string, opts={"title": "Dataset"})
def log_implementation(self, params):
if self.disabled:
return
implementation_string = ""
for param, value in params.items():
implementation_string += "<b>%s</b>: %s\n" % (param, value)
implementation_string = implementation_string.replace("\n", "<br>")
self.implementation_string = implementation_string
self.implementation_win = self.vis.text(
implementation_string, opts={"title": "Training implementation"})
def draw_projections(self,
embeds,
utterances_per_speaker,
step,
out_fpath=None,
max_speakers=16):
if self.disabled:
return
max_speakers = min(max_speakers, len(colormap))
embeds = embeds[:max_speakers * utterances_per_speaker]
n_speakers = len(embeds) // utterances_per_speaker
ground_truth = np.repeat(np.arange(n_speakers), utterances_per_speaker)
colors = [colormap[i] for i in ground_truth]
reducer = umap.UMAP()
projected = reducer.fit_transform(embeds)
plt.scatter(projected[:, 0], projected[:, 1], c=colors)
plt.gca().set_aspect("equal", "datalim")
plt.title("UMAP projection (step %d)" % step)
if out_fpath is not None:
plt.savefig(out_fpath)
plt.clf()
self.experiment.log_image(out_fpath, step=step)
def main(args, config=None, init_distributed=False):
utils.import_user_module(args)
experiment = None
if config:
experiment = ExistingExperiment(
api_key=config["api_key"],
previous_experiment=config["experiment_key"],
auto_output_logging=None,
)
assert (
args.max_tokens is not None or args.max_sentences is not None
), "Must specify batch size either with --max-tokens or --max-sentences"
# Initialize CUDA and distributed training
if torch.cuda.is_available() and not args.cpu:
torch.cuda.set_device(args.device_id)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if init_distributed:
args.distributed_rank = distributed_utils.distributed_init(args)
if distributed_utils.is_master(args):
checkpoint_utils.verify_checkpoint_directory(args.save_dir)
print(args)
if experiment:
experiment.log_parameters(vars(args),
prefix="Device {} :: ".format(
args.device_id))
# Setup task, e.g., translation, language modeling, etc.
task = tasks.setup_task(args)
# Load valid dataset (we load training data below, based on the latest checkpoint)
for valid_sub_split in args.valid_subset.split(","):
task.load_dataset(valid_sub_split, combine=False, epoch=0)
# Build model and criterion
model = task.build_model(args)
criterion = task.build_criterion(args)
print(model)
print("| model {}, criterion {}".format(args.arch,
criterion.__class__.__name__))
print("| num. model params: {} (num. trained: {})".format(
sum(p.numel() for p in model.parameters()),
sum(p.numel() for p in model.parameters() if p.requires_grad),
))
if experiment:
experiment.log_parameters(
{
"criterion":
criterion.__class__.__name__,
"num. model params":
sum(p.numel() for p in model.parameters()),
"num. trained params":
sum(p.numel() for p in model.parameters() if p.requires_grad),
},
prefix="Device {} :: ".format(args.device_id),
)
# Build trainer
trainer = Trainer(args, task, model, criterion)
print("| training on {} GPUs".format(args.distributed_world_size))
print("| max tokens per GPU = {} and max sentences per GPU = {}".format(
args.max_tokens, args.max_sentences))
# Load the latest checkpoint if one is available and restore the
# corresponding train iterator
extra_state, epoch_itr = checkpoint_utils.load_checkpoint(args, trainer)
# Train until the learning rate gets too small
max_epoch = args.max_epoch or math.inf
max_update = args.max_update or math.inf
lr = trainer.get_lr()
train_meter = StopwatchMeter()
train_meter.start()
valid_subsets = args.valid_subset.split(",")
while (lr > args.min_lr and epoch_itr.epoch < max_epoch
and trainer.get_num_updates() < max_update):
# train for one epoch
train(args, trainer, task, epoch_itr, experiment)
if (not args.disable_validation
and epoch_itr.epoch % args.validate_interval == 0):
valid_losses = validate(args, trainer, task, epoch_itr,
valid_subsets, experiment)
else:
valid_losses = [None]
# only use first validation loss to update the learning rate
lr = trainer.lr_step(epoch_itr.epoch, valid_losses[0])
# save checkpoint
if epoch_itr.epoch % args.save_interval == 0:
checkpoint_utils.save_checkpoint(args, trainer, epoch_itr,
valid_losses[0])
reload_dataset = ":" in getattr(args, "data", "")
# sharded data: get train iterator for next epoch
epoch_itr = trainer.get_train_iterator(epoch_itr.epoch,
load_dataset=reload_dataset)
train_meter.stop()
print("| done training in {:.1f} seconds".format(train_meter.sum))
if experiment:
experiment.log_metrics(
{
"valid_loss": valid_losses[0],
"lr": lr
},
prefix="Device {} ".format(args.device_id),
)
if args.resume:
experiment = ExistingExperiment(
api_key=args.api_key,
previous_experiment=checkpoint['experiment_key'],
auto_param_logging=False,
auto_metric_logging=False,
parse_args=False)
else:
experiment = Experiment(
api_key=args.api_key,
project_name=project_name,
auto_param_logging=False,
auto_metric_logging=False,
parse_args=False)
experiment.log_other('experiment_name', experiment_name)
experiment.log_parameters(vars(args))
for k in hyperparameters:
if type(hyperparameters[k]) == dict:
experiment.log_parameters(hyperparameters[k], prefix=k)
else:
experiment.log_parameter(k, hyperparameters[k])
# Mapping: {'Cat': 0, 'Dog': 1}
try:
dataset = torchvision.datasets.ImageFolder(root='./trainset')
except:
import zipfile
zip_ref = zipfile.ZipFile('trainset.zip', 'r')
zip_ref.extractall()
zip_ref.close()
dataset = torchvision.datasets.ImageFolder(root='./trainset')
def main(opts):
"""
Opts prevalence:
1. Load file specified in args.default (or shared/trainer/defaults.yaml
if none is provided)
2. Update with file specified in args.config (or no update if none is provided)
3. Update with parsed command-line arguments
e.g.
`python train.py args.config=config/large-lr.yaml data.loaders.batch_size=10`
loads defaults, overrides with values in large-lr.yaml and sets batch_size to 10
"""
# -----------------------------
# ----- Parse arguments -----
# -----------------------------
hydra_opts = Dict(OmegaConf.to_container(opts))
args = hydra_opts.pop("args", None)
auto_resumed = {}
config_path = args.config
if hydra_opts.train.resume:
out_ = str(env_to_path(hydra_opts.output_path))
config_path = Path(out_) / "opts.yaml"
if not config_path.exists():
config_path = None
print("WARNING: could not reuse the opts in {}".format(out_))
default = args.default or Path(
__file__).parent / "shared/trainer/defaults.yaml"
# -----------------------
# ----- Load opts -----
# -----------------------
opts = load_opts(config_path, default=default, commandline_opts=hydra_opts)
if args.resume:
opts.train.resume = True
opts.jobID = os.environ.get("SLURM_JOBID")
opts.slurm_partition = os.environ.get("SLURM_JOB_PARTITION")
opts.output_path = str(env_to_path(opts.output_path))
print("Config output_path:", opts.output_path)
exp = comet_previous_id = None
# -------------------------------
# ----- Check output_path -----
# -------------------------------
# Auto-continue if same slurm job ID (=job was requeued)
if not opts.train.resume and opts.train.auto_resume:
print("\n\nTrying to auto-resume...")
existing_path = find_existing_training(opts)
if existing_path is not None and existing_path.exists():
auto_resumed["original output_path"] = str(opts.output_path)
auto_resumed["existing_path"] = str(existing_path)
opts.train.resume = True
opts.output_path = str(existing_path)
# Still not resuming: creating new output path
if not opts.train.resume:
opts.output_path = str(get_increased_path(opts.output_path))
Path(opts.output_path).mkdir(parents=True, exist_ok=True)
# Copy the opts's sbatch_file to output_path
copy_run_files(opts)
# store git hash
opts.git_hash = get_git_revision_hash()
opts.git_branch = get_git_branch()
if not args.no_comet:
# ----------------------------------
# ----- Set Comet Experiment -----
# ----------------------------------
if opts.train.resume:
# Is resuming: get existing comet exp id
assert Path(
opts.output_path).exists(), "Output_path does not exist"
comet_previous_id = get_existing_comet_id(opts.output_path)
# Continue existing experiment
if comet_previous_id is None:
print("WARNING could not retreive previous comet id")
print(f"from {opts.output_path}")
else:
print("Continuing previous experiment", comet_previous_id)
auto_resumed["continuing exp id"] = comet_previous_id
exp = ExistingExperiment(previous_experiment=comet_previous_id,
**comet_kwargs)
print("Comet Experiment resumed")
if exp is None:
# Create new experiment
print("Starting new experiment")
exp = Experiment(project_name="climategan", **comet_kwargs)
exp.log_asset_folder(
str(Path(__file__).parent / "climategan"),
recursive=True,
log_file_name=True,
)
exp.log_asset(str(Path(__file__)))
# Log note
if args.note:
exp.log_parameter("note", args.note)
# Merge and log tags
if args.comet_tags or opts.comet.tags:
tags = set([f"branch:{opts.git_branch}"])
if args.comet_tags:
tags.update(args.comet_tags)
if opts.comet.tags:
tags.update(opts.comet.tags)
opts.comet.tags = list(tags)
print("Logging to comet.ml with tags", opts.comet.tags)
exp.add_tags(opts.comet.tags)
# Log all opts
exp.log_parameters(flatten_opts(opts))
if auto_resumed:
exp.log_text("\n".join(f"{k:20}: {v}"
for k, v in auto_resumed.items()))
# allow some time for comet to get its url
sleep(1)
# Save comet exp url
url_path = get_increased_path(Path(opts.output_path) / "comet_url.txt")
with open(url_path, "w") as f:
f.write(exp.url)
# Save config file
opts_path = get_increased_path(Path(opts.output_path) / "opts.yaml")
with (opts_path).open("w") as f:
yaml.safe_dump(opts.to_dict(), f)
pprint("Running model in", opts.output_path)
# -------------------
# ----- Train -----
# -------------------
trainer = Trainer(opts, comet_exp=exp, verbose=1)
trainer.logger.time.start_time = time()
trainer.setup()
trainer.train()
# -----------------------------
# ----- End of training -----
# -----------------------------
pprint("Done training")
kill_job(opts.jobID)
class CometLogger():
def __init__(self, disabled, is_existing=False, prev_exp_key=None):
"""
Handles logging of experiment to comet and also persistence to local file system.
Supports resumption of stopped experiments.
"""
if not is_existing:
self.experiment = Experiment(api_key=COMET_API_KEY,
workspace=COMET_WORKSPACE,
project_name=PROJECT_NAME,
disabled=disabled)
else:
if prev_exp_key is None:
raise ValueError("Requested existing experiment, but no key provided")
print("Continuing existing experiment with key: ", prev_exp_key)
self.experiment = ExistingExperiment(api_key=COMET_API_KEY,
workspace=COMET_WORKSPACE,
project_name=PROJECT_NAME,
disabled=disabled,
previous_experiment=prev_exp_key)
self.disabled = disabled
self.name = None
def get_experiment_key(self):
return self.experiment.get_key()[:9]
def add_tag(self, tag):
self.experiment.add_tag(tag)
def log_metric(self, name, value, step=None):
self.experiment.log_metric(name, value, step=step)
def log_metrics(self, metrics_dict, prefix, step=None):
self.experiment.log_metrics(metrics_dict, prefix=prefix, step=step)
def log_params(self, params_dict):
self.experiment.log_parameters(params_dict)
def set_name(self, name_str):
self.experiment.set_name(name_str)
self.name = name_str
def save_act_grads(self, log_dict):
"""Save a dictionary of activation/gradients records to disk"""
assert isinstance(log_dict, dict)
if self.name is None:
warnings.warn("Experiment name not set, not saving")
return
# Save the log dictionary
file_name = f"./.{self.name}.record"
with open(file_name, 'wb') as f:
pickle.dump(log_dict, f)
# TODO: need to rewrite before can be used for MNIST.
def draw_projections(self, embeds, utterances_per_speaker, step, out_fpath=None,
max_speakers=16):
import umap
if self.disabled:
return
max_speakers = min(max_speakers, len(colormap))
embeds = embeds[:max_speakers * utterances_per_speaker]
n_speakers = len(embeds) // utterances_per_speaker
ground_truth = np.repeat(np.arange(n_speakers), utterances_per_speaker)
colors = [colormap[i] for i in ground_truth]
reducer = umap.UMAP()
projected = reducer.fit_transform(embeds)
plt.scatter(projected[:, 0], projected[:, 1], c=colors)
plt.gca().set_aspect("equal", "datalim")
plt.title("UMAP projection (step %d)" % step)
if out_fpath is not None:
plt.savefig(out_fpath)
plt.clf()
self.experiment.log_image(out_fpath, step=step)
else:
experiment = Experiment(api_key=args.api_key,
project_name='comp767_project',
parse_args=False,
auto_metric_logging=False)
else:
raise Exception
else:
if args.api_key:
experiment = Experiment(api_key=args.api_key,
project_name='comp767_project',
parse_args=False,
auto_metric_logging=False)
_, experiment_name = split(dirname(realpath(__file__)))
experiment.log_other('experiment_name', experiment_name)
experiment.log_parameters(HYPERPARAMETERS)
num_steps = 0
best_score = 0
last_evaluation = 0
epsilon = HYPERPARAMETERS['epsilon_start']
print('=> num_steps: {}, best_score: {}, epsilon: {}, last_evaluation: {}\n'.
format(num_steps, best_score, epsilon, last_evaluation))
target_net = DQN(env.action_space.n).to(device)
target_net.load_state_dict(policy_net.state_dict())
target_net.eval()
start_time = time.time()
memory = ReplayMemory(HYPERPARAMETERS['memory_size'])
while True:
observation = reset(env)