class CometMLLogger:
def __init__(self):
self.experiment = Experiment(api_key="iU4f44llKnowZwmrEo9wfR2ch",
project_name="general",
workspace="yahyaalaamassoud",
log_code=False,
log_graph=False)
def log_params(self, params: Dict[str, int]):
self.experiment.log_parameters(params)
def log_metric(self, metric_name, metric_val, step=None):
self.experiment.log_metric(metric_name, metric_val, step=step)
def log_metrics(self, metrics: Dict[str, float], step=None):
self.experiment.log_metrics(metrics, step=step)
def log_figure(self, figure_name: str, step: str):
self.experiment.log_image(image_data=self.__savefig(),
name=figure_name,
step=step,
overwrite=False)
def __savefig(self):
buf = io.BytesIO()
plt.savefig(buf, format='png')
buf.seek(0)
return Image.open(io.BytesIO(buf.getvalue()))
def fit_validate(exp_params, k, data_path, write_path, others=None, custom_tag=''):
"""Fit model and compute metrics on train and validation set. Intended for hyperparameter search.
Only logs final metrics and scatter plot of final embedding.
Args:
exp_params(dict): Parameter dict. Should at least have keys model_name, dataset_name & random_state. Other
keys are assumed to be model parameters.
k(int): Fold identifier.
data_path(str): Data directory.
write_path(str): Where to write temp files.
others(dict): Other things to log to Comet experiment.
custom_tag(str): Custom tag for comet experiment.
"""
# Comet experiment
exp = Experiment(parse_args=False)
exp.disable_mp()
custom_tag += '_validate'
exp.add_tag(custom_tag)
exp.log_parameters(exp_params)
if others is not None:
exp.log_others(others)
# Parse experiment parameters
model_name, dataset_name, random_state, model_params = parse_params(exp_params)
# Fetch and split dataset.
data_train = getattr(grae.data, dataset_name)(split='train', random_state=random_state, data_path=data_path)
data_train, data_val = data_train.validation_split(random_state=FOLD_SEEDS[k])
# Model
m = getattr(grae.models, model_name)(random_state=FOLD_SEEDS[k], **model_params)
m.write_path = write_path
m.data_val = data_val
with exp.train():
m.fit(data_train)
# Log plot
m.comet_exp = exp
m.plot(data_train, data_val, title=f'{model_name} : {dataset_name}')
# Probe embedding
prober = EmbeddingProber()
prober.fit(model=m, dataset=data_train, mse_only=True)
train_z, train_metrics = prober.score(data_train, is_train=True)
# Log train metrics
exp.log_metrics(train_metrics)
with exp.validate():
val_z, val_metrics = prober.score(data_val)
# Log train metrics
exp.log_metrics(val_metrics)
# Log marker to mark successful experiment
exp.log_other('success', 1)
def log_metrics(metrics: dict, comet_logger: Experiment, epoch: int,
context_val: bool):
if context_val:
with comet_logger.validate():
comet_logger.log_metrics(metrics, epoch=epoch)
else:
with comet_logger.train():
comet_logger.log_metrics(metrics, epoch=epoch)
def generate_categories():
# capture the config path from the run arguments then process the json configuration file
try:
args = get_args()
config = process_config(args.config)
except ValueError:
print("Missing or invalid arguments")
exit(0)
print("Logging experiment name: {name}".format(
name=config.experiment.experiment_name))
experiment = Experiment(api_key=config.experiment.api_key,
project_name=config.experiment.project_name,
workspace=config.experiment.workspace)
experiment.set_name(config.experiment.experiment_name)
print('Creating the data loader...')
data_loader = DataLoader(config.defects_summarizer.paths)
train_data, test_data = data_loader.get_data()
print('Creating the Preprocessor...')
preprocessor = CorexPreprocessor(train_data, config)
preprocessor.prepare_data()
print('Loading and evaluating the Model...')
model = CorexModel(config.defects_summarizer, preprocessor, seed=False)
trainer = CorexTrainer(model, preprocessor.get_data())
trainer.train()
trainer.generate_topics()
top_docs_df = trainer.get_top_documents(
config.defects_summarizer.evaluate.extract_topics,
preprocessor.get_raw_corpus(),
config.defects_summarizer.evaluate.extraction_quantile,
labels=True)
top_docs_df.to_csv(config.defects_summarizer.paths.save_data_path)
print('Saving the trained topic model...')
model.save()
print('Preprocessing the summarizer...')
summary_preprocessor = TextRankPreprocessor(
top_docs_df, n_docs=config.defects_summarizer.evaluate.n_docs)
summary_preprocessor.prepare_data()
print('Loading and evaluating the summarizer...')
summary_model = TextRankModel(config)
summary_trainer = TextRankTrainer(summary_model, summary_preprocessor)
avg_prec, avg_recall, avg_f1 = summary_trainer.train_and_evaluate(
test_data)
# Log the rest of the experiment
metrics = {"precision": avg_prec, "recall": avg_recall, "f1": avg_f1}
experiment.log_metrics(metrics)
experiment.log_model(
name=config.experiment.model_name,
file_or_folder=config.labels_generator.paths.save_model_path)
def pretrain(self, save_filename, patience):
''' Train & validate the model on full training dataset (all pilots) from scratch.
Inputs:
- save_filename: (string: 'weights_save_filename.h5').
- patience: Number of epochs without improvement before training stops (int).
Output:
- hist: Contains loss, acc, val_loss, val_acc metrics over epochs. (dictionnary)
'''
X_train, y_train, X_valid, y_valid, X_test, y_test = [], [], [], [], [], []
if self.log_pilots:
experiment = Experiment(api_key='cSZq9kuH2I87ezvm2dEWTx6op', project_name='All pilots', log_code=False, auto_param_logging=False)
for pilot_idx in range(1, self.n_pilots + 1):
X_train_pilot, y_train_pilot, X_valid_pilot, y_valid_pilot, X_test_pilot, y_test_pilot = self.load_data(pilot_idx, valid_ratio=0.2)
# Stacking training/validation datasets of each pilot into a big dataset
if len(X_train)==0 and len(X_valid)==0:
X_train = X_train_pilot
y_train = y_train_pilot
X_valid = X_valid_pilot
y_valid = y_valid_pilot
X_test = X_test_pilot
y_test = y_test_pilot
else:
X_train = np.concatenate([X_train, X_train_pilot], axis=0)
y_train = np.concatenate([y_train, y_train_pilot], axis=0)
X_valid = np.concatenate([X_valid, X_valid_pilot], axis=0)
y_valid = np.concatenate([y_valid, y_valid_pilot], axis=0)
X_test = np.concatenate([X_test, X_test_pilot], axis=0)
y_test = np.concatenate([y_test, y_test_pilot], axis=0)
# Shuffle
X_train, y_train = shuffle(X_train, y_train, random_state=0)
X_valid, y_valid = shuffle(X_valid, y_valid, random_state=0)
X_test, y_test = shuffle(X_test, y_test, random_state=0)
# Build model
self.model = self.build_model(load_weights=False)
# Train on all pilots
hist, _ = self.train(X_train, y_train, X_valid, y_valid, save_filename, patience)
# Get some metrics
score = self.test('all', save_filename, X_train, y_train, X_test, y_test, False)
if self.log_pilots:
experiment.log_metrics({'Test accuracy' : score})
experiment.end()
return hist
def generate_topics():
# capture the config path from the run arguments then process the json configuration file
try:
args = get_args()
config = process_config(args.config)
except ValueError:
print("Missing or invalid arguments")
exit(0)
print("Logging experiment name: {name}".format(
name=config.experiment.experiment_name))
experiment = Experiment(api_key=config.experiment.api_key,
project_name=config.experiment.project_name,
workspace=config.experiment.workspace)
experiment.set_name(config.experiment.experiment_name)
params = config.labels_generator.model
experiment.log_parameters(params)
print('Creating the data loader...')
data_loader = DataLoader(config.labels_generator.paths)
data = data_loader.get_data()
print('Creating the Preprocessor...')
preprocessor = CorexPreprocessor(data, config)
preprocessor.prepare_data()
print('Creating and training the Model...')
model = CorexModel(config, preprocessor)
trainer = CorexTrainer(model, preprocessor.get_data())
trainer.train()
print('Evaluating the model...')
coherence_lst, avg_coherence = trainer.evaluate(preprocessor.get_data(),
preprocessor.get_corpus())
trainer.generate_topics()
print("Coherence score: {score_lst} \nAvg coherence score: {avg_score}".
format(score_lst=coherence_lst, avg_score=avg_coherence))
print('Saving the trained model...')
model.save()
# Log the rest of the experiment
metrics = {"coherence": avg_coherence}
experiment.log_metrics(metrics)
experiment.log_model(
name=config.experiment.model_name,
file_or_folder=config.labels_generator.paths.save_model_path)
class CometLogger(LightningLoggerBase):
def __init__(self, *args, **kwargs):
super(CometLogger, self).__init__()
self.experiment = CometExperiment(*args, **kwargs)
@rank_zero_only
def log_hyperparams(self, params):
self.experiment.log_parameters(vars(params))
@rank_zero_only
def log_metrics(self, metrics, step_num):
# self.experiment.set_epoch(self, metrics.get('epoch', 0))
self.experiment.log_metrics(metrics)
@rank_zero_only
def finalize(self, status):
self.experiment.end()
class Logger:
def __init__(self, send_logs, tags, parameters):
self.send_logs = send_logs
if self.send_logs:
self.experiment = Experiment(api_key="OZwyhJHyqzPZgHEpDFL1zxhyI",
project_name="drilling-the-hole",
workspace="wwydmanski")
self.sent_mb = 0
if self.send_logs:
if tags is not None:
self.experiment.add_tags(tags)
if parameters is not None:
self.experiment.log_parameters(parameters)
def begin_logging(self, episode_count, steps_per_ep):
if self.send_logs:
self.experiment.log_parameter("Episode count", episode_count)
self.experiment.log_parameter("Max steps per episode",
steps_per_ep)
def log_round(self, actions, reward, cumulative_reward, angle, loss, step):
if self.send_logs:
self.experiment.log_metric("Round reward", reward, step=step)
self.experiment.log_metric("Per-ep reward",
cumulative_reward,
step=step)
self.experiment.log_metric("Action 1", actions[0], step=step)
self.experiment.log_metric("Action 2", actions[1], step=step)
self.experiment.log_metric("Current angle", angle, step=step)
self.experiment.log_metrics(loss, step=step)
def log_episode(self, cumulative_reward, state, step):
if self.send_logs:
self.experiment.log_metric("Angle", state[0], step=step)
self.experiment.log_metric("Goal", state[1], step=step)
self.experiment.log_metric("Cumulative reward",
cumulative_reward,
step=step)
def end(self):
if self.send_logs:
self.experiment.end()
def _log_final_metrics(self, experiment: Experiment, model: Sequential,
data: DataSets,
preprocessing_fnc: Callable[[np.ndarray],
np.ndarray]):
scores = model.evaluate(preprocessing_fnc(data.x_train),
data.y_train,
verbose=2)
experiment.log_metrics({
"loss": scores[0],
"acc": scores[1]
},
prefix="train")
scores = model.evaluate(preprocessing_fnc(data.x_dev),
data.y_dev,
verbose=2)
experiment.log_metrics({
"loss": scores[0],
"acc": scores[1]
},
prefix="dev")
timer = ElapsedTime("Test prediction")
with timer:
scores = model.evaluate(preprocessing_fnc(data.x_test),
data.y_test,
verbose=2)
experiment.log_metric("test_inference_time", timer.elapsed_time_ms)
experiment.log_metrics({
"loss": scores[0],
"acc": scores[1]
},
prefix="test")
class BaseLogger:
def __init__(self, log_interval, train_len):
self.log_interval = log_interval
self.train_len = train_len
self.metrics = {}
self.__cometml_api_key = os.environ.get("COMETML_API_KEY")
if self.__cometml_api_key:
self.experiment = Experiment(self.__cometml_api_key,
project_name="sirius-adversarial-attack")
def log_test(self, engine):
self.metrics = engine.state.metrics
logging.info('----------------------------------------')
logging.info(f'TEST: Epoch:[{engine.state.epoch}]')
logging.info(f', '.join([
f'{name}: {self.metrics[name]:.5f}' for name in self.metrics
]))
logging.info(f'----------------------------------------')
if self.__cometml_api_key:
self.experiment.log_metrics(self.metrics, prefix='Test')
def log_train(self, engine):
if engine.state.iteration % self.train_len % self.log_interval == 0:
logging.info("Epoch[{}] Iteration[{}/{}] Loss: {:.5f}"
.format(engine.state.epoch, engine.state.iteration %
self.train_len if engine.state.iteration
% self.train_len else self.train_len,
self.train_len, engine.state.output))
if self.__cometml_api_key:
logs_dict = {"loss": engine.state.output}
self.experiment.log_metrics(logs_dict,
step=engine.state.iteration,
epoch=engine.state.epoch,
prefix='Train')
def log_hparams(self, hparams_dict):
hparams_dict.update(self.metrics)
if self.__cometml_api_key:
self.experiment.log_parameters(hparams_dict)
class CometLogger(BaseLogger):
def __init__(self, experiment_id=None):
self.experiment = Experiment(auto_metric_logging=False)
if experiment_id is not None:
self.experiment.log_parameter('experiment_id', experiment_id)
def add_scalar(self, name, value, step):
self.experiment.log_metric(name, value, epoch=step)
def log_parameters(self, params_dict):
self.experiment.log_parameters(params_dict)
def log_metrics(self, metrics_dict, epoch):
self.experiment.log_metrics(metrics_dict, epoch=epoch)
def add_text(self, name, text):
self.experiment.log_text(f'{name}: {text}')
def set_context_prefix(self, prefix):
self.experiment.context = prefix
def reset_context_prefix(self):
self.experiment.context = None
def log(self, experiment=None):
''' Export all logs in the Comet.ml environment.
See https://www.comet.ml/ for more details
'''
# Initialize Comet.ml experience (naming, tags) for automatic logging
project_name = 'Optimization' if self.comet_optimize else 'Summary'
experiment_name = '{} - {} '.format(self.model_name, str(self.batch_size)) + ('ES+' if self.train_after_es else '')
experiment_tags = [ self.model_name, self.monitor_val ] + (['ES+'] if self.train_after_es else []) + (['Pre-train'] if self.pretraining else [])
if experiment == None:
experiment = Experiment(api_key='cSZq9kuH2I87ezvm2dEWTx6op', project_name=project_name, log_code=False, auto_param_logging=False, auto_metric_logging=False)
experiment.set_name(experiment_name)
experiment.add_tags(experiment_tags)
# Export hyperparameters
experiment.log_parameters(self.dataloader_params)
experiment.log_parameters(self.training_params)
# Export metrics values
experiment.log_metrics({'Average accuracy' : np.mean(self.test_score['accuracy']), 'Std accuracy' : np.std(self.test_score['accuracy'])})
# Export metrics graphs for each pilot (accuracy, loss, confusion matrix)
[ experiment.log_figure(figure_name='Confusion matrix {}'.format(pilot_idx), figure=plot_cm(self.conf_matrices, pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)]
[ experiment.log_figure(figure_name='Loss pilot {}'.format(pilot_idx), figure=plot_loss(self.histories[pilot_idx-1], pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)]
fig, ax = plt.subplots(figsize=(10,6))
plot_full_barchart(self.test_score, n_pilots=self.n_pilots, title=' {} ConvNet model'.format(self.model_name), fig=fig)
experiment.log_figure(figure_name='Accuracy barchart', figure=fig)
if self.train_after_es:
[ experiment.log_figure(figure_name='Loss pilot {} (ES+)'.format(pilot_idx), figure=plot_loss(self.histories_es[pilot_idx-1], pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)]
# Export model weights for each pilot
[ experiment.log_asset('{}{}.h5'.format(self.weights_savename_prefix, pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)]
experiment.end()
class CometMLLogger(ExperimentLogger):
def __init__(self, provider_args: EasyDict, config, **kwargs):
self.experiment = Experiment(api_key=provider_args.api_key,
project_name=provider_args.project_name,
workspace=provider_args.workspace,
auto_param_logging=False,
auto_metric_logging=False)
super().__init__(config)
self.run_key = self.experiment.get_key()
self.log_url = self.experiment.url
def log_on_hyperparameters(self, config: EasyDict):
hyper_params = {}
if config is not None:
hyper_params['model'] = config.model
hyper_params['trainer'] = config.trainer
if 'train' in config.dataset and 'augmentations' in config.dataset.train:
hyper_params[
'augmentations'] = config.dataset.train.augmentations
self.experiment.log_parameters(flatten(hyper_params, reducer='path'))
def log_on_step_update(self, metrics_log: dict):
step = metrics_log['step']
metrics_log.pop('step')
self.experiment.log_metrics(metrics_log, step=step)
def log_on_epoch_update(self, metrics_log: dict):
epoch = metrics_log['epoch']
metrics_log.pop('epoch')
self.experiment.log_metrics(metrics_log, epoch=epoch)
def log_on_model_save(self, file_log: dict):
pass
def log_on_validation_result(self, metrics_log: dict):
epoch = metrics_log['epoch']
metrics_log.pop('epoch')
self.experiment.log_metrics(metrics_log, epoch=epoch)
g_opt_op, g_current_loss_tensor = g_train_optimizer_ops[sizes.index(current_resolution)]
elif current_mode == 'stabilize':
d_opt_op, d_current_loss_tensor = d_stabilize_optimizer_ops[sizes.index(current_resolution)]
g_opt_op, g_current_loss_tensor = g_stabilize_optimizer_ops[sizes.index(current_resolution)]
for i in range(d_steps):
try:
_, D_l = session.run([d_opt_op, d_current_loss_tensor])
except tf.errors.InvalidArgumentError as e:
print('jpeg error: ' + str(e))
for i in range(g_steps):
_, G_l = session.run([g_opt_op, g_current_loss_tensor])
if step % 10 == 0:
experiment.log_metrics({'d_loss': D_l, 'g_loss': G_l, 'current_resolution': current_resolution,
'current_mode': (0 if current_mode == 'train' else 1), 'time_to_res_schedule_update': current_resolution_schedule_period_length - (time.time() - last_schedule_update_time)})
# experiment.log_metric("d_loss", D_l)
# experiment.log_metric("g_loss", G_l)
# experiment.log_metric("current_resolution", current_resolution)
# experiment.log_metric("current_mode", 0 if current_mode == 'train' else 1)
if np.isnan(D_l) or np.isnan(G_l):
print('loss is NaN.')
exit()
if step % 1000 == 0:
print('epoch: {} step: {} G_loss: {} D_loss: {}'.format(epoch, step, G_l, D_l))
# Save figure
sampled_images = session.run([samples_for_all_resolutions[sizes.index(current_resolution)]])[0]
def train(self):
# comet_ml
# Create an experiment
experiment = Experiment(api_key="B6hzNydshIpZSG2Xi9BDG9gdG",
project_name="glow-mnist", workspace="voletiv")
hparams_dict = self.hparams_dict()
experiment.log_parameters(hparams_dict)
# set to training state
self.graph.train()
self.global_step = self.loaded_step
# begin to train
for epoch in range(self.n_epoches):
print("epoch", epoch)
progress = tqdm(self.data_loader)
for i_batch, batch in enumerate(progress):
experiment.set_step(self.global_step)
# update learning rate
lr = self.lrschedule["func"](global_step=self.global_step,
**self.lrschedule["args"])
for param_group in self.optim.param_groups:
param_group['lr'] = lr
self.optim.zero_grad()
# log
if self.global_step % self.scalar_log_gaps == 0:
# self.writer.add_scalar("lr/lr", lr, self.global_step)
experiment.log_metrics({"lr": lr, "epoch": epoch+i_batch/len(self.data_loader)})
# get batch data
for k in batch:
batch[k] = batch[k].to(self.data_device)
x = batch["x"]
y = None
y_onehot = None
if self.y_condition:
if self.y_criterion == "multi-classes":
assert "y_onehot" in batch, "multi-classes ask for `y_onehot` (torch.FloatTensor onehot)"
y_onehot = batch["y_onehot"]
elif self.y_criterion == "single-class":
assert "y" in batch, "single-class ask for `y` (torch.LongTensor indexes)"
y = batch["y"]
y_onehot = thops.onehot(y, num_classes=self.y_classes)
# at first time, initialize ActNorm
if self.global_step == 0:
self.graph(x[:self.batch_size // len(self.devices), ...],
y_onehot[:self.batch_size // len(self.devices), ...] if y_onehot is not None else None)
# parallel
if len(self.devices) > 1 and not hasattr(self.graph, "module"):
print("[Parallel] move to {}".format(self.devices))
self.graph = torch.nn.parallel.DataParallel(self.graph, self.devices, self.devices[0])
# forward phase
z, nll, y_logits = self.graph(x=x, y_onehot=y_onehot)
# loss_generative
loss_generative = Glow.loss_generative(nll)
# loss_classes
loss_classes = 0
if self.y_condition:
loss_classes = (Glow.loss_multi_classes(y_logits, y_onehot)
if self.y_criterion == "multi-classes" else
Glow.loss_class(y_logits, y))
# total loss
loss = loss_generative + loss_classes * self.weight_y
# log
if self.global_step % self.scalar_log_gaps == 0:
# self.writer.add_scalar("loss/loss_generative", loss_generative, self.global_step)
experiment.log_metrics({"loss_generative": loss_generative})
if self.y_condition:
# self.writer.add_scalar("loss/loss_classes", loss_classes, self.global_step)
experiment.log_metrics({"loss_classes": loss_classes, "total_loss": loss})
# backward
self.graph.zero_grad()
self.optim.zero_grad()
loss.backward()
# operate grad
if self.max_grad_clip is not None and self.max_grad_clip > 0:
torch.nn.utils.clip_grad_value_(self.graph.parameters(), self.max_grad_clip)
if self.max_grad_norm is not None and self.max_grad_norm > 0:
grad_norm = torch.nn.utils.clip_grad_norm_(self.graph.parameters(), self.max_grad_norm)
if self.global_step % self.scalar_log_gaps == 0:
# self.writer.add_scalar("grad_norm/grad_norm", grad_norm, self.global_step)
experiment.log_metrics({"grad_norm": grad_norm})
# step
self.optim.step()
# checkpoints
if self.global_step % self.checkpoints_gap == 0 and self.global_step > 0:
save(global_step=self.global_step,
graph=self.graph,
optim=self.optim,
pkg_dir=self.checkpoints_dir,
is_best=True,
max_checkpoints=self.max_checkpoints)
# plot images
if self.global_step % self.plot_gaps == 0:
img = self.graph(z=z, y_onehot=y_onehot, reverse=True)
# img = torch.clamp(img, min=0, max=1.0)
if self.y_condition:
if self.y_criterion == "multi-classes":
y_pred = torch.sigmoid(y_logits)
elif self.y_criterion == "single-class":
y_pred = thops.onehot(torch.argmax(F.softmax(y_logits, dim=1), dim=1, keepdim=True),
self.y_classes)
y_true = y_onehot
# plot images
# self.writer.add_image("0_reverse/{}".format(bi), torch.cat((img[bi], batch["x"][bi]), dim=1), self.global_step)
vutils.save_image(torch.stack([torch.cat((img[bi], batch["x"][bi]), dim=1) for bi in range(min([len(img), self.n_image_samples]))]), '/tmp/vikramvoleti.png', nrow=10)
experiment.log_image('/tmp/vikramvoleti_rev.png', file_name="0_reverse")
# plot preds
# for bi in range(min([len(img), self.n_image_samples])):
# # wandb.log({"0_reverse_{}".format(bi): [wandb.Image(torch.cat((img[bi], batch["x"][bi]), dim=1), caption="0_reverse/{}".format(bi))]}, step=self.global_step)
# if self.y_condition:
# # self.writer.add_image("1_prob/{}".format(bi), plot_prob([y_pred[bi], y_true[bi]], ["pred", "true"]), self.global_step)
# wandb.log({"1_prob_{}".format(bi): [wandb.Image(plot_prob([y_pred[bi], y_true[bi]], ["pred", "true"]))]}, step=self.global_step)
# inference
if hasattr(self, "inference_gap"):
if self.global_step % self.inference_gap == 0:
try:
img = self.graph(z=None, y_onehot=inference_y_onehot, eps_std=0.5, reverse=True)
except NameError:
inference_y_onehot = torch.zeros_like(y_onehot, device=torch.device('cpu'))
for i in range(inference_y_onehot.size(0)):
inference_y_onehot[i, (i % inference_y_onehot.size(1))] = 1.
# now
inference_y_onehot = inference_y_onehot.to(y_onehot.device)
img = self.graph(z=None, y_onehot=inference_y_onehot, eps_std=0.5, reverse=True)
# grid
vutils.save_image(img[:min([len(img), self.n_image_samples])], '/tmp/vikramvoleti.png', nrow=10)
experiment.log_image('/tmp/vikramvoleti_sam.png', file_name="1_samples")
# img = torch.clamp(img, min=0, max=1.0)
# for bi in range(min([len(img), n_images])):
# # self.writer.add_image("2_sample/{}".format(bi), img[bi], self.global_step)
# wandb.log({"2_sample_{}".format(bi): [wandb.Image(img[bi])]}, step=self.global_step)
if self.global_step == 0:
subprocess.run('nvidia-smi')
# global step
self.global_step += 1
parameters['launch_epoch'] = epoch
disable_flag = 1
sample_count = len(train_batched)
else:
if save:
torch.save(model.state_dict(), model_name)
best_idx = epoch
best_test_F, new_test_F, _ = evaluating_batch(model, test_batched,
best_test_F)
all_F.append([0.0, new_dev_F, new_test_F])
sys.stdout.flush()
print('Epoch %d : train/dev/test : %.2f / %.2f / %.2f - %d' %
(epoch, new_train_F, new_dev_F, new_test_F, best_idx))
model.train(True)
adjust_learning_rate(optimizer,
lr=learning_rate /
(1 + 0.05 * sample_count / len(train_data)))
metrics['new_train_F'] = new_train_F
metrics['new_test_F'] = new_test_F
metrics['new_dev_F'] = new_dev_F
experiment.log_metrics(metrics)
experiment.set_step(epoch + 1)
print(time.time() - t)
def train(args, use_comet : bool = True):
data_cls = funcs[args['dataset']]
model_cls = funcs[args['model']]
network = funcs[args['network']]
print ('[INFO] Getting dataset...')
data = data_cls()
data.load_data()
(x_train, y_train), (x_test, y_test) = (data.x_train, data.y_train), (data.x_test, data.y_test)
classes = data.mapping
# #Used for testing only
# x_train = x_train[:100, :, :]
# y_train = y_train[:100, :]
# x_test = x_test[:100, :, :]
# y_test = y_test[:100, :]
# print ('[INFO] Training shape: ', x_train.shape, y_train.shape)
# print ('[INFO] Test shape: ', x_test.shape, y_test.shape)
# #delete these lines
# distribute 90% test 10% val dataset with equal class distribution
(x_test, x_valid, y_test, y_valid) = train_test_split(x_test, y_test, test_size=0.2, random_state=42)
print ('[INFO] Training shape: ', x_train.shape, y_train.shape)
print ('[INFO] Validation shape: ', x_valid.shape, y_valid.shape)
print ('[INFO] Test shape: ', x_test.shape, y_test.shape)
print ('[INFO] Setting up the model..')
if args['network'] == 'lstmctc':
network_args = {'backbone' : args['backbone'],
'seq_model' : args['seq'],
'bi' : args['bi']
}
model = model_cls(network, data_cls, network_args)
else:
model = model_cls(network, data_cls)
print (model)
dataset = dict({
'x_train' : x_train,
'y_train' : y_train,
'x_valid' : x_valid,
'y_valid' : y_valid,
'x_test' : x_test,
'y_test' : y_test
})
if use_comet and args['find_lr'] == False:
#create an experiment with your api key
experiment = Experiment(api_key='WVBNRAfMLCBWslJAAsffxM4Gz',
project_name='iam_lines',
auto_param_logging=False)
print ('[INFO] Starting Training...')
#will log metrics with the prefix 'train_'
with experiment.train():
_ = train_model(
model,
dataset,
batch_size=args['batch_size'],
epochs=args['epochs'],
name=args['network']
)
print ('[INFO] Starting Testing...')
#will log metrics with the prefix 'test_'
with experiment.test():
score = model.evaluate(dataset, int(args['batch_size']))
print(f'[INFO] Test evaluation: {score*100}...')
metrics = {
'accuracy':score
}
experiment.log_metrics(metrics)
experiment.log_parameters(args)
experiment.log_dataset_hash(x_train) #creates and logs a hash of your data
experiment.end()
elif use_comet and args['find_lr'] == True:
_ = train_model(
model,
dataset,
batch_size=args['batch_size'],
epochs=args['epochs'],
FIND_LR=args['find_lr'],
name=args['network']
)
else :
print ('[INFO] Starting Training...')
train_model(
model,
dataset,
batch_size=args['batch_size'],
epochs=args['epochs'],
name=args['network']
)
print ('[INFO] Starting Testing...')
score = model.evaluate(dataset, args['batch_size'])
print(f'[INFO] Test evaluation: {score*100}...')
if args['weights']:
model.save_weights()
if args['save_model']:
model.save_model()
def main(model_name, pose_refine, exp: Experiment):
config, m = load_model(model_name)
test_set = get_dataset(config)
params_path = os.path.join(LOG_PATH, str(model_name),
"preprocess_params.pkl")
transform = SaveableCompose.from_file(params_path, test_set, globals())
test_set.transform = transform
assert isinstance(transform.transforms[1].normalizer, MeanNormalize3D)
normalizer3d = transform.transforms[1].normalizer
post_process_func = get_postprocessor(config, test_set, normalizer3d)
prefix = "R" if pose_refine else "NR"
prefix = f"mupo_{prefix}"
# logger = TemporalMupotsEvaluator(
# m,
# test_set,
# config["model"]["loss"],
# True,
# post_process3d=post_process_func,
# prefix="mupo_NR",
# orient_norm=None # config["orient_norm"]
# )
logger = TemporalTestEvaluator(
m,
test_set,
config["model"]["loss"],
True,
post_process3d=post_process_func,
prefix="mpi_NR",
orient_norm=None # config["orient_norm"]
)
logger.eval(calculate_scale_free=not pose_refine, verbose=not pose_refine)
exp.log_metrics(logger.losses_to_log)
# pred_3d = unstack_mpi3dhp_poses(test_set, logger)
# print("\n%13s R-PCK R-AUC A-PCK A-AUC" % "")
# print("%13s: " % "all poses", end="")
# keys = ["R-PCK", "R-AUC", "A-PCK", "A-AUC"]
# values = []
# for relative in [True, False]:
# pcks, aucs = mpii_3dhp.eval_poses(
# relative,
# "annot3" if config["pose3d_scaling"] == "normal" else "univ_annot3",
# pred_3d,
# )
# pck = np.mean(list(pcks.values()))
# auc = np.mean(list(aucs.values()))
# values.append(pck)
# values.append(auc)
# print(" %4.1f %4.1f " % (pck, auc), end="")
# print()
# exp.log_metrics({f"{prefix}-{k}": v for k, v in zip(keys, values)})
if pose_refine:
refine_config = load("../models/pose_refine_config.json")
pred = np.concatenate([logger.preds[i] for i in range(1, 21)])
pred = optimize_poses(pred, test_set, refine_config)
l = StackedArrayAllMupotsEvaluator(pred, test_set, True, prefix=prefix)
l.eval(calculate_scale_free=True, verbose=True)
exp.log_metrics(l.losses_to_log)
pred_by_seq = {}
for seq in range(1, 21):
inds = test_set.index.seq_num == seq
pred_by_seq[seq] = pred[inds]
pred_2d, pred_3d = unstack_mupots_poses(test_set, pred_by_seq)
else:
pred_2d, pred_3d = unstack_mupots_poses(test_set, logger.preds)
exp.log_metrics(logger.losses_to_log)
print("\nR-PCK R-AUC A-PCK A-AUC")
keys = ["R-PCK", "R-AUC", "A-PCK", "A-AUC"]
values = []
for relative in [True, False]:
pcks, aucs = mupots_3d.eval_poses(
False,
relative,
"annot3"
if config["pose3d_scaling"] == "normal" else "univ_annot3",
pred_2d,
pred_3d,
keep_matching=True,
)
pck = np.mean(list(pcks.values()))
auc = np.mean(list(aucs.values()))
values.append(pck)
values.append(auc)
print(" %4.1f %4.1f " % (pck, auc), end="")
print()
exp.log_metrics({f"{prefix}-{k}": v for k, v in zip(keys, values)})
if t % valid_inc == 0:
test(valid_dist_dataset, args.beta)
step_time = round(time.time() - start_time, 1)
metrics = {
metric_name: log(metric)
for metric_name, metric in trainer.metrics.items()
}
metrics['step_time'] = step_time
# validation plotting
progbar.add(valid_inc, [('Train Loss', metrics['train_loss']),
('Validation Loss', metrics['valid_loss']),
('Time (s)', step_time)])
#Plot on Comet
experiment.log_metrics(metrics, step=t)
# Plot on WandB
wandb.log(metrics, step=t)
if (t + 1) % save_inc == 0:
trainer.save_weights(model_path,
run_id=wandb.run.id,
experiment_key=experiment.get_key())
if not args.gcbc and not args.images:
z_enc, z_plan = produce_cluster_fig(next(plotting_dataset),
encoder,
planner,
TEST_DATA_PATHS[0],
num_take=dl.batch_size // 4)
#Comet
class AKMCS:
"""
Create AK-MCS model (Active Kriging - Monte Carlo Simulation) for reliability analysis
Args:
krigobj (object): Kriging object for AKMCS analysis
akmcsInfo (dict): Dictionary that contains AKMCS model information.
detail akmcsInfo:
- akmcsInfo["init_samp"] (nparray): Initial Monte-Carlo population
- akmcsInfo["maxupdate"] (int): Maximum number of update. Defaults to 120
- akmcsInfo["problem"] (str): Type of case
Returns:
updatedX (nparray): updated samples.
minUiter (nparray): minimum U value for each iteration
"""
def __init__(self, krigobj, akmcsInfo):
"""
Initialize akmcs
Args:
krigobj (object): Kriging object for AKMCS analysis
akmcsInfo (dict): Dictionary that contains AKMCS model information.
detail akmcsInfo:
- akmcsInfo["init_samp"] (nparray): Initial Monte-Carlo population
- akmcsInfo["maxupdate"] (int): Maximum number of update. Defaults to 120
- akmcsInfo["problem"] (str): Type of case
"""
akmcsInfo = akmcsInfocheck(akmcsInfo)
self.krigobj = krigobj
self.akmcsInfo = akmcsInfo
self.init_samp = akmcsInfo['init_samp']
self.maxupdate = akmcsInfo['maxupdate']
self.nsamp = np.size(self.init_samp, axis=0)
self.Gx = np.zeros(shape=[self.nsamp,1])
self.sigmaG = np.zeros(shape=[1,self.nsamp])
self.stop_criteria = 100 #assign large number
self.logging = None
def run(self, autoupdate=True, disp=True, savedatato=None, logging=False, saveimageto=None, plotdatapos=None,
plotdataneg=None, loggingAPIkey=None, logname=None, logworkspace=None):
"""
Run AKMCS analysis
Args:
autoupdate (bool): Perform automatic update on design space or not. Default to True.
disp (bool): Display progress or not. Default to True.
savedatato (str): Filename to save update data. e.g.: 'filename.csv'
Return:
None
"""
#logging
if logging:
if loggingAPIkey is None or logname is None or logworkspace is None:
raise ValueError('Logging is turned on, APIkey, project and workspace must be specified.')
self.logging = Experiment(api_key=loggingAPIkey,
project_name=logname, workspace=logworkspace)
if savedatato is not None:
self.logging.set_name(savedatato)
else:
pass
else:
pass
# Calculate Gx and SigmaG
# Split init_samp to avoid memory error
krig_initsamp = self.krigobj.KrigInfo['X']
t1 = time.time()
if self.nsamp < 10000:
self.Gx,self.sigmaG = self.krigobj.predict(self.init_samp, ['pred','s'])
else:
run_times = int(np.ceil(self.nsamp/10000))
for i in range(run_times):
start = i * 10000
stop = (i+1) * 10000
if i != (run_times - 1):
self.Gx[start:stop, :], self.sigmaG[:,start:stop] = \
self.krigobj.predict(self.init_samp[start:stop, :], ['pred','s'])
else:
self.Gx[start:, :], self.sigmaG[:,start:] = \
self.krigobj.predict(self.init_samp[start:, :], ['pred','s'])
t2 = time.time()
# Calculate probability of failure
self.Pf = self.pfcalc()
# Calculate learning function U
self.lfucalc()
self.stopcrit()
self.updateX = np.array([self.xnew])
self.minUiter = np.array([self.minU])
if disp:
print(f"Done iter no: 0, Pf: {self.Pf}, minU: {self.minU}")
# Update samples automatically
while autoupdate:
labeladded = False
for i in range(self.maxupdate):
# Evaluate new samples and append into Kriging object information
t = time.time()
ynew = evaluate(self.xnew, type=self.akmcsInfo['problem'])
self.krigobj.KrigInfo['y'] = np.vstack((self.krigobj.KrigInfo['y'],ynew))
self.krigobj.KrigInfo['X'] = np.vstack((self.krigobj.KrigInfo['X'], self.xnew))
self.krigobj.KrigInfo['nsamp'] += 1
# standardize model and train updated kriging model
t3 = time.time()
self.krigobj.standardize()
self.krigobj.train(disp=False)
t4 = time.time()
# Calculate Gx and SigmaG
# Split init_samp to avoid memory error
if self.nsamp < 10000:
self.Gx, self.sigmaG = self.krigobj.predict(self.init_samp, ['pred', 's'])
else:
run_times = int(np.ceil(self.nsamp / 10000))
for ii in range(run_times):
start = ii * 10000
stop = (ii + 1) * 10000
if ii != (run_times - 1):
self.Gx[start:stop, :], self.sigmaG[:, start:stop] = \
self.krigobj.predict(self.init_samp[start:stop, :], ['pred', 's'])
else:
self.Gx[start:, :], self.sigmaG[:, start:] = \
self.krigobj.predict(self.init_samp[start:, :], ['pred', 's'])
t5 = time.time()
# Calculate Pf, COV and LFU
self.Pf = self.pfcalc()
self.cov = self.covpf()
self.lfucalc()
self.stopcrit()
t6 = time.time()
# Update variables
self.updateX = np.vstack((self.updateX,self.xnew))
self.minUiter = np.vstack((self.minUiter,self.minU))
elapsed = time.time() - t
if disp:
print(f"iter no: {i+1}, Pf: {self.Pf}, stopcrit: {self.stop_criteria}, time(s): {elapsed}, "
f"ynew: {ynew}")
if logging:
self.logging.log_parameter('krigtype',self.krigobj.KrigInfo['type'])
outdict = {"Prob_fail":self.Pf,
"stopcrit":self.stop_criteria,
"time(s)":elapsed
}
self.logging.log_metrics(outdict,step=i+1)
if savedatato is not None:
temparray = np.array([i,self.Pf,self.stop_criteria,elapsed])
if i == 0:
totaldata = temparray[:]
else:
totaldata = np.vstack((totaldata,temparray))
filename = savedatato
np.savetxt(filename, totaldata, delimiter=',', header='iter,Pf,stopcrit,time(s)')
else:
pass
if saveimageto is not None:
imagefile = saveimageto + str(i) + ".PNG"
title = "Pf = " + str(self.Pf)
plt.figure(0, figsize=[10, 9])
if not labeladded:
plt.scatter(plotdatapos[:, 0], plotdatapos[:, 1], c='yellow', label='Feasible')
plt.scatter(plotdataneg[:, 0], plotdataneg[:, 1], c='cyan', label='Infeasible')
plt.scatter(krig_initsamp[:, 0], krig_initsamp[:, 1], c='red', label='Initial Kriging Population')
plt.scatter(self.updateX[:, 0], self.updateX[:, 1], s=75, c='black', marker='x', label='Update')
labeladded = True
else:
plt.scatter(plotdatapos[:, 0], plotdatapos[:, 1], c='yellow')
plt.scatter(plotdataneg[:, 0], plotdataneg[:, 1], c='cyan')
plt.scatter(krig_initsamp[:, 0], krig_initsamp[:, 1], c='red')
plt.scatter(self.updateX[:, 0], self.updateX[:, 1], s=75, c='black', marker='x')
plt.xlabel('X1', fontsize=18)
plt.ylabel('X2', fontsize=18)
plt.tick_params(axis='both', which='both', labelsize=16)
plt.legend(loc=1, prop={'size': 15})
plt.title(title,fontdict={'fontsize':20})
plt.savefig(imagefile, format='png')
else:
pass
# Break condition
if self.stop_criteria <= 0.05 and i >= 15:
break
else:
pass
print(f"COV: {self.cov}")
if self.cov <= 0.05:
break
else:
pass
break # temporary break for debugging, delete/comment this line later
def pfcalc(self):
nGless = len([i for i in self.Gx if i <= 0])
nsamp = np.size(self.init_samp, axis=0)
Pf = nGless / nsamp
return Pf
def covpf(self):
nmc = np.size(self.init_samp, axis=0)
if self.Pf == 0:
cov = 1000
else:
cov = np.sqrt((1 - self.Pf) / (self.Pf * nmc))
return cov
def lfucalc(self):
self.U = abs(self.Gx) / self.sigmaG.reshape(-1,1)
self.minU = np.min(self.U)
minUloc = np.argmin(self.U)
self.xnew = self.init_samp[minUloc,:]
def stopcrit(self):
nsamp = np.size(self.init_samp, axis=0)
temp1 = self.Gx - 1.96 * self.sigmaG.reshape(-1,1)
temp2 = self.Gx + 1.96 * self.sigmaG.reshape(-1,1)
pfp = len([i for i in temp1 if i <= 0])/nsamp
pfn = len([i for i in temp2 if i <= 0])/nsamp
pf0 = len([i for i in self.Gx if i <= 0])/nsamp
if pf0 == 0:
self.stop_criteria = 100
else:
self.stop_criteria = (pfp-pfn)/pf0
class Logger:
def __init__(self, send_logs, tags, parameters, experiment=None):
self.stations = 5
self.send_logs = send_logs
if self.send_logs:
if experiment is None:
json_loc = glob.glob("./**/comet_token.json")[0]
with open(json_loc, "r") as f:
kwargs = json.load(f)
self.experiment = Experiment(**kwargs)
else:
self.experiment = experiment
self.sent_mb = 0
self.speed_window = deque(maxlen=100)
self.step_time = None
self.current_speed = 0
if self.send_logs:
if tags is not None:
self.experiment.add_tags(tags)
if parameters is not None:
self.experiment.log_parameters(parameters)
def begin_logging(self, episode_count, steps_per_ep, sigma, theta, step_time):
self.step_time = step_time
if self.send_logs:
self.experiment.log_parameter("Episode count", episode_count)
self.experiment.log_parameter("Steps per episode", steps_per_ep)
self.experiment.log_parameter("theta", theta)
self.experiment.log_parameter("sigma", sigma)
def log_round(self, states, reward, cumulative_reward, info, loss, observations, step):
self.experiment.log_histogram_3d(states, name="Observations", step=step)
info = [[j for j in i.split("|")] for i in info]
info = np.mean(np.array(info, dtype=np.float32), axis=0)
try:
# round_mb = np.mean([float(i.split("|")[0]) for i in info])
round_mb = info[0]
except Exception as e:
print(info)
print(reward)
raise e
self.speed_window.append(round_mb)
self.current_speed = np.mean(np.asarray(self.speed_window)/self.step_time)
self.sent_mb += round_mb
# CW = np.mean([float(i.split("|")[1]) for i in info])
CW = info[1]
# stations = np.mean([float(i.split("|")[2]) for i in info])
self.stations = info[2]
fairness = info[3]
if self.send_logs:
self.experiment.log_metric("Round reward", np.mean(reward), step=step)
self.experiment.log_metric("Per-ep reward", np.mean(cumulative_reward), step=step)
self.experiment.log_metric("Megabytes sent", self.sent_mb, step=step)
self.experiment.log_metric("Round megabytes sent", round_mb, step=step)
self.experiment.log_metric("Chosen CW", CW, step=step)
self.experiment.log_metric("Station count", self.stations, step=step)
self.experiment.log_metric("Current throughput", self.current_speed, step=step)
self.experiment.log_metric("Fairness index", fairness, step=step)
for i, obs in enumerate(observations):
self.experiment.log_metric(f"Observation {i}", obs, step=step)
self.experiment.log_metrics(loss, step=step)
def log_episode(self, cumulative_reward, speed, step):
if self.send_logs:
self.experiment.log_metric("Cumulative reward", cumulative_reward, step=step)
self.experiment.log_metric("Speed", speed, step=step)
self.sent_mb = 0
self.last_speed = speed
self.speed_window = deque(maxlen=100)
self.current_speed = 0
def end(self):
if self.send_logs:
self.experiment.end()
def experiment():
"""Run finetuning label shift exp"""
args = get_args(None)
name = "Finetune {}:{}:{} {}:{} {}:{}:{}:{} {}{}v{}".format(
args.dataset,
args.dataset_cap,
args.warmstart_ratio,
args.shift_strategy,
args.dirichlet_alpha,
args.shift_correction,
args.rlls_reg,
args.rlls_lambda,
args.lr,
"IW " if args.train_iw else "NOIW ",
"ITIW " if args.iterative_iw else "NOITIW ",
args.version,
)
# Initialize comet.ml
if args.log:
comet_api = api.API(api_key=comet_ml_key)
exps = comet_api.get_experiments(
"ericzhao28",
project_name="active-label-shift-adaptation",
pattern=name)
for exp in exps:
if exp.get_name() == name:
raise ValueError("EXP EXISTS!")
logger = Experiment(comet_ml_key,
project_name="active-label-shift-adaptation")
logger.set_name(name)
logger.log_parameters(vars(args))
# Seed the experiment
seed = args.seed
torch.manual_seed(seed)
np.random.seed(seed)
random.seed(seed)
print("Running seed ", seed)
torch.cuda.set_device(args.device)
assert torch.cuda.is_available()
# Shuffle dataset
dataset = get_datasets(args)
dataset.label_ptrs(np.arange(dataset.online_len()))
# Train h0
net_cls = get_net_cls(args)
network = net_cls(args.num_cls).to(args.device)
def log_fn(epoch):
network.eval()
accuracy = evaluate(network,
dataset.iterate(args.infer_batch_size,
False,
split="test"),
args.device,
args,
label_weights=dataset.label_weights)
print(accuracy)
logger.log_metrics(accuracy, prefix="initial", step=epoch)
train(network,
dataset=dataset,
epochs=args.initial_epochs,
args=args,
log_fn=log_fn)
# Get source shift corrections
lsmse = label_shift(network, dataset, args)
def log_fn_shifted(epoch):
network.eval()
if args.iterative_iw:
label_shift(network, dataset, args)
accuracy = evaluate(network,
dataset.iterate(args.infer_batch_size,
False,
split="test"),
args.device,
args,
label_weights=dataset.label_weights)
print(accuracy)
logger.log_metrics(accuracy, prefix="shifted", step=epoch)
train(network,
dataset=dataset,
epochs=args.initial_epochs,
args=args,
log_fn=log_fn_shifted)
if args.iterative_iw:
lsmse = label_shift(network, dataset, args)
logger.log_metrics({"IW MSE": lsmse}, prefix="initial")
if not os.path.exists('models'):
os.mkdir('models')
epoch = 0
for epoch in range(int(iterations)):
# while i < int(iterations):
for sample in tqdm(data_loader):
images, labels = sample['images'].to(device), sample['labels'].to(device)
logits = model(images)
labels = labels.view(int(bs)*int(seq_length)) ##??
loss = criterion(logits, labels)
optimizer.zero_grad()
loss.backward()
losses.update(loss.item(), images.size(0))
optimizer.step()
print('epoch: {}\tLoss: {loss.val:.4f} ({loss.avg:.4f})'.format(epoch, loss=losses))
epoch += 1
if epoch % it_save == 0:
torch.save({'optimizer_state_dict': optimizer.state_dict(),
'model_state_dict': model.state_dict()}, 'models/swingnet_{}.pth.tar'.format(epoch))
if epoch == iterations:
break
experiment.log_metrics("train_loss", losses, step=epoch)
def train(config):
experiment = Experiment(api_key="Q8LzfxMlAfA3ABWwq9fJDoR6r",
project_name="hotpot",
workspace="fan-luo")
experiment.set_name(config.run_name)
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.dev_eval_file, "r") as fh:
dev_eval_file = json.load(fh)
with open(config.idx2word_file, 'r') as fh:
idx2word_dict = json.load(fh)
config.save = '{}-{}'.format(config.save, time.strftime("%Y%m%d-%H%M%S"))
create_exp_dir(
config.save,
scripts_to_save=['run.py', 'model.py', 'util.py', 'sp_model.py'])
def logging(s, print_=True, log_=True):
if print_:
print(s)
if log_:
with open(os.path.join(config.save, 'log.txt'), 'a+') as f_log:
f_log.write(s + '\n')
logging('Config')
for k, v in config.__dict__.items():
logging(' - {} : {}'.format(k, v))
logging("Building model...")
train_buckets = get_buckets(config.train_record_file)
dev_buckets = get_buckets(config.dev_record_file)
def build_train_iterator():
return DataIterator(train_buckets, config.batch_size,
config.para_limit, config.ques_limit,
config.char_limit, True, config.sent_limit)
def build_dev_iterator():
return DataIterator(dev_buckets, config.batch_size, config.para_limit,
config.ques_limit, config.char_limit, False,
config.sent_limit)
if config.sp_lambda > 0:
model = SPModel(config, word_mat, char_mat)
else:
model = Model(config, word_mat, char_mat)
logging('nparams {}'.format(
sum([p.nelement() for p in model.parameters() if p.requires_grad])))
ori_model = model.cuda()
model = nn.DataParallel(ori_model)
print("next(model.parameters()).is_cuda: " +
str(next(model.parameters()).is_cuda))
print("next(ori_model.parameters()).is_cuda: " +
str(next(ori_model.parameters()).is_cuda))
lr = config.init_lr
optimizer = optim.SGD(filter(lambda p: p.requires_grad,
model.parameters()),
lr=config.init_lr)
cur_patience = 0
total_loss = 0
total_ans_loss = 0
total_sp_loss = 0
global_step = 0
best_dev_F1 = None
stop_train = False
start_time = time.time()
eval_start_time = time.time()
model.train()
for epoch in range(10000):
for data in build_train_iterator():
context_idxs = Variable(data['context_idxs'])
ques_idxs = Variable(data['ques_idxs'])
context_char_idxs = Variable(data['context_char_idxs'])
ques_char_idxs = Variable(data['ques_char_idxs'])
context_lens = Variable(data['context_lens'])
y1 = Variable(data['y1'])
y2 = Variable(data['y2'])
q_type = Variable(data['q_type'])
is_support = Variable(data['is_support'])
start_mapping = Variable(data['start_mapping'])
end_mapping = Variable(data['end_mapping'])
all_mapping = Variable(data['all_mapping'])
logit1, logit2, predict_type, predict_support = model(
context_idxs,
ques_idxs,
context_char_idxs,
ques_char_idxs,
context_lens,
start_mapping,
end_mapping,
all_mapping,
return_yp=False)
loss_1 = (nll_sum(predict_type, q_type) + nll_sum(logit1, y1) +
nll_sum(logit2, y2)) / context_idxs.size(0)
loss_2 = nll_average(predict_support.view(-1, 2),
is_support.view(-1))
loss = loss_1 + config.sp_lambda * loss_2
optimizer.zero_grad()
loss.backward()
optimizer.step()
total_loss += loss.data[0]
total_ans_loss += loss_1.data[0]
total_sp_loss += loss_2.data[0]
global_step += 1
if global_step % config.period == 0:
cur_loss = total_loss / config.period
cur_ans_loss = total_ans_loss / config.period
cur_sp_loss = total_sp_loss / config.period
elapsed = time.time() - start_time
logging(
'| epoch {:3d} | step {:6d} | lr {:05.5f} | ms/batch {:5.2f} | train loss {:8.3f} | answer loss {:8.3f} | supporting facts loss {:8.3f} '
.format(epoch, global_step, lr,
elapsed * 1000 / config.period, cur_loss,
cur_ans_loss, cur_sp_loss))
experiment.log_metrics(
{
'train loss': cur_loss,
'train answer loss': cur_ans_loss,
'train supporting facts loss': cur_sp_loss
},
step=global_step)
total_loss = 0
total_ans_loss = 0
total_sp_loss = 0
start_time = time.time()
if global_step % config.checkpoint == 0:
model.eval()
metrics = evaluate_batch(build_dev_iterator(), model, 0,
dev_eval_file, config)
model.train()
logging('-' * 89)
logging(
'| eval {:6d} in epoch {:3d} | time: {:5.2f}s | dev loss {:8.3f} | answer loss {:8.3f} | supporting facts loss {:8.3f} | EM {:.4f} | F1 {:.4f}'
.format(global_step // config.checkpoint, epoch,
time.time() - eval_start_time, metrics['loss'],
metrics['ans_loss'], metrics['sp_loss'],
metrics['exact_match'], metrics['f1']))
logging('-' * 89)
experiment.log_metrics(
{
'dev loss': metrics['loss'],
'dev answer loss': metrics['ans_loss'],
'dev supporting facts loss': metrics['sp_loss'],
'EM': metrics['exact_match'],
'F1': metrics['f1']
},
step=global_step)
eval_start_time = time.time()
dev_F1 = metrics['f1']
if best_dev_F1 is None or dev_F1 > best_dev_F1:
best_dev_F1 = dev_F1
torch.save(ori_model.state_dict(),
os.path.join(config.save, 'model.pt'))
cur_patience = 0
else:
cur_patience += 1
if cur_patience >= config.patience:
lr /= 2.0
for param_group in optimizer.param_groups:
param_group['lr'] = lr
if lr < config.init_lr * 1e-2:
stop_train = True
break
cur_patience = 0
if stop_train: break
logging('best_dev_F1 {}'.format(best_dev_F1))
logger.info("Get prediction...")
y_pred = clf.predict(X_test_scaled)
print("\nResults\nConfusion matrix \n {}".format(confusion_matrix(y_test, y_pred)))
f1 = f1_score(y_test, y_pred)
precision = precision_score(y_test, y_pred)
recall = recall_score(y_test, y_pred)
print("F1 score is {:6.3f}".format(f1))
print("Precision score is {:6.3f}".format(precision))
print("Recall score is {:6.3f}".format(recall))
# these will be logged to your sklearn-demos project on Comet.ml
params = {
"random_state": random_state,
"model_type": "logreg",
"scaler": "standard scaler",
"param_grid": str(param_grid),
"stratify": True,
}
metrics = {"f1": f1, "recall": recall, "precision": precision}
logger.info("Logging params, meta info to commet.ml ...")
exp.log_dataset_hash(X_train_scaled)
exp.log_parameters(params)
exp.log_metrics(metrics)
logger.info("Done...")
class Reptile(Task):
"""
A meta-learning task that teaches an agent over a set of other tasks
"""
def __init__(self,
data_handler,
load_key=None,
sender=True,
receiver=True,
image_captioner=True,
image_selector=False,
track_results=True):
self.sess = Agent.sess
self.N = 1 # number of steps taken for each task - should be > 1
self.S = SenderAgent()
self.R = ReceiverAgent(*self.S.get_output())
self.IC = ImageCaptioner()
# self.IS = ImageSelector()
self.S.all_agents_initialized(load_key)
self.R.all_agents_initialized(load_key)
self.train_metrics = {}
self.val_metrics = {}
self.experiment = Experiment(api_key='1jl4lQOnJsVdZR6oekS6WO5FI',
project_name='Reptile',
auto_param_logging=False,
auto_metric_logging=False,
disabled=(not track_results))
self.params = {}
self.params.update(Agent.get_params())
self.params.update(data_handler.get_params())
self.experiment.log_parameters(self.params)
self.T = {}
if image_captioner:
self.ic = ImageCaptioning(self.IC,
experiment=self.experiment,
track_results=False)
self.T["Image Captioner"] = lambda img, capts: self.ic.train_batch(
(img, capts), mode="train")
if image_selector:
self.is_ = ImageSelection(self.IS,
experiment=self.experiment,
track_results=False)
self.T["Image Selector"] = lambda img, capts: self.is_.train_batch(
(img, capts), mode="train")
if sender or receiver:
self.rg = ReferentialGame(self.S,
self.R,
experiment=self.experiment,
track_results=False)
if receiver:
self.T["Receiver"] = lambda img, capts: self.rg.train_batch(
img, mode="receiver_train")
if sender:
self.T["Sender"] = lambda img, capts: self.rg.train_batch(
img, mode="sender_train")
# Initialize TF
variables_to_initialize = tf.global_variables()
if load_key is not None:
dont_initialize = []
if SenderAgent.loaded:
dont_initialize += SenderAgent.get_all_weights()
if ReceiverAgent.loaded:
dont_initialize += ReceiverAgent.get_all_weights()
if ImageCaptioner.loaded:
dont_initialize += ImageCaptioner.get_all_weights()
variables_to_initialize = [
v for v in tf.global_variables() if v not in dont_initialize
]
# REMOVE LATER
#variables_to_initialize += ImageCaptioner.optimizer.variables()
Agent.sess.run(tf.variables_initializer(variables_to_initialize))
self.sender_shared_state = VariableState(
self.sess, SenderAgent.get_shared_weights())
self.receiver_shared_state = VariableState(
self.sess, ReceiverAgent.get_shared_weights())
self.sender_own_state = VariableState(self.sess,
SenderAgent.get_weights())
self.receiver_own_state = VariableState(self.sess,
ReceiverAgent.get_weights())
# print(SenderAgent.get_shared_weights())
# print(ReceiverAgent.get_shared_weights())
# print(SenderAgent.get_weights())
# print(ReceiverAgent.get_weights())
# print(tf.trainable_variables())
self.shared_states = {
"shared_sender": self.sender_shared_state,
"shared_receiver": self.receiver_shared_state
}
self.own_states = {
"own_sender": self.sender_own_state,
"own_receiver": self.receiver_own_state
}
shared_average = []
for k, v in self.shared_states.items():
shared_average.append(v.export_variables())
shared_average = np.mean(shared_average, axis=0)
self.set_weights(new_shared_weights=shared_average)
self.dh = data_handler
with open(
"{}/data/csv_loss_{}.csv".format(project_path,
self.experiment.get_key()),
'w+') as csv_loss_file:
csv_loss_file.write(
"Image Captioner Loss,Image Selector Loss,Sender Loss,Receiver Loss\n"
)
with open(
"{}/data/csv_accuracy_{}.csv".format(
project_path, self.experiment.get_key()),
'w+') as csv_acc_file:
csv_acc_file.write(
"Image Captioner Loss,Image Selector Loss,Sender Loss,Receiver Loss\n"
)
self.step = 0
def get_diff(self, a, b):
diff = 0.
if isinstance(a, (np.ndarray, np.generic)):
return np.sum(np.abs(a - b))
elif isinstance(a, list):
for i in range(len(a)):
diff += self.get_diff(a[i], b[i])
elif isinstance(a, dict):
for k in a:
diff += self.get_diff(a[k], b[k])
return diff
def set_weights(self, new_own_weights=None, new_shared_weights=None):
if new_own_weights is not None:
for k, s in self.own_states.items():
s.import_variables(new_own_weights[k])
if new_shared_weights is not None:
for k, s in self.shared_states.items():
s.import_variables(new_shared_weights)
def train_epoch(self, e, mode=None):
self.dh.set_params(distractors=0)
image_gen = self.dh.get_images(return_captions=True, mode="train")
# Get current variables
start_vars = {
k: s.export_variables()
for k, s in self.own_states.items()
}
start_vars["shared"] = self.shared_states[
"shared_sender"].export_variables()
while True:
try:
# Save current variables
old_own = {
k: s.export_variables()
for k, s in self.own_states.items()
}
new_own = {k: [] for k, s in self.own_states.items()}
old_shared = self.shared_states[
"shared_sender"].export_variables()
new_shared = []
# For each task
for task in ["Image Captioner", "Sender", "Receiver"]:
# parameter setup to not waste data
if task in ["Sender", "Receiver", "Image Selector"]:
self.dh.set_params(distractors=Agent.D)
else:
self.dh.set_params(distractors=0)
# Run task n times
for _ in range(self.N):
images, captions = next(image_gen)
acc, loss = self.T[task](images, captions)
self.train_metrics[task + " Accuracy"] = acc
self.train_metrics[task + " Loss"] = loss
# Store new variables
[
new_own[k].append(s.export_variables())
for k, s in self.own_states.items()
]
[
new_shared.append(s.export_variables())
for k, s in self.shared_states.items()
]
# Reset to old variables for next task
[
s.import_variables(old_own[k])
for k, s in self.own_states.items()
]
[
s.import_variables(old_shared)
for k, s in self.shared_states.items()
]
self.step += 1
self.experiment.set_step(self.step)
self.experiment.log_metrics(self.train_metrics)
# Average new variables
new_own = {
k: interpolate_vars(old_own[k], average_vars(new_own[k]),
0.2)
for k, s in self.own_states.items()
}
new_shared = interpolate_vars(old_shared,
average_vars(new_shared), 0.2)
# Set variables to new variables
self.set_weights(new_own_weights=new_own,
new_shared_weights=new_shared)
except StopIteration:
break
# Get change in weights
end_vars = {
k: s.export_variables()
for k, s in self.own_states.items()
}
end_vars["shared"] = self.shared_states[
"shared_sender"].export_variables()
weight_diff = self.get_diff(start_vars, end_vars)
#self.experiment.set_step(e)
self.val_metrics["Weight Change"] = weight_diff
self.experiment.log_metrics(self.val_metrics)
# Log data to a csv
with open("{}/data/csv_loss_{}.csv".format(project_path, self.experiment.get_key()), 'a') as csv_loss_file, \
open("{}/data/csv_accuracy_{}.csv".format(project_path, self.experiment.get_key()), 'a') as csv_acc_file:
losses = []
accs = []
for task in ["Image Captioner", "Sender", "Receiver"]:
losses.append(str(self.train_metrics[task + " Loss"]))
accs.append(str(self.train_metrics[task + " Accuracy"]))
csv_loss_file.write(",".join(losses))
csv_loss_file.write("\n")
csv_acc_file.write(",".join(accs))
csv_acc_file.write("\n")
return 0, weight_diff
D_G_z2 = output.mean().item()
# Update G
optimizerG.step()
"""
Loss_D - discriminator loss calculated as the sum of losses for the all real and all fake batches (log(D(x))+log(D(G(z)))).
Loss_G - generator loss calculated as log(D(G(z)))
D(x) - the average output (across the batch) of the discriminator for the all real batch.
This should start close to 1 then theoretically converge to 0.5 when G gets better. Think about why this is.
D(G(z)) - average discriminator outputs for the all fake batch. The first number is before D is updated and the second number
is after D is updated. These numbers should start near 0 and converge to 0.5 as G gets better. Think about why this is.
"""
# Output training stats
if i % 10 == 0:
print('step:', steps, ' epoch:', epoch)
experiment.log_metrics({'Loss_D': errD.item(), 'Loss_G': errG.item(), 'D(x)': D_x, 'D(G(z1))': D_G_z1, 'D(G(z2))': D_G_z2})
if (steps % 100 == 0) or ((epoch == num_epochs-1) and (i == len(dataloader)-1)):
fixed_noise = torch.randn(3, nz, 1, 1, device=device)
with torch.no_grad():
fake = netG(fixed_noise).detach().cpu().numpy()
plot = plt.figure(figsize=(20, 10))
for m in range(3):
plt.subplot(1, 3, m + 1)
plt.imshow((fake[m].transpose((1, 2, 0))+1)/2)
experiment.log_figure(figure_name='epoch_{}_{}'.format(epoch, steps))
plt.close()
steps += 1
# person and ball metrics
metrics = {
'val_presision': precision.mean(),
'val_precision_person': precision[0].mean(),
'val_precision_ball': precision[1].mean(),
'val_recall': recall.mean(),
'val_recall_person': recall[0].mean(),
'val_recall_ball': recall[1].mean(),
'val_mAP': AP.mean(),
'val_AP_person': AP[0].mean(),
'val_AP_ball': AP[1].mean(),
'val_f1': f1.mean()
}
experiment.log_metrics(metrics, step=epoch)
# if epoch % opt.checkpoint_interval == 0:
# torch.save(model.state_dict(), f"checkpoints/yolov3_ckpt_%d.pth" % epoch)
# print("Save model: ", f"checkpoints/yolov3_ckpt_%d.pth" % epoch)
# plot image
# if epoch % 10 == 0:
# model.eval()
# cv2_img = cv2.imread("data/obj/20191201F-netvsYSCC_00049.jpg")
# cv2_img = cv2.cvtColor(cv2_img, cv2.COLOR_BGR2RGB)
# img = Image.open("data/obj/20191201F-netvsYSCC_00049.jpg")
# demo_img = transforms.ToTensor()(Image.open("data/obj/20191201F-netvsYSCC_00049.jpg").convert('RGB'))
# demo_diff = transforms.ToTensor()(Image.open("data/difference/20191201F-netvsYSCC/20191201F-netvsYSCC_00049.jpg").convert('L'))
# demo_img = demo_img * demo_diff + demo_img
def eval(
model: LiSSModel,
dataset: UnalignedDataset,
exp: Experiment,
total_iters: int = 0,
nb_ims: int = 30,
):
liss = model.opt.model == "liss"
metrics = {}
print(f"----------- Evaluation {total_iters} ----------")
with torch.no_grad():
data = {
"translation": {
"A": {
"rec": None,
"idt": None,
"real": None,
"fake": None
},
"B": {
"rec": None,
"idt": None,
"real": None,
"fake": None
},
}
}
force = set(["identity", "translation"])
if liss:
for t in model.tasks:
tmp = {}
if t.eval_visuals_pred or t.log_type == "acc":
tmp["pred"] = None
if t.eval_visuals_target or t.log_type == "acc":
tmp["target"] = None
data[t.key] = {domain: deepcopy(tmp) for domain in "AB"}
force |= set(model.tasks.keys)
losses = {
k: []
for k in dir(model) if k.startswith("loss_")
and isinstance(getattr(model, k), torch.Tensor)
}
for i, b in enumerate(dataset):
# print(f"\rEval batch {i}", end="")
model.set_input(b)
model.forward(force=force)
model.backward_G(losses_only=True, force=force)
for k in dir(model):
if k.startswith("loss_") and isinstance(
getattr(model, k), torch.Tensor):
if k not in losses:
losses[k] = []
losses[k].append(getattr(model, k).detach().cpu().item())
if liss:
for t in model.tasks:
for domain in "AB":
for dtype in data[t.key][domain]:
if (t.log_type != "acc"
and data[t.key][domain][dtype] is not None
and
len(data[t.key][domain][dtype]) >= nb_ims):
continue
v = model.get(
f"{domain}_{t.key}_{dtype}").detach().cpu()
if data[t.key][domain][dtype] is None:
data[t.key][domain][dtype] = v
else:
data[t.key][domain][dtype] = torch.cat(
[data[t.key][domain][dtype], v],
dim=0,
)
# -------------------------
# ----- Translation -----
# -------------------------
if (data["translation"]["A"]["real"] is None
or len(data["translation"]["A"]["real"]) < nb_ims):
for domain in "AB":
for dtype in ["real", "fake", "rec", "idt"]:
dom = domain
if dtype in {"fake", "idt"}:
dom = swap_domain(domain)
v = model.get(f"{dom}_{dtype}").detach().cpu()
if data["translation"][domain][dtype] is None:
data["translation"][domain][dtype] = v
else:
data["translation"][domain][dtype] = torch.cat(
[data["translation"][domain][dtype], v], dim=0)
# print(
# f"{domain} {dtype} {len(data['translation'][domain][dtype])}"
# )
for task in data:
if task != "translation" and model.tasks[task].log_type != "vis":
continue
for domain in data[task]:
for i, v in data[task][domain].items():
data[task][domain][i] = torch.cat(list(v[:nb_ims].permute(
0, 2, 3, 1)),
axis=1)
log_images = int(data["translation"]["A"]["real"].shape[1] /
data["translation"]["A"]["real"].shape[0])
im_size = data["translation"]["A"]["real"].shape[0]
ims = {"A": None, "B": None}
data_keys = ["translation"]
translation_keys = ["real", "fake", "rec", "idt"]
data_keys += [task for task in data if task not in data_keys]
for task in data_keys:
if task != "translation" and model.tasks[task].log_type != "vis":
continue
for domain in "AB":
im_types = (translation_keys if task == "translation" else list(
data[task][domain].keys()))
for im_type in im_types:
v = data[task][domain][im_type].float()
if task == "depth":
v = to_min1_1(v)
v = v.repeat((1, 1, 3))
v = v + 1
v = v / 2
if ims[domain] is None:
ims[domain] = v
else:
ims[domain] = torch.cat([ims[domain], v], dim=0)
# ------------------------
# ----- Comet Logs -----
# ------------------------
for i in range(0, log_images, 5):
k = i + 5
exp.log_image(
ims["A"][:, i * im_size:k * im_size, :].numpy(),
"test_A_{}_{}_rfcidg".format(i * 5, (i + 1) * 5 - 1),
step=total_iters,
)
exp.log_image(
ims["B"][:, i * im_size:k * im_size, :].numpy(),
"test_B_{}_{}_rfcidg".format(i * 5, (i + 1) * 5 - 1),
step=total_iters,
)
if liss:
test_losses = {
"test_" + ln: np.mean(losses["loss_" + ln])
for t in model.tasks for ln in t.loss_names
}
test_accs = {
f"test_G_{domain}_{t.key}_acc":
np.mean(data[t.key][domain]["pred"].max(-1)[1].numpy() == data[
t.key][domain]["target"].numpy())
for domain in "AB" for t in model.tasks if t.log_type == "acc"
}
if liss:
exp.log_metrics(test_losses, step=total_iters)
exp.log_metrics(test_accs, step=total_iters)
for t in model.tasks:
if t.log_type != "acc":
continue
for domain in "AB":
target = data[t.key][domain]["target"].numpy()
pred = data[t.key][domain]["pred"].numpy()
exp.log_confusion_matrix(
get_one_hot(target, t.output_dim),
pred,
file_name=
f"confusion_{domain}_{t.key}_{total_iters}.json",
title=f"confusion_{domain}_{t.key}_{total_iters}.json",
)
metrics = {k + "_loss": v for k, v in test_losses.items()}
metrics.update(test_accs)
print("----------- End Evaluation----------")
return metrics
class _Experiment(object):
def __init__(self, name):
config_data = read_file_in_dir('./', name + '.json')
if config_data is None:
raise Exception("Configuration file doesn't exist: ", name)
self.name = config_data['experiment_name']
dataset_config = config_data['dataset']
data_percentage = dataset_config['data_percentage']
batch_size = dataset_config['batch_size']
num_workers = dataset_config['num_workers']
data_files = dataset_config['data_file_path']
experiment_config = config_data['experiment']
self.epochs = experiment_config['num_epochs']
learning_rate = experiment_config['learning_rate']
model_config = config_data['model']
self.is_vae = model_config['is_vae']
hidden_size = model_config['hidden_size']
embedding_size = model_config['embedding_size']
self.is_variational = model_config['is_variational']
self.is_conditional = model_config['is_conditional']
class_label = model_config['class_label']
generation_config = config_data['generation']
max_length = generation_config['max_length']
prediction_type = ("Stochastic",
"Deterministic")[generation_config["deterministic"]]
temperature = generation_config['temperature']
self.experiment_dir = os.path.join(ROOT_STATS_DIR, self.name)
self.log_comet = config_data['comet']['log_comet']
if self.log_comet:
self.experiment = Experiment(
api_key="CaoCCUZVjE0gXKA3cbtMKSSKL",
project_name="image-captioning-251b",
workspace="keshav919",
)
# Load Datasets
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
self.train_loader, self.val_loader, self.test_loader = getDataloaders(
data_files,
max_length,
batch_size,
num_workers,
tokenizer,
data_percentage,
class_label=class_label)
# Setup Experiment
self.current_epoch = 0
self.training_losses = []
self.bleu1_t = [] # Geeling: log bleu scores
self.bleu4_t = []
self.bleu1_v = [] # Keshav: log bleu scores
self.bleu4_v = []
self.val_losses = []
self.best_loss = float('inf')
self.best_model = None # Save your best model in this field and use this in test method.
if config_data['generation']:
tags = [self.name, self.is_variational, prediction_type]
hyper_params = {
"Epochs": self.epochs,
"Batch Size": batch_size,
"Learning Rate": learning_rate,
"Hidden Size": hidden_size,
"Embedding Size": embedding_size,
"Max Length": max_length,
"Temperature": temperature
}
if self.log_comet:
self.experiment.add_tags(tags)
self.experiment.log_parameters(hyper_params)
# Initialize Model
self.tokenizer = tokenizer
self.vocab_size = tokenizer.vocab_size
self.model = getModel(config_data, self.vocab_size)
# TODO: need to add KL divergence
self.criterion = torch.nn.CrossEntropyLoss()
self.optimizer = optim.Adam(self.model.parameters(), lr=learning_rate)
self.init_model()
self.load_experiment() # Load Experiment Data if available
def load_experiment(self):
"""
Loads the experiment data if exists to resume training from last saved checkpoint.
"""
os.makedirs(ROOT_STATS_DIR, exist_ok=True)
# Since we use comet, all our metrics are logged there rather than these directories.
# Create the dir just for test output
os.makedirs(self.experiment_dir, exist_ok=True)
def init_model(self):
"""
Gets GPU ready to use
"""
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
self.model.to(self.device)
self.criterion.to(self.device)
def loss_function(self, raw_outputs, hypothesis, mu, logvar):
ceLoss = self.criterion(raw_outputs, hypothesis)
if self.is_variational:
klLoss = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
klLoss /= len(raw_outputs)
ceLoss += klLoss
return ceLoss
def run(self):
start_epoch = self.current_epoch
for epoch in range(
start_epoch,
self.epochs): # loop over the dataset multiple times
start_time = datetime.now()
self.current_epoch = epoch
if self.is_vae:
########################## VAE ##############################
train_loss, bleu1_scores_t, bleu4_scores_t = self.train_vae()
if self.log_comet:
self.experiment.log_metrics({'Train_Loss': train_loss},
epoch=epoch)
self.experiment.log_metrics(
{'Train_Metric/BLEU-1': bleu1_scores_t}, epoch=epoch)
self.experiment.log_metrics(
{'Train_Metric/BLEU-4': bleu4_scores_t}, epoch=epoch)
val_loss, bleu1_scores_v, bleu4_scores_v = self.val_vae()
if self.log_comet:
self.experiment.log_metrics({'Val_Loss': val_loss},
epoch=epoch)
self.experiment.log_metrics(
{'Val_Metric/BLEU-1': bleu1_scores_v}, epoch=epoch)
self.experiment.log_metrics(
{'Val_Metric/BLEU-4': bleu4_scores_v}, epoch=epoch)
else:
########################## BERT ##############################
train_loss, train_accu = self.train_bert()
if self.log_comet:
self.experiment.log_metrics({'Train_Loss': train_loss},
epoch=epoch)
self.experiment.log_metrics({'Train_Accu': train_accu},
epoch=epoch)
val_loss, val_accu = self.val_bert()
if self.log_comet:
self.experiment.log_metrics({'Val_Loss': val_loss},
epoch=epoch)
self.experiment.log_metrics({'Val_Accu': val_accu},
epoch=epoch)
# Early stopping
if val_loss < self.best_loss:
self.best_loss = val_loss
torch.save(self.model, './saved_models/{}'.format(self.name))
if self.is_vae:
self.test_vae()
else:
self.test_bert()
########################## VAE ##############################
def train_vae(self):
self.model.train()
training_loss = 0
bleu1_scores = 0.0
bleu4_scores = 0.0
print_iter = 150
if_counter = 0
for i, (prem, hyp, _, _, lab) in enumerate(self.train_loader):
self.model.zero_grad()
prem = prem.long().to(self.device)
hyp = hyp.long().to(self.device)
lab = lab.to(self.device)
# Forward pass
preds, raw_outputs, mu, logvar = self.model(
prem,
hyp,
lab,
self.device,
is_teacher_forcing_on=True,
skip_generation=i % print_iter != 0,
is_conditional=self.is_conditional)
# Calculate loss and perform backprop
loss = self.loss_function(raw_outputs[:, 1:].permute(0, 2, 1),
hyp[:, 1:], mu, logvar)
loss.backward()
self.optimizer.step()
# Log the training loss
training_loss += loss.item()
# View deterministic predictions
if i % print_iter == 0:
if_counter += 1
# Get the sentence without the <start> and <end> and other tags
clean_premise_text, _ = clean_caption(prem, self.tokenizer)
clean_preds_text, _ = clean_caption(preds, self.tokenizer)
clean_targets_text, _ = clean_caption(hyp, self.tokenizer)
# Calculate bleu scores
b1 = calculate_bleu(bleu1, clean_preds_text,
clean_targets_text)
b4 = calculate_bleu(bleu4, clean_preds_text,
clean_targets_text)
bleu1_scores += (b1 / len(clean_preds_text))
bleu4_scores += (b4 / len(clean_preds_text))
print(self.current_epoch, i, ": ------ TRAIN ------")
print("------ Actual Premise ------")
print(clean_premise_text[0])
print("------ Actual Hypothesis ------")
print(clean_targets_text[0])
print("------ Predicted Hypothesis ------")
print(clean_preds_text[0])
print()
return training_loss / (
i + 1), bleu1_scores / if_counter, bleu4_scores / if_counter
def val_vae(self):
self.model.eval()
val_loss = 0
bleu1_scores = 0.0
bleu4_scores = 0.0
print_iter = 150
if_counter = 0
with torch.no_grad():
for i, (prem, hyp, _, _, lab) in enumerate(self.val_loader):
prem = prem.long().to(self.device)
hyp = hyp.long().to(self.device)
lab = lab.to(self.device)
# Forward pass
preds, raw_outputs, mu, logvar = self.model(
prem,
hyp,
lab,
self.device,
is_teacher_forcing_on=True,
skip_generation=i % print_iter != 0,
is_conditional=self.is_conditional)
# Calculate loss and perform backprop
loss = self.loss_function(raw_outputs[:, 1:].permute(0, 2, 1),
hyp[:, 1:], mu, logvar)
# Log the training loss
val_loss += loss.item()
# View deterministic predictions
if i % print_iter == 0:
if_counter += 1
# Get the sentence without the <start> and <end> and other tags
clean_premise_text, _ = clean_caption(prem, self.tokenizer)
clean_preds_text, _ = clean_caption(preds, self.tokenizer)
clean_targets_text, _ = clean_caption(hyp, self.tokenizer)
# Calculate bleu scores
b1 = calculate_bleu(bleu1, clean_preds_text,
clean_targets_text)
b4 = calculate_bleu(bleu4, clean_preds_text,
clean_targets_text)
bleu1_scores += (b1 / len(clean_preds_text))
bleu4_scores += (b4 / len(clean_preds_text))
print(self.current_epoch, i, ": ------ VALIDATION ------")
print("------ Actual Premise ------")
print(clean_premise_text[0])
print("------ Actual Hypothesis ------")
print(clean_targets_text[0])
print("------ Predicted Hypothesis ------")
print(clean_preds_text[0])
print()
return val_loss / (
i + 1), bleu1_scores / if_counter, bleu4_scores / if_counter
def test_vae(self):
self.model = torch.load('./saved_models/{}'.format(self.name))
self.model.eval()
test_loss = 0
bleu1_scores = 0.0
bleu4_scores = 0.0
print_iter = 150
predicted = []
premises = []
labels = []
language = []
with torch.no_grad():
for i, (prem, hyp, _, _, lab) in enumerate(self.test_loader):
prem = prem.long().to(self.device)
hyp = hyp.long().to(self.device)
lab = lab.to(self.device)
# Forward pass
_, raw_outputs, mu, logvar = self.model(
prem,
hyp,
lab,
self.device,
is_teacher_forcing_on=True,
is_conditional=self.is_conditional)
preds, _, _, _ = self.model(prem,
hyp,
lab,
self.device,
is_teacher_forcing_on=False,
is_conditional=self.is_conditional)
# Calculate loss and perform backprop
loss = self.loss_function(raw_outputs[:, 1:].permute(0, 2, 1),
hyp[:, 1:], mu, logvar)
# Log the training loss
test_loss += loss.item()
# Get the sentence without the <start> and <end> and other tags
clean_premise_text, clean_premise_joined = clean_caption(
prem, self.tokenizer)
clean_preds_text, clean_preds_joined = clean_caption(
preds, self.tokenizer)
clean_targets_text, _ = clean_caption(hyp, self.tokenizer)
predicted = predicted + clean_preds_joined
premises = premises + clean_premise_joined
labels = labels + lab.tolist()
language = language + ['en'] * len(clean_premise_joined)
# Calculate bleu scores
b1 = calculate_bleu(bleu1, clean_preds_text,
clean_targets_text)
b4 = calculate_bleu(bleu4, clean_preds_text,
clean_targets_text)
bleu1_scores += (b1 / len(clean_preds_text))
bleu4_scores += (b4 / len(clean_preds_text))
if i % print_iter == 0:
print(i, ": ------ TEST ------")
print("------ Actual Premise ------")
print(clean_premise_text[0])
print("------ Actual Hypothesis ------")
print(clean_targets_text[0])
print("------ Predicted Hypothesis ------")
print(clean_preds_text[0])
print()
bleu1_scores = bleu1_scores / (i + 1)
bleu4_scores = bleu4_scores / (i + 1)
test_loss = test_loss / (i + 1)
result_str = "Test Performance: Loss: {}, Bleu1: {}, Bleu4: {}".format(
test_loss, bleu1_scores, bleu4_scores)
self.log(result_str)
if self.log_comet and self.is_vae:
self.experiment.log_metrics({'Test_Loss': test_loss})
self.experiment.log_metrics({'Test_Metric/BLEU-1': bleu1_scores})
self.experiment.log_metrics({'Test_Metric/BLEU-4': bleu4_scores})
ds = {
"premise": premises,
"hypothesis": predicted,
"label": labels,
"lang_abv": language
}
df = pd.DataFrame(
ds, columns=["premise", "hypothesis", "label", "lang_abv"])
df.to_csv("predicted_vae" + self.name + ".csv", index=False)
return test_loss, bleu1_scores, bleu4_scores
########################## BERT ##############################
def train_bert(self):
self.model.train()
training_loss = 0
print_iter = 50
total_pred = 0
correct_pred = 0
for i, (prem_id, hyp_id, prem_att_mask, hypo_att_mask,
lab) in enumerate(self.train_loader):
self.model.zero_grad()
# Push data to GPU
# (Model is pushed to GPU in line 127)
prem_id = prem_id.long().to(self.device)
hyp_id = hyp_id.long().to(self.device)
prem_att_mask = prem_att_mask.long().to(self.device)
hypo_att_mask = hypo_att_mask.long().to(self.device)
lab = lab.long().to(self.device)
# Prepare data as inputs to bert
input_ids = torch.cat((prem_id, hyp_id), dim=1).long()
attention_mask = torch.cat((prem_att_mask, hypo_att_mask),
dim=1).long()
label = lab.unsqueeze(1).long()
# Forward pass
outputs = self.model(input_ids,
attention_mask=attention_mask,
labels=label)
# Calculate loss and perform backprop
loss = outputs.loss
loss.backward()
self.optimizer.step()
# Log the training loss
training_loss += loss.item()
# Get predicted labels
predicted = torch.argmax(outputs.logits, 1)
# calculate val accuracy = correct predictions/total predictions
for j in range(len(lab)):
total_pred += 1
if lab[j] == predicted[j]:
correct_pred += 1
accu = correct_pred / total_pred
# View deterministic predictions
if i % print_iter == 0:
print(self.current_epoch, i, ": ------ TRAIN ------")
print("------ Actual Labels ------")
print(lab)
print("------ Predicted Labels ------")
print(predicted)
print("Current training accuracy: ", accu)
# debugging code
# if i==print_iter+1:
# print("i is: ", i)
# sys.exit()
return training_loss / (i + 1), accu
def val_bert(self):
self.model.eval()
val_loss = 0
print_iter = 50
total_pred = 0
correct_pred = 0
with torch.no_grad():
for i, (prem_id, hyp_id, prem_att_mask, hypo_att_mask,
lab) in enumerate(self.val_loader):
self.model.zero_grad()
# Push data to GPU
prem_id = prem_id.long().to(self.device)
hyp_id = hyp_id.long().to(self.device)
prem_att_mask = prem_att_mask.long().to(self.device)
hypo_att_mask = hypo_att_mask.long().to(self.device)
lab = lab.long().to(self.device)
# Prepare data as inputs to bert
input_ids = torch.cat((prem_id, hyp_id), dim=1)
attention_mask = torch.cat((prem_att_mask, hypo_att_mask),
dim=1)
label = lab.unsqueeze(1)
# Forward pass
outputs = self.model(input_ids,
attention_mask=attention_mask,
labels=label)
# Calculate loss and perform backprop
loss = outputs.loss
# Log the val loss
val_loss += loss.item()
# Get predicted labels
predicted = torch.argmax(outputs.logits, 1)
# calculate val accuracy = correct predictions/total predictions
for j in range(len(lab)):
total_pred += 1
if lab[j] == predicted[j]:
correct_pred += 1
accu = correct_pred / total_pred
# View deterministic predictions
if i % print_iter == 0:
print(self.current_epoch, i, ": ------ VAL ------")
print("------ Actual Labels ------")
print(lab)
print("------ Predicted Labels ------")
print(predicted)
print("Current validation accuracy: ", accu)
return val_loss / (i + 1), accu
def test_bert(self):
self.model.eval()
test_loss = 0
print_iter = 50
total_pred = 0
correct_pred = 0
for i, (prem_id, hyp_id, prem_att_mask, hypo_att_mask,
lab) in enumerate(self.test_loader):
self.model.zero_grad()
# Push data to GPU
prem_id = prem_id.long().to(self.device)
hyp_id = hyp_id.long().to(self.device)
prem_att_mask = prem_att_mask.long().to(self.device)
hypo_att_mask = hypo_att_mask.long().to(self.device)
lab = lab.long().to(self.device)
# Prepare data as inputs to bert
input_ids = torch.cat((prem_id, hyp_id), dim=1)
attention_mask = torch.cat((prem_att_mask, hypo_att_mask), dim=1)
label = lab.unsqueeze(1)
# Forward pass
outputs = self.model(input_ids,
attention_mask=attention_mask,
labels=label)
# Calculate loss and perform backprop
loss = outputs.loss
# Log the test loss
test_loss += loss.item()
# Get predicted labels
predicted = torch.argmax(outputs.logits, 1)
# calculate test accuracy = correct predictions/total predictions
for j in range(len(lab)):
total_pred += 1
if lab[j] == predicted[j]:
correct_pred += 1
accu = correct_pred / total_pred
# View deterministic predictions
if i % print_iter == 0:
print(self.current_epoch, i, ": ------ TEST ------")
print("------ Actual Labels ------")
print(lab)
print("------ Predicted Labels ------")
print(predicted)
print("Current test accuracy: ", accu)
if self.log_comet and (not self.is_vae):
self.experiment.log_metrics({'Test_Loss': test_loss})
self.experiment.log_metrics({'Test_Accu': accu})
return test_loss / (i + 1), accu
########################## Log ##############################
def save_model(self):
root_model_path = os.path.join(self.experiment_dir, 'latest_model.pt')
model_dict = self.model.state_dict()
state_dict = {
'model': model_dict,
'optimizer': self.optimizer.state_dict()
}
torch.save(state_dict, root_model_path)
def __record_stats(self, train_loss, bleu1_t, bleu4_t, val_loss, bleu1_v,
bleu4_v):
self.training_losses.append(train_loss)
self.bleu1_t.append(bleu1_t)
self.bleu4_t.append(bleu4_t)
self.val_losses.append(val_loss)
self.bleu1_v.append(bleu1_v)
self.bleu4_v.append(bleu4_v)
self.plot_stats()
write_to_file_in_dir(self.experiment_dir, 'training_losses.txt',
self.__training_losses)
write_to_file_in_dir(self.experiment_dir, 'val_losses.txt',
self.val_losses)
write_to_file_in_dir(self.experiment_dir, 'bleu1.txt', self.bleu1)
write_to_file_in_dir(self.experiment_dir, 'bleu4.txt', self.bleu4)
def log(self, log_str, file_name=None):
print(log_str)
log_to_file_in_dir(self.experiment_dir, 'all.log', log_str)
if file_name is not None:
log_to_file_in_dir(self.experiment_dir, file_name, log_str)
def log_epoch_stats(self, start_time):
time_elapsed = datetime.now() - start_time
time_to_completion = time_elapsed * (self.epochs - self.current_epoch -
1)
train_loss = self.training_losses[self.current_epoch]
val_loss = self.val_losses[self.current_epoch]
summary_str = "Epoch: {}, Train Loss: {}, Val Loss: {}, Took {}, ETA: {}\n"
summary_str = summary_str.format(self.current_epoch + 1, train_loss,
val_loss, str(time_elapsed),
str(time_to_completion))
self.log(summary_str, 'epoch.log')
