def test_dummylogger_noop_method_calls():
"""Test that the DummyLogger methods can be called with arbitrary arguments."""
logger = DummyLogger()
logger.log_hyperparams("1", 2, three="three")
logger.log_metrics("1", 2, three="three")
def train(**args):
params = EasyDict(args)
params.gpu = int(params.gpu)
config = ConfigParser()
config.read('config.ini')
if params.datasets == ['all']:
params.datasets = ['imdb', 'amazon', 'yelp', 'rottentomatoes', 'hotel']
is_tokenizer_length_dataset_specific = Models(params.model) == Models.distilbert and (
params.tokenizer_length is None or params.tokenizer_length)
is_number_prototypes_dataset_specific = Models(params.model) == Models.protoconv and (
params.pc_number_of_prototypes is None or params.pc_number_of_prototypes == -1)
is_sep_loss_dataset_specific = Models(params.model) == Models.protoconv and (
params.pc_sep_loss_weight is None or params.pc_sep_loss_weight == -1)
if_ce_loss_dataset_specific = Models(params.model) == Models.protoconv and (
params.pc_ce_loss_weight is None or params.pc_ce_loss_weight == -1)
for dataset in params.datasets:
params.data_set = dataset
seed_everything(params.seed)
if is_tokenizer_length_dataset_specific:
params.tokenizer_length = dataset_tokens_length[params.data_set]
if is_number_prototypes_dataset_specific:
params.pc_number_of_prototypes = dataset_to_number_of_prototypes[params.data_set]
if is_sep_loss_dataset_specific:
params.pc_sep_loss_weight = dataset_to_separation_loss[params.data_set]
if if_ce_loss_dataset_specific:
weight = 1 - (params.pc_cls_loss_weight + params.pc_sep_loss_weight + params.pc_l1_loss_weight)
assert weight > 0, f'Weight {weight} of cross entropy loss cannot be less or equal to 0'
params.pc_ce_loss_weight = weight
logger = DummyLogger()
if params.logger:
comet_config = EasyDict(config['cometml'])
project_name = params.project_name if params.project_name else comet_config.projectname
logger = CometLogger(api_key=comet_config.apikey, project_name=project_name,
workspace=comet_config.workspace)
# logger.experiment.log_code(folder='src')
logger.log_hyperparams(params)
base_callbacks = [LearningRateMonitor(logging_interval='epoch')]
df_dataset = pd.read_csv(f'data/{params.data_set}/tokenized_data.csv')
n_splits = get_n_splits(dataset=df_dataset, x_label='text', y_label='label', folds=params.fold)
log_splits(n_splits, logger)
embeddings = GloVe('42B', cache=params.cache) if Models(params.model) != Models.distilbert else None
best_models_scores, number_of_prototypes = [], []
for fold_id, (train_index, val_index, test_index) in enumerate(n_splits):
i = str(fold_id)
model_checkpoint = ModelCheckpoint(
filepath='checkpoints/fold_' + i + '_{epoch:02d}-{val_loss_' + i + ':.4f}-{val_acc_' + i + ':.4f}',
save_weights_only=True, save_top_k=1, monitor='val_acc_' + i,
period=params.pc_project_prototypes_every_n
)
early_stop = EarlyStopping(monitor=f'val_loss_{i}', patience=10, verbose=True, mode='min', min_delta=0.005)
callbacks = deepcopy(base_callbacks) + [model_checkpoint, early_stop]
lit_module = model_to_litmodule[params.model]
train_df, valid_df = df_dataset.iloc[train_index + val_index], df_dataset.iloc[test_index]
model, train_loader, val_loader, *utils = lit_module.from_params_and_dataset(train_df, valid_df, params,
fold_id, embeddings)
trainer = Trainer(auto_lr_find=params.find_lr, logger=logger, max_epochs=params.epoch, callbacks=callbacks,
gpus=params.gpu, deterministic=True, fast_dev_run=params.fast_dev_run,
num_sanity_val_steps=0)
trainer.tune(model, train_dataloader=train_loader, val_dataloaders=val_loader)
trainer.fit(model, train_dataloader=train_loader, val_dataloaders=val_loader)
for absolute_path in model_checkpoint.best_k_models.keys():
logger.experiment.log_model(Path(absolute_path).name, absolute_path)
if model_checkpoint.best_model_score:
best_models_scores.append(model_checkpoint.best_model_score.tolist())
logger.log_metrics({'best_model_score_' + i: model_checkpoint.best_model_score.tolist()}, step=0)
if Models(params.model) == Models.protoconv and model_checkpoint.best_model_path:
best_model = lit_module.load_from_checkpoint(model_checkpoint.best_model_path)
saved_number_of_prototypes = sum(best_model.enabled_prototypes_mask.tolist())
number_of_prototypes.append(saved_number_of_prototypes)
logger.log_hyperparams({
f'saved_prototypes_{fold_id}': saved_number_of_prototypes,
f'best_model_path_{fold_id}': str(Path(model_checkpoint.best_model_path).name)
})
if params.pc_visualize:
data_visualizer = DataVisualizer(best_model)
logger.experiment.log_html(f'<h1>Split {fold_id}</h1><br> <h3>Prototypes:</h3><br>'
f'{data_visualizer.visualize_prototypes()}<br>')
logger.experiment.log_figure(f'Prototypes similarity_{fold_id}',
data_visualizer.visualize_similarity().figure)
logger.experiment.log_html(f'<h3>Random prediction explanations:</h3><br>'
f'{data_visualizer.visualize_random_predictions(val_loader, n=15)}')
if len(best_models_scores) >= 1:
avg_best, std_best = float(np.mean(np.array(best_models_scores))), float(
np.std(np.array(best_models_scores)))
table_entry = f'{avg_best:.3f} ($\pm${std_best:.3f})'
logger.log_hyperparams({
'avg_best_scores': avg_best,
'std_best_scores': std_best,
'table_entry': table_entry
})
if len(number_of_prototypes) >= 1:
logger.log_hyperparams({'avg_saved_prototypes': float(np.mean(np.array(number_of_prototypes)))})
logger.experiment.end()
