def run_evaluator_on_latest_model(dataset, config='lstm'): config = configurations[config](dataset) latest_model = get_latest_model( os.path.join(config['training']['basepath'], config['training']['exp_dirname'])) evaluator = Evaluator(dataset, latest_model, _type=dataset.trainer_type) _ = evaluator.evaluate(dataset.test_data, save_results=True) return evaluator
def run_evaluator_on_latest_model(dataset, config="lstm"): print("EVALUATING LATEST MODEL") config = configurations[config](dataset) latest_model = get_latest_model( os.path.join(config["training"]["basepath"], config["training"]["exp_dirname"]) ) evaluator = Evaluator(dataset, latest_model, _type=dataset.trainer_type) _ = evaluator.evaluate(dataset.test_data, save_results=True) return evaluator
def generate_graphs_on_latest_model(dataset, config='lstm'): config = configurations[config](dataset) latest_model = get_latest_model( os.path.join(config['training']['basepath'], config['training']['exp_dirname'])) evaluator = Evaluator(dataset, latest_model, _type=dataset.trainer_type) _ = evaluator.evaluate(dataset.test_data, save_results=False) generate_graphs(dataset, config['training']['exp_dirname'], evaluator.model, test_data=dataset.test_data)
def train_dataset(dataset, args, config='lstm') : config = generate_config(dataset, args, config) trainer = Trainer(dataset, args, config=config) #go ahead and save model dirname = trainer.model.save_values(save_model=False) print("DIRECTORY:", dirname) if args.adversarial : trainer.train_adversarial(dataset.train_data, dataset.test_data, args) else : trainer.train_standard(dataset.train_data, dataset.test_data, args, save_on_metric=dataset.save_on_metric) print('####################################') print("TEST RESULTS FROM BEST MODEL") evaluator = Evaluator(dataset, trainer.model.dirname, args) _ = evaluator.evaluate(dataset.test_data, save_results=True) return trainer, evaluator
def train_dataset(dataset, config='lstm'): try: config = configurations[config](dataset) trainer = Trainer(dataset, config=config, _type=dataset.trainer_type) trainer.train(dataset.train_data, dataset.dev_data, n_iters=8, save_on_metric=dataset.save_on_metric) evaluator = Evaluator(dataset, trainer.model.dirname, _type=dataset.trainer_type) _ = evaluator.evaluate(dataset.test_data, save_results=True) return trainer, evaluator except: return
def generate_graphs_on_latest_model(dataset, config="lstm"): print("GENERATING GRAPHS FOR EXPERIMENT ON LATEST MODEL") config = configurations[config](dataset) latest_model = get_latest_model( os.path.join(config["training"]["basepath"], config["training"]["exp_dirname"]) ) evaluator = Evaluator(dataset, latest_model, _type=dataset.trainer_type) _ = evaluator.evaluate(dataset.test_data, save_results=False) generate_graphs( dataset, config["training"]["exp_dirname"], evaluator.model, test_data=dataset.test_data, )
def train_dataset(dataset, config='lstm'): config = configurations[config](dataset) trainer = Trainer(dataset, config=config, _type=dataset.trainer_type) if hasattr(dataset, 'n_iter'): n_iters = dataset.n_iter else: n_iters = 8 trainer.train(dataset.train_data, dataset.dev_data, n_iters=n_iters, save_on_metric=dataset.save_on_metric) evaluator = Evaluator(dataset, trainer.model.dirname, _type=dataset.trainer_type) _ = evaluator.evaluate(dataset.test_data, save_results=True) return trainer, evaluator
def train_dataset_and_get_lime_explanations(dataset, encoders, num_iters=15): for e in encoders: config = configurations[e](dataset) trainer = Trainer(dataset, config=config, _type=dataset.trainer_type) trainer.train(dataset.train_data, dataset.dev_data, n_iters=num_iters, save_on_metric=dataset.save_on_metric) evaluator = Evaluator(dataset, trainer.model.dirname, _type=dataset.trainer_type) predictions, attentions = evaluator.evaluate(dataset.test_data, save_results=True) lime_explanations = trainer.model.get_lime_explanations( dataset.test_data.X) return predictions, attentions, lime_explanations
def train_dataset_and_get_gradient(dataset, encoders, num_iters=15): for e in encoders: config = configurations[e](dataset) trainer = Trainer(dataset, config=config, _type=dataset.trainer_type) trainer.train(dataset.train_data, dataset.dev_data, n_iters=num_iters, save_on_metric=dataset.save_on_metric) evaluator = Evaluator(dataset, trainer.model.dirname, _type=dataset.trainer_type) predictions, attentions = evaluator.evaluate(dataset.test_data, save_results=True) grads = evaluator.gradient_experiment_get_grads(dataset.test_data) from Trainers.PlottingBC import process_grads process_grads(grads) return predictions, attentions, grads
def train_dataset(dataset, config="lstm"): print("STARTING TRAINING") config = configurations[config](dataset) trainer = Trainer(dataset, config=config, _type=dataset.trainer_type) if hasattr(dataset, "n_iter"): n_iters = dataset.n_iter else: n_iters = 8 trainer.train( dataset.train_data, dataset.dev_data, n_iters=n_iters, save_on_metric=dataset.save_on_metric, ) evaluator = Evaluator(dataset, trainer.model.dirname, _type=dataset.trainer_type) _ = evaluator.evaluate(dataset.test_data, save_results=True) return trainer, evaluator
def train_dataset_and_get_atn_map(dataset, encoders, num_iters=15): for e in encoders: config = configurations[e](dataset) trainer = Trainer(dataset, config=config, _type=dataset.trainer_type) trainer.train(dataset.train_data, dataset.dev_data, n_iters=num_iters, save_on_metric=dataset.save_on_metric) train_losses = trainer.model.train_losses evaluator = Evaluator(dataset, trainer.model.dirname, _type=dataset.trainer_type) predictions, attentions = evaluator.evaluate(dataset.test_data, save_results=True) # evaluator = Evaluator(dataset, trainer.model.last_epch_dirname, # _type=dataset.trainer_type) predictions_lst_epch, attentions_lst_epch = evaluator.evaluate( dataset.test_data) return predictions, attentions, predictions_lst_epch, attentions_lst_epch, train_losses
def eval_swa_model(dataset, top_lvl_models_dir): dirs = [ d for d in os.listdir(top_lvl_models_dir) if 'enc.th' in os.listdir(os.path.join(top_lvl_models_dir, d)) ] Model = BC.Model swa = Model.init_from_config(os.path.join(top_lvl_models_dir, dirs[0])) swa.dirname = os.path.join(top_lvl_models_dir, dirs[0]) i = 1 for new_model_dir in dirs[1:]: new_model = BC.Model.init_from_config( os.path.join(top_lvl_models_dir, new_model_dir)) adding_params(swa.encoder, new_model.encoder, 1.0 / (i + 1)) adding_params(swa.decoder, new_model.decoder, 1.0 / (i + 1)) i += 1 # divide_all_params(swa.encoder, len(dirs)) # divide_all_params(swa.decoder, len(dirs)) evaluator = Evaluator(dataset, os.path.join(top_lvl_models_dir, dirs[0]), _type=dataset.trainer_type) evaluator.model = swa _ = evaluator.evaluate(dataset.test_data, save_results=True) return evaluator
def train_dataset_and_temp_scale(dataset, encoders): for e in encoders: config = configurations[e](dataset) trainer = Trainer(dataset, config=config, _type=dataset.trainer_type) trainer.train(dataset.train_data, dataset.dev_data, n_iters=8, save_on_metric=dataset.save_on_metric) evaluator = Evaluator(dataset, trainer.model.dirname, _type=dataset.trainer_type) print("Temperature-scaling..") orig_model = evaluator.model dev_x_tensor = BatchHolder(dataset.dev_data.X).seq dev_x_tensor_lengths = BatchHolder(dataset.dev_data.X).lengths dev_x_tensor_masks = BatchHolder(dataset.dev_data.X).masks valid_dataset = TensorDataset( dev_x_tensor, dev_x_tensor_lengths, dev_x_tensor_masks, torch.from_numpy(np.array(dataset.dev_data.y))) valid_loader = DataLoader(valid_dataset, batch_size=1) scaled_model = ModelWithTemperature(orig_model) scaled_model.set_temperature(valid_loader)
def run_evaluator_on_specific_model(dataset, model_path, config='lstm'): config = configurations[config](dataset) evaluator = Evaluator(dataset, model_path, _type=dataset.trainer_type) evaluator.model.temperature = config['training']['temperature'] _ = evaluator.evaluate(dataset.test_data, save_results=True) return evaluator