def __init__(self, logger, config): if torch.cuda.is_available(): self.device = torch.device('cuda') else: self.device = torch.device('cpu') self.logger = logger self.train_config = registry.instantiate(TrainConfig, config['train']) self.data_random = random_state.RandomContext( self.train_config.data_seed) self.model_random = random_state.RandomContext( self.train_config.model_seed) self.init_random = random_state.RandomContext( self.train_config.init_seed) with self.init_random: # 0. Construct preprocessors self.model_preproc = registry.instantiate(registry.lookup( 'model', config['model']).Preproc, config['model'], unused_keys=('name', )) self.model_preproc.load() # 1. Construct model self.model = registry.construct('model', config['model'], unused_keys=('encoder_preproc', 'decoder_preproc'), preproc=self.model_preproc, device=self.device) self.model.to(self.device)
def __init__(self, logger, config, gpu): if torch.cuda.is_available(): self.device = torch.device('cuda:{}'.format(gpu)) else: self.device = torch.device('cpu') random.seed(1) numpy.random.seed(1) torch.manual_seed(1) torch.cuda.manual_seed_all(1) torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True self.logger = logger self.train_config = registry.instantiate(TrainConfig, config['train']) self.train_config.eval_every_n = 500 self.train_config.save_every_n = 500 self.data_random = random_state.RandomContext( self.train_config.data_seed) self.model_random = random_state.RandomContext( self.train_config.model_seed) self.init_random = random_state.RandomContext( self.train_config.init_seed) with self.init_random: # 0. Construct preprocessors self.model_preproc = registry.instantiate(registry.lookup( 'model', config['model']).Preproc, config['model'], unused_keys=('name', )) self.model_preproc.load() # 1. Construct model self.model = registry.construct('model', config['model'], unused_keys=('encoder_preproc', 'decoder_preproc'), preproc=self.model_preproc, device=self.device) self.model.to(self.device)
def finetune(self, config, model_load_dir, model_save_dir, infer_output_path, beam_size, output_history, use_heuristic): random_seeds = [i for i in range(3)] orig_data = registry.construct('dataset', self.config['data']['val']) databases = orig_data.get_databases() for seed in random_seeds: data_random = random_state.RandomContext(seed) print("seed:", seed) metrics_list = [] batch_1_scores = [] no_grad_scores = [] batch_32_scores = [] n_2_scores = [] with data_random: # print("No grad") # no_grad_infer_output_path = infer_output_path + "no_grad/no_grad.infer" # os.makedirs(os.path.dirname(no_grad_infer_output_path), exist_ok=False) # print(no_grad_infer_output_path) # for database in databases: # # self.finetune_on_database(no_grad_infer_output_path, database, config, model_load_dir, # beam_size, output_history, use_heuristic, metrics_list, no_grad_scores, # take_grad_steps=False, batch_size="1") # print("No grad scores", no_grad_scores) # print("average", self.aggregate_score(no_grad_scores)) no_grad_scores = [ ('dog_kennels', 0.5, 82), ('flight_2', 0.5875, 80), ('pets_1', 0.4523809523809524, 42), ('concert_singer', 0.5333333333333333, 45), ('museum_visit', 0.4444444444444444, 18), ('battle_death', 0.5625, 16), ('student_transcripts_tracking', 0.48717948717948717, 78), ('singer', 0.7333333333333333, 30), ('cre_Doc_Template_Mgt', 0.7023809523809523, 84), ('world_1', 0.19166666666666668, 120), ('employee_hire_evaluation', 0.8421052631578947, 38), ('network_1', 0.6428571428571429, 56), ('poker_player', 0.875, 40), ('real_estate_properties', 0.25, 4), ('course_teach', 0.7333333333333333, 30), ('voter_1', 0.4666666666666667, 15), ('wta_1', 0.5, 62), ('orchestra', 0.85, 40), ('car_1', 0.32608695652173914, 92), ('tvshow', 0.6612903225806451, 62) ] average = self.aggregate_score(no_grad_scores) no_grad_scores.append(("average", average)) # self.plot(no_grad_scores, "no_grad_scores.png", "no grad scores") # print("No grad scores", no_grad_scores) # print("average", average) # # print("batch size 1") # batch_1_infer_output_path = infer_output_path + "seed_"+str(seed)+"/batch_1/batch_1.infer" # os.makedirs(os.path.dirname(batch_1_infer_output_path), exist_ok=False) # print(batch_1_infer_output_path) # for database in databases: # self.finetune_on_database(batch_1_infer_output_path, database, config, model_load_dir, # beam_size, output_history, use_heuristic, metrics_list, batch_1_scores, # take_grad_steps=True, batch_size="1") # average = self.aggregate_score(batch_1_scores) # batch_1_scores.append(("average", average)) # self.plot(batch_1_scores, "batch_1_scores_seed_"+str(seed)+".png", "batch size 1 scores seed "+ str(seed)) # print("batch size 1 scores", batch_1_scores) # print("average", average) # # print("batch size 32") # batch_32_infer_output_path = infer_output_path + "seed_"+str(seed)+"/batch_32/batch_32.infer" # os.makedirs(os.path.dirname(batch_32_infer_output_path), exist_ok=False) # print(batch_32_infer_output_path) # for database in databases: # self.finetune_on_database(batch_32_infer_output_path, database, config, model_load_dir, # beam_size, output_history, use_heuristic, metrics_list, batch_32_scores, # take_grad_steps=True, batch_size="32") # average = self.aggregate_score(batch_32_scores) # batch_32_scores.append(("average", average)) # self.plot(batch_32_scores, "batch_32_scores_seed_"+str(seed)+".png", "batch size 32 scores seed " + str(seed)) # print("batch size 32 scores", batch_32_scores) # print("average",average) print("n^2") n_2_infer_output_path = infer_output_path + "seed_" + str( seed) + "/n_2/n_2.infer" os.makedirs(os.path.dirname(n_2_infer_output_path), exist_ok=False) print(n_2_infer_output_path) for database in databases: self.finetune_on_database(n_2_infer_output_path, database, config, model_load_dir, beam_size, output_history, use_heuristic, metrics_list, n_2_scores, take_grad_steps=True, batch_size="n^2") average = self.aggregate_score(n_2_scores) n_2_scores.append(("average", average)) self.plot(n_2_scores, "n_2_scores_no_repeat_seed_" + str(seed) + ".png", "batch n^2 scores no repeat seed " + str(seed)) print("n^2 scores", n_2_scores) print("average", average) # print("Score on entire validation set:") # self.finetune_on_database(infer_output_path, None, config, model_load_dir, # beam_size, output_history, use_heuristic, metrics_list, scores, take_grad_steps=False) print("") print("changes") # print("batch size 1 changes") # self.plot(self.get_change(no_grad_scores, batch_1_scores), # "batch_size_1_changes_seed_"+str(seed)+".png", # "batch size 1 score changes") # print(self.get_change(no_grad_scores, batch_1_scores)) # # print("batch size 32 changes") # self.plot(self.get_change(no_grad_scores, batch_32_scores), # "batch_size_32_changes_seed_" + str(seed) + ".png", # "batch size 32 score changes") # print(self.get_change(no_grad_scores, batch_32_scores)) print("batch size n^2 changes") self.plot( self.get_change(no_grad_scores, n_2_scores), "batch_size_n_2_no_repeat_changes_seed_" + str(seed) + ".png", "batch size n^2 score changes with no repeat queries") print(self.get_change(no_grad_scores, n_2_scores))