def __init__(self, logger, config):
if torch.cuda.is_available():
self.device = torch.device('cuda')
else:
self.device = torch.device('cpu')
self.logger = logger
self.train_config = registry.instantiate(TrainConfig, config['train'])
self.data_random = random_state.RandomContext(
self.train_config.data_seed)
self.model_random = random_state.RandomContext(
self.train_config.model_seed)
self.init_random = random_state.RandomContext(
self.train_config.init_seed)
with self.init_random:
# 0. Construct preprocessors
self.model_preproc = registry.instantiate(registry.lookup(
'model', config['model']).Preproc,
config['model'],
unused_keys=('name', ))
self.model_preproc.load()
# 1. Construct model
self.model = registry.construct('model',
config['model'],
unused_keys=('encoder_preproc',
'decoder_preproc'),
preproc=self.model_preproc,
device=self.device)
self.model.to(self.device)
def __init__(self, logger, config, gpu):
if torch.cuda.is_available():
self.device = torch.device('cuda:{}'.format(gpu))
else:
self.device = torch.device('cpu')
random.seed(1)
numpy.random.seed(1)
torch.manual_seed(1)
torch.cuda.manual_seed_all(1)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
self.logger = logger
self.train_config = registry.instantiate(TrainConfig, config['train'])
self.train_config.eval_every_n = 500
self.train_config.save_every_n = 500
self.data_random = random_state.RandomContext(
self.train_config.data_seed)
self.model_random = random_state.RandomContext(
self.train_config.model_seed)
self.init_random = random_state.RandomContext(
self.train_config.init_seed)
with self.init_random:
# 0. Construct preprocessors
self.model_preproc = registry.instantiate(registry.lookup(
'model', config['model']).Preproc,
config['model'],
unused_keys=('name', ))
self.model_preproc.load()
# 1. Construct model
self.model = registry.construct('model',
config['model'],
unused_keys=('encoder_preproc',
'decoder_preproc'),
preproc=self.model_preproc,
device=self.device)
self.model.to(self.device)
def finetune(self, config, model_load_dir, model_save_dir,
infer_output_path, beam_size, output_history, use_heuristic):
random_seeds = [i for i in range(3)]
orig_data = registry.construct('dataset', self.config['data']['val'])
databases = orig_data.get_databases()
for seed in random_seeds:
data_random = random_state.RandomContext(seed)
print("seed:", seed)
metrics_list = []
batch_1_scores = []
no_grad_scores = []
batch_32_scores = []
n_2_scores = []
with data_random:
# print("No grad")
# no_grad_infer_output_path = infer_output_path + "no_grad/no_grad.infer"
# os.makedirs(os.path.dirname(no_grad_infer_output_path), exist_ok=False)
# print(no_grad_infer_output_path)
# for database in databases:
#
# self.finetune_on_database(no_grad_infer_output_path, database, config, model_load_dir,
# beam_size, output_history, use_heuristic, metrics_list, no_grad_scores,
# take_grad_steps=False, batch_size="1")
# print("No grad scores", no_grad_scores)
# print("average", self.aggregate_score(no_grad_scores))
no_grad_scores = [
('dog_kennels', 0.5, 82), ('flight_2', 0.5875, 80),
('pets_1', 0.4523809523809524, 42),
('concert_singer', 0.5333333333333333, 45),
('museum_visit', 0.4444444444444444, 18),
('battle_death', 0.5625, 16),
('student_transcripts_tracking', 0.48717948717948717, 78),
('singer', 0.7333333333333333, 30),
('cre_Doc_Template_Mgt', 0.7023809523809523, 84),
('world_1', 0.19166666666666668, 120),
('employee_hire_evaluation', 0.8421052631578947, 38),
('network_1', 0.6428571428571429, 56),
('poker_player', 0.875, 40),
('real_estate_properties', 0.25, 4),
('course_teach', 0.7333333333333333, 30),
('voter_1', 0.4666666666666667, 15), ('wta_1', 0.5, 62),
('orchestra', 0.85, 40),
('car_1', 0.32608695652173914, 92),
('tvshow', 0.6612903225806451, 62)
]
average = self.aggregate_score(no_grad_scores)
no_grad_scores.append(("average", average))
# self.plot(no_grad_scores, "no_grad_scores.png", "no grad scores")
# print("No grad scores", no_grad_scores)
# print("average", average)
#
# print("batch size 1")
# batch_1_infer_output_path = infer_output_path + "seed_"+str(seed)+"/batch_1/batch_1.infer"
# os.makedirs(os.path.dirname(batch_1_infer_output_path), exist_ok=False)
# print(batch_1_infer_output_path)
# for database in databases:
# self.finetune_on_database(batch_1_infer_output_path, database, config, model_load_dir,
# beam_size, output_history, use_heuristic, metrics_list, batch_1_scores,
# take_grad_steps=True, batch_size="1")
# average = self.aggregate_score(batch_1_scores)
# batch_1_scores.append(("average", average))
# self.plot(batch_1_scores, "batch_1_scores_seed_"+str(seed)+".png", "batch size 1 scores seed "+ str(seed))
# print("batch size 1 scores", batch_1_scores)
# print("average", average)
#
# print("batch size 32")
# batch_32_infer_output_path = infer_output_path + "seed_"+str(seed)+"/batch_32/batch_32.infer"
# os.makedirs(os.path.dirname(batch_32_infer_output_path), exist_ok=False)
# print(batch_32_infer_output_path)
# for database in databases:
# self.finetune_on_database(batch_32_infer_output_path, database, config, model_load_dir,
# beam_size, output_history, use_heuristic, metrics_list, batch_32_scores,
# take_grad_steps=True, batch_size="32")
# average = self.aggregate_score(batch_32_scores)
# batch_32_scores.append(("average", average))
# self.plot(batch_32_scores, "batch_32_scores_seed_"+str(seed)+".png", "batch size 32 scores seed " + str(seed))
# print("batch size 32 scores", batch_32_scores)
# print("average",average)
print("n^2")
n_2_infer_output_path = infer_output_path + "seed_" + str(
seed) + "/n_2/n_2.infer"
os.makedirs(os.path.dirname(n_2_infer_output_path),
exist_ok=False)
print(n_2_infer_output_path)
for database in databases:
self.finetune_on_database(n_2_infer_output_path,
database,
config,
model_load_dir,
beam_size,
output_history,
use_heuristic,
metrics_list,
n_2_scores,
take_grad_steps=True,
batch_size="n^2")
average = self.aggregate_score(n_2_scores)
n_2_scores.append(("average", average))
self.plot(n_2_scores,
"n_2_scores_no_repeat_seed_" + str(seed) + ".png",
"batch n^2 scores no repeat seed " + str(seed))
print("n^2 scores", n_2_scores)
print("average", average)
# print("Score on entire validation set:")
# self.finetune_on_database(infer_output_path, None, config, model_load_dir,
# beam_size, output_history, use_heuristic, metrics_list, scores, take_grad_steps=False)
print("")
print("changes")
# print("batch size 1 changes")
# self.plot(self.get_change(no_grad_scores, batch_1_scores),
# "batch_size_1_changes_seed_"+str(seed)+".png",
# "batch size 1 score changes")
# print(self.get_change(no_grad_scores, batch_1_scores))
#
# print("batch size 32 changes")
# self.plot(self.get_change(no_grad_scores, batch_32_scores),
# "batch_size_32_changes_seed_" + str(seed) + ".png",
# "batch size 32 score changes")
# print(self.get_change(no_grad_scores, batch_32_scores))
print("batch size n^2 changes")
self.plot(
self.get_change(no_grad_scores, n_2_scores),
"batch_size_n_2_no_repeat_changes_seed_" + str(seed) +
".png",
"batch size n^2 score changes with no repeat queries")
print(self.get_change(no_grad_scores, n_2_scores))