def __init__(self, opt, emb_matrix=None):
print(">> Current Model: StudentNetwork")
super(StudentNetwork, self).__init__()
if opt['base_mode'] == 0: # start from scratch
print(" Do not use base model.")
self.inst_encoder = BaseNetwork(opt=opt, emb_matrix=emb_matrix)
else:
self.base_model_file = opt['save_dir'] + '/' + opt['base_id'] + '/best_model.pt'
self.base_opt = torch_utils.load_config(self.base_model_file)
if opt['base_mode'] == 1: # load & fine tune
print(" Fine-tune base model.")
inst_base_model = BaseModel(self.base_opt)
inst_base_model.load(self.base_model_file)
self.inst_encoder = inst_base_model.model
elif opt['base_mode'] == 2: # load & fix pre-trained
print(" Fix pre-trained base model.")
inst_base_model = BaseModel(self.base_opt)
inst_base_model.load(self.base_model_file)
inst_base_model = inst_base_model.model
for param in inst_base_model.parameters():
param.requires_grad = False
inst_base_model.eval()
self.inst_encoder = inst_base_model
else:
print('Illegal Parameter (base_mode).')
assert False
self.pe_emb = nn.Embedding(constant.MAX_LEN * 2 + 1, opt['pe_dim_attn'])
self.ner_emb = nn.Embedding(constant.NER_NUM, opt['ner_dim_attn'])
self.attn_layer = MultiAspectAttention(opt)
self.final_linear = nn.Linear(2 * opt['hidden_dim'], opt['num_class'])
self.opt = opt
self.init_weights()
def __init__(self, opt, emb_matrix=None):
print(">> Current Model: TeacherNetwork")
super(TeacherNetwork, self).__init__()
if opt['base_mode'] == 0: # start from scratch
print(" Do not use base model.")
self.inst_encoder = BaseNetwork(opt=opt, emb_matrix=emb_matrix)
else:
self.base_model_file = opt['save_dir'] + '/' + opt[
'base_id'] + '/best_model.pt'
self.base_opt = torch_utils.load_config(self.base_model_file)
if opt['base_mode'] == 1: # load & fine tune
print(" Fine-tune base model.")
inst_base_model = BaseModel(self.base_opt)
inst_base_model.load(self.base_model_file)
self.inst_encoder = inst_base_model.model
elif opt['base_mode'] == 2: # load & fix pre-trained
print(" Fix pre-trained base model.")
inst_base_model = BaseModel(self.base_opt)
inst_base_model.load(self.base_model_file)
inst_base_model = inst_base_model.model
for param in inst_base_model.parameters():
param.requires_grad = False
inst_base_model.eval()
self.inst_encoder = inst_base_model
else:
print('Illegal Parameter (base_mode).')
assert False
self.rel_matrix = nn.Embedding(
opt['num_class'],
opt['num_class'],
padding_idx=constant.LABEL_TO_ID['no_relation'])
self.opt = opt
self.init_weights()
def evaluate_model(evalparams):
torch.manual_seed(evalparams.seed)
random.seed(1234)
if evalparams.cpu:
evalparams.cuda = False
elif evalparams.cud:
torch.cuda.manual_seed(args.seed)
# load opt
print(evalparams.model_dir, evalparams.model)
# model_file = evalparams.model_dir + "/" + evalparams.model
model_file = 'best_model.pt'
print("Loading model from {}".format(model_file))
opt = torch_utils.load_config(model_file)
model = RelationModel(opt)
model.load(model_file)
# load vocab
vocab_file = evalparams.model_dir + '/vocab.pkl'
vocab = Vocab(vocab_file, load=True)
assert opt[
'vocab_size'] == vocab.size, "Vocab size must match that in the saved model."
# load data
data_file = opt['data_dir'] + '/{}.json'.format(evalparams.dataset)
print("Loading data from {} with batch size {}...".format(
data_file, opt['batch_size']))
batch = DataLoader(data_file,
opt['batch_size'],
opt,
vocab,
evaluation=True)
helper.print_config(opt)
id2label = dict([(v, k) for k, v in constant.LABEL_TO_ID.items()])
predictions = []
all_probs = []
for i, b in enumerate(batch):
preds, probs, _ = model.predict(b)
predictions += preds
all_probs += probs
predictions = [id2label[p] for p in predictions]
p, r, f1 = scorer.score(batch.gold(), predictions, verbose=True)
# save probability scores
if len(evalparams.out) > 0:
helper.ensure_dir(os.path.dirname(evalparams.out))
with open(evalparams.out, 'wb') as outfile:
pickle.dump(all_probs, outfile)
print("Prediction scores saved to {}.".format(evalparams.out))
print("Evaluation ended.")
return (batch.gold(), predictions, model)
def load_best_model(model_dir, model_type="predictor"):
model_file = model_dir + "/best_model.pt"
print("Loading model from {}".format(model_file))
model_opt = torch_utils.load_config(model_file)
if model_type == "predictor":
predictor = Predictor(model_opt)
model = Trainer(model_opt, predictor, model_type=model_type)
else:
selector = Selector(model_opt)
model = Trainer(model_opt, selector, model_type=model_type)
model.load(model_file)
helper.print_config(model_opt)
return model
def __init__(self, model_files):
self.models = []
for model_file in model_files:
opt = torch_utils.load_config(model_file)
model = GCNClassifier(opt)
checkpoint = self.get_checkpoint(model_file)
model.load_state_dict(checkpoint['model'])
if opt['cuda']:
model.cuda()
self.models.append(model)
def load_data(args):
data_path = args.data_dir + '/' + args.dataset + '.json'
with open(data_path, 'r') as f:
data = json.load(f)
f.close()
# load opt
model_file = args.model_dir + '/' + args.model
print("Loading model from {}".format(model_file))
opt = torch_utils.load_config(model_file)
model = RelationModel(opt)
model.load(model_file)
# load vocab
vocab_file = args.model_dir + '/vocab.pkl'
vocab = Vocab(vocab_file, load=True)
assert opt[
'vocab_size'] == vocab.size, "Vocab size must match that in the saved model."
# helper.print_config(opt)
id2label = dict([(v, k) for k, v in constant.LABEL_TO_ID.items()])
def modelfn(inp):
batch = DataLoader(json.dumps([inp]),
3,
opt,
vocab,
evaluation=True,
load_from_file=False)
predictions = []
all_probs = []
for i, b in enumerate(batch):
preds, probs, _ = model.predict(b)
predictions += preds
all_probs += probs
predictions = [id2label[p] for p in predictions]
return all_probs[0], predictions
sim_dict = load_similarity_dict()
return data, sim_dict, modelfn
def __init__(self,
model_stuff_list: List[ModelStuff],
biassed_prediction=None):
self.id2label = dict([(v, k) for k, v in constant.LABEL_TO_ID.items()])
self.models = OrderedDict()
self.biassed_prediction = biassed_prediction
for model_stuff in model_stuff_list:
self.models[model_stuff.representation] = []
for model_file in model_stuff.files:
opt = torch_utils.load_config(model_file)
model = GCNClassifier(opt)
checkpoint = self.get_checkpoint(model_file)
model.load_state_dict(checkpoint['model'])
if opt['cuda']:
model.cuda()
self.models[model_stuff.representation].append(model)
def get_biased_model_class_probs(args):
# load opt
model_file = args.model_dir + "/" + args.model
print("Loading model from {}".format(model_file))
opt = torch_utils.load_config(model_file)
model = RelationModel(opt)
model.load(model_file)
# load vocab
vocab_file = args.model_dir + "/vocab.pkl"
vocab = Vocab(vocab_file, load=True)
assert opt[
"vocab_size"] == vocab.size, "Vocab size must match that in the saved model."
opt["vocab_size"] = vocab.size
emb_file = opt["vocab_dir"] + "/embedding.npy"
emb_matrix = np.load(emb_file)
assert emb_matrix.shape[0] == vocab.size
assert emb_matrix.shape[1] == opt["emb_dim"]
# load data
data_file = args.data_dir + "/{}".format(args.data_name)
print("Loading data from {} with batch size {}...".format(
data_file, opt["batch_size"]))
batch = DataLoader(data_file,
opt["batch_size"],
opt,
vocab,
evaluation=True)
# helper.print_config(opt)
id2label = dict([(v, k) for k, v in constant.LABEL_TO_ID.items()])
all_probs = []
for i, b in enumerate(batch):
preds, probs, _ = model.predict(b)
all_probs.append(probs)
return all_probs
vocab.save(model_save_dir + '/vocab.pkl')
file_logger = helper.FileLogger(
model_save_dir + '/' + opt['log'],
header="# epoch\ttrain_loss\tdev_loss\tdev_score\tbest_dev_score")
# print model info
helper.print_config(opt)
# model
if not opt['load']:
trainer = GCNTrainer(opt, emb_matrix=emb_matrix)
else:
# load pretrained model
model_file = opt['model_file']
print("Loading model from {}".format(model_file))
model_opt = torch_utils.load_config(model_file)
model_opt['optim'] = opt['optim']
trainer = GCNTrainer(model_opt)
trainer.load(model_file)
id2label = dict([(v, k) for k, v in label2id.items()])
dev_score_history = []
current_lr = opt['lr']
global_step = 0
global_start_time = time.time()
format_str = '{}: step {}/{} (epoch {}/{}), loss = {:.6f} ({:.3f} sec/batch), lr: {:.6f}'
max_steps = len(train_batch) * opt['num_epoch']
# start training
for epoch in range(1, opt['num_epoch'] + 1):
parser.add_argument('--seed', type=int, default=1234)
parser.add_argument('--cuda', type=bool, default=torch.cuda.is_available())
parser.add_argument('--cpu', action='store_true')
args = parser.parse_args()
torch.manual_seed(args.seed)
random.seed(1234)
if args.cpu:
args.cuda = False
elif args.cuda:
torch.cuda.manual_seed(args.seed)
# load opt
model_p_file = args.p_dir + '/' + args.p_model
print("Loading predictor from {}".format(model_p_file))
opt_p = torch_utils.load_config(model_p_file)
predictor = Predictor(opt_p)
model_p = PredictorTrainer(opt_p, predictor)
model_p.load(model_p_file)
model_s_file = args.s_dir + '/' + args.s_model
print("Loading selector from {}".format(model_s_file))
opt_s = torch_utils.load_config(model_s_file)
selector = Predictor(opt_s)
model_s = SelectorTrainer(opt_s, selector)
model_s.load(model_s_file)
# load vocab
TOKEN = data.Field(sequential=True,
batch_first=True,
lower=True,
def transre_search(ffn, connect, hidden_dim, trans_layers, multi_heads,
ffn_ex_size, initial, final):
opt['weighted'] = False
opt['rnn'] = False
opt['ffn'] = ffn
opt['connect'] = connect
opt['hidden_dim'] = hidden_dim
opt['trans_layers'] = trans_layers
opt['multi_heads'] = multi_heads
opt['ffn_ex_size'] = ffn_ex_size
opt['initial'] = initial
opt['final'] = final
id = opt['id'] if len(opt['id']) > 1 else '0' + opt['id']
model_name =str (opt['optim']) + '_' + str (opt['lr']) + str (ffn) + '_' +str(connect)+"_"\
+ str (hidden_dim) + '_' + str (trans_layers) + '_' + str (multi_heads) + '_' + \
str (ffn_ex_size)+'_'+str(initial)+'_'+str(final)
model_name = model_name + '' + str(opt['memo'])
model_name = str(id) + "_" + model_name
model_save_dir = opt['save_dir'] + '/' + model_name
opt['model_save_dir'] = model_save_dir
helper.ensure_dir(model_save_dir, verbose=True)
# save config
helper.save_config(opt, model_save_dir + '/config.json', verbose=True)
vocab.save(model_save_dir + '/vocab.pkl')
file_logger = helper.FileLogger(
model_save_dir + '/' + opt['log'],
header="# epoch\ttrain_loss\tdev_loss\tdev_score\tbest_dev_score")
helper.print_config(opt)
if not opt['load']:
trainer = TransTrainer(opt, emb_matrix=emb_matrix)
else:
# load pre-train model
model_file = opt['model_file']
print("Loading model from {}".format(model_file))
model_opt = torch_utils.load_config(model_file)
model_opt['optim'] = opt['optim']
trainer = TransTrainer(model_opt)
trainer.load(model_file)
id2label = dict([(v, k) for k, v in label2id.items()
]) # the classification result
dev_score_history = []
dev_loss_history = []
current_lr = opt['lr']
global_step = 0
format_str = '{}: step {}/{} (epoch {}/{}), loss = {:.6f} ({:.3f} sec/batch), lr: {:.6f}'
max_steps = len(train_batch) * opt['num_epoch']
best_result = "unknown"
file_logger.log(str(opt['memo']))
for epoch in range(1, opt['num_epoch'] + 1):
train_loss = 0
epoch_start_time = time.time()
for i, batch in enumerate(train_batch):
start_time = time.time()
global_step += 1
loss, norm = trainer.update(batch)
train_loss += loss
if global_step % opt['log_step'] == 0:
duration = time.time() - start_time
print(
format_str.format(datetime.now(), global_step, max_steps,
epoch, opt['num_epoch'], loss, duration,
current_lr))
print("Evaluating on dev set...")
predictions = []
dev_loss = 0
for i, batch in enumerate(dev_batch):
preds, _, loss, _ = trainer.predict(batch)
predictions += preds
dev_loss += loss
predictions = [id2label[p] for p in predictions]
train_loss = train_loss / train_batch.num_examples * opt[
'batch_size'] # avg loss per batch
dev_loss = dev_loss / dev_batch.num_examples * opt['batch_size']
acc, dev_p, dev_r, dev_f1 = scorer.score(dev_batch.gold(), predictions)
print(
"epoch {}: train_loss = {:.6f}, dev_loss = {:.6f}, dev_f1 = {:.4f}"
.format(epoch, train_loss, dev_loss, dev_f1))
dev_score = dev_f1
file_logger.log("{}\t{:.3f}\t{:.6f}\t{:.6f}\t{:.4f}\t{:.4f}".format(
epoch, acc, train_loss, dev_loss, dev_score,
max([dev_score] + dev_score_history)))
# save
model_file = model_save_dir + '/checkpoint_epoch_{}.pt'.format(epoch)
trainer.save(model_file, epoch)
if epoch == 1 or dev_score > max(dev_score_history):
copyfile(model_file, model_save_dir + '/best_model.pt')
best_result = (model_name, dev_score)
print("new best model saved.")
file_logger.log(
"new best model saved at epoch {}: {:.2f}\t{:.2f}\t{:.2f}".
format(epoch, dev_p * 100, dev_r * 100, dev_score * 100))
if epoch % opt['save_epoch'] != 0:
os.remove(model_file)
# lr schedule
if len(dev_score_history
) > opt['decay_epoch'] and dev_score <= dev_score_history[
-1] and opt['optim'] in ['sgd', 'adagrad', 'adadelta']:
current_lr *= opt['lr_decay']
trainer.update_lr(current_lr)
dev_score_history += [dev_score]
dev_loss_history += [dev_loss]
epoch_end_time = time.time()
print("epoch time {:.3f}".format(epoch_end_time - epoch_start_time))
return best_result
from torch.autograd import Variable
from sklearn import metrics
from utils import constant, helper, torch_utils
from utils import helper
from data.loader import DataLoader,read_tsv,split_test_data
from model.trainer import MyTrainer
parser = argparse.ArgumentParser()
parser.add_argument('--data_dir', type=str, default='dataset', help='Dictionary of the test file.')
parser.add_argument('--test_filename', type=str, default='test.tsv', help='Name of the test file.')
parser.add_argument('--model_dir', type=str, default='saved_models/grained/best_model.pt', help='Directory of the model.')
args = parser.parse_args()
print("Loading model from {}".format(args.model_dir))
opt = torch_utils.load_config(args.model_dir)
helper.print_config(opt)
model = MyTrainer(opt)
model.load(args.model_dir)
print("Loading data from {} with batch size {}...".format(os.path.join(args.data_dir, args.test_filename), opt['batch_size']))
# split_test_data for multi
if opt['type'] == 'multi':
split_test_data(opt['coarse_name'])
is_multi_eval = False
if opt['type'] == 'multi':
is_multi_eval = True
test_batch = DataLoader(os.path.join(args.data_dir, args.test_filename), opt['batch_size'], opt, is_multi_eval)
helper.save_config(opt, model_save_dir + '/config.json', verbose=True)
vocab.save(model_save_dir + '/vocab.pkl')
file_logger = helper.FileLogger(
model_save_dir + '/' + opt['log'],
header="# epoch\ttrain_loss\tdev_loss\tdev_f1\ttest_loss\ttest_f1")
# print model info
helper.print_config(opt)
# model
base_outputs = ['placeholder' for _ in range(0, len(train_batch))]
if opt['base_mode'] == 3:
base_outputs = []
base_model_file = opt['save_dir'] + '/' + opt['base_id'] + '/best_model.pt'
print("Loading base model from {}".format(base_model_file))
base_opt = torch_utils.load_config(base_model_file)
base_model = BaseModel(opt=base_opt)
base_model.load(base_model_file)
base_model.model.eval()
for _, batch in enumerate(train_batch):
inputs = [b.cuda() for b in batch[:10]
] if opt['cuda'] else [b for b in batch[:10]]
base_logits, _, _ = base_model.model(inputs)
base_outputs.append([base_logits.data.cpu().numpy()])
teacher_outputs = []
if opt['use_teacher']:
teacher_model_file = opt['save_dir'] + '/' + opt[
'teacher_id'] + '/best_model.pt'
print("Loading teacher model from {}".format(teacher_model_file))
teacher_opt = torch_utils.load_config(teacher_model_file)
def main():
args = get_parser()
# set seed and prepare for training
torch.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
if args.cpu:
args.cuda = False
elif args.cuda:
torch.cuda.manual_seed(args.seed)
init_time = time.time()
# make opt
opt = vars(args)
TEXT, train_batch, dev_batch = load_data(opt['batch_size'],
device='cuda:0')
vocab = TEXT.vocab
opt['vocab_size'] = len(vocab.stoi)
emb_matrix = vocab.vectors
assert emb_matrix.shape[0] == opt['vocab_size']
assert emb_matrix.shape[1] == opt['emb_dim']
model_id = opt['id'] if len(opt['id']) > 1 else '0' + opt['id']
model_save_dir = opt['save_dir'] + '/' + str(model_id)
opt['model_save_dir'] = model_save_dir
helper.ensure_dir(model_save_dir, verbose=True)
# save config
path = os.path.join(model_save_dir, 'config.json')
helper.save_config(opt, path, verbose=True)
# vocab.save(os.path.join(model_save_dir, 'vocab.pkl'))
file_logger = helper.FileLogger(
os.path.join(model_save_dir, opt['log']),
header="# epoch\ttrain_loss\tdev_loss\tdev_score\tbest_dev_score")
# print model info
helper.print_config(opt)
# Build Model
if not opt['load']:
trainer = LSTMTrainer(opt, emb_matrix)
else:
model_file = opt['model_file']
print("Loading model from {}".format(model_file))
model_opt = torch_utils.load_config(model_file)
model_opt['optim'] = opt['optim']
trainer = LSTMTrainer(model_opt)
trainer.load(model_file)
dev_score_history = []
current_lr = opt['lr']
global_step = 0
global_start_time = time.time()
format_str = '{}: step {}/{} (epoch {}/{}), loss = {:.6f} ({:.3f} sec/batch), lr: {:.6f}'
max_steps = len(train_batch) * opt['num_epoch']
# start training
for epoch in range(1, opt['num_epoch'] + 1):
train_loss = 0
for i, batch in enumerate(train_batch):
start_time = time.time()
global_step += 1
loss = trainer.update(batch)
train_loss += loss
if global_step % opt['log_step'] == 0:
duration = time.time() - start_time
print(format_str.format(datetime.now(), global_step, max_steps, epoch, \
opt['num_epoch'], loss, duration, current_lr))
# eval on dev
print("Evaluating on dev set ...")
predictions = []
golds = []
dev_loss = 0.0
for i, batch in enumerate(dev_batch):
preds, probs, labels, loss = trainer.predict(batch)
predictions += preds
golds += labels
dev_loss += loss
train_loss = train_loss / len(train_batch)
dev_loss = dev_loss / len(dev_batch)
# print(golds)
# print(predictions)
print(accuracy_score(golds, predictions))
dev_roc = roc_auc_score(golds, predictions)
print(
"epoch {}: train loss = {:.6f}, dev loss = {:.6f}, dev roc = {:.4f}"
.format(epoch, train_loss, dev_loss, dev_roc))
dev_score = dev_roc
file_logger.log("{}\t{:.6f}\t{:.6f}\t{:.4f}\t{:.4f}".format(
epoch, train_loss, dev_loss, dev_score,
max([dev_score] + dev_score_history)))
# save model
model_file = os.path.join(model_save_dir,
"checkpoint_epoch_{}.py".format(epoch))
trainer.save(model_file, epoch)
if epoch == 1 or dev_score > max(dev_score_history):
copyfile(model_file, model_save_dir + '/best_model.pt')
print("new best model saved.")
file_logger.log("new best model saved at epoch {}: {:.2f}"\
.format(epoch, dev_score*100))
if epoch % opt['save_epoch'] != 0:
os.remove(model_file)
if len(dev_score_history) > opt['decay_epoch'] and dev_score <= dev_score_history[-1] and \
opt['optim'] in ['sgd', 'adagrad', 'adadelta']:
current_lr *= opt['lr_decay']
trainer.update_lr(current_lr)
dev_score_history += [dev_score]
print("")
print("Training ended with {} epochs.".format(epoch))
