def load_test_model(opt, dummy_opt, model_path=None):
if model_path is None:
model_path = opt.models[0]
checkpoint = torch.load(model_path,
map_location=lambda storage, loc: storage)
fields = inputters.load_fields_from_vocab(
checkpoint['vocab'], data_type=opt.data_type)
model_opt = checkpoint['opt']
for arg in dummy_opt:
if arg not in model_opt:
model_opt.__dict__[arg] = dummy_opt[arg]
#if not hasattr(model_opt, 'model_mode'):
model_opt.model_mode = opt.model_mode
model_opt.model_mode2 = opt.model_mode2
model_opt.model_ffn_mode = opt.model_ffn_mode
print(
"[onmt.model_builder.py] model_opt.model_mode: {}, model_opt.model_mode2: {}, model_opt.model_ffn_mode: {}"
.format(model_opt.model_mode, model_opt.model_mode2, model_opt.model_ffn_mode)
)
model = build_base_model(model_opt, fields, use_gpu(opt), checkpoint)
model.eval()
model.generator.eval()
return fields, model, model_opt
def load_lm_bias_test_model(opt, dummy_opt, model_path=None):
if model_path is None:
model_path = opt.models[0]
checkpoint = torch.load(model_path,
map_location=lambda storage, loc: storage)
fields = inputters.load_fields_from_vocab(checkpoint['vocab'],
data_type=opt.data_type)
model_opt = checkpoint['opt']
for arg in dummy_opt:
if arg not in model_opt:
model_opt.__dict__[arg] = dummy_opt[arg]
lm_out_checkpoint = torch.load(opt.lm_out,
map_location=lambda storage, loc: storage)
lm_in_checkpoint = torch.load(opt.lm_in,
map_location=lambda storage, loc: storage)
model = build_lm_bias_base_model(model_opt, fields, use_gpu(opt),
checkpoint, lm_out_checkpoint,
lm_in_checkpoint)
model.eval()
model.generator.eval()
model.lm_out.eval()
model.lm_out.generator.eval()
model.lm_in.eval()
model.lm_in.generator.eval()
return fields, model, model_opt
def load_test_model(opt,
dummy_opt,
FeatureValues,
FeatureTensors,
FeatureTypes,
FeaturesList,
FeatureNames,
FTInfos,
FeatureTypesNames,
SimulationLanguages,
model_path=None):
if model_path is None:
model_path = opt.models[0]
checkpoint = torch.load(model_path,
map_location=lambda storage, loc: storage)
fields = inputters.load_fields_from_vocab(checkpoint['vocab'],
data_type=opt.data_type)
model_opt = checkpoint['opt']
for arg in dummy_opt:
if arg not in model_opt:
model_opt.__dict__[arg] = dummy_opt[arg]
#TODO: delete all parameters related to WALS: FeatureValues, FeatureTensors, FeatureTypes, FeaturesList, FeatureNames, FTInfos, FeatureTypesNames, SimulationLanguages
#TODO: include four numpy arrays (from wals.npz)
model = build_base_model(model_opt, fields, use_gpu(opt), FeatureValues,
FeatureTensors, FeatureTypes, FeaturesList,
FeatureNames, FTInfos, FeatureTypesNames,
SimulationLanguages, checkpoint)
model.eval()
model.generator.eval()
return fields, model, model_opt
def load_test_model(opt, dummy_opt, model_path=None):
if model_path is None:
model_path = opt.models[0]
checkpoint = torch.load(model_path,
map_location=lambda storage, loc: storage)
fields = inputters.load_fields_from_vocab(checkpoint['vocab'],
data_type=opt.data_type)
if opt.data_type == 'text':
if opt.use_port != "":
for (k, f) in fields.items():
if k == "src" or k == "tgt":
f.use_vocab = False
f.dtype = torch.float
f.sequential = False
f.include_lengths = False
model_opt = checkpoint['opt']
if model_opt.rnn_size != -1:
model_opt.enc_rnn_size = model_opt.rnn_size
model_opt.dec_rnn_size = model_opt.rnn_size
if model_opt.model_type == 'text' and \
model_opt.enc_rnn_size != model_opt.dec_rnn_size:
raise AssertionError("""We do not support different encoder and
decoder rnn sizes for translation now.""")
for arg in dummy_opt:
if arg not in model_opt:
model_opt.__dict__[arg] = dummy_opt[arg]
model = build_base_model(model_opt, fields, use_gpu(opt), checkpoint)
model.eval()
if model.generator is not None:
model.generator.eval()
return fields, model, model_opt
def load_test_model(opt, dummy_opt, model_path=None):
if model_path is None:
model_path = opt.models[0]
checkpoint = torch.load(model_path,
map_location=lambda storage, loc: storage)
vocab = checkpoint['vocab']
if inputters.old_style_vocab(vocab):
fields = inputters.load_fields_from_vocab(vocab, opt.data_type)
else:
fields = vocab
model_opt = checkpoint['opt']
for arg in dummy_opt:
if arg not in model_opt:
model_opt.__dict__[arg] = dummy_opt[arg]
# changed to my_build_base_model by wchen
if 'hr' in model_opt.encoder_type or 'hr' in model_opt.decoder_type or 'CatSeqD' in model_opt.decoder_type:
model = my_build_base_model(model_opt, fields, use_gpu(opt),
checkpoint)
else:
model = build_base_model(model_opt, fields, use_gpu(opt), checkpoint)
model.eval()
model.generator.eval()
return fields, model, model_opt
def load_test_model(opt, dummy_opt, model_path=None):
if model_path is None:
model_path = opt.models[0]
checkpoint = torch.load(model_path,
map_location=lambda storage, loc: storage)
fields = inputters.load_fields_from_vocab(checkpoint['vocab'],
data_type=opt.data_type)
model_opt = checkpoint['opt']
for arg in dummy_opt:
if arg not in model_opt:
model_opt.__dict__[arg] = dummy_opt[arg]
model = build_base_model(model_opt, fields, use_gpu(opt), checkpoint)
# now build the generator
alpha_lookup = {'softmax': 1.0, 'tsallis15': 1.5, 'sparsemax': 2.0}
gen_alpha = alpha_lookup.get(model_opt.generator_function,
model_opt.loss_alpha)
assert opt.k == 0 or opt.bisect_iter == 0, \
"Bisection and topk are mutually exclusive ! !"
if gen_alpha == 1.0:
gen_func = nn.LogSoftmax(dim=-1)
elif gen_alpha == 2.0:
if opt.k > 0:
gen_func = onmt.modules.sparse_activations.LogSparsemaxTopK(
dim=-1, k=opt.k)
elif opt.bisect_iter > 0:
gen_func = onmt.modules.sparse_activations.LogSparsemaxBisect(
n_iter=opt.bisect_iter)
else:
gen_func = onmt.modules.sparse_activations.LogSparsemax(dim=-1)
elif gen_alpha == 1.5 and opt.bisect_iter == 0:
if opt.k > 0:
gen_func = onmt.modules.sparse_activations.LogTsallis15TopK(
dim=-1, k=opt.k)
else:
gen_func = onmt.modules.sparse_activations.LogTsallis15(dim=-1)
else:
# generic tsallis with bisection
assert opt.bisect_iter > 0, "Must use bisection with alpha != 1,1.5,2"
gen_func = onmt.modules.sparse_activations.LogTsallisBisect(
alpha=gen_alpha, n_iter=opt.bisect_iter)
# if model.generator is a Sequential, this unpacks the linear layer from
# inside it so it can be combined with the translation-time output
# function.
# In practice model.generator is always an nn.Sequential instance, but
# it should work if you just replace it with a linear layer.
gen_weights = model.generator[0] if \
isinstance(model.generator, nn.Sequential) else model.generator
generator = nn.Sequential(gen_weights, gen_func)
model.generator = generator
print(model.generator)
model.eval()
model.generator.eval()
return fields, model, model_opt
def load_test_model(opt, dummy_opt):
""" Load model for Inference """
checkpoint = torch.load(opt.model,
map_location=lambda storage, loc: storage)
fields = inputters.load_fields_from_vocab(checkpoint['vocab'],
data_type=opt.data_type)
model_opt = checkpoint['opt']
for arg in dummy_opt:
if arg not in model_opt:
model_opt.__dict__[arg] = dummy_opt[arg]
model = build_base_model(model_opt, fields, use_gpu(opt), checkpoint)
model.eval()
model.generator.eval()
return fields, model, model_opt
def load_test_model(opt, dummy_opt, model_path=None):
if model_path is None:
model_path = opt.models[0] # 'cnndm_sent_step_50.pt'
# model_path add
checkpoint = torch.load(model_path,
map_location=lambda storage, loc: storage)
fields = inputters.load_fields_from_vocab(checkpoint['vocab'],
data_type=opt.data_type)
model_opt = checkpoint['opt']
for arg in dummy_opt:
if arg not in model_opt:
model_opt.__dict__[arg] = dummy_opt[arg]
model = build_base_model(model_opt, fields, use_gpu(opt), checkpoint)
model.eval()
model.generator.eval()
return fields, model, model_opt
def load_test_model(opt, dummy_opt, model_path=None):
if model_path is None:
model_path = opt.models[0]
checkpoint = torch.load(
model_path,
map_location=lambda storage, loc: storage) #map_location='cpu')
fields = inputters.load_fields_from_vocab(checkpoint['vocab'],
data_type=opt.data_type)
model_opt = checkpoint['opt']
# model_opt.gpuid=-1
for arg in dummy_opt:
if arg not in model_opt:
model_opt.__dict__[arg] = dummy_opt[arg]
model = build_base_model(model_opt, fields, use_gpu(model_opt), checkpoint)
# if torch.cuda.device_count() > 1:
# model = nn.DataParallel(model)
model.eval()
model.generator.eval()
return fields, model, model_opt
def load_test_model(opt, dummy_opt, model_path=None):
if model_path is None:
model_path = opt.models[0]
checkpoint = torch.load(model_path,
map_location=lambda storage, loc: storage)
fields = inputters.load_fields_from_vocab(checkpoint['vocab'],
data_type=opt.data_type)
model_opt = checkpoint['opt']
if model_opt.rnn_size != -1:
model_opt.enc_rnn_size = model_opt.rnn_size
model_opt.dec_rnn_size = model_opt.rnn_size
if model_opt.model_type == 'text' and \
model_opt.enc_rnn_size != model_opt.dec_rnn_size:
raise AssertionError("""We do not support different encoder and
decoder rnn sizes for translation now.""")
for arg in dummy_opt:
if arg not in model_opt:
model_opt.__dict__[arg] = dummy_opt[arg]
model = build_base_model(model_opt, fields, use_gpu(opt), checkpoint)
model.eval()
model.generator.eval()
return fields, model, model_opt
def __init__(self, model_file, temp_dir, logger, silent=False):
self.model_file = model_file
self.temp_dir = temp_dir
if not silent:
logger.info("Start loading the model")
# checkpoint = torch.load(self.model_file,
# map_location=lambda storage, location: 'cpu')
checkpoint = torch.load(self.model_file, map_location='cpu')
# fields = inputters.load_fields_from_vocab(checkpoint['vocab'])
fields = inputters.load_fields_from_vocab(checkpoint['vocab'])
model_opt = checkpoint['opt']
# OpenNMT changed their configuration...
model_opt.enc_rnn_size = model_opt.rnn_size
model_opt.dec_rnn_size = model_opt.rnn_size
# model_opt.max_relative_positions = 0 # default is 0
# model_opt.model_dtype = 'fp32' # not half model
model = build_base_model(model_opt,
fields,
gpu=False,
checkpoint=checkpoint)
model.eval()
model.generator.eval()
self.model = model
self.fields = fields
self.model_opt = model_opt
if not os.path.exists(pjoin(self.temp_dir, "l2e")):
os.makedirs(pjoin(self.temp_dir, "l2e"))
if not silent:
logger.info("Model built")
def load_test_model(opt, dummy_opt, model_path=None):
if model_path is None:
model_path = opt.models[0]
checkpoint = torch.load(model_path,
map_location=lambda storage, loc: storage)
fields = inputters.load_fields_from_vocab(checkpoint['vocab'],
data_type=opt.data_type)
model_opt = checkpoint['opt']
for arg in dummy_opt:
if arg not in model_opt:
model_opt.__dict__[arg] = dummy_opt[arg]
# 用旧的opt训练的模型，没有一些新的参数选项，需要从当前opt继承
# 如 arch
for arg in opt.__dict__:
if arg not in model_opt:
model_opt.__dict__[arg] = opt.__dict__[arg]
model = build_base_model(model_opt, fields, use_gpu(opt), checkpoint)
model.to(f"cuda:{opt.gpu}")
model.eval()
model.generator.eval()
return fields, model, model_opt
def gen_re_files2(inputdir,vocab_file_path,data_type):
'''
:param inputdir: e.g. absolute path
:param vocab_file_path: e.g. 'data/GCN_DATA_DELEX.vocab.pt'
:param data_type: e.g. 'gcn'
:return:
'''
part = 'train'
inputdir = inputdir + '/'
triple_file_path = inputdir+part+'-webnlg-all-delex.triple'
lex_file_path = inputdir+part+'-webnlg-all-delex.lex'
savedir = inputdir+'RE/'
if not os.path.exists(savedir):
os.mkdir(savedir)
relation2id_file_path = savedir+vocab_file_path.split('/')[-1].split('.')[0]+'_relation2id.json'
word2id_file_path = savedir+vocab_file_path.split('/')[-1].split('.')[0]+'_word2id.json'
triples = open(triple_file_path,'r').readlines()
lexes = open(lex_file_path,'r').readlines()
fields = inputters.load_fields_from_vocab(
torch.load(vocab_file_path), data_type)
src_vocab = fields['src'].vocab
tgt_vocab = fields['tgt'].vocab
# Build relation2id.json
relation2id = {}
relation2id['None']=0
cnt = 1
for i in range(len(triples)):
triple = triples[i]
for p in triple.split('< TSP >'):
for idx, word in enumerate(p.split('|')):
word = word.strip().replace(' ','_')
if idx == 1 and word not in relation2id.keys():
relation2id[word] = cnt
cnt = cnt + 1
with open(relation2id_file_path,'w+') as f:
json.dump(relation2id,f,indent=4)
print(relation2id_file_path+' done')
# Build word2id.json
word2id={}
for idx, word in enumerate(tgt_vocab.itos):
word2id[word] = idx
with open(word2id_file_path,'w+') as f:
json.dump(word2id,f,indent=4)
print(word2id_file_path + ' done')
# Build train.json/dev.json/test.json
parts = ['train','dev','test']
inputdir = inputdir + 'RE/'
for part in parts:
triples_id = []
lexes_id = []
for i in range(len(triples)):
triple = triples[i]
lex = lexes[i]
tmpt = {}
# tmpt = []
tmpl = []
for p in triple.split('< TSP >'):
for idx, word in enumerate(p.split('|')):
word = word.strip().replace(' ','_')
if idx == 1:
tmpt[0]=word
# tmpt.append(relation2id[word])
elif idx == 0:
tmpt[1]=word
elif idx == 2:
tmpt[2]=word
# else:
# try:
# pos = lex.split(' ').index(word)
# except:
# pos = -1
# tmpt.append(pos + 1)
for word in lex.split(' '):
word = word.strip()
try:
tmpl.append(word2id[word])
except:
tmpl.append(word2id['<unk>'])
triples_id.append(tmpt.values())
# triples_id.append(tmpt)
lexes_id.append(tmpl)
# dct = [lexes_id, triples_id]
# with open(savedir+part + '.json', 'w+') as f:
# json.dump(dct, f)
# print(savedir+part + '.json done')
with open(savedir+part + '.txt', 'w+') as f:
for line in triples_id:
f.write(' '.join(line)+'\n')