def reload(path, params):
"""
Create a sentence embedder from a pretrained model.
"""
# reload model
reloaded = torch.load(path)
state_dict = reloaded['model']
# handle models from multi-GPU checkpoints
if 'checkpoint' in path:
state_dict = {(k[7:] if k.startswith('module.') else k): v
for k, v in state_dict.items()}
# reload dictionary and model parameters
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'],
reloaded['dico_counts'])
pretrain_params = AttrDict(reloaded['params'])
pretrain_params.n_words = len(dico)
pretrain_params.bos_index = dico.index(BOS_WORD)
pretrain_params.eos_index = dico.index(EOS_WORD)
pretrain_params.pad_index = dico.index(PAD_WORD)
pretrain_params.unk_index = dico.index(UNK_WORD)
pretrain_params.mask_index = dico.index(MASK_WORD)
# build model and reload weights
model = TransformerModel(pretrain_params, dico, True, True)
model.load_state_dict(state_dict)
model.eval()
# adding missing parameters
params.max_batch_size = 0
return MyModel(model, dico, pretrain_params, params)
def load_xlm_embeddings(path, model_name="model"):
"""
Load all xlm embeddings
Params:
path:
model_name: model name in the reloaded path, "model" for pretrained xlm encoder; "encoder" for encoder of translation model "decoder" for decoder of translation model
"""
reloaded = torch.load(path)
assert model_name in ["model", "encoder", "decoder"]
state_dict = reloaded[model_name]
# handle models from multi-GPU checkpoints
state_dict = {(k[7:] if k.startswith('module.') else k): v
for k, v in state_dict.items()}
# reload dictionary and model parameters
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'],
reloaded['dico_counts'])
pretrain_params = AttrDict(reloaded['params'])
pretrain_params.n_words = len(dico)
pretrain_params.bos_index = dico.index(BOS_WORD)
pretrain_params.eos_index = dico.index(EOS_WORD)
pretrain_params.pad_index = dico.index(PAD_WORD)
pretrain_params.unk_index = dico.index(UNK_WORD)
pretrain_params.mask_index = dico.index(MASK_WORD)
# build model and reload weights
if model_name != "decoder":
model = TransformerModel(pretrain_params, dico, True, True)
else:
model = TransformerModel(pretrain_params, dico, False, True)
model.load_state_dict(state_dict)
return model.embeddings.weight.data, dico
def initialize_model():
"""
"""
print('launching model')
chemin = getcwd()
curPath = chemin if "xlm" in chemin else (os.path.join(getcwd(), 'xlm'))
onlyfiles = [f for f in listdir(chemin) if isfile(join(chemin, f))]
print(onlyfiles)
print(os.path.normpath(os.path.join(getcwd(),
'./mlm_tlm_xnli15_1024.pth')))
model_path = os.path.normpath(
os.path.join(getcwd(), './mlm_tlm_xnli15_1024.pth'))
reloaded = torch.load(model_path)
# print('allez le model')
# response = requests.get(url)
# print('response downloaded')
# f = io.BytesIO(response.content)
# reloaded = torch.load(f)
# print('file downloaded')
# reloaded = Reloaded.serve()
params = AttrDict(reloaded['params'])
print("Supported languages: %s" % ", ".join(params.lang2id.keys()))
# build dictionary / update parameters
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'],
reloaded['dico_counts'])
params.n_words = len(dico)
params.bos_index = dico.index(BOS_WORD)
params.eos_index = dico.index(EOS_WORD)
params.pad_index = dico.index(PAD_WORD)
params.unk_index = dico.index(UNK_WORD)
params.mask_index = dico.index(MASK_WORD)
# build model / reload weights
model = TransformerModel(params, dico, True, True)
model.load_state_dict(reloaded['model'])
# bpe = fastBPE.fastBPE(
# path.normpath(path.join(curPath, "./codes_xnli_15") ),
# path.normpath(path.join(curPath, "./vocab_xnli_15") ) )
print('fin lecture')
return model, params, dico
def reload_ar_checkpoint(path):
""" Reload autoregressive params, dictionary, model from a given path """
# Load dictionary/model/datasets first
reloaded = torch.load(path)
params = AttrDict(reloaded['params'])
# build dictionary / update parameters
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'],
reloaded['dico_counts'])
params.n_words = len(dico)
params.n_langs = 1
params.bos_index = dico.index(BOS_WORD)
params.eos_index = dico.index(EOS_WORD)
params.pad_index = dico.index(PAD_WORD)
params.unk_index = dico.index(UNK_WORD)
params.mask_index = dico.index(MASK_WORD)
# build Transformer model
model = TransformerModel(params, is_encoder=False, with_output=True)
model.load_state_dict(reloaded['model'])
return params, dico, model
def reload_checkpoint(path):
""" Reload params, dictionary, model from a given path """
# Load dictionary/model/datasets first
reloaded = torch.load(path)
params = AttrDict(reloaded['params'])
print("Supported languages: %s" % ", ".join(params.lang2id.keys()))
# build dictionary / update parameters
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'],
reloaded['dico_counts'])
params.n_words = len(dico)
params.bos_index = dico.index(BOS_WORD)
params.eos_index = dico.index(EOS_WORD)
params.pad_index = dico.index(PAD_WORD)
params.unk_index = dico.index(UNK_WORD)
params.mask_index = dico.index(MASK_WORD)
# build model / reload weights
model = TransformerModel(params, dico, True, True)
model.load_state_dict(reloaded['model'])
return params, dico, model
def load_model(params):
# check parameters
assert os.path.isdir(params.data_path)
assert os.path.isfile(params.model_path)
reloaded = torch.load(params.model_path)
encoder_model_params = AttrDict(reloaded['enc_params'])
decoder_model_params = AttrDict(reloaded['dec_params'])
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'],
reloaded['dico_counts'])
params.n_langs = encoder_model_params['n_langs']
params.id2lang = encoder_model_params['id2lang']
params.lang2id = encoder_model_params['lang2id']
params.n_words = len(dico)
params.bos_index = dico.index(BOS_WORD)
params.eos_index = dico.index(EOS_WORD)
params.pad_index = dico.index(PAD_WORD)
params.unk_index = dico.index(UNK_WORD)
params.mask_index = dico.index(MASK_WORD)
encoder = TransformerModel(encoder_model_params,
dico,
is_encoder=True,
with_output=False)
decoder = TransformerModel(decoder_model_params,
dico,
is_encoder=False,
with_output=True)
def _process_state_dict(state_dict):
return {(k[7:] if k.startswith('module.') else k): v
for k, v in state_dict.items()}
encoder.load_state_dict(_process_state_dict(reloaded['encoder']))
decoder.load_state_dict(_process_state_dict(reloaded['decoder']))
return encoder, decoder, dico
#%%
# model_path = '/checkpoint/guismay/dumped/clm_test3/10431904/periodic-23.pth'
model_path = '/checkpoint/guismay/dumped/clm_test3/10431904/periodic-23.pth'
reloaded = torch.load(model_path)
params = AttrDict(reloaded['params'])
print("Supported languages: %s" % ", ".join(params.lang2id.keys()))
#%% [markdown]
# ## Build dictionary / update parameters / build model
#%%
# build dictionary / update parameters
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'],
reloaded['dico_counts'])
assert params.n_words == len(dico)
assert params.bos_index == dico.index(BOS_WORD)
assert params.eos_index == dico.index(EOS_WORD)
assert params.pad_index == dico.index(PAD_WORD)
assert params.unk_index == dico.index(UNK_WORD)
assert params.mask_index == dico.index(MASK_WORD)
# build model / reload weights
model = TransformerModel(params, dico, True, True)
model.load_state_dict(reloaded['model'])
model.cuda()
model.eval()
#%%
#%%
FASTBPE_PATH = '/private/home/guismay/tools/fastBPE/fast'
def main(params):
# initialize the experiment
logger = initialize_exp(params)
# generate parser / parse parameters
parser = get_parser()
params = parser.parse_args()
torch.manual_seed(
params.seed
) # Set random seed. NB: Multi-GPU also needs torch.cuda.manual_seed_all(params.seed)
assert (params.sample_temperature
== 0) or (params.beam_size == 1), 'Cannot sample with beam search.'
assert params.amp <= 1, f'params.amp == {params.amp} not yet supported.'
reloaded = torch.load(params.model_path)
model_params = AttrDict(reloaded['params'])
logger.info("Supported languages: %s" %
", ".join(model_params.lang2id.keys()))
# update dictionary parameters
for name in [
'n_words', 'bos_index', 'eos_index', 'pad_index', 'unk_index',
'mask_index'
]:
setattr(params, name, getattr(model_params, name))
# build dictionary / build encoder / build decoder / reload weights
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'],
reloaded['dico_counts'])
encoder = TransformerModel(model_params,
dico,
is_encoder=True,
with_output=False).cuda().eval()
decoder = TransformerModel(model_params,
dico,
is_encoder=False,
with_output=True).cuda().eval()
if all([k.startswith('module.') for k in reloaded['encoder'].keys()]):
reloaded['encoder'] = {
k[len('module.'):]: v
for k, v in reloaded['encoder'].items()
}
encoder.load_state_dict(reloaded['encoder'])
if all([k.startswith('module.') for k in reloaded['decoder'].keys()]):
reloaded['decoder'] = {
k[len('module.'):]: v
for k, v in reloaded['decoder'].items()
}
decoder.load_state_dict(reloaded['decoder'])
if params.amp != 0:
models = apex.amp.initialize([encoder, decoder],
opt_level=('O%i' % params.amp))
encoder, decoder = models
params.src_id = model_params.lang2id[params.src_lang]
params.tgt_id = model_params.lang2id[params.tgt_lang]
# read sentences from stdin
src_sent = []
for line in sys.stdin.readlines():
assert len(line.strip().split()) > 0
src_sent.append(line)
logger.info("Read %i sentences from stdin. Translating ..." %
len(src_sent))
# f = io.open(params.output_path, 'w', encoding='utf-8')
hypothesis = [[] for _ in range(params.beam_size)]
for i in range(0, len(src_sent), params.batch_size):
# prepare batch
word_ids = [
torch.LongTensor([dico.index(w) for w in s.strip().split()])
for s in src_sent[i:i + params.batch_size]
]
lengths = torch.LongTensor([len(s) + 2 for s in word_ids])
batch = torch.LongTensor(lengths.max().item(),
lengths.size(0)).fill_(params.pad_index)
batch[0] = params.eos_index
for j, s in enumerate(word_ids):
if lengths[j] > 2: # if sentence not empty
batch[1:lengths[j] - 1, j].copy_(s)
batch[lengths[j] - 1, j] = params.eos_index
langs = batch.clone().fill_(params.src_id)
# encode source batch and translate it
encoded = encoder('fwd',
x=batch.cuda(),
lengths=lengths.cuda(),
langs=langs.cuda(),
causal=False)
encoded = encoded.transpose(0, 1)
max_len = int(1.5 * lengths.max().item() + 10)
if params.beam_size == 1:
decoded, dec_lengths = decoder.generate(
encoded,
lengths.cuda(),
params.tgt_id,
max_len=max_len,
sample_temperature=(None if params.sample_temperature == 0 else
params.sample_temperature))
else:
decoded, dec_lengths, all_hyp_strs = decoder.generate_beam(
encoded,
lengths.cuda(),
params.tgt_id,
beam_size=params.beam_size,
length_penalty=params.length_penalty,
early_stopping=params.early_stopping,
max_len=max_len,
output_all_hyps=True)
# hypothesis.extend(convert_to_text(decoded, dec_lengths, dico, params))
# convert sentences to words
for j in range(decoded.size(1)):
# remove delimiters
sent = decoded[:, j]
delimiters = (sent == params.eos_index).nonzero().view(-1)
assert len(delimiters) >= 1 and delimiters[0].item() == 0
sent = sent[1:] if len(delimiters) == 1 else sent[1:delimiters[1]]
# output translation
source = src_sent[i + j].strip().replace('<unk>', '<<unk>>')
target = " ".join([dico[sent[k].item()] for k in range(len(sent))
]).replace('<unk>', '<<unk>>')
if params.beam_size == 1:
hypothesis[0].append(target)
else:
for hyp_rank in range(params.beam_size):
print(
all_hyp_strs[j]
[hyp_rank if hyp_rank < len(all_hyp_strs[j]) else -1])
hypothesis[hyp_rank].append(
all_hyp_strs[j]
[hyp_rank if hyp_rank < len(all_hyp_strs[j]) else -1])
sys.stderr.write("%i / %i: %s -> %s\n" %
(i + j, len(src_sent), source.replace(
'@@ ', ''), target.replace('@@ ', '')))
# f.write(target + "\n")
# f.close()
# export sentences to reference and hypothesis files / restore BPE segmentation
save_dir, split = params.output_path.rsplit('/', 1)
for hyp_rank in range(len(hypothesis)):
hyp_name = f'hyp.st={params.sample_temperature}.bs={params.beam_size}.lp={params.length_penalty}.es={params.early_stopping}.seed={params.seed if (len(hypothesis) == 1) else str(hyp_rank)}.{params.src_lang}-{params.tgt_lang}.{split}.txt'
hyp_path = os.path.join(save_dir, hyp_name)
with open(hyp_path, 'w', encoding='utf-8') as f:
f.write('\n'.join(hypothesis[hyp_rank]) + '\n')
restore_segmentation(hyp_path)
# evaluate BLEU score
if params.ref_path:
bleu = eval_moses_bleu(params.ref_path, hyp_path)
logger.info("BLEU %s %s : %f" % (hyp_path, params.ref_path, bleu))
def main():
parser.add_argument("--input", type=str, default="", help="input file")
parser.add_argument("--model", type=str, default="", help="model path")
parser.add_argument("--spm_model",
type=str,
default="",
help="spm model path")
parser.add_argument("--batch_size",
type=int,
default=64,
help="batch size")
parser.add_argument("--max_words", type=int, default=100, help="max words")
parser.add_argument("--cuda", type=str, default="True", help="use cuda")
parser.add_argument("--output", type=str, default="", help="output file")
args = parser.parse_args()
# Reload a pretrained model
reloaded = torch.load(args.model)
params = AttrDict(reloaded['params'])
# Reload the SPM model
spm_model = spm.SentencePieceProcessor()
spm_model.Load(args.spm_model)
# cuda
assert args.cuda in ["True", "False"]
args.cuda = eval(args.cuda)
# build dictionary / update parameters
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'],
reloaded['dico_counts'])
params.n_words = len(dico)
params.bos_index = dico.index(BOS_WORD)
params.eos_index = dico.index(EOS_WORD)
params.pad_index = dico.index(PAD_WORD)
params.unk_index = dico.index(UNK_WORD)
params.mask_index = dico.index(MASK_WORD)
# build model / reload weights
model = TransformerModel(params, dico, True, True)
reloaded['model'] = OrderedDict({
key.replace('module.', ''): reloaded['model'][key]
for key in reloaded['model']
})
model.load_state_dict(reloaded['model'])
model.eval()
if args.cuda:
model.cuda()
# load sentences
sentences = []
with open(args.input) as f:
for line in f:
line = spm_model.EncodeAsPieces(line.rstrip())
line = line[:args.max_words - 1]
sentences.append(line)
# encode sentences
embs = []
for i in range(0, len(sentences), args.batch_size):
batch = sentences[i:i + args.batch_size]
lengths = torch.LongTensor([len(s) + 1 for s in batch])
bs, slen = len(batch), lengths.max().item()
assert slen <= args.max_words
x = torch.LongTensor(slen, bs).fill_(params.pad_index)
for k in range(bs):
sent = torch.LongTensor([params.eos_index] +
[dico.index(w) for w in batch[k]])
x[:len(sent), k] = sent
if args.cuda:
x = x.cuda()
lengths = lengths.cuda()
with torch.no_grad():
embedding = model('fwd',
x=x,
lengths=lengths,
langs=None,
causal=False).contiguous()[0].cpu()
embs.append(embedding)
# save embeddings
torch.save(torch.cat(embs, dim=0).squeeze(0), args.output)
def main(params):
# initialize the experiment
logger = initialize_exp(params)
parser = get_parser()
params = parser.parse_args()
models_path = params.model_path.split(',')
# generate parser / parse parameters
models_reloaded = []
for model_path in models_path:
models_reloaded.append(torch.load(model_path))
model_params = AttrDict(models_reloaded[0]['params'])
logger.info("Supported languages: %s" %
", ".join(model_params.lang2id.keys()))
# update dictionary parameters
for name in [
'n_words', 'bos_index', 'eos_index', 'pad_index', 'unk_index',
'mask_index'
]:
setattr(params, name, getattr(model_params, name))
# build dictionary / build encoder / build decoder / reload weights
dico = Dictionary(models_reloaded[0]['dico_id2word'],
models_reloaded[0]['dico_word2id'],
models_reloaded[0]['dico_counts'])
params.src_id = model_params.lang2id[params.src_lang]
params.tgt_id = model_params.lang2id[params.tgt_lang]
encoders = []
decoders = []
def package_module(modules):
state_dict = OrderedDict()
for k, v in modules.items():
if k.startswith('module.'):
state_dict[k[7:]] = v
else:
state_dict[k] = v
return state_dict
for reloaded in models_reloaded:
encoder = TransformerModel(model_params,
dico,
is_encoder=True,
with_output=True).to(params.device).eval()
decoder = TransformerModel(model_params,
dico,
is_encoder=False,
with_output=True).to(params.device).eval()
encoder.load_state_dict(package_module(reloaded['encoder']))
decoder.load_state_dict(package_module(reloaded['decoder']))
# float16
if params.fp16:
assert torch.backends.cudnn.enabled
encoder = network_to_half(encoder)
decoder = network_to_half(decoder)
encoders.append(encoder)
decoders.append(decoder)
#src_sent = ['Poly@@ gam@@ ie statt Demokratie .']
src_sent = []
for line in sys.stdin.readlines():
assert len(line.strip().split()) > 0
src_sent.append(line)
f = io.open(params.output_path, 'w', encoding='utf-8')
for i in range(0, len(src_sent), params.batch_size):
# prepare batch
word_ids = [
torch.LongTensor([dico.index(w) for w in s.strip().split()])
for s in src_sent[i:i + params.batch_size]
]
lengths = torch.LongTensor([len(s) + 2 for s in word_ids])
batch = torch.LongTensor(lengths.max().item(),
lengths.size(0)).fill_(params.pad_index)
batch[0] = params.eos_index
for j, s in enumerate(word_ids):
if lengths[j] > 2: # if sentence not empty
batch[1:lengths[j] - 1, j].copy_(s)
batch[lengths[j] - 1, j] = params.eos_index
langs = batch.clone().fill_(params.src_id)
# encode source batch and translate it
encodeds = []
for encoder in encoders:
encoded = encoder('fwd',
x=batch.to(params.device),
lengths=lengths.to(params.device),
langs=langs.to(params.device),
causal=False)
encoded = encoded.transpose(0, 1)
encodeds.append(encoded)
assert encoded.size(0) == lengths.size(0)
decoded, dec_lengths = generate_beam(
decoders,
encodeds,
lengths.to(params.device),
params.tgt_id,
beam_size=params.beam,
length_penalty=params.length_penalty,
early_stopping=False,
max_len=int(1.5 * lengths.max().item() + 10),
params=params)
# convert sentences to words
for j in range(decoded.size(1)):
# remove delimiters
sent = decoded[:, j]
delimiters = (sent == params.eos_index).nonzero().view(-1)
assert len(delimiters) >= 1 and delimiters[0].item() == 0
sent = sent[1:] if len(delimiters) == 1 else sent[1:delimiters[1]]
# output translation
source = src_sent[i + j].strip()
target = " ".join([dico[sent[k].item()] for k in range(len(sent))])
sys.stderr.write("%i / %i: %s -> %s\n" %
(i + j, len(src_sent), source, target))
f.write(target + "\n")
f.close()
class Translate():
def __init__(self,
model_path,
tgt_lang,
src_lang,
dump_path="./dumped/",
exp_name="translate",
exp_id="test",
batch_size=32):
# parse parameters
parser = argparse.ArgumentParser(description="Translate sentences")
# main parameters
parser.add_argument("--dump_path",
type=str,
default=dump_path,
help="Experiment dump path")
parser.add_argument("--exp_name",
type=str,
default=exp_name,
help="Experiment name")
parser.add_argument("--exp_id",
type=str,
default=exp_id,
help="Experiment ID")
parser.add_argument("--batch_size",
type=int,
default=batch_size,
help="Number of sentences per batch")
# model / output paths
parser.add_argument("--model_path",
type=str,
default=model_path,
help="Model path")
# parser.add_argument("--max_vocab", type=int, default=-1, help="Maximum vocabulary size (-1 to disable)")
# parser.add_argument("--min_count", type=int, default=0, help="Minimum vocabulary count")
# source language / target language
parser.add_argument("--src_lang",
type=str,
default=src_lang,
help="Source language")
parser.add_argument("--tgt_lang",
type=str,
default=tgt_lang,
help="Target language")
parser.add_argument('-d',
"--text",
type=str,
default="",
nargs='+',
help="Text to be translated")
params = parser.parse_args()
assert params.src_lang != '' and params.tgt_lang != '' and params.src_lang != params.tgt_lang
# initialize the experiment
logger = initialize_exp(params)
# On a pas de GPU
#reloaded = torch.load(params.model_path)
reloaded = torch.load(params.model_path,
map_location=torch.device('cpu'))
model_params = AttrDict(reloaded['params'])
self.supported_languages = model_params.lang2id.keys()
logger.info("Supported languages: %s" %
", ".join(self.supported_languages))
# update dictionary parameters
for name in [
'n_words', 'bos_index', 'eos_index', 'pad_index', 'unk_index',
'mask_index'
]:
try:
setattr(params, name, getattr(model_params, name))
except AttributeError:
key = list(model_params.meta_params.keys())[0]
attr = getattr(model_params.meta_params[key], name)
setattr(params, name, attr)
setattr(model_params, name, attr)
# build dictionary / build encoder / build decoder / reload weights
self.dico = Dictionary(reloaded['dico_id2word'],
reloaded['dico_word2id'],
reloaded['dico_counts'])
#self.encoder = TransformerModel(model_params, dico, is_encoder=True, with_output=True).cuda().eval()
self.encoder = TransformerModel(model_params,
self.dico,
is_encoder=True,
with_output=True).eval()
#self.decoder = TransformerModel(model_params, dico, is_encoder=False, with_output=True).cuda().eval()
self.decoder = TransformerModel(model_params,
self.dico,
is_encoder=False,
with_output=True).eval()
self.encoder.load_state_dict(reloaded['encoder'])
self.decoder.load_state_dict(reloaded['decoder'])
params.src_id = model_params.lang2id[params.src_lang]
params.tgt_id = model_params.lang2id[params.tgt_lang]
self.model_params = model_params
self.params = params
def translate(self, src_sent=[]):
flag = False
if type(src_sent) == str:
src_sent = [src_sent]
flag = True
tgt_sent = []
for i in range(0, len(src_sent), self.params.batch_size):
# prepare batch
word_ids = [
torch.LongTensor(
[self.dico.index(w) for w in s.strip().split()])
for s in src_sent[i:i + self.params.batch_size]
]
lengths = torch.LongTensor([len(s) + 2 for s in word_ids])
batch = torch.LongTensor(lengths.max().item(),
lengths.size(0)).fill_(
self.params.pad_index)
batch[0] = self.params.eos_index
for j, s in enumerate(word_ids):
if lengths[j] > 2: # if sentence not empty
batch[1:lengths[j] - 1, j].copy_(s)
batch[lengths[j] - 1, j] = self.params.eos_index
langs = batch.clone().fill_(self.params.src_id)
# encode source batch and translate it
#encoded = self.encoder('fwd', x=batch.cuda(), lengths=lengths.cuda(), langs=langs.cuda(), causal=False)
encoded = self.encoder('fwd',
x=batch,
lengths=lengths,
langs=langs,
causal=False)
encoded = encoded.transpose(0, 1)
#decoded, dec_lengths = self.decoder.generate(encoded, lengths.cuda(), self.params.tgt_id, max_len=int(1.5 * lengths.max().item() + 10))
decoded, dec_lengths = self.decoder.generate(
encoded,
lengths,
self.params.tgt_id,
max_len=int(1.5 * lengths.max().item() + 10))
# convert sentences to words
for j in range(decoded.size(1)):
# remove delimiters
sent = decoded[:, j]
delimiters = (sent == self.params.eos_index).nonzero().view(-1)
assert len(delimiters) >= 1 and delimiters[0].item() == 0
sent = sent[1:] if len(
delimiters) == 1 else sent[1:delimiters[1]]
# output translation
source = src_sent[i + j].strip()
target = " ".join(
[self.dico[sent[k].item()] for k in range(len(sent))])
sys.stderr.write("%i / %i: %s -> %s\n" %
(i + j, len(src_sent), source, target))
tgt_sent.append(target)
if flag:
return tgt_sent[0]
return tgt_sent
def main(params):
# initialize the experiment
logger = initialize_exp(params)
# generate parser / parse parameters
parser = get_parser()
params = parser.parse_args()
reloaded = torch.load(params.model_path)
model_params = AttrDict(reloaded['params'])
logger.info("Supported languages: %s" % ", ".join(model_params.lang2id.keys()))
# update dictionary parameters
for name in ['n_words', 'bos_index', 'eos_index', 'pad_index', 'unk_index', 'mask_index']:
setattr(params, name, getattr(model_params, name))
# build dictionary / build encoder / build decoder / reload weights
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'], reloaded['dico_counts'])
encoder = TransformerModel(model_params, dico, is_encoder=True, with_output=True).cuda().eval()
decoder = TransformerModel(model_params, dico, is_encoder=False, with_output=True).cuda().eval()
encoder.load_state_dict(reloaded['encoder'])
decoder.load_state_dict(reloaded['decoder'])
params.src_id = model_params.lang2id[params.src_lang]
params.tgt_id = model_params.lang2id[params.tgt_lang]
# float16
if params.fp16:
assert torch.backends.cudnn.enabled
encoder = network_to_half(encoder)
decoder = network_to_half(decoder)
# read sentences from stdin
src_sent = []
for line in sys.stdin.readlines():
assert len(line.strip().split()) > 0
src_sent.append(line)
logger.info("Read %i sentences from stdin. Translating ..." % len(src_sent))
f = io.open(params.output_path, 'w', encoding='utf-8')
for i in range(0, len(src_sent), params.batch_size):
# prepare batch
word_ids = [torch.LongTensor([dico.index(w) for w in s.strip().split()])
for s in src_sent[i:i + params.batch_size]]
lengths = torch.LongTensor([len(s) + 2 for s in word_ids])
batch = torch.LongTensor(lengths.max().item(), lengths.size(0)).fill_(params.pad_index)
batch[0] = params.eos_index
for j, s in enumerate(word_ids):
if lengths[j] > 2: # if sentence not empty
batch[1:lengths[j] - 1, j].copy_(s)
batch[lengths[j] - 1, j] = params.eos_index
langs = batch.clone().fill_(params.src_id)
# encode source batch and translate it
encoded = encoder('fwd', x=batch.cuda(), lengths=lengths.cuda(), langs=langs.cuda(), causal=False)
encoded = encoded.transpose(0, 1)
decoded, dec_lengths = decoder.generate(encoded, lengths.cuda(), params.tgt_id, max_len=int(1.5 * lengths.max().item() + 10))
# convert sentences to words
for j in range(decoded.size(1)):
# remove delimiters
sent = decoded[:, j]
delimiters = (sent == params.eos_index).nonzero().view(-1)
assert len(delimiters) >= 1 and delimiters[0].item() == 0
sent = sent[1:] if len(delimiters) == 1 else sent[1:delimiters[1]]
# output translation
source = src_sent[i + j].strip()
target = " ".join([dico[sent[k].item()] for k in range(len(sent))])
sys.stderr.write("%i / %i: %s -> %s\n" % (i + j, len(src_sent), source, target))
f.write(target + "\n")
f.close()
def main(params):
# initialize the experiment
logger = initialize_exp(params)
# generate parser / parse parameters
parser = get_parser()
params = parser.parse_args()
reloaded = torch.load(params.model_path)
model_params = AttrDict(reloaded['params'])
logger.info("Supported languages: %s" %
", ".join(model_params.lang2id.keys()))
# update dictionary parameters
for name in [
'n_words', 'bos_index', 'eos_index', 'pad_index', 'unk_index',
'mask_index'
]:
setattr(params, name, getattr(model_params, name))
# build dictionary / build encoder / build decoder / reload weights
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'],
reloaded['dico_counts'])
encoder = TransformerModel(model_params,
dico,
is_encoder=True,
with_output=True).cuda().eval()
encoder.load_state_dict(reloaded['encoder'])
decoder = None
# decoder = TransformerModel(model_params, dico, is_encoder=False, with_output=True).cuda().eval()
# decoder.load_state_dict(reloaded['decoder'])
params.src_id = model_params.lang2id[params.src_lang]
params.tgt_id = model_params.lang2id[params.tgt_lang]
# read sentences from stdin
src_sent = []
for line in sys.stdin.readlines():
assert len(line.strip().split()) > 0
src_sent.append(line)
logger.info("Read %i sentences from stdin. Translating ..." %
len(src_sent))
all_encodings = []
# For each sentence...
for i in range(0, len(src_sent), params.batch_size):
# prepare batch
word_ids = [
torch.LongTensor([dico.index(w) for w in s.strip().split()])
for s in src_sent[i:i + params.batch_size]
]
lengths = torch.LongTensor([len(s) + 2 for s in word_ids])
batch = torch.LongTensor(lengths.max().item(),
lengths.size(0)).fill_(params.pad_index)
batch[0] = params.eos_index
for j, s in enumerate(word_ids):
if lengths[j] > 2: # if sentence not empty
batch[1:lengths[j] - 1, j].copy_(s)
batch[lengths[j] - 1, j] = params.eos_index
langs = batch.clone().fill_(params.src_id)
# encode source batch and translate it, deal with padding
encodings = encoderouts(encoder, batch, lengths, langs)
# batch is actually in original order, append each sent to all_encodings
for idx in encodings:
all_encodings.append(idx.cpu().numpy())
# Save all encodings to npy
np.save(params.output_path, np.stack(all_encodings))
def main(params):
# generate parser / parse parameters
parser = get_parser()
params = parser.parse_args()
reloaded = torch.load(params.model_path)
model_params = AttrDict(reloaded['params'])
# update dictionary parameters
for name in ['src_n_words', 'tgt_n_words', 'bos_index', 'eos_index', 'pad_index', 'unk_index', 'mask_index']:
setattr(params, name, getattr(model_params, name))
# build dictionary / build encoder / build decoder / reload weights
source_dico = Dictionary(reloaded['source_dico_id2word'], reloaded['source_dico_word2id'])
target_dico = Dictionary(reloaded['target_dico_id2word'], reloaded['target_dico_word2id'])
encoder = TransformerEncoder(model_params, source_dico, with_output=False).cuda().eval()
decoder = TransformerDecoder(model_params, target_dico, with_output=True).cuda().eval()
encoder.load_state_dict(reloaded['encoder'])
decoder.load_state_dict(reloaded['decoder'])
# read sentences from stdin
table_lines = []
table_inf = open(params.table_path, 'r', encoding='utf-8')
for table_line in table_inf:
table_lines.append(table_line)
outf = io.open(params.output_path, 'w', encoding='utf-8')
for i in range(0, len(table_lines), params.batch_size):
# prepare batch
enc_x1_ids = []
enc_x2_ids = []
enc_x3_ids = []
enc_x4_ids = []
for table_line in table_lines[i:i + params.batch_size]:
record_seq = [each.split('|') for each in table_line.split()]
assert all([len(x) == 4 for x in record_seq])
enc_x1_ids.append(torch.LongTensor([source_dico.index(x[0]) for x in record_seq]))
enc_x2_ids.append(torch.LongTensor([source_dico.index(x[1]) for x in record_seq]))
enc_x3_ids.append(torch.LongTensor([source_dico.index(x[2]) for x in record_seq]))
enc_x4_ids.append(torch.LongTensor([source_dico.index(x[3]) for x in record_seq]))
enc_xlen = torch.LongTensor([len(x) + 2 for x in enc_x1_ids])
enc_x1 = torch.LongTensor(enc_xlen.max().item(), enc_xlen.size(0)).fill_(params.pad_index)
enc_x1[0] = params.eos_index
enc_x2 = torch.LongTensor(enc_xlen.max().item(), enc_xlen.size(0)).fill_(params.pad_index)
enc_x2[0] = params.eos_index
enc_x3 = torch.LongTensor(enc_xlen.max().item(), enc_xlen.size(0)).fill_(params.pad_index)
enc_x3[0] = params.eos_index
enc_x4 = torch.LongTensor(enc_xlen.max().item(), enc_xlen.size(0)).fill_(params.pad_index)
enc_x4[0] = params.eos_index
for j, (s1,s2,s3,s4) in enumerate(zip(enc_x1_ids, enc_x2_ids, enc_x3_ids, enc_x4_ids)):
if enc_xlen[j] > 2: # if sentence not empty
enc_x1[1:enc_xlen[j] - 1, j].copy_(s1)
enc_x2[1:enc_xlen[j] - 1, j].copy_(s2)
enc_x3[1:enc_xlen[j] - 1, j].copy_(s3)
enc_x4[1:enc_xlen[j] - 1, j].copy_(s4)
enc_x1[enc_xlen[j] - 1, j] = params.eos_index
enc_x2[enc_xlen[j] - 1, j] = params.eos_index
enc_x3[enc_xlen[j] - 1, j] = params.eos_index
enc_x4[enc_xlen[j] - 1, j] = params.eos_index
enc_x1 = enc_x1.cuda()
enc_x2 = enc_x2.cuda()
enc_x3 = enc_x3.cuda()
enc_x4 = enc_x4.cuda()
enc_xlen = enc_xlen.cuda()
# encode source batch and translate it
encoder_output = encoder('fwd', x1=enc_x1, x2=enc_x2, x3=enc_x3, x4=enc_x4, lengths=enc_xlen)
encoder_output = encoder_output.transpose(0, 1)
# max_len = int(1.5 * enc_xlen.max().item() + 10)
max_len = 602
if params.beam_size <= 1:
decoded, dec_lengths = decoder.generate(encoder_output, enc_xlen, max_len=max_len)
elif params.beam_size > 1:
decoded, dec_lengths = decoder.generate_beam(encoder_output, enc_xlen, params.beam_size,
params.length_penalty, params.early_stopping, max_len=max_len)
for j in range(decoded.size(1)):
# remove delimiters
sent = decoded[:, j]
delimiters = (sent == params.eos_index).nonzero().view(-1)
assert len(delimiters) >= 1 and delimiters[0].item() == 0
sent = sent[1:] if len(delimiters) == 1 else sent[1:delimiters[1]]
# output translation
source = table_lines[i + j].strip()
target = " ".join([target_dico[sent[k].item()] for k in range(len(sent))])
sys.stderr.write("%i / %i: %s\n" % (i + j, len(table_lines), target))
outf.write(target + "\n")
outf.close()
def main(params):
# generate parser / parse parameters
#parser = get_parser()
#params = parser.parse_args()
reloaded = torch.load(params.model_path)
model_params = AttrDict(reloaded['params'])
# update dictionary parameters
for name in [
'src_n_words', 'tgt_n_words', 'bos_index', 'eos_index',
'pad_index', 'unk_index', 'mask_index'
]:
setattr(params, name, getattr(model_params, name))
# print(f'src {getattr(model_params, "src_n_words")}')
# print(f'tgt {getattr(model_params, "tgt_n_words")}')
# build dictionary / build encoder / build decoder / reload weights
source_dico = Dictionary(reloaded['source_dico_id2word'],
reloaded['source_dico_word2id'])
target_dico = Dictionary(reloaded['target_dico_id2word'],
reloaded['target_dico_word2id'])
# originalDecoder = reloaded['decoder'].copy()
encoder = TransformerEncoder(model_params, source_dico,
with_output=False).cuda().eval()
encoder.load_state_dict(reloaded['encoder'])
decoder = TransformerDecoder(model_params, target_dico,
with_output=True).cuda().eval()
decoder.load_state_dict(reloaded['decoder'])
# read sentences from stdin
table_lines = []
title_lines = []
table_inf = open(params.table_path, 'r', encoding='utf-8')
for table_line in table_inf:
table_lines.append(table_line)
with open(params.title_path, 'r', encoding='utf-8') as title_inf:
for title_line in title_inf:
title_lines.append(title_line)
assert len(title_lines) == len(table_lines)
outf = io.open(params.output_path, 'w', encoding='utf-8')
fillers = [
'in', 'the', 'and', 'or', 'an', 'as', 'can', 'be', 'a', ':', '-', 'to',
'but', 'is', 'of', 'it', 'on', '.', 'at', '(', ')', ',', 'with'
]
for i in range(0, len(table_lines), params.batch_size):
# prepare batch
"""valueLengths = []
xLabelLengths = []
yLabelLengths = []
titleLengths = []"""
enc_x1_ids = []
enc_x2_ids = []
enc_x3_ids = []
enc_x4_ids = []
for table_line, title_line in zip(table_lines[i:i + params.batch_size],
title_lines[i:i +
params.batch_size]):
record_seq = [each.split('|') for each in table_line.split()]
assert all([len(x) == 4 for x in record_seq])
enc_x1_ids.append(
torch.LongTensor([source_dico.index(x[0])
for x in record_seq]))
enc_x2_ids.append(
torch.LongTensor([source_dico.index(x[1])
for x in record_seq]))
enc_x3_ids.append(
torch.LongTensor([source_dico.index(x[2])
for x in record_seq]))
enc_x4_ids.append(
torch.LongTensor([source_dico.index(x[3])
for x in record_seq]))
xLabel = record_seq[1][0].split('_')
yLabel = record_seq[0][0].split('_')
"""cleanXLabel = len([item for item in xLabel if item not in fillers])
cleanYLabel = len([item for item in yLabel if item not in fillers])
cleanTitle = len([word for word in title_line.split() if word not in fillers])
xLabelLengths.append(cleanXLabel)
yLabelLengths.append(cleanYLabel)
titleLengths.append(cleanTitle)
valueLengths.append(round(len(record_seq)/2))"""
enc_xlen = torch.LongTensor([len(x) + 2 for x in enc_x1_ids])
enc_x1 = torch.LongTensor(enc_xlen.max().item(),
enc_xlen.size(0)).fill_(params.pad_index)
enc_x1[0] = params.eos_index
enc_x2 = torch.LongTensor(enc_xlen.max().item(),
enc_xlen.size(0)).fill_(params.pad_index)
enc_x2[0] = params.eos_index
enc_x3 = torch.LongTensor(enc_xlen.max().item(),
enc_xlen.size(0)).fill_(params.pad_index)
enc_x3[0] = params.eos_index
enc_x4 = torch.LongTensor(enc_xlen.max().item(),
enc_xlen.size(0)).fill_(params.pad_index)
enc_x4[0] = params.eos_index
for j, (s1, s2, s3, s4) in enumerate(
zip(enc_x1_ids, enc_x2_ids, enc_x3_ids, enc_x4_ids)):
if enc_xlen[j] > 2: # if sentence not empty
enc_x1[1:enc_xlen[j] - 1, j].copy_(s1)
enc_x2[1:enc_xlen[j] - 1, j].copy_(s2)
enc_x3[1:enc_xlen[j] - 1, j].copy_(s3)
enc_x4[1:enc_xlen[j] - 1, j].copy_(s4)
enc_x1[enc_xlen[j] - 1, j] = params.eos_index
enc_x2[enc_xlen[j] - 1, j] = params.eos_index
enc_x3[enc_xlen[j] - 1, j] = params.eos_index
enc_x4[enc_xlen[j] - 1, j] = params.eos_index
enc_x1 = enc_x1.cuda()
enc_x2 = enc_x2.cuda()
enc_x3 = enc_x3.cuda()
enc_x4 = enc_x4.cuda()
enc_xlen = enc_xlen.cuda()
# encode source batch and translate it
encoder_output = encoder('fwd',
x1=enc_x1,
x2=enc_x2,
x3=enc_x3,
x4=enc_x4,
lengths=enc_xlen)
encoder_output = encoder_output.transpose(0, 1)
max_len = 602
if params.beam_size <= 1:
decoded, dec_lengths = decoder.generate(encoder_output,
enc_xlen,
max_len=max_len)
elif params.beam_size > 1:
decoded, dec_lengths = decoder.generate_beam(encoder_output,
enc_xlen,
params.beam_size,
params.length_penalty,
params.early_stopping,
max_len=max_len)
for j in range(decoded.size(1)):
# remove delimiters
sent = decoded[:, j]
delimiters = (sent == params.eos_index).nonzero().view(-1)
assert len(delimiters) >= 1 and delimiters[0].item() == 0
sent = sent[1:] if len(delimiters) == 1 else sent[1:delimiters[1]]
# print(sent)
# output translation
# source = table_lines[i + j].strip()
# print(source)
tokens = []
for k in range(len(sent)):
ids = sent[k].item()
#if ids in removedDict:
# print('index error')
word = target_dico[ids]
tokens.append(word)
target = " ".join(tokens)
sys.stderr.write("%i / %i: %s\n" %
(i + j, len(table_lines), target))
outf.write(target + "\n")
outf.close()
def run_xnlg():
params = get_params()
# initialize the experiment / build sentence embedder
logger = initialize_exp(params)
if params.tokens_per_batch > -1:
params.group_by_size = True
# check parameters
assert os.path.isdir(params.data_path)
assert os.path.isfile(params.model_path)
# tasks
params.transfer_tasks = params.transfer_tasks.split(',')
assert len(params.transfer_tasks) > 0
assert all([task in TASKS for task in params.transfer_tasks])
reloaded = torch.load(params.model_path)
model_params = AttrDict(reloaded['params'])
logger.info("Supported languages: %s" %
", ".join(model_params.lang2id.keys()))
params.n_langs = model_params['n_langs']
params.id2lang = model_params['id2lang']
params.lang2id = model_params['lang2id']
if "enc_params" in reloaded:
encoder_model_params = AttrDict(reloaded["enc_params"])
elif params.n_enc_layers == model_params.n_layers or params.n_enc_layers == 0:
encoder_model_params = model_params
else:
encoder_model_params = AttrDict(reloaded['params'])
encoder_model_params.n_layers = params.n_enc_layers
assert model_params.n_layers is not encoder_model_params.n_layers
if "dec_params" in reloaded:
decoder_model_params = AttrDict(reloaded["dec_params"])
elif params.n_dec_layers == model_params.n_layers or params.n_dec_layers == 0:
decoder_model_params = model_params
else:
decoder_model_params = AttrDict(reloaded['params'])
decoder_model_params.n_layers = params.n_dec_layers
assert model_params.n_layers is not decoder_model_params.n_layers
params.encoder_model_params = encoder_model_params
params.decoder_model_params = decoder_model_params
if params.emb_dim != -1:
encoder_model_params.emb_dim = params.emb_dim
decoder_model_params.emb_dim = params.emb_dim
# build dictionary / build encoder / build decoder / reload weights
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'],
reloaded['dico_counts'])
for p in [params, encoder_model_params, decoder_model_params]:
p.n_words = len(dico)
p.bos_index = dico.index(BOS_WORD)
p.eos_index = dico.index(EOS_WORD)
p.pad_index = dico.index(PAD_WORD)
p.unk_index = dico.index(UNK_WORD)
p.mask_index = dico.index(MASK_WORD)
encoder = TransformerModel(encoder_model_params,
dico,
is_encoder=True,
with_output=False)
decoder = TransformerModel(decoder_model_params,
dico,
is_encoder=False,
with_output=True)
def _process_state_dict(state_dict):
return {(k[7:] if k.startswith('module.') else k): v
for k, v in state_dict.items()}
if params.no_init == "all":
logger.info("All Models will not load state dict.!!!")
elif params.reload_emb != "":
logger.info("Reloading embedding from %s ..." % params.reload_emb)
word2id, embeddings = read_txt_embeddings(logger, params.reload_emb)
set_pretrain_emb(logger, encoder, dico, word2id, embeddings)
set_pretrain_emb(logger, decoder, dico, word2id, embeddings)
else:
if "model" in reloaded:
if params.no_init != "encoder":
encoder.load_state_dict(_process_state_dict(reloaded['model']),
strict=False)
if params.no_init != "decoder":
decoder.load_state_dict(_process_state_dict(reloaded['model']),
strict=False)
else:
if params.no_init != "encoder":
encoder.load_state_dict(_process_state_dict(
reloaded['encoder']),
strict=False)
if params.no_init != "decoder":
decoder.load_state_dict(
_process_state_dict(reloaded['decoder']))
scores = {}
# run
for task in params.transfer_tasks:
if task == "XQG":
XQG_v3(encoder, decoder, scores, dico, params).run()
elif task == "XSumm":
XSumm(encoder, decoder, scores, dico, params).run()
def main():
# Load pre-trained model
model_path = './models/mlm_tlm_xnli15_1024.pth'
reloaded = torch.load(model_path)
params = AttrDict(reloaded['params'])
# build dictionary / update parameters
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'],
reloaded['dico_counts'])
params.n_words = len(dico)
params.bos_index = dico.index(BOS_WORD)
params.eos_index = dico.index(EOS_WORD)
params.pad_index = dico.index(PAD_WORD)
params.unk_index = dico.index(UNK_WORD)
params.mask_index = dico.index(MASK_WORD)
# build model / reload weights
model = TransformerModel(params, dico, True, True)
#model.cuda() #if using GPU
model.load_state_dict(reloaded['model'])
""" """
with open(args.filename, "r") as f:
sentence_list = f.readlines()[args.sn[0]:args.sn[1]]
# remove new line symbols
for i in range(0, len(sentence_list)):
sentence_list[i] = sentence_list[i].replace("\n", "")
# save as dataframe and add language tokens
sentence_df = pd.DataFrame(sentence_list)
sentence_df.columns = ['sentence']
sentence_df['language'] = 'en'
# match xlm format
sentences = list(zip(sentence_df.sentence, sentence_df.language))(sentence,
language)
""" from XLM repo """
# add </s> sentence delimiters
sentences = [(('</s> %s </s>' % sent.strip()).split(), lang)
for sent, lang in sentences]
# Create batch
bs = len(sentences)
slen = max([len(sent) for sent, _ in sentences])
word_ids = torch.LongTensor(slen, bs).fill_(params.pad_index)
for i in range(len(sentences)):
sent = torch.LongTensor([dico.index(w) for w in sentences[i][0]])
word_ids[:len(sent), i] = sent
lengths = torch.LongTensor([len(sent) for sent, _ in sentences])
langs = torch.LongTensor([params.lang2id[lang] for _, lang in sentences
]).unsqueeze(0).expand(slen, bs)
#if using GPU:
#word_ids=word_ids.cuda()
#lengths=lengths.cuda()
#langs=langs.cuda()
tensor = model('fwd',
x=word_ids,
lengths=lengths,
langs=langs,
causal=False).contiguous()
print(tensor.size())
# The variable tensor is of shape (sequence_length, batch_size, model_dimension).
# tensor[0] is a tensor of shape (batch_size, model_dimension) that corresponds to the first hidden state of the last layer of each sentence.
# This is this vector that we use to finetune on the GLUE and XNLI tasks.
""" """
torch.save(tensor[0], args.o)
from src.utils import AttrDict
from src.data.dictionary import Dictionary, BOS_WORD, EOS_WORD, PAD_WORD, UNK_WORD, MASK_WORD
from src.model.transformer import TransformerModel
logger = getLogger()
# NOTE: remember to replace the model path here
model_path = './dumped/XLM_bora_es/abcedf/checkpoint.pth'
reloaded = torch.load(model_path)
params = AttrDict(reloaded['params'])
print("Supported languages: %s" % ", ".join(params.lang2id.keys()))
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'], reloaded['dico_counts'])
params.n_words = len(dico)
params.bos_index = dico.index(BOS_WORD)
params.eos_index = dico.index(EOS_WORD)
params.pad_index = dico.index(PAD_WORD)
params.unk_index = dico.index(UNK_WORD)
params.mask_index = dico.index(MASK_WORD)
# build model / reload weights
model = TransformerModel(params, dico, True, True)
model.eval()
model.load_state_dict(reloaded['model'])
codes = "./data/processed/XLM_bora_es/60k/codes" # path to the codes of the model
fastbpe = os.path.join(os.getcwd(), 'tools/fastBPE/fast')
def to_bpe(sentences):
def main(params):
# initialize the experiment
logger = initialize_exp(params)
# generate parser / parse parameters
parser = get_parser()
params = parser.parse_args()
reloaded = torch.load(params.model_path)
model_params = AttrDict(reloaded['params'])
logger.info("Supported languages: %s" % ", ".join(model_params.lang2id.keys()))
# update dictionary parameters
for name in ['n_words', 'bos_index', 'eos_index', 'pad_index', 'unk_index', 'mask_index']:
setattr(params, name, getattr(model_params, name))
# build dictionary / build encoder / build decoder / reload weights
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'], reloaded['dico_counts'])
encoder = TransformerModel(model_params, dico, is_encoder=True, with_output=True).cuda().eval()
decoder = TransformerModel(model_params, dico, is_encoder=False, with_output=True).cuda().eval()
encoder.load_state_dict(reloaded['encoder'])
decoder.load_state_dict(reloaded['decoder'])
params.src_id = model_params.lang2id[params.src_lang]
params.tgt_id = model_params.lang2id[params.tgt_lang]
logger.info("encoder: {}".format(encoder))
logger.info("decoder: {}".format(decoder))
# read sentences from stdin
src_sent = []
with open(params.sentences_path, 'r') as file1:
for line in file1:
src_sent.append(line)
logger.info("Read %i sentences from sentences file.Writing them to a src file. Translating ..." % len(src_sent))
f = io.open(params.output_path + 'src_sent', 'w', encoding='utf-8')
for sentence in src_sent:
f.write(sentence + "\n")
f.close()
logger.info("Wrote them to a src file")
f = io.open(params.output_path, 'w', encoding='utf-8')
for i in range(0, len(src_sent), params.batch_size):
# prepare batch
word_ids = [torch.LongTensor([dico.index(w) for w in s.strip().split()])
for s in src_sent[i:i + params.batch_size]]
lengths = torch.LongTensor([len(s) + 2 for s in word_ids])
batch = torch.LongTensor(lengths.max().item(), lengths.size(0)).fill_(params.pad_index)
batch[0] = params.eos_index
for j, s in enumerate(word_ids):
if lengths[j] > 2: # if sentence not empty
batch[1:lengths[j] - 1, j].copy_(s)
batch[lengths[j] - 1, j] = params.eos_index
langs = batch.clone().fill_(params.src_id)
# encode source batch and translate it
encoded, _ = encoder('fwd', x=batch.cuda(), lengths=lengths.cuda(), langs=langs.cuda(), causal=False,
encoder_only=False, extra_adapters_flag=True)
encoded = encoded.transpose(0, 1)
# decoded, dec_lengths = decoder.generate(encoded, lengths.cuda(), params.tgt_id, max_len=int(1.5 * lengths.max().item() + 10))
decoded, dec_lengths = decoder.generate_beam(
encoded, lengths.cuda(), params.tgt_id, beam_size=params.beam_size, length_penalty=params.length_penalty,
early_stopping=params.early_stopping, max_len=int(1.5 * lengths.cuda().max().item() + 10),
extra_adapters_flag=True)
# convert sentences to words
for j in range(decoded.size(1)):
# remove delimiters
sent = decoded[:, j]
delimiters = (sent == params.eos_index).nonzero().view(-1)
assert len(delimiters) >= 1 and delimiters[0].item() == 0
sent = sent[1:] if len(delimiters) == 1 else sent[1:delimiters[1]]
# output translation
source = src_sent[i + j].strip()
target = " ".join([dico[sent[k].item()] for k in range(len(sent))])
#logger.info("%i / %i: %s -> %s\n" % (i + j, len(src_sent), source, target))
if (i+j)%10000 == 0:
logger.info("Translation of %i / %i:\n Source sentence: %s \n Translation: %s\n" % (i + j, len(src_sent), source, target))
# sys.stderr.write("%i / %i: %s -> %s\n" % (i + j, len(src_sent), source, target))
f.write(target + "\n")
f.close()
import subprocess
from torch.nn.modules.distance import CosineSimilarity
cm = CosineSimilarity(dim=0)
#On initialise le modèle
model_path = './mlm_tlm_xnli15_1024.pth'
reloaded = torch.load(model_path)
params = AttrDict(reloaded['params'])
print("Supported languages: %s" % ", ".join(params.lang2id.keys()))
# build dictionary / update parameters
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'],
reloaded['dico_counts'])
params.n_words = len(dico)
params.bos_index = dico.index(BOS_WORD)
params.eos_index = dico.index(EOS_WORD)
params.pad_index = dico.index(PAD_WORD)
params.unk_index = dico.index(UNK_WORD)
params.mask_index = dico.index(MASK_WORD)
# build model / reload weights
model = TransformerModel(params, dico, True, True)
model.load_state_dict(reloaded['model'])
local = False
if os.environ["TERM_PROGRAM"] == "Apple_Terminal":
local = True
#On crée les fonctions intermédiaires
import os
import torch
from src.utils import AttrDict
from src.data.dictionary import Dictionary, BOS_WORD, EOS_WORD, PAD_WORD, UNK_WORD, MASK_WORD
from src.model.transformer import TransformerModel
model_path = '/Users/pengjie/Documents/code/XLM/model/mlm_tlm_xnli15_1024.pth'
reloaded = torch.load(model_path)
params = AttrDict(reloaded['params'])
# build dictionary / update parameters
dico = Dictionary(reloaded['dico_id2word'], reloaded['dico_word2id'],
reloaded['dico_counts'])
params.n_words = len(dico)
params.bos_index = dico.index(BOS_WORD)
params.eos_index = dico.index(EOS_WORD)
params.pad_index = dico.index(PAD_WORD)
params.unk_index = dico.index(UNK_WORD)
params.mask_index = dico.index(MASK_WORD)
# build model / reload weights
model = TransformerModel(params, dico, True, True)
model.eval()
model.load_state_dict(reloaded['model'])
# list of (sentences, lang)
sentences = [
('the following secon@@ dary charac@@ ters also appear in the nov@@ el .',
'en'),
('les zones rurales offr@@ ent de petites routes , a deux voies .', 'fr'),
