def read_train_data(file):
print('read training set')
# pairs_tensor = []
pairs_li = []
lines = 0 # 一共多少个训练数据
for line in open(file, 'r', encoding='utf-8').readlines():
lines += 1
try:
source, target = line.split('==')
except:
print('format mismatch in dataset: ', line.split('=='))
continue
if len(source.split(' - ')) == 4: # poem_1031k_theme
source = source + ' - ' + source.split(' - ')[0]
source_words = ('START1 ' + source + ' END1').split(' ')
target = target.replace('\n', '')
target_words = target.replace('\t', ' / ').split(' ') + [
'/'
] + target.split('\t')[0].split(' ') # 用5个句子训练
source_ids = [
word2id.get(word, vocab_size - 1) for word in source_words
] # default = 4776 '-' PAD?
target_ids = [
word2id.get(word, vocab_size - 1) for word in target_words
]
target_ids.append(EOS_token) # 没有SOS_token
if len(target_ids) == 40: # 只考虑七言
pairs_li.append([source_ids, target_ids])
print('training set size:', len(pairs_li))
return pairs_li # tmp
def predict(self, data, cangtou, predict_param):
hard_rhyme = predict_param['hard_rhyme']
with torch.no_grad():
encoder_hidden = self.encoder.initHidden(1)
input_sentence, input_length = data
encoder_outputs, encoder_hidden = self.encoder(
input_sentence, [input_length], encoder_hidden)
# 将encoder_outputs padding至INPUT_MAX_LENGTH 因为attention中已经固定此维度大小为INPUT_MAX_LENGTH
encoder_outputs_padded = torch.zeros(1,
self.input_max_len,
self.encoder.hidden_size,
device=device)
for b in range(1):
for ei in range(input_length):
encoder_outputs_padded[b, ei] = encoder_outputs[b, ei]
decoder_input = torch.tensor([[SOS_token]],
device=device) # 第一个input是START
decoder_hidden = encoder_hidden # Use last hidden state from encoder to start decoder
sen_len = 7 # 暂时
sen_num = 4
decoded_words = []
for i in range(sen_num):
for j in range(sen_len):
position = torch.tensor([[i, j]],
dtype=torch.float,
device=device)
decoder_output, decoder_hidden, decoder_attention = self.decoder(
decoder_input, decoder_hidden, encoder_outputs_padded,
position)
if j == 0 and cangtou and i < len(cangtou):
top_word = cangtou[i]
top_id = torch.LongTensor(
[word2id.get(top_word, vocab_size - 1)])
else:
top_id, top_word = get_next_word(decoder_output.data,
decoded_words,
hard_rhyme=hard_rhyme)
if top_word == 'N':
print('cannot meet requirements')
break
decoded_words.append(top_word)
decoder_input = top_id.reshape(
(1, 1)).detach() # detach from history as input
position = torch.tensor([[i, 7]],
dtype=torch.float,
device=device)
tmp_decoder_output, tmp_decoder_hidden, tmp_decoder_attention = self.decoder(
decoder_input, decoder_hidden, encoder_outputs_padded,
position)
decoder_hidden = tmp_decoder_hidden
decoder_input = torch.tensor([[2]], device=device) # '/'作为输入
return decoded_words
def predict(self, data, cangtou, predict_param):
with torch.no_grad():
encoder_hidden = self.encoder.initHidden(1)
sen_len = 7 # 暂时
sen_num = 4
input_sentence, input_length = data
decoded_words = [id2word[str(id.item())] for id in input_sentence[0]]
decoder_input = torch.tensor([[2]], device=device) # 第一个input是/ # Jun24
for i in range(sen_num-1):
encoder_outputs, encoder_hidden = self.encoder(input_sentence, [input_length], encoder_hidden)
# 将encoder_outputs padding至INPUT_MAX_LENGTH 因为attention中已经固定此维度大小为INPUT_MAX_LENGTH
encoder_outputs_padded = torch.zeros(1, self.input_max_len, self.encoder.hidden_size,
device=device)
for b in range(1):
for ei in range(input_length):
encoder_outputs_padded[b, ei] = encoder_outputs[b, ei]
decoder_hidden = encoder_hidden # Use last hidden state from encoder to start decoder
for j in range(sen_len):
position = torch.tensor([[i, j]], dtype=torch.float, device=device)
decoder_output, decoder_hidden, decoder_attention = self.decoder(
decoder_input, decoder_hidden, encoder_outputs_padded, position)
if j == 0 and cangtou and i < len(cangtou):
top_word = cangtou[i]
top_id = torch.LongTensor([word2id.get(top_word, vocab_size - 1)])
else:
top_id, top_word = get_next_word(decoder_output.data, decoded_words)
if top_word == 'N':
print('cannot meet requirements')
break
decoded_words.append(top_word)
decoder_input = top_id.reshape((1, 1)).detach() # detach from history as input
li = [word2id[word] for word in decoded_words]
for k in range(self.input_max_len-len(li)):
li.append(0)
input_sentence = torch.tensor([li], dtype=torch.long, device=device)
input_length = torch.tensor([len(decoded_words)], dtype=torch.long, device=device)
decoder_input = torch.tensor([[2]], device=device) # '/'作为输入
return decoded_words
def read_nL21L_train_data(file):
pairs = []
for line in open(file, 'r', encoding='utf-8').readlines():
source, target = line.split('==')
target = target.replace('\n', '')
sentences = target.split('\t')
idss = []
for sentence in sentences:
words = sentence.split(' ')
ids = [word2id.get(word, vocab_size - 1) for word in words]
ids.append(2) # 包括target行 每行都加了/
idss.append(ids)
p1 = [idss[0], idss[1]]
p2 = [idss[0] + idss[1], idss[2]]
p3 = [idss[0] + idss[1] + idss[2], idss[3]]
pairs.append(p1)
pairs.append(p2)
pairs.append(p3)
return pairs
def read_nL21L_eval_data(file):
print('read eval set (nL21L)')
input_li = []
lines = []
targets = []
for line in open(file, 'r', encoding='utf-8').readlines():
line = line.replace('\n', '')
line, target = line.split('==')
lines.append('kw')
target = target.replace('\n', '')
l1, l2, l3, l4 = target.split('\t')
words = l1.split(' ')
ids = [word2id.get(word, vocab_size - 1) for word in words]
input_li.append(ids)
target = target.replace(' ', '').replace('\t', '/')
targets.append(target)
print('read test set done')
return input_li, lines, targets
def read_BERT_train_data(dataset):
dataset_path = 'resource/dataset/' + dataset + '.txt'
encodes_path = 'BERT_as_service/encodes/enc_li_' + dataset + '.json'
target_ids_li = []
pairs = []
for line in open(dataset_path, 'r', encoding='utf-8').readlines():
source, target = line.split('==')
target = target.replace('\n', '')
target_words = target.replace('\t', ' / ').split(' ') + [
'/'
] + target.split('\t')[0].split(' ') # 用5个句子训练
target_ids = [
word2id.get(word, vocab_size - 1) for word in target_words
]
target_ids.append(EOS_token) # 没有SOS_token
target_ids_li.append(target_ids)
with open(encodes_path, 'r', encoding='utf-8') as f:
source_enc_li = json.load(f)
for i in range(len(target_ids_li)):
pairs.append([source_enc_li[i], target_ids_li[i]])
print('training set size:', len(pairs))
return pairs
def line2ids(line): # for testset
input_words = line.split(' ')
input_ids = [word2id.get(word, vocab_size - 1)
for word in input_words] # default = 4776 '-' ?
input_ids.append(EOS_token)
return input_ids