def get_data(dataset,
folder,
train_source_file,
train_target_file,
dev_source_file,
dev_target_file,
tokenization,
pickle_path=""):
#pickle_dump = "preprocess_{}.p".format(
# "word" if word_token else "ngram_" + str(enc_ngram_order_tokenization))
pickle_dump = "data/" + dataset + "_preprocessed.pickle" #os.path.join(folder, dataset + "_preprocessed.pickle")
if not os.path.exists(pickle_dump):
helper.preprocess_and_save_data(os.path.join(folder,
train_source_file),
os.path.join(folder,
train_target_file),
os.path.join(folder, dev_source_file),
os.path.join(folder, dev_target_file),
text_to_ids,
tokenization,
out_file=pickle_dump)
else:
print('Loads datasets from {}'.format(pickle_dump))
return helper.load_preprocess(file=pickle_dump)
def translate(translate_sentence='he saw a old yellow truck .'):
_, (source_vocab_to_int,
target_vocab_to_int), (source_int_to_vocab,
target_int_to_vocab) = helper.load_preprocess()
load_path = helper.load_params()
translate_sentence = sentence_to_seq(translate_sentence,
source_vocab_to_int)
loaded_graph = tf.Graph()
with tf.Session(graph=loaded_graph) as sess:
# Load saved model
loader = tf.train.import_meta_graph(load_path + '.meta')
loader.restore(sess, load_path)
input_data = loaded_graph.get_tensor_by_name('input:0')
logits = loaded_graph.get_tensor_by_name('logits:0')
keep_prob = loaded_graph.get_tensor_by_name('keep_prob:0')
translate_logits = sess.run(logits, {
input_data: [translate_sentence],
keep_prob: 1.0
})[0]
print('Input')
print(' Word Ids: {}'.format([i for i in translate_sentence]))
print(' English Words: {}'.format(
[source_int_to_vocab[i] for i in translate_sentence]))
print('\nPrediction')
print(' Word Ids: {}'.format(
[i for i in np.argmax(translate_logits, 1)]))
print(' French Words: {}'.format(
[target_int_to_vocab[i] for i in np.argmax(translate_logits, 1)]))
def get_test(test_source_file, test_target_file, pickle):
_, _, (source_vocab_to_int, target_vocab_to_int), (
source_int_to_vocab,
target_int_to_vocab) = helper.load_preprocess(pickle)
test_source_text = helper.load_data(test_source_file).lower().split("\n")
test_target_text = helper.load_data(test_target_file).lower().split("\n")
return text_to_ids(test_source_text, test_target_text, source_vocab_to_int,
target_vocab_to_int)
def predecir(self, frase):
_, (source_vocab_to_int, target_vocab_to_int), (
source_int_to_vocab,
target_int_to_vocab) = helper.load_preprocess()
load_path = helper.load_params()
tests.test_sentence_to_seq(sentence_to_seq)
translate_sentence = frase
pIngles = translate_sentence
translate_sentence = sentence_to_seq(translate_sentence,
source_vocab_to_int)
loaded_graph = tf.Graph()
with tf.Session(graph=loaded_graph) as sess:
# Load saved model
loader = tf.train.import_meta_graph(load_path + '.meta')
loader.restore(sess, load_path)
input_data = loaded_graph.get_tensor_by_name('input:0')
logits = loaded_graph.get_tensor_by_name('predictions:0')
target_sequence_length = loaded_graph.get_tensor_by_name(
'target_sequence_length:0')
source_sequence_length = loaded_graph.get_tensor_by_name(
'source_sequence_length:0')
keep_prob = loaded_graph.get_tensor_by_name('keep_prob:0')
translate_logits = sess.run(
logits, {
input_data: [translate_sentence] * batch_size,
target_sequence_length:
[len(translate_sentence) * 2] * batch_size,
source_sequence_length:
[len(translate_sentence)] * batch_size,
keep_prob: 1.0
})[0]
"""
print('Input')
print(' Word Ids: {}'.format([i for i in translate_sentence]))
print(' English Words: {}'.format([source_int_to_vocab[i] for i in translate_sentence]))
print('\nPrediction')
print(' Word Ids: {}'.format([i for i in translate_logits]))
print(' Spanish Words: {}'.format(" ".join([target_int_to_vocab[i] for i in translate_logits])))
"""
variableRetornar = format(" ".join(
[target_int_to_vocab[i] for i in translate_logits]))
print('Resultado de ', pIngles)
print(variableRetornar)
miTxt = open("BorderOut\\IA\\respuesta.txt", 'w')
miTxt.write(variableRetornar)
miTxt.close()
return variableRetornar
def apply_model(d):
gen_length = 30
if request.method == 'POST':
content = request.json
gen_length = content['gen_length']
_, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()
prime_word = random.choice(int_to_vocab)
print (prime_word)
seq_length, load_dir = helper.load_params()
loaded_graph = tf.Graph()
with tf.Session(graph=loaded_graph) as sess:
# Load saved model
loader = tf.train.import_meta_graph(load_dir + '.meta')
loader.restore(sess, load_dir)
# Get Tensors from loaded model
input_text, initial_state, final_state, probs = get_tensors(loaded_graph)
# Sentences generation setup
gen_sentences = [prime_word]
prev_state = sess.run(initial_state, {input_text: np.array([[1]])})
print (gen_sentences)
# Generate sentences
for n in range(gen_length):
# Dynamic Input
dyn_input = [[vocab_to_int[word] for word in gen_sentences[-seq_length:]]]
dyn_seq_length = len(dyn_input[0])
# Get Prediction
probabilities, prev_state = sess.run(
[probs, final_state],
{input_text: dyn_input, initial_state: prev_state})
pred_word = pick_word(probabilities[dyn_seq_length-1], int_to_vocab)
gen_sentences.append(pred_word)
data = massage_results(gen_sentences, token_dict)
if (len(data) > 2):
wod_return = random.choice(data[1:-1])
else:
wod_return = data[0]
def genScript():
_, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()
#print("vocab to int : ",vocab_to_int)
seq_length, load_dir = helper.load_params()
gen_length = 200
prime_word = 'bart_simpson'
#'bart_simpson' # 'moe_szyslak'
loaded_graph = tf.Graph()
with tf.Session(graph=loaded_graph) as sess:
loader = tf.train.import_meta_graph('save.meta')
loader.restore(sess, load_dir)
input_text, initial_state, final_state, probs = getTensors(
loaded_graph)
gen_sentences = [prime_word + ':']
prev_state = sess.run(initial_state, {input_text: np.array([[1]])})
# Generate sentences
for n in range(gen_length):
dyn_input = [[
vocab_to_int[word] for word in gen_sentences[-seq_length:]
]]
dyn_seq_length = len(dyn_input[0])
probabilities, prev_state = sess.run([probs, final_state], {
input_text: dyn_input,
initial_state: prev_state
})
pred_word = pick_word(probabilities[0][dyn_seq_length - 1],
int_to_vocab)
gen_sentences.append(pred_word)
# remove tokens
tv_script = ' '.join(gen_sentences)
for key, token in token_dict.items():
ending = ' ' if key in ['\n', '(', '"'] else ''
tv_script = tv_script.replace(' ' + token.lower(), key)
tv_script = tv_script.replace('\n ', '\n')
tv_script = tv_script.replace('( ', '(')
print(tv_script)
def writeSongs(load_dir, seq_length):
int_text, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()
# 生成文本的长度
gen_length = 500
# 文章开头的字，指定一个即可，这个字必须是在训练词汇列表中的
prime_word = u'我'
loaded_graph = tf.Graph()
with tf.Session(graph=loaded_graph) as sess:
# 加载保存过的session
loader = tf.train.import_meta_graph(load_dir + '.meta')
loader.restore(sess, load_dir)
# 通过名称获取缓存的tensor
input_text, initial_state, final_state, probs = get_tensors(
loaded_graph)
# 准备开始生成文本
gen_sentences = [prime_word]
prev_state = sess.run(initial_state, {input_text: np.array([[1]])})
# 开始生成文本
for n in range(gen_length):
dyn_input = [[
vocab_to_int[word] for word in gen_sentences[-seq_length:]
]]
dyn_seq_length = len(dyn_input[0])
probabilities, prev_state = sess.run([probs, final_state], {
input_text: dyn_input,
initial_state: prev_state
})
pred_word = pick_word(probabilities[dyn_seq_length - 1],
int_to_vocab)
gen_sentences.append(pred_word)
# 将标点符号还原
novel = ''.join(gen_sentences)
for key, token in token_dict.items():
ending = ' ' if key in ['\n', '（', '“'] else ''
novel = novel.replace(token.lower(), key)
novel = novel.replace('\n ', '\n')
novel = novel.replace('（ ', '（')
print(novel)
def predict():
# Check for a GPU
train_on_gpu = torch.cuda.is_available()
if not train_on_gpu:
print('No GPU found. Please use a GPU to train your neural network.')
_, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()
class RNN(nn.Module):
def __init__(self,
vocab_size,
output_size,
embedding_dim,
hidden_dim,
n_layers,
dropout=0.5):
"""
Initialize the PyTorch RNN Module
:param vocab_size: The number of input dimensions of the neural network (the size of the vocabulary)
:param output_size: The number of output dimensions of the neural network
:param embedding_dim: The size of embeddings, should you choose to use them
:param hidden_dim: The size of the hidden layer outputs
:param dropout: dropout to add in between LSTM/GRU layers
"""
super(RNN, self).__init__()
# TODO: Implement function
# set class variables
self.output_size = output_size
self.n_layers = n_layers
self.hidden_dim = hidden_dim
# define model layers
# embedding and LSTM layers
self.embed = nn.Embedding(vocab_size, embedding_dim)
self.lstm = nn.LSTM(embedding_dim,
hidden_dim,
n_layers,
dropout=dropout,
batch_first=True)
# linear layer
self.fc = nn.Linear(hidden_dim, output_size)
def forward(self, nn_input, hidden):
"""
Forward propagation of the neural network
:param nn_input: The input to the neural network
:param hidden: The hidden state
:return: Two Tensors, the output of the neural network and the latest hidden state
"""
# TODO: Implement function
batch_size = nn_input.size(0)
# embeddings and lstm_out
embeds = self.embed(nn_input)
lstm_out, hidden = self.lstm(embeds, hidden)
# stack up lstm outputs
lstm_out = lstm_out.contiguous().view(-1, self.hidden_dim)
# dropout and fully-connected layer
output = self.fc(lstm_out)
# reshape to be batch_size first
output = output.view(batch_size, -1, self.output_size)
out = output[:, -1] # get last batch of labels
# return one batch of output word scores and the hidden state
return out, hidden
def init_hidden(self, batch_size):
'''
Initialize the hidden state of an LSTM/GRU
:param batch_size: The batch_size of the hidden state
:return: hidden state of dims (n_layers, batch_size, hidden_dim)
'''
# Implement function
# initialize hidden state with zero weights, and move to GPU if available
# initialize hidden state with zero weights, and move to GPU if available
weight = next(self.parameters()).data
if (train_on_gpu):
hidden = (weight.new(self.n_layers, batch_size,
self.hidden_dim).zero_().cuda(),
weight.new(self.n_layers, batch_size,
self.hidden_dim).zero_().cuda())
else:
hidden = (weight.new(self.n_layers, batch_size,
self.hidden_dim).zero_(),
weight.new(self.n_layers, batch_size,
self.hidden_dim).zero_())
return hidden
# Model parameters
# Vocab size
vocab_size = len(vocab_to_int)
# Output size
output_size = vocab_size
# Embedding Dimension
embedding_dim = 256
# Hidden Dimension
hidden_dim = 256
# Number of RNN Layers
n_layers = 2
sequence_length = 8
trained_rnn = RNN(vocab_size,
output_size,
embedding_dim,
hidden_dim,
n_layers,
dropout=0.5)
trained_rnn.load_state_dict(
torch.load('trained_rnn1.pt', map_location=torch.device('cpu')))
"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
import torch.nn.functional as F
def generate(rnn,
prime_id,
int_to_vocab,
token_dict,
pad_value,
predict_len=100):
"""
Generate text using the neural network
:param decoder: The PyTorch Module that holds the trained neural network
:param prime_id: The word id to start the first prediction
:param int_to_vocab: Dict of word id keys to word values
:param token_dict: Dict of puncuation tokens keys to puncuation values
:param pad_value: The value used to pad a sequence
:param predict_len: The length of text to generate
:return: The generated text
"""
rnn.eval()
# create a sequence (batch_size=1) with the prime_id
current_seq = np.full((1, sequence_length), pad_value)
current_seq[-1][-1] = prime_id
predicted = [int_to_vocab[prime_id]]
for _ in range(predict_len):
if train_on_gpu:
current_seq = torch.LongTensor(current_seq).cuda()
else:
current_seq = torch.LongTensor(current_seq)
# initialize the hidden state
hidden = rnn.init_hidden(current_seq.size(0))
# get the output of the rnn
output, _ = rnn(current_seq, hidden)
# get the next word probabilities
p = F.softmax(output, dim=1).data
if (train_on_gpu):
p = p.cpu() # move to cpu
# use top_k sampling to get the index of the next word
top_k = 5
p, top_i = p.topk(top_k)
top_i = top_i.numpy().squeeze()
# select the likely next word index with some element of randomness
p = p.numpy().squeeze()
word_i = np.random.choice(top_i, p=p / p.sum())
# retrieve that word from the dictionary
word = int_to_vocab[word_i]
predicted.append(word)
# the generated word becomes the next "current sequence" and the cycle can continue
current_seq = np.roll(current_seq, -1, 1)
current_seq[-1][-1] = word_i
gen_sentences = ' '.join(predicted)
# Replace punctuation tokens
for key, token in token_dict.items():
ending = ' ' if key in ['\n', '(', '"'] else ''
gen_sentences = gen_sentences.replace(' ' + token.lower(), key)
gen_sentences = gen_sentences.replace('\n ', '\n')
gen_sentences = gen_sentences.replace('( ', '(')
# return all the sentences
return gen_sentences
if (request.method == 'POST'):
message = request.form['message']
gen_length = 400 # modify the length to your preference
prime_word = message # name for starting the script
pad_word = helper.SPECIAL_WORDS['PADDING']
generated_script = generate(trained_rnn,
vocab_to_int[prime_word + ':'],
int_to_vocab, token_dict,
vocab_to_int[pad_word], gen_length)
return render_template('result.html', prediction=generated_script)
ipython = get_ipython()
if '__IPYTHON__' in globals():
ipython.magic('load_ext autoreload')
ipython.magic('autoreload 1')
import numpy as np
import modelling as MD
from keras.preprocessing.sequence import pad_sequences
import helper
loaded_model = MD.load_model()
loaded_model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy', 'MSE'])
x, y, x_tk, y_tk = helper.load_preprocess()
###############################################################
"""
y_id_to_word = {value: key for key, value in y_tk.word_index.items()}
y_id_to_word[0] = '<PAD>'
sentence1 = 'the july is chilly'
sentence2 = 'he saw a old yellow truck'
sentence1 = [x_tk.word_index[word] for word in sentence1.split()]
sentence1 = pad_sequences([sentence1], maxlen=x.shape[-1], padding='post')
sentence2 = [x_tk.word_index[word] for word in sentence2.split()]
sentence2 = pad_sequences([sentence2], maxlen=x.shape[-1], padding='post')
sentences = np.array([sentence1[0],sentence2[0]])
#sentences = np.array([sentence[0], x[0]])
def train():
# Number of Epochs
epochs = 8
# Batch Size
batch_size = 512
# RNN Size
rnn_size = 512
# Number of Layers
num_layers = 2
# Embedding Size
encoding_embedding_size = 200
decoding_embedding_size = 200
# Learning Rate
learning_rate = 0.01
# Dropout Keep Probability
keep_probability = 0.5
display_step = 1
save_path = 'checkpoints/dev'
(source_int_text, target_int_text), (source_vocab_to_int, target_vocab_to_int), _ = helper.load_preprocess()
max_target_sentence_length = max([len(sentence) for sentence in source_int_text])
train_graph = tf.Graph()
with train_graph.as_default():
input_data, targets, lr, keep_prob, target_sequence_length, max_target_sequence_length, source_sequence_length = model_inputs()
# sequence_length = tf.placeholder_with_default(max_target_sentence_length, None, name='sequence_length')
input_shape = tf.shape(input_data)
train_logits, inference_logits = seq2seq_model(tf.reverse(input_data, [-1]),
targets,
keep_prob,
batch_size,
source_sequence_length,
target_sequence_length,
max_target_sequence_length,
len(source_vocab_to_int),
len(target_vocab_to_int),
encoding_embedding_size,
decoding_embedding_size,
rnn_size,
num_layers,
target_vocab_to_int)
training_logits = tf.identity(train_logits.rnn_output, name='logits')
inference_logits = tf.identity(inference_logits.sample_id, name='predictions')
masks = tf.sequence_mask(target_sequence_length, max_target_sequence_length, dtype=tf.float32, name='masks')
with tf.name_scope("optimization"):
# Loss function
cost = tf.contrib.seq2seq.sequence_loss(
training_logits,
targets,
masks)
# Optimizer
optimizer = tf.train.AdamOptimizer(lr)
# Gradient Clipping
gradients = optimizer.compute_gradients(cost)
capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gradients if grad is not None]
train_op = optimizer.apply_gradients(capped_gradients)
# Split data to training and validation sets
train_source = source_int_text[batch_size:]
train_target = target_int_text[batch_size:]
valid_source = source_int_text[:batch_size]
valid_target = target_int_text[:batch_size]
(valid_sources_batch, valid_targets_batch, valid_sources_lengths, valid_targets_lengths) = next(
get_batches(valid_source,
valid_target,
batch_size,
source_vocab_to_int['<PAD>'],
target_vocab_to_int['<PAD>']))
with tf.Session(graph=train_graph) as sess:
sess.run(tf.global_variables_initializer())
for epoch_i in range(epochs):
for batch_i, (source_batch, target_batch, sources_lengths, targets_lengths) in enumerate(
get_batches(train_source, train_target, batch_size,
source_vocab_to_int['<PAD>'],
target_vocab_to_int['<PAD>'])):
_, loss = sess.run(
[train_op, cost],
{input_data: source_batch,
targets: target_batch,
lr: learning_rate,
target_sequence_length: targets_lengths,
source_sequence_length: sources_lengths,
keep_prob: keep_probability})
if batch_i % display_step == 0 and batch_i > 0:
batch_train_logits = sess.run(
inference_logits,
{input_data: source_batch,
source_sequence_length: sources_lengths,
target_sequence_length: targets_lengths,
keep_prob: 1.0})
batch_valid_logits = sess.run(
inference_logits,
{input_data: valid_sources_batch,
source_sequence_length: valid_sources_lengths,
target_sequence_length: valid_targets_lengths,
keep_prob: 1.0})
train_acc = get_accuracy(target_batch, batch_train_logits)
valid_acc = get_accuracy(valid_targets_batch, batch_valid_logits)
print(
'Epoch {:>3} Batch {:>4}/{} - Train Accuracy: {:>6.4f}, Validation Accuracy: {:>6.4f}, Loss: {:>6.4f}'
.format(epoch_i, batch_i, len(source_int_text) // batch_size, train_acc, valid_acc, loss))
# Save Model
saver = tf.train.Saver()
saver.save(sess, save_path)
print('Model Trained and Saved')
helper.save_params(save_path)
def checkpoint():
_, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()
trained_rnn = helper.load_model('./save/trained_rnn')
return trained_rnn
def run(username, prime_word, gen_length):
# directory for saving the model
save_dir = './save'
_, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()
seq_length, load_dir = helper.load_params()
loaded_graph = tf.Graph()
with tf.Session(graph=loaded_graph) as sess:
# Load saved model
loader = tf.train.import_meta_graph(load_dir + '.meta')
loader.restore(sess, load_dir)
# Get Tensors from loaded model
input_text = loaded_graph.get_tensor_by_name('input:0')
#input_text_rev=loaded_graph.get_tensor_by_name('input_rev:0')
initial_state = loaded_graph.get_tensor_by_name('initial_state:0')
final_state = loaded_graph.get_tensor_by_name('final_state:0')
probs = loaded_graph.get_tensor_by_name('probs_f:0')
# Sentences generation setup
gen_sentences = [prime_word] #changed
prev_state = sess.run(
initial_state,
{input_text: np.array([[1]])}) #,input_text_rev: np.array([[1]])})
# Generate sentences
for n in range(gen_length):
# Dynamic Input
dyn_input = [[
vocab_to_int[word] for word in gen_sentences[-seq_length:]
]]
dyn_seq_length = len(dyn_input[0])
# Get Prediction
probabilities, prev_state = sess.run([probs, final_state], {
input_text: dyn_input,
initial_state: prev_state,
})
index = np.argmax(probabilities[0][dyn_seq_length - 1]) ###
pred_word = int_to_vocab[index]
gen_sentences.append(pred_word)
# Remove tokens
quote_script = ' '.join(gen_sentences)
for key, token in token_dict.items():
ending = ' ' if key in ['\n', '(', '"'] else ''
quote_script = quote_script.replace(' ' + token.lower(), key)
quote_script = quote_script.replace('\n ', '\n')
quote_script = quote_script.replace('( ', '(')
quote_data = quote_script.split("\n")
quote_data = quote_data[1:-1]
res = []
for i in quote_data:
res.append(i)
for i in quote_data:
if len(i) <= 139 - len(username) and len(
i) > 9: # "i have" not in i:
res.append(i) # remove it
#final list of result is in "res"
choice = random.randint(1, 1000)
choice_fin = choice % len(res)
#return username+" "+res[choice_fin]
return res
def main():
#tests.test_build_nn(build_nn)
preprocess_and_save_data(data_dir, token_lookup, create_lookup_tables)
int_text, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()
#build graph to run the train
train_graph = tf.Graph()
with train_graph.as_default():
vocab_size = len(int_to_vocab)
input_text, targets, lr = get_inputs()
input_data_shape = tf.shape(input_text)
cell, initial_state = get_init_cell(input_data_shape[0], rnn_size)
print("cell is: ", cell)
print("rnn_size is: ", rnn_size)
print("input_text is: ", input_text)
print("vocab_size is: ", vocab_size)
print("input_data_shape is: ", input_data_shape)
print("embed_dim is: ", embed_dim)
logits, final_state = build_nn(cell, rnn_size, input_text, vocab_size,
embed_dim)
print("logits is: ", logits)
# Probabilities for generating words
probs = tf.nn.softmax(logits, name='probs')
print("probs is: ", probs)
# Loss function
cost = seq2seq.sequence_loss(
logits, targets,
tf.ones([input_data_shape[0], input_data_shape[1]]))
print("cost is: ", cost)
# Optimizer
optimizer = tf.train.AdamOptimizer(lr)
# Gradient Clipping (google it)
gradients = optimizer.compute_gradients(cost)
capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var)
for grad, var in gradients if grad is not None]
train_op = optimizer.apply_gradients(capped_gradients)
batches = get_batches(int_text, batch_size, seq_length)
with tf.Session(graph=train_graph) as sess:
sess.run(tf.global_variables_initializer())
for epoch_i in range(num_epochs):
state = sess.run(initial_state, {input_text: batches[0][0]})
for batch_i, (x, y) in enumerate(batches):
feed = {
input_text: x,
targets: y,
initial_state: state,
lr: learning_rate
}
train_loss, state, _ = sess.run([cost, final_state, train_op],
feed)
if (epoch_i * len(batches) +
batch_i) % show_every_n_batches == 0:
print('Epoch {:>3} Batch {:>4}/{} train_loss = {:.3f}'.
format(epoch_i, batch_i, len(batches), train_loss))
# save model
saver = tf.train.Saver()
saver.save(sess, save_dir)
print('Model Trained and Saved')
tests.test_text_to_ids(text_to_ids)
"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
helper.preprocess_and_save_data(source_path, target_path, text_to_ids,
PREPROCESS_PATH)
"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
import numpy as np
import helper
import problem_unittests as tests
(source_int_text, target_int_text), (
source_vocab_to_int,
target_vocab_to_int), _ = helper.load_preprocess(PREPROCESS_PATH)
"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
from distutils.version import LooseVersion
import warnings
import tensorflow as tf
# Check TensorFlow Version
assert LooseVersion(tf.__version__) >= LooseVersion(
'1.0'), 'Please use TensorFlow version 1.0 or newer'
print('TensorFlow Version: {}'.format(tf.__version__))
# Check for a GPU
if not tf.test.gpu_device_name():
warnings.warn(
tokens['\n'] = '<NEW_LINE>'
#tokens[':'] = '<COLON>'
return tokens
tests.test_tokenize(token_lookup)
#################################################
## Pre-process all the data and save it
#################################################
# pre-process training data
helper.preprocess_and_save_data(data_dir, token_lookup, create_lookup_tables)
#################################################
## Check Point
#################################################
int_text, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()
#################################################
## Build the Neural Network
#################################################
# Check for a GPU
train_on_gpu = torch.cuda.is_available()
if not train_on_gpu:
print('No GPU found. Please use a GPU to train your neural network.')
if train_on_gpu:
print("CUDA Device:", torch.cuda.get_device_name(0))
print("Memory Allocated:", round(torch.cuda.memory_allocated(0)/1024**3,1), "GB")
def batch_data(words, sequence_length, batch_size):
"""
"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
import tensorflow as tf
import numpy as np
import helper
import problem_unittests as tests
_, (source_vocab_to_int,
target_vocab_to_int), (source_int_to_vocab,
target_int_to_vocab) = helper.load_preprocess()
load_path = helper.load_params()
batch_size = 128 * 4
#%% Sentence to Sequence
def sentence_to_seq(sentence, vocab_to_int):
"""
Convert a sentence to a sequence of ids
:param sentence: String
:param vocab_to_int: Dictionary to go from the words to an id
:return: List of word ids
"""
sentence_ids = [
vocab_to_int.get(word.lower(), vocab_to_int['<UNK>'])
for word in sentence.split()
]
return sentence_ids
def Train(embed_dim=512,
num_epochs=20,
learning_rate=0.01,
seq_length=10,
rnn_size=700,
batch_size=100):
data_dir = './data/simpsons/moes_tavern_lines.txt'
text = helper.load_data(data_dir)
# Ignore notice, since we don't use it for analysing the data
text = text[81:]
helper.preprocess_and_save_data(data_dir, token_lookup,
create_lookup_tables)
int_text, _, int_to_vocab, _ = helper.load_preprocess()
show_every_n_batches = 50
train_graph = tf.Graph()
with train_graph.as_default():
vocab_size = len(int_to_vocab)
input_text, targets, lr = get_inputs()
input_data_shape = tf.shape(input_text)
cell, initial_state = get_init_cell(input_data_shape[0], rnn_size)
logits, final_state = build_nn(cell, rnn_size, input_text, vocab_size,
embed_dim)
# Probabilities for generating words
tf.nn.softmax(logits, name='probs')
# Loss function
cost = seq2seq.sequence_loss(
logits, targets,
tf.ones([input_data_shape[0], input_data_shape[1]]))
# Optimizer
optimizer = tf.train.AdamOptimizer(lr)
# Gradient Clipping
gradients = optimizer.compute_gradients(cost)
capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var)
for grad, var in gradients if grad is not None]
train_op = optimizer.apply_gradients(capped_gradients)
batches = get_batches(int_text, batch_size, seq_length)
with tf.Session(graph=train_graph) as sess:
sess.run(tf.global_variables_initializer())
for epoch_i in range(num_epochs):
state = sess.run(initial_state, {input_text: batches[0][0]})
for batch_i, (x, y) in enumerate(batches):
feed = {
input_text: x,
targets: y,
initial_state: state,
lr: learning_rate
}
train_loss, state, _ = sess.run([cost, final_state, train_op],
feed)
# Show every <show_every_n_batches> batches
if (epoch_i * len(batches) +
batch_i) % show_every_n_batches == 0:
print('Epoch {:>3} Batch {:>4}/{} train_loss = {:.3f}'.
format(epoch_i, batch_i, len(batches), train_loss))
# Save Model
saver = tf.train.Saver()
saver.save(sess, "./save")
print('Model Trained and Saved')
# Save parameters for checkpoint
helper.save_params((seq_length, "./save"))
"!": "||EXCL_MARK||",
"?": "||QUEST_MARK||",
"(": "||L_PARENTH||",
")": "||R_PARENTH||",
"--": "||DASH||",
"\n": "||RETURN||"
}
for key, token in tokens.items():
text = text.replace(key, ' {} '.format(token.lower()))
lines = text.split(' ||period|| ')
first_words = list(set([line.split(" ")[0] for line in lines]))
_, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()
seq_length, _ = helper.load_params()
# Длина генерируемой последовательности
gen_length = 10
phrases = 10
def get_tensors(loaded_graph):
inputs = loaded_graph.get_tensor_by_name('input:0')
initial_state = loaded_graph.get_tensor_by_name('initial_state:0')
final_state = loaded_graph.get_tensor_by_name('final_state:0')
probs = loaded_graph.get_tensor_by_name('probs:0')
return inputs, initial_state, final_state, probs
def train():
(source_int_text,
target_int_text), (source_vocab_to_int,
target_vocab_to_int), _ = helper.load_preprocess()
# Check TensorFlow Version
assert LooseVersion(tf.__version__) >= LooseVersion(
'1.0'), 'Please use TensorFlow version 1.0 or newer'
print('TensorFlow Version: {}'.format(tf.__version__))
# Check for a GPU
if not tf.test.gpu_device_name():
warnings.warn(
'No GPU found. Please use a GPU to train your neural network.')
else:
print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))
epochs = 3
# Batch Size
batch_size = 128
# RNN Size
rnn_size = 256
# Number of Layers
num_layers = 2
# Embedding Size
encoding_embedding_size = 200
decoding_embedding_size = 200
# Learning Rate
learning_rate = 0.001
# Dropout Keep Probability
keep_probability = 0.5
# ### Build the Graph
save_path = 'ckpt'
(source_int_text,
target_int_text), (source_vocab_to_int,
target_vocab_to_int), _ = helper.load_preprocess()
max_target_sentence_length = max(
[len(sentence) for sentence in source_int_text])
train_graph = tf.Graph()
with train_graph.as_default():
input_data, targets, lr, keep_prob = model_inputs()
sequence_length = tf.placeholder_with_default(
max_target_sentence_length, None, name='sequence_length')
input_shape = tf.shape(input_data)
train_logits, inference_logits = seq2seq_model(
tf.reverse(input_data, [-1]), targets, keep_prob,
batch_size, sequence_length, len(source_vocab_to_int),
len(target_vocab_to_int), encoding_embedding_size,
decoding_embedding_size, rnn_size, num_layers, target_vocab_to_int)
tf.identity(inference_logits, 'logits')
with tf.name_scope("optimization"):
# Loss function
cost = tf.contrib.seq2seq.sequence_loss(
train_logits, targets,
tf.ones([input_shape[0], sequence_length]))
# Optimizer
optimizer = tf.train.AdamOptimizer(lr)
# Gradient Clipping
gradients = optimizer.compute_gradients(cost)
capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var)
for grad, var in gradients if grad is not None]
train_op = optimizer.apply_gradients(capped_gradients)
def get_accuracy(target, logits):
"""
Calculate accuracy
"""
max_seq = max(target.shape[1], logits.shape[1])
if max_seq - target.shape[1]:
target = np.pad(target_batch,
[(0, 0), (0, max_seq - target_batch.shape[1]),
(0, 0)], 'constant')
if max_seq - batch_train_logits.shape[1]:
logits = np.pad(logits, [(0, 0), (0, max_seq - logits.shape[1]),
(0, 0)], 'constant')
return np.mean(np.equal(target, np.argmax(logits, 2)))
train_source = source_int_text[batch_size:]
train_target = target_int_text[batch_size:]
valid_source = helper.pad_sentence_batch(source_int_text[:batch_size])
valid_target = helper.pad_sentence_batch(target_int_text[:batch_size])
with tf.Session(graph=train_graph) as sess:
sess.run(tf.global_variables_initializer())
for epoch_i in range(epochs):
for batch_i, (source_batch, target_batch) in enumerate(
helper.batch_data(train_source, train_target, batch_size)):
start_time = time.time()
_, loss = sess.run(
[train_op, cost], {
input_data: source_batch,
targets: target_batch,
lr: learning_rate,
sequence_length: target_batch.shape[1],
keep_prob: keep_probability
})
batch_train_logits = sess.run(inference_logits, {
input_data: source_batch,
keep_prob: 1.0
})
batch_valid_logits = sess.run(inference_logits, {
input_data: valid_source,
keep_prob: 1.0
})
train_acc = get_accuracy(target_batch, batch_train_logits)
valid_acc = get_accuracy(np.array(valid_target),
batch_valid_logits)
end_time = time.time()
print(
'Epoch {:>3} Batch {:>4}/{} - Train Accuracy: {:>6.3f}, Validation Accuracy: {:>6.3f}, Loss: {:>6.3f}'
.format(epoch_i, batch_i,
len(source_int_text) // batch_size, train_acc,
valid_acc, loss))
# Save Model
saver = tf.train.Saver()
saver.save(sess, save_path)
print('Model Trained and Saved')
helper.save_params(save_path)
# # 检查点 # # 这是你的第一个检查点。如果你什么时候决定再回到该记事本，或需要重新启动该记事本，可以从这里继续。预处理的数据已保存到磁盘上。 # In[5]: """ DON'T MODIFY ANYTHING IN THIS CELL """ import numpy as np import helper (source_int_text, target_int_text), (source_vocab_to_int, target_vocab_to_int), _ = helper.load_preprocess() # ### 检查 TensorFlow 版本，确认可访问 GPU # # 这一检查步骤，可以确保你使用的是正确版本的 TensorFlow，并且能够访问 GPU。 # # In[7]: """ DON'T MODIFY ANYTHING IN THIS CELL """ from distutils.version import LooseVersion import warnings
def traducir(frase):
# Number of Epochs
epochs = 10
# Batch Size
batch_size = 512
# RNN Size
rnn_size = 128
# Number of Layers
num_layers = 2
# Embedding Size
encoding_embedding_size = 128
decoding_embedding_size = 128
# Learning Rate
learning_rate = 0.001
# Dropout Keep Probability
keep_probability = 0.55
display_step = True
_, (source_vocab_to_int,
target_vocab_to_int), (source_int_to_vocab,
target_int_to_vocab) = helper.load_preprocess()
load_path = helper.load_params()
tests.test_sentence_to_seq(sentence_to_seq)
translate_sentence = frase
pIngles = translate_sentence
translate_sentence = sentence_to_seq(translate_sentence,
source_vocab_to_int)
loaded_graph = tf.Graph()
with tf.Session(graph=loaded_graph) as sess:
# Load saved model
loader = tf.train.import_meta_graph(load_path + '.meta')
loader.restore(sess, load_path)
input_data = loaded_graph.get_tensor_by_name('input:0')
logits = loaded_graph.get_tensor_by_name('predictions:0')
target_sequence_length = loaded_graph.get_tensor_by_name(
'target_sequence_length:0')
source_sequence_length = loaded_graph.get_tensor_by_name(
'source_sequence_length:0')
keep_prob = loaded_graph.get_tensor_by_name('keep_prob:0')
translate_logits = sess.run(
logits, {
input_data: [translate_sentence] * batch_size,
target_sequence_length:
[len(translate_sentence) * 2] * batch_size,
source_sequence_length: [len(translate_sentence)] * batch_size,
keep_prob: 1.0
})[0]
"""
print('Input')
print(' Word Ids: {}'.format([i for i in translate_sentence]))
print(' English Words: {}'.format([source_int_to_vocab[i] for i in translate_sentence]))
print('\nPrediction')
print(' Word Ids: {}'.format([i for i in translate_logits]))
print(' Spanish Words: {}'.format(" ".join([target_int_to_vocab[i] for i in translate_logits])))
"""
variableRetornar = format(" ".join(
[target_int_to_vocab[i] for i in translate_logits]))
return variableRetornar
helper.preprocess_and_save_data(source_path, target_path, text_to_ids)
# # 检查点
#
# 这是你的第一个检查点。如果你什么时候决定再回到该记事本，或需要重新启动该记事本，可以从这里继续。预处理的数据已保存到磁盘上。
# In[5]:
"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
import numpy as np
import helper
(source_int_text,
target_int_text), (source_vocab_to_int,
target_vocab_to_int), _ = helper.load_preprocess()
# ### 检查 TensorFlow 版本，确认可访问 GPU
#
# 这一检查步骤，可以确保你使用的是正确版本的 TensorFlow，并且能够访问 GPU。
#
# In[6]:
"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
from distutils.version import LooseVersion
import warnings
import tensorflow as tf
# Check TensorFlow Version
def trainModel():
"""
From make-tweet.py
"""
_, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()
seq_length, load_dir = helper.load_params()
print('Done')
def get_tensors(loaded_graph):
"""
Get input, initial state, final state, and probabilities tensor from <loaded_graph>
"""
InputTensor = loaded_graph.get_tensor_by_name("input:0")
InitialStateTensor = loaded_graph.get_tensor_by_name("initial_state:0")
FinalStateTensor = loaded_graph.get_tensor_by_name("final_state:0")
ProbsTensor = loaded_graph.get_tensor_by_name("probs:0")
return InputTensor, InitialStateTensor, FinalStateTensor, ProbsTensor
def pick_word(probabilities, int_to_vocab):
"""
Pick the next word in the generated text
"""
# TODO: Implement Function
#one_hot_encoded = np.argmax(probabilities, axis=0)
one_hot_encoded = np.random.choice(len(int_to_vocab),p=probabilities)
next_word = int_to_vocab[one_hot_encoded]
return next_word
# ## Generate TV Script
# This will generate the TV script for you. Set `gen_length` to the length of TV script you want to generate.
gen_length = 30 # length of a tweet
#prime_word = '@'
choice = np.random.choice(len(int_to_vocab))
prime_word = int_to_vocab[choice]
prime_word
loaded_graph = tf.Graph()
with tf.Session(graph=loaded_graph) as sess:
# Load saved model
loader = tf.train.import_meta_graph(load_dir + '.meta')
loader.restore(sess, load_dir)
# Get Tensors from loaded model
input_text, initial_state, final_state, probs = get_tensors(loaded_graph)
# Sentences generation setup
gen_sentences = [prime_word ]# + ':']
prev_state = sess.run(initial_state, {input_text: np.array([[1]])})
# Generate sentences
for n in range(gen_length):
# Dynamic Input
dyn_input = [[vocab_to_int[word] for word in gen_sentences[-seq_length:]]]
dyn_seq_length = len(dyn_input[0])
# Get Prediction
probabilities, prev_state = sess.run(
[probs, final_state],
{input_text: dyn_input, initial_state: prev_state})
pred_word = pick_word(probabilities[dyn_seq_length-1], int_to_vocab)
gen_sentences.append(pred_word)
# Remove tokens
tweet = ' '.join(gen_sentences)
for key, token in token_dict.items():
ending = ' ' if key in ['\n', '(', '"'] else ''
tweet = tweet.replace(' ' + token.lower(), key)
tweet = tweet.replace('\n ', '\n')
tweet = tweet.replace('( ', '(')
tweet = tweet.replace('\\', '')
tweet = tweet.replace('\/', '')
tweet = tweet.replace('\"', '')
tweet = tweet.replace(' ', ' ')
return tweet
