def translate(translate_sentence='he saw a old yellow truck .'): _, (source_vocab_to_int, target_vocab_to_int), (source_int_to_vocab, target_int_to_vocab) = helper.load_preprocess() load_path = helper.load_params() translate_sentence = sentence_to_seq(translate_sentence, source_vocab_to_int) loaded_graph = tf.Graph() with tf.Session(graph=loaded_graph) as sess: # Load saved model loader = tf.train.import_meta_graph(load_path + '.meta') loader.restore(sess, load_path) input_data = loaded_graph.get_tensor_by_name('input:0') logits = loaded_graph.get_tensor_by_name('logits:0') keep_prob = loaded_graph.get_tensor_by_name('keep_prob:0') translate_logits = sess.run(logits, { input_data: [translate_sentence], keep_prob: 1.0 })[0] print('Input') print(' Word Ids: {}'.format([i for i in translate_sentence])) print(' English Words: {}'.format( [source_int_to_vocab[i] for i in translate_sentence])) print('\nPrediction') print(' Word Ids: {}'.format( [i for i in np.argmax(translate_logits, 1)])) print(' French Words: {}'.format( [target_int_to_vocab[i] for i in np.argmax(translate_logits, 1)]))
def predecir(self, frase): _, (source_vocab_to_int, target_vocab_to_int), ( source_int_to_vocab, target_int_to_vocab) = helper.load_preprocess() load_path = helper.load_params() tests.test_sentence_to_seq(sentence_to_seq) translate_sentence = frase pIngles = translate_sentence translate_sentence = sentence_to_seq(translate_sentence, source_vocab_to_int) loaded_graph = tf.Graph() with tf.Session(graph=loaded_graph) as sess: # Load saved model loader = tf.train.import_meta_graph(load_path + '.meta') loader.restore(sess, load_path) input_data = loaded_graph.get_tensor_by_name('input:0') logits = loaded_graph.get_tensor_by_name('predictions:0') target_sequence_length = loaded_graph.get_tensor_by_name( 'target_sequence_length:0') source_sequence_length = loaded_graph.get_tensor_by_name( 'source_sequence_length:0') keep_prob = loaded_graph.get_tensor_by_name('keep_prob:0') translate_logits = sess.run( logits, { input_data: [translate_sentence] * batch_size, target_sequence_length: [len(translate_sentence) * 2] * batch_size, source_sequence_length: [len(translate_sentence)] * batch_size, keep_prob: 1.0 })[0] """ print('Input') print(' Word Ids: {}'.format([i for i in translate_sentence])) print(' English Words: {}'.format([source_int_to_vocab[i] for i in translate_sentence])) print('\nPrediction') print(' Word Ids: {}'.format([i for i in translate_logits])) print(' Spanish Words: {}'.format(" ".join([target_int_to_vocab[i] for i in translate_logits]))) """ variableRetornar = format(" ".join( [target_int_to_vocab[i] for i in translate_logits])) print('Resultado de ', pIngles) print(variableRetornar) miTxt = open("BorderOut\\IA\\respuesta.txt", 'w') miTxt.write(variableRetornar) miTxt.close() return variableRetornar
def run(): """ This method is the main entry point for this processing block """ ensure_data_directories_exist() params: dict = load_params() input_metadata: FeatureCollection = load_metadata() pol_processor = SNAPPolarimetry(params) result, outfile, outfile_pol = pol_processor.process(input_metadata, params) save_metadata(result) if params['mask'] is not None: pol_processor.post_process(outfile, outfile_pol) pol_processor.rename_final_stack(outfile, outfile_pol)
def apply_model(d): gen_length = 30 if request.method == 'POST': content = request.json gen_length = content['gen_length'] _, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess() prime_word = random.choice(int_to_vocab) print (prime_word) seq_length, load_dir = helper.load_params() loaded_graph = tf.Graph() with tf.Session(graph=loaded_graph) as sess: # Load saved model loader = tf.train.import_meta_graph(load_dir + '.meta') loader.restore(sess, load_dir) # Get Tensors from loaded model input_text, initial_state, final_state, probs = get_tensors(loaded_graph) # Sentences generation setup gen_sentences = [prime_word] prev_state = sess.run(initial_state, {input_text: np.array([[1]])}) print (gen_sentences) # Generate sentences for n in range(gen_length): # Dynamic Input dyn_input = [[vocab_to_int[word] for word in gen_sentences[-seq_length:]]] dyn_seq_length = len(dyn_input[0]) # Get Prediction probabilities, prev_state = sess.run( [probs, final_state], {input_text: dyn_input, initial_state: prev_state}) pred_word = pick_word(probabilities[dyn_seq_length-1], int_to_vocab) gen_sentences.append(pred_word) data = massage_results(gen_sentences, token_dict) if (len(data) > 2): wod_return = random.choice(data[1:-1]) else: wod_return = data[0]
def run(): """ This method is the main entry point for this processing block """ ensure_data_directories_exist() params: dict = load_params() input_metadata: FeatureCollection = load_metadata() pol_processor = SNAPPolarimetry(params) result, out_dict = pol_processor.process(input_metadata, params) save_metadata(result) for out_id in out_dict: if params["mask"] is not None: pol_processor.post_process(out_dict[out_id]["out_path"], out_dict[out_id]["z"]) pol_processor.rename_final_stack(out_dict[out_id]["out_path"], out_dict[out_id]["z"])
def genScript(): _, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess() #print("vocab to int : ",vocab_to_int) seq_length, load_dir = helper.load_params() gen_length = 200 prime_word = 'bart_simpson' #'bart_simpson' # 'moe_szyslak' loaded_graph = tf.Graph() with tf.Session(graph=loaded_graph) as sess: loader = tf.train.import_meta_graph('save.meta') loader.restore(sess, load_dir) input_text, initial_state, final_state, probs = getTensors( loaded_graph) gen_sentences = [prime_word + ':'] prev_state = sess.run(initial_state, {input_text: np.array([[1]])}) # Generate sentences for n in range(gen_length): dyn_input = [[ vocab_to_int[word] for word in gen_sentences[-seq_length:] ]] dyn_seq_length = len(dyn_input[0]) probabilities, prev_state = sess.run([probs, final_state], { input_text: dyn_input, initial_state: prev_state }) pred_word = pick_word(probabilities[0][dyn_seq_length - 1], int_to_vocab) gen_sentences.append(pred_word) # remove tokens tv_script = ' '.join(gen_sentences) for key, token in token_dict.items(): ending = ' ' if key in ['\n', '(', '"'] else '' tv_script = tv_script.replace(' ' + token.lower(), key) tv_script = tv_script.replace('\n ', '\n') tv_script = tv_script.replace('( ', '(') print(tv_script)
def train(self, path_params='best_params.json', path_model='model.h5', plot_chart=False, handmade_params=None): if os.path.exists(path_params) and not handmade_params: layers, average_pooling, batch, epochs, learning_rate = load_params( path_params) else: layers, average_pooling, batch, epochs, learning_rate = handmade_params layers = layer_combination[layers] print( fg('green') + 'Parameters : layers : {0}, average_pooling : {1}, batch : {2}, epochs : ' '{3}, ' 'learning rate : {4}'.format(layers, average_pooling, batch, epochs, learning_rate)) sudoku_model = SudokuBreaker(layers=layers, average_pooling=average_pooling) id_mlflow = random.randint(1, 2542314) if self.custom: sudoku_model.fit_custom(self.train_x, self.train_y, self.val_x, self.val_y, batch=batch, epochs=epochs, learning_rate=learning_rate, id_mlflow=id_mlflow) else: sudoku_model.fit_inbuilt(self.train_x, self.train_y, self.val_x, self.val_y, batch=batch, epochs=epochs, learning_rate=learning_rate) sudoku_model.save(path_model) if plot_chart: plot(sudoku_model.hist)
# # 检查点 # In[18]: """ DON'T MODIFY ANYTHING IN THIS CELL """ import tensorflow as tf import numpy as np import helper import problem_unittests as tests _, (source_vocab_to_int, target_vocab_to_int), (source_int_to_vocab, target_int_to_vocab) = helper.load_preprocess() load_path = helper.load_params() # ## 句子到序列 # # 要向模型提供要翻译的句子,你首先需要预处理该句子。实现函数 `sentence_to_seq()` 以预处理新的句子。 # # - 将句子转换为小写形式 # - 使用 `vocab_to_int` 将单词转换为 id # - 如果单词不在词汇表中,将其转换为`<UNK>` 单词 id # In[19]: def sentence_to_seq(sentence, vocab_to_int): """ Convert a sentence to a sequence of ids
# Save parameters for checkpoint helper.save_params((seq_length, save_dir)) # # Checkpoint # In[60]: """ DON'T MODIFY ANYTHING IN THIS CELL """ import tensorflow as tf import numpy as np import helper import problem_unittests as tests _, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess() seq_length, load_dir = helper.load_params() # ## Implement Generate Functions # ### Get Tensors # Get tensors from `loaded_graph` using the function [`get_tensor_by_name()`](https://www.tensorflow.org/api_docs/python/tf/Graph#get_tensor_by_name). Get the tensors using the following names: # - "input:0" # - "initial_state:0" # - "final_state:0" # - "probs:0" # # Return the tensors in the following tuple `(InputTensor, InitialStateTensor, FinalStateTensor, ProbsTensor)` # In[61]: def get_tensors(loaded_graph):
def run(username, prime_word, gen_length): # directory for saving the model save_dir = './save' _, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess() seq_length, load_dir = helper.load_params() loaded_graph = tf.Graph() with tf.Session(graph=loaded_graph) as sess: # Load saved model loader = tf.train.import_meta_graph(load_dir + '.meta') loader.restore(sess, load_dir) # Get Tensors from loaded model input_text = loaded_graph.get_tensor_by_name('input:0') #input_text_rev=loaded_graph.get_tensor_by_name('input_rev:0') initial_state = loaded_graph.get_tensor_by_name('initial_state:0') final_state = loaded_graph.get_tensor_by_name('final_state:0') probs = loaded_graph.get_tensor_by_name('probs_f:0') # Sentences generation setup gen_sentences = [prime_word] #changed prev_state = sess.run( initial_state, {input_text: np.array([[1]])}) #,input_text_rev: np.array([[1]])}) # Generate sentences for n in range(gen_length): # Dynamic Input dyn_input = [[ vocab_to_int[word] for word in gen_sentences[-seq_length:] ]] dyn_seq_length = len(dyn_input[0]) # Get Prediction probabilities, prev_state = sess.run([probs, final_state], { input_text: dyn_input, initial_state: prev_state, }) index = np.argmax(probabilities[0][dyn_seq_length - 1]) ### pred_word = int_to_vocab[index] gen_sentences.append(pred_word) # Remove tokens quote_script = ' '.join(gen_sentences) for key, token in token_dict.items(): ending = ' ' if key in ['\n', '(', '"'] else '' quote_script = quote_script.replace(' ' + token.lower(), key) quote_script = quote_script.replace('\n ', '\n') quote_script = quote_script.replace('( ', '(') quote_data = quote_script.split("\n") quote_data = quote_data[1:-1] res = [] for i in quote_data: res.append(i) for i in quote_data: if len(i) <= 139 - len(username) and len( i) > 9: # "i have" not in i: res.append(i) # remove it #final list of result is in "res" choice = random.randint(1, 1000) choice_fin = choice % len(res) #return username+" "+res[choice_fin] return res
def trainModel(): """ From make-tweet.py """ _, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess() seq_length, load_dir = helper.load_params() print('Done') def get_tensors(loaded_graph): """ Get input, initial state, final state, and probabilities tensor from <loaded_graph> """ InputTensor = loaded_graph.get_tensor_by_name("input:0") InitialStateTensor = loaded_graph.get_tensor_by_name("initial_state:0") FinalStateTensor = loaded_graph.get_tensor_by_name("final_state:0") ProbsTensor = loaded_graph.get_tensor_by_name("probs:0") return InputTensor, InitialStateTensor, FinalStateTensor, ProbsTensor def pick_word(probabilities, int_to_vocab): """ Pick the next word in the generated text """ # TODO: Implement Function #one_hot_encoded = np.argmax(probabilities, axis=0) one_hot_encoded = np.random.choice(len(int_to_vocab),p=probabilities) next_word = int_to_vocab[one_hot_encoded] return next_word # ## Generate TV Script # This will generate the TV script for you. Set `gen_length` to the length of TV script you want to generate. gen_length = 30 # length of a tweet #prime_word = '@' choice = np.random.choice(len(int_to_vocab)) prime_word = int_to_vocab[choice] prime_word loaded_graph = tf.Graph() with tf.Session(graph=loaded_graph) as sess: # Load saved model loader = tf.train.import_meta_graph(load_dir + '.meta') loader.restore(sess, load_dir) # Get Tensors from loaded model input_text, initial_state, final_state, probs = get_tensors(loaded_graph) # Sentences generation setup gen_sentences = [prime_word ]# + ':'] prev_state = sess.run(initial_state, {input_text: np.array([[1]])}) # Generate sentences for n in range(gen_length): # Dynamic Input dyn_input = [[vocab_to_int[word] for word in gen_sentences[-seq_length:]]] dyn_seq_length = len(dyn_input[0]) # Get Prediction probabilities, prev_state = sess.run( [probs, final_state], {input_text: dyn_input, initial_state: prev_state}) pred_word = pick_word(probabilities[dyn_seq_length-1], int_to_vocab) gen_sentences.append(pred_word) # Remove tokens tweet = ' '.join(gen_sentences) for key, token in token_dict.items(): ending = ' ' if key in ['\n', '(', '"'] else '' tweet = tweet.replace(' ' + token.lower(), key) tweet = tweet.replace('\n ', '\n') tweet = tweet.replace('( ', '(') tweet = tweet.replace('\\', '') tweet = tweet.replace('\/', '') tweet = tweet.replace('\"', '') tweet = tweet.replace(' ', ' ') return tweet
def traducir(frase): # Number of Epochs epochs = 10 # Batch Size batch_size = 512 # RNN Size rnn_size = 128 # Number of Layers num_layers = 2 # Embedding Size encoding_embedding_size = 128 decoding_embedding_size = 128 # Learning Rate learning_rate = 0.001 # Dropout Keep Probability keep_probability = 0.55 display_step = True _, (source_vocab_to_int, target_vocab_to_int), (source_int_to_vocab, target_int_to_vocab) = helper.load_preprocess() load_path = helper.load_params() tests.test_sentence_to_seq(sentence_to_seq) translate_sentence = frase pIngles = translate_sentence translate_sentence = sentence_to_seq(translate_sentence, source_vocab_to_int) loaded_graph = tf.Graph() with tf.Session(graph=loaded_graph) as sess: # Load saved model loader = tf.train.import_meta_graph(load_path + '.meta') loader.restore(sess, load_path) input_data = loaded_graph.get_tensor_by_name('input:0') logits = loaded_graph.get_tensor_by_name('predictions:0') target_sequence_length = loaded_graph.get_tensor_by_name( 'target_sequence_length:0') source_sequence_length = loaded_graph.get_tensor_by_name( 'source_sequence_length:0') keep_prob = loaded_graph.get_tensor_by_name('keep_prob:0') translate_logits = sess.run( logits, { input_data: [translate_sentence] * batch_size, target_sequence_length: [len(translate_sentence) * 2] * batch_size, source_sequence_length: [len(translate_sentence)] * batch_size, keep_prob: 1.0 })[0] """ print('Input') print(' Word Ids: {}'.format([i for i in translate_sentence])) print(' English Words: {}'.format([source_int_to_vocab[i] for i in translate_sentence])) print('\nPrediction') print(' Word Ids: {}'.format([i for i in translate_logits])) print(' Spanish Words: {}'.format(" ".join([target_int_to_vocab[i] for i in translate_logits]))) """ variableRetornar = format(" ".join( [target_int_to_vocab[i] for i in translate_logits])) return variableRetornar
"?": "||QUEST_MARK||", "(": "||L_PARENTH||", ")": "||R_PARENTH||", "--": "||DASH||", "\n": "||RETURN||" } for key, token in tokens.items(): text = text.replace(key, ' {} '.format(token.lower())) lines = text.split(' ||period|| ') first_words = list(set([line.split(" ")[0] for line in lines])) _, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess() seq_length, _ = helper.load_params() # Длина генерируемой последовательности gen_length = 10 phrases = 10 def get_tensors(loaded_graph): inputs = loaded_graph.get_tensor_by_name('input:0') initial_state = loaded_graph.get_tensor_by_name('initial_state:0') final_state = loaded_graph.get_tensor_by_name('final_state:0') probs = loaded_graph.get_tensor_by_name('probs:0') return inputs, initial_state, final_state, probs def pick_word(probabilities, int_to_vocab):
"?": "||QUEST_MARK||", "(": "||L_PARENTH||", ")": "||R_PARENTH||", "--": "||DASH||", "\n": "||RETURN||" } for key, token in tokens.items(): text = text.replace(key, ' {} '.format(token.lower())) lines = text.split(' ||period|| ') first_words = list(set([line.split(" ")[0] for line in lines])) _, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess() seq_length, _ = helper.load_params() # Длина генерируемой последовательности gen_length = 10 phrases = 10 def get_tensors(loaded_graph): inputs = loaded_graph.get_tensor_by_name('input:0') initial_state = loaded_graph.get_tensor_by_name('initial_state:0') final_state = loaded_graph.get_tensor_by_name('final_state:0') probs = loaded_graph.get_tensor_by_name('probs:0') return inputs, initial_state, final_state, probs