def next_feed_word(start, end): batch = sentences[start:end] ''' max_encoder_length = 0 for b in batch: if len(b) > max_encoder_length: max_encoder_length = len(b) #batch = [[3,4,5,2,6,7,8,9],[6,7,3,4,5],[2,2,4,5,6]] max_decoder_length = max_encoder_length + 3 ''' encoder_inputs_, encoder_input_lengths_ = helpers.batch(batch) decoder_targets_, _ = helpers.batch( #[(sequence) + [EOS] + [PAD] * 2 for sequence in batch] [(sequence) + [EOS] + [PAD] * (max_decoder_length - len(sequence) - 1) for sequence in batch]) return { model.encoder_inputs: encoder_inputs_, model.encoder_inputs_length: encoder_input_lengths_, model.decoder_targets: decoder_targets_, model.decoder_lengths: [max_decoder_length for v in encoder_input_lengths_] }
def next_feed(source_batches, target_batches, encoder_inputs, encoder_inputs_length, decoder_targets, batch_num, parameters, learning_rate): # get transpose of source_batches[batch_num] #print('next_feed:',helpers.batch(source_batches[batch_num])) encoder_inputs_, encoder_input_lengths_ = helpers.batch( source_batches[batch_num]) # get max input sequence length max_input_length = max(encoder_input_lengths_) # target word is max character_changing_num character longer than source word # get transpose of target_batches[i] and put an EOF and PAD at the end decoder_targets_, _ = helpers.batch([ (sequence) + [parameters.EOS] + [parameters.PAD] * ((max_input_length + parameters.character_changing_num - 1) - len(sequence)) for sequence in target_batches[batch_num] ]) return { encoder_inputs: encoder_inputs_, encoder_inputs_length: encoder_input_lengths_, decoder_targets: decoder_targets_ #, #learning_rate: parameters.learning_rate }
def next_feed(partition): if partition == "train": sample, target = sess.run(train_next_batch) elif partition == "valid": sample, target = sess.run(valid_next_batch) else: sample, target = sess.run(test_next_batch) sample = process_data(sample) target = process_data(target) encoder_inputs_, encoder_input_lengths_ = helpers.batch(sample) decoder_targets_, _ = helpers.batch([(sequence) + [EOS] + [PAD] * 2 for sequence in target]) if partition == "test": return { encoder_inputs: encoder_inputs_, encoder_inputs_length: encoder_input_lengths_, decoder_targets: decoder_targets_, }, target else: return { encoder_inputs: encoder_inputs_, encoder_inputs_length: encoder_input_lengths_, decoder_targets: decoder_targets_, }
def get_translation(eng_sentence): batch_size_ = 1 words = nltk.word_tokenize(eng_sentence) wbw_words = [eng_fr_dic.get(w, "UNK") for w in words] print("wbw> ", " ".join([w for w in wbw_words])) #if word by word, then first get french word and then tokenize #if wbw, source_to_token dictionary will map french words t token batch_source = [[source_to_token.get(w, 2) for w in wbw_words]] batch_target = [[1] * len(batch_source)] encoder_inputs_, encoder_input_lengths_ = helpers.batch( batch_source, reverse_input=False) decoder_targets_, decoder_inputs_length_ = helpers.batch( [(sequence) + [EOS] + [PAD] * (2) for sequence in batch_target], reverse_input=False) feed = { encoder_inputs: encoder_inputs_, encoder_inputs_length: encoder_input_lengths_, decoder_targets: decoder_targets_, decoder_inputs_length: decoder_inputs_length_ } predict_ = sess.run(decoder_prediction, feed) output_str = [token_to_target[out] for out in predict_.T[0]] output_str = " ".join([s for s in output_str]) return output_str
def process(input_val, output_val): encoder_inputs_, _ = helpers.batch(input_val, 50) decoder_targets_, _ = helpers.batch(output_val, 50) return { model.encoder_inputs: encoder_inputs_, model.encoder_inputs_length: np.full(len(input_val), 50), model.decoder_targets: decoder_targets_, }
def next_feed(): next_batch = next(batches) encoder_inputs_, encoder_input_lengths_ = helpers.batch(next_batch) decoder_targets_, _ = helpers.batch([sequence + [EOS] + [PAD] * 2 for sequence in next_batch]) return { encoder_inputs: encoder_inputs_, encoder_inputs_length: encoder_input_lengths_, decoder_targets: decoder_targets_, }
def input_string(str): encoder_inputs_, encoder_input_lengths_ = helpers.batch(str) decoder_targets_, _ = helpers.batch([(sequence) + [EOS] + [PAD] * 2 for sequence in str]) return { encoder_inputs: encoder_inputs_, encoder_inputs_length: encoder_input_lengths_, decoder_targets: decoder_targets_, }
def make_inference_inputs(self, input_seq): inputs_, inputs_length_ = helpers.batch(input_seq) targets_, targets_length_ = helpers.batch(input_seq) return { self.encoder_inputs: inputs_, self.encoder_inputs_length: inputs_length_, self.decoder_targets: targets_, self.decoder_targets_length: targets_length_, }
def make_train_inputs(self, input_seq, target_seq): inputs_, inputs_length_ = helpers.batch(input_seq) targets_, targets_length_ = helpers.batch(target_seq) return { self.is_training: True, self.encoder_inputs: inputs_, self.encoder_inputs_length: inputs_length_, self.decoder_targets: targets_, self.decoder_targets_length: targets_length_, }
def next_feed(): batch = next(batches) label_batch = next(label_batches) encoder_inputs_, encoder_input_lengths_ = helpers.batch(batch) decoder_targets_, _ = helpers.batch(label_batch, max_sequence_length=30) return { encoder_inputs: encoder_inputs_, encoder_inputs_length: encoder_input_lengths_, decoder_targets: decoder_targets_, }
def make_train_inputs(self, input_seq, target_seq): inputs_, inputs_length_ = helpers.batch(input_seq) targets_, targets_length_ = helpers.batch(target_seq) return { self.encoder_inputs: inputs_, self.encoder_inputs_length: inputs_length_, self.decoder_targets: targets_, self.decoder_targets_length: targets_length_, }
def next_input(l, u): en_inputs, en_input_len = helpers.batch(eng_seq[l:u], 84) # print(en_input_len) decoder_target, _ = helpers.batch( [sequence + [EOS] for sequence in Cze_Seq[l:u]], 94) return { en_input: en_inputs, en_in_len: en_input_len, out_tar: decoder_target, }
def next_feed(): batch = next(batches) encoder_inputs_, _ = helpers.batch(batch) decoder_targets_, _ = helpers.batch([(sequence) for sequence in batch]) decoder_inputs_, _ = helpers.batch([(sequence) for sequence in batch]) return { encoder_inputs: encoder_inputs_, decoder_inputs: decoder_inputs_, decoder_targets: decoder_targets_, }
def next_feed(): batch = next(batches) encoder_inputs_, encoder_input_lengths_ = helpers.batch(batch) decoder_targets_, _ = helpers.batch([(sequence) + [EOS] + [PAD] * 2 for sequence in batch]) return { encoder_input: encoder_inputs_, encoder_input_length: encoder_input_lengths_, decoder_target: decoder_targets_, }
def test_feed(batch_inp, batch_out): encoder_inputs_, encoder_input_lengths_ = helpers.batch( batch_inp, max_sequence_length=1051) decoder_targets_, _ = helpers.batch([(sequence) + [EOS] for sequence in batch_out], max_sequence_length=1051) return { encoder_inputs: encoder_inputs_, encoder_inputs_length: encoder_input_lengths_, decoder_targets: decoder_targets_, }
def next_input(batch): encoder_inputs_, encoder_input_lengths_ = helpers.batch(batch) decoder_targets_, _ = helpers.batch([(sequence) + [EOS] + [PAD] * 2 for sequence in batch]) return { encoder_inputs: encoder_inputs_, encoder_inputs_length: encoder_input_lengths_, decoder_targets: decoder_targets_, }
def next_feed(start): batch = h.createBatch(sentences, start, batch_size) encoder_inputs_, encoder_input_lengths_ = helpers.batch(batch) decoder_targets_, _ = helpers.batch([(sequence) + [EOS] + [PAD] * 2 for sequence in batch]) return { encoder_inputs: encoder_inputs_, encoder_inputs_length: encoder_input_lengths_, decoder_targets: decoder_targets_, }
def make_train_inputs(self, input_seq, target_seq, learning_rate, keep_prob): inputs_, inputs_length_ = helpers.batch(input_seq) targets_, targets_length_ = helpers.batch(target_seq) return { self.encoder_inputs: inputs_, self.encoder_inputs_length: inputs_length_, self.decoder_targets: targets_, self.decoder_targets_length: targets_length_, self.learning_rate: learning_rate, self.keep_prob: keep_prob, }
def next_feed(): next_batch = next(batches) encoder_inputs_, _ = helpers.batch(next_batch) decoder_targets_, _ = helpers.batch( [sequence + [EOS] for sequence in next_batch]) decoder_inputs_, _ = helpers.batch([[EOS] + sequence for sequence in next_batch]) return { encoder_inputs: encoder_inputs_, decoder_inputs: decoder_inputs_, decoder_targets: decoder_targets_, }
def next_feed(): batch = next(batches) #batch中装入的是100个队列,可以理解为不同的句子 encoder_inputs_, _ = helpers.batch(batch) #转换成一样长度的向量,_中记录了有多少个不为0 decoder_targets_, _ = helpers.batch( #添加句子结束标记,此处EOS定义为1 [(sequence) + [EOS] for sequence in batch]) decoder_inputs_, _ = helpers.batch([[EOS] + (sequence) for sequence in batch]) return { encoder_inputs: encoder_inputs_, #[8,100] decoder_inputs: decoder_inputs_, #[9,100],以EOS开头 decoder_targets: decoder_targets_, #[9,100],以EOS结尾 }
def next_feed(train_features_batches, train_labels_batches): train_features_batche = train_features_batches train_labels_batche = train_labels_batches encoder_inputs_, encoder_input_lengths_ = helpers.batch( train_features_batche) decoder_targets_, _ = helpers.batch([(sequence) + [EOS] + [PAD] * 2 for sequence in train_labels_batche]) return { encoder_inputs: encoder_inputs_, encoder_inputs_length: encoder_input_lengths_, decoder_targets: decoder_targets_, }
def next_feed(sequences): EOS = 1 encoder_inputs_, _ = helpers.batch(sequences) decoder_targets_, _ = helpers.batch([(sequence) + [EOS] for sequence in sequences]) decoder_inputs_, _ = helpers.batch([[EOS] + (sequence) for sequence in sequences]) return { encoder_inputs: encoder_inputs_, decoder_inputs: decoder_inputs_, decoder_targets: decoder_targets_, }
def make_train_inputs(self, input_seq, target_seq): inputs_, inputs_length_ = helpers.batch(input_seq) targets_, targets_length_ = helpers.batch(target_seq) '''print(targets_length_) print(targets_) print(type(targets_length_) , type(inputs_length_))''' return { self.encoder_inputs: inputs_, self.encoder_inputs_length: inputs_length_, self.decoder_targets: targets_, self.decoder_targets_length: targets_length_ #self.embedding_placeholder: self.embedding_pp }
def next_feed(batches, from_idx, to_idx): ###input: ? -> output: dict batch = batches[from_idx:to_idx] encoder_inputs_, _ = helpers.batch(batch) batch1 = batches[from_idx + 1:to_idx + 1] decoder_targets_, _ = helpers.batch([(sequence) + [EOS] for sequence in batch1]) decoder_inputs_, _ = helpers.batch([[EOS] + (sequence) for sequence in batch1]) return { encoder_inputs: encoder_inputs_, decoder_inputs: decoder_inputs_, decoder_targets: decoder_targets_, }
def next_feed(): batch = next(batches) encoder_inputs_, _ = helpers.batch(batch) decoder_targets_, _ = helpers.batch( [(sequence) + [EOS] for sequence in batch ] ) decoder_inputs_, _ = helpers.batch( [ [EOS] + (sequence) for sequence in batch ] ) return { seq2seq_model.encoder_inputs: encoder_inputs_, seq2seq_model.decoder_inputs: decoder_inputs_, seq2seq_model.decoder_targets: decoder_targets_, }
def do_prediction(sess, sequence): encoder_inputs_, _ = helpers.batch(sequence) decoder_inputs_, _ = helpers.batch([[0] + (sequence) for sequence in sequence]) #return(sess.run(self.prediction, feed_dict={ return (sess.run( tf.get_collection('prediction'), feed_dict={ tf.get_default_graph().get_tensor_by_name("encoder_inputs:0"): encoder_inputs_, tf.get_default_graph().get_tensor_by_name("decoder_inputs:0"): decoder_inputs_ }))
def next_feed(): print("next feed") print("--------------------------------") batch = next(batches) print(batch) encoder_inputs_, encoder_input_lengths_ = helpers.batch(batch) decoder_targets_, _ = helpers.batch([(sequence) + [EOS] + [PAD] * 2 for sequence in batch]) return { 'encoder_inputs': encoder_inputs_, 'encoder_inputs_length': encoder_input_lengths_, 'decoder_targets': decoder_targets_, }
def next_feed(): encoder_inputs_, _ = helpers.batch(batch_) encoder_targets_, _ = helpers.batch([(sequence[1:]) + [EOS] for sequence in batch_]) #print("inputs") #print(encoder_inputs_) #print("target..") #print(encoder_targets_) return { encoder_inputs: encoder_inputs_, encoder_targets: encoder_targets_, }
def next_test_feed(): (src_batch, tgt_batch) = (test_src_list, test_tgt_list) # pprint.pprint(batch) encoder_inputs_, encoder_input_lengths_ = helpers.batch(src_batch) decoder_targets_, decoder_targets_lengths_ = helpers.batch( [sequence + [EOS] + [PAD] * 2 for sequence in tgt_batch]) # pprint.pprint(decoder_targets_lengths_) # sys.exit() return { encoder_inputs: encoder_inputs_, encoder_inputs_length: encoder_input_lengths_, decoder_targets: decoder_targets_, decoder_targets_length: decoder_targets_lengths_, }
def next_feed(): batch = next(batches) encoder_inputs_, _ = helpers.batch(batch) decoder_targets_, _ = helpers.batch( [(sequence) + [EOS] for sequence in batch] ) decoder_inputs_, _ = helpers.batch( [[EOS] + (sequence) for sequence in batch] ) return { encoder_inputs: encoder_inputs_, decoder_inputs: decoder_inputs_, decoder_targets: decoder_targets_, sequence_len: [len(encoder_inputs_)]*batch_size, }
def main(): collection_uri, team_project, test_run_id, access_token = process_args() print "Got args", collection_uri, team_project, test_run_id, access_token worker_count = 10 q = Queue() print "Main thread starting workers" for i in range(worker_count): worker = UploadWorker(q, i, collection_uri, team_project, test_run_id, access_token) worker.daemon = True worker.start() print "Beginning reading of test results." all_results = read_results(os.getcwd()) batch_size = 1000 batches = batch(all_results, batch_size) print "Uploading results in batches of size {}".format(batch_size) for b in batches: q.put(b) print "Main thread finished queueing batches" q.join() print "Main thread exiting"
def make_inference_inputs(self, input_seq): inputs_, inputs_length_ = helpers.batch(input_seq) return { self.encoder_inputs: inputs_, self.encoder_inputs_length: inputs_length_, }