def next_feed_word(start, end):
batch = sentences[start:end]
'''
max_encoder_length = 0
for b in batch:
if len(b) > max_encoder_length:
max_encoder_length = len(b)
#batch = [[3,4,5,2,6,7,8,9],[6,7,3,4,5],[2,2,4,5,6]]
max_decoder_length = max_encoder_length + 3
'''
encoder_inputs_, encoder_input_lengths_ = helpers.batch(batch)
decoder_targets_, _ = helpers.batch(
#[(sequence) + [EOS] + [PAD] * 2 for sequence in batch]
[(sequence) + [EOS] + [PAD] *
(max_decoder_length - len(sequence) - 1)
for sequence in batch])
return {
model.encoder_inputs:
encoder_inputs_,
model.encoder_inputs_length:
encoder_input_lengths_,
model.decoder_targets:
decoder_targets_,
model.decoder_lengths:
[max_decoder_length for v in encoder_input_lengths_]
}
def next_feed(source_batches, target_batches, encoder_inputs,
encoder_inputs_length, decoder_targets, batch_num, parameters,
learning_rate):
# get transpose of source_batches[batch_num]
#print('next_feed:',helpers.batch(source_batches[batch_num]))
encoder_inputs_, encoder_input_lengths_ = helpers.batch(
source_batches[batch_num])
# get max input sequence length
max_input_length = max(encoder_input_lengths_)
# target word is max character_changing_num character longer than source word
# get transpose of target_batches[i] and put an EOF and PAD at the end
decoder_targets_, _ = helpers.batch([
(sequence) + [parameters.EOS] + [parameters.PAD] *
((max_input_length + parameters.character_changing_num - 1) -
len(sequence)) for sequence in target_batches[batch_num]
])
return {
encoder_inputs: encoder_inputs_,
encoder_inputs_length: encoder_input_lengths_,
decoder_targets: decoder_targets_ #,
#learning_rate: parameters.learning_rate
}
def next_feed(partition):
if partition == "train":
sample, target = sess.run(train_next_batch)
elif partition == "valid":
sample, target = sess.run(valid_next_batch)
else:
sample, target = sess.run(test_next_batch)
sample = process_data(sample)
target = process_data(target)
encoder_inputs_, encoder_input_lengths_ = helpers.batch(sample)
decoder_targets_, _ = helpers.batch([(sequence) + [EOS] + [PAD] * 2
for sequence in target])
if partition == "test":
return {
encoder_inputs: encoder_inputs_,
encoder_inputs_length: encoder_input_lengths_,
decoder_targets: decoder_targets_,
}, target
else:
return {
encoder_inputs: encoder_inputs_,
encoder_inputs_length: encoder_input_lengths_,
decoder_targets: decoder_targets_,
}
def get_translation(eng_sentence):
batch_size_ = 1
words = nltk.word_tokenize(eng_sentence)
wbw_words = [eng_fr_dic.get(w, "UNK") for w in words]
print("wbw> ", " ".join([w for w in wbw_words]))
#if word by word, then first get french word and then tokenize
#if wbw, source_to_token dictionary will map french words t token
batch_source = [[source_to_token.get(w, 2) for w in wbw_words]]
batch_target = [[1] * len(batch_source)]
encoder_inputs_, encoder_input_lengths_ = helpers.batch(
batch_source, reverse_input=False)
decoder_targets_, decoder_inputs_length_ = helpers.batch(
[(sequence) + [EOS] + [PAD] * (2) for sequence in batch_target],
reverse_input=False)
feed = {
encoder_inputs: encoder_inputs_,
encoder_inputs_length: encoder_input_lengths_,
decoder_targets: decoder_targets_,
decoder_inputs_length: decoder_inputs_length_
}
predict_ = sess.run(decoder_prediction, feed)
output_str = [token_to_target[out] for out in predict_.T[0]]
output_str = " ".join([s for s in output_str])
return output_str
def process(input_val, output_val):
encoder_inputs_, _ = helpers.batch(input_val, 50)
decoder_targets_, _ = helpers.batch(output_val, 50)
return {
model.encoder_inputs: encoder_inputs_,
model.encoder_inputs_length: np.full(len(input_val), 50),
model.decoder_targets: decoder_targets_,
}
def next_feed():
next_batch = next(batches)
encoder_inputs_, encoder_input_lengths_ = helpers.batch(next_batch)
decoder_targets_, _ = helpers.batch([sequence + [EOS] + [PAD] * 2 for sequence in next_batch])
return {
encoder_inputs: encoder_inputs_,
encoder_inputs_length: encoder_input_lengths_,
decoder_targets: decoder_targets_,
}
def input_string(str):
encoder_inputs_, encoder_input_lengths_ = helpers.batch(str)
decoder_targets_, _ = helpers.batch([(sequence) + [EOS] + [PAD] * 2
for sequence in str])
return {
encoder_inputs: encoder_inputs_,
encoder_inputs_length: encoder_input_lengths_,
decoder_targets: decoder_targets_,
}
def make_inference_inputs(self, input_seq):
inputs_, inputs_length_ = helpers.batch(input_seq)
targets_, targets_length_ = helpers.batch(input_seq)
return {
self.encoder_inputs: inputs_,
self.encoder_inputs_length: inputs_length_,
self.decoder_targets: targets_,
self.decoder_targets_length: targets_length_,
}
def make_train_inputs(self, input_seq, target_seq):
inputs_, inputs_length_ = helpers.batch(input_seq)
targets_, targets_length_ = helpers.batch(target_seq)
return {
self.is_training: True,
self.encoder_inputs: inputs_,
self.encoder_inputs_length: inputs_length_,
self.decoder_targets: targets_,
self.decoder_targets_length: targets_length_,
}
def next_feed():
batch = next(batches)
label_batch = next(label_batches)
encoder_inputs_, encoder_input_lengths_ = helpers.batch(batch)
decoder_targets_, _ = helpers.batch(label_batch, max_sequence_length=30)
return {
encoder_inputs: encoder_inputs_,
encoder_inputs_length: encoder_input_lengths_,
decoder_targets: decoder_targets_,
}
def make_train_inputs(self, input_seq, target_seq):
inputs_, inputs_length_ = helpers.batch(input_seq)
targets_, targets_length_ = helpers.batch(target_seq)
return {
self.encoder_inputs: inputs_,
self.encoder_inputs_length: inputs_length_,
self.decoder_targets: targets_,
self.decoder_targets_length: targets_length_,
}
def next_input(l, u):
en_inputs, en_input_len = helpers.batch(eng_seq[l:u], 84)
# print(en_input_len)
decoder_target, _ = helpers.batch(
[sequence + [EOS] for sequence in Cze_Seq[l:u]], 94)
return {
en_input: en_inputs,
en_in_len: en_input_len,
out_tar: decoder_target,
}
def next_feed():
batch = next(batches)
encoder_inputs_, _ = helpers.batch(batch)
decoder_targets_, _ = helpers.batch([(sequence) for sequence in batch])
decoder_inputs_, _ = helpers.batch([(sequence) for sequence in batch])
return {
encoder_inputs: encoder_inputs_,
decoder_inputs: decoder_inputs_,
decoder_targets: decoder_targets_,
}
def next_feed():
batch = next(batches)
encoder_inputs_, encoder_input_lengths_ = helpers.batch(batch)
decoder_targets_, _ = helpers.batch([(sequence) + [EOS] + [PAD] * 2
for sequence in batch])
return {
encoder_input: encoder_inputs_,
encoder_input_length: encoder_input_lengths_,
decoder_target: decoder_targets_,
}
def test_feed(batch_inp, batch_out):
encoder_inputs_, encoder_input_lengths_ = helpers.batch(
batch_inp, max_sequence_length=1051)
decoder_targets_, _ = helpers.batch([(sequence) + [EOS]
for sequence in batch_out],
max_sequence_length=1051)
return {
encoder_inputs: encoder_inputs_,
encoder_inputs_length: encoder_input_lengths_,
decoder_targets: decoder_targets_,
}
def next_input(batch):
encoder_inputs_, encoder_input_lengths_ = helpers.batch(batch)
decoder_targets_, _ = helpers.batch([(sequence) + [EOS] + [PAD] * 2
for sequence in batch])
return {
encoder_inputs: encoder_inputs_,
encoder_inputs_length: encoder_input_lengths_,
decoder_targets: decoder_targets_,
}
def next_feed(start):
batch = h.createBatch(sentences, start, batch_size)
encoder_inputs_, encoder_input_lengths_ = helpers.batch(batch)
decoder_targets_, _ = helpers.batch([(sequence) + [EOS] + [PAD] * 2
for sequence in batch])
return {
encoder_inputs: encoder_inputs_,
encoder_inputs_length: encoder_input_lengths_,
decoder_targets: decoder_targets_,
}
def make_train_inputs(self, input_seq, target_seq, learning_rate,
keep_prob):
inputs_, inputs_length_ = helpers.batch(input_seq)
targets_, targets_length_ = helpers.batch(target_seq)
return {
self.encoder_inputs: inputs_,
self.encoder_inputs_length: inputs_length_,
self.decoder_targets: targets_,
self.decoder_targets_length: targets_length_,
self.learning_rate: learning_rate,
self.keep_prob: keep_prob,
}
def next_feed():
next_batch = next(batches)
encoder_inputs_, _ = helpers.batch(next_batch)
decoder_targets_, _ = helpers.batch(
[sequence + [EOS] for sequence in next_batch])
decoder_inputs_, _ = helpers.batch([[EOS] + sequence
for sequence in next_batch])
return {
encoder_inputs: encoder_inputs_,
decoder_inputs: decoder_inputs_,
decoder_targets: decoder_targets_,
}
def next_feed():
batch = next(batches) #batch中装入的是100个队列，可以理解为不同的句子
encoder_inputs_, _ = helpers.batch(batch) #转换成一样长度的向量，_中记录了有多少个不为0
decoder_targets_, _ = helpers.batch( #添加句子结束标记，此处EOS定义为1
[(sequence) + [EOS] for sequence in batch])
decoder_inputs_, _ = helpers.batch([[EOS] + (sequence)
for sequence in batch])
return {
encoder_inputs: encoder_inputs_, #[8,100]
decoder_inputs: decoder_inputs_, #[9,100],以EOS开头
decoder_targets: decoder_targets_, #[9,100]，以EOS结尾
}
def next_feed(train_features_batches, train_labels_batches):
train_features_batche = train_features_batches
train_labels_batche = train_labels_batches
encoder_inputs_, encoder_input_lengths_ = helpers.batch(
train_features_batche)
decoder_targets_, _ = helpers.batch([(sequence) + [EOS] + [PAD] * 2
for sequence in train_labels_batche])
return {
encoder_inputs: encoder_inputs_,
encoder_inputs_length: encoder_input_lengths_,
decoder_targets: decoder_targets_,
}
def next_feed(sequences):
EOS = 1
encoder_inputs_, _ = helpers.batch(sequences)
decoder_targets_, _ = helpers.batch([(sequence) + [EOS]
for sequence in sequences])
decoder_inputs_, _ = helpers.batch([[EOS] + (sequence)
for sequence in sequences])
return {
encoder_inputs: encoder_inputs_,
decoder_inputs: decoder_inputs_,
decoder_targets: decoder_targets_,
}
def make_train_inputs(self, input_seq, target_seq):
inputs_, inputs_length_ = helpers.batch(input_seq)
targets_, targets_length_ = helpers.batch(target_seq)
'''print(targets_length_)
print(targets_)
print(type(targets_length_) , type(inputs_length_))'''
return {
self.encoder_inputs: inputs_,
self.encoder_inputs_length: inputs_length_,
self.decoder_targets: targets_,
self.decoder_targets_length: targets_length_
#self.embedding_placeholder: self.embedding_pp
}
def next_feed(batches, from_idx, to_idx): ###input: ? -> output: dict
batch = batches[from_idx:to_idx]
encoder_inputs_, _ = helpers.batch(batch)
batch1 = batches[from_idx + 1:to_idx + 1]
decoder_targets_, _ = helpers.batch([(sequence) + [EOS]
for sequence in batch1])
decoder_inputs_, _ = helpers.batch([[EOS] + (sequence)
for sequence in batch1])
return {
encoder_inputs: encoder_inputs_,
decoder_inputs: decoder_inputs_,
decoder_targets: decoder_targets_,
}
def next_feed():
batch = next(batches)
encoder_inputs_, _ = helpers.batch(batch)
decoder_targets_, _ = helpers.batch(
[(sequence) + [EOS] for sequence in batch ]
)
decoder_inputs_, _ = helpers.batch(
[ [EOS] + (sequence) for sequence in batch ]
)
return {
seq2seq_model.encoder_inputs: encoder_inputs_,
seq2seq_model.decoder_inputs: decoder_inputs_,
seq2seq_model.decoder_targets: decoder_targets_,
}
def do_prediction(sess, sequence):
encoder_inputs_, _ = helpers.batch(sequence)
decoder_inputs_, _ = helpers.batch([[0] + (sequence)
for sequence in sequence])
#return(sess.run(self.prediction, feed_dict={
return (sess.run(
tf.get_collection('prediction'),
feed_dict={
tf.get_default_graph().get_tensor_by_name("encoder_inputs:0"):
encoder_inputs_,
tf.get_default_graph().get_tensor_by_name("decoder_inputs:0"):
decoder_inputs_
}))
def next_feed():
print("next feed")
print("--------------------------------")
batch = next(batches)
print(batch)
encoder_inputs_, encoder_input_lengths_ = helpers.batch(batch)
decoder_targets_, _ = helpers.batch([(sequence) + [EOS] + [PAD] * 2
for sequence in batch])
return {
'encoder_inputs': encoder_inputs_,
'encoder_inputs_length': encoder_input_lengths_,
'decoder_targets': decoder_targets_,
}
def next_feed():
encoder_inputs_, _ = helpers.batch(batch_)
encoder_targets_, _ = helpers.batch([(sequence[1:]) + [EOS]
for sequence in batch_])
#print("inputs")
#print(encoder_inputs_)
#print("target..")
#print(encoder_targets_)
return {
encoder_inputs: encoder_inputs_,
encoder_targets: encoder_targets_,
}
def next_test_feed():
(src_batch, tgt_batch) = (test_src_list, test_tgt_list)
# pprint.pprint(batch)
encoder_inputs_, encoder_input_lengths_ = helpers.batch(src_batch)
decoder_targets_, decoder_targets_lengths_ = helpers.batch(
[sequence + [EOS] + [PAD] * 2 for sequence in tgt_batch])
# pprint.pprint(decoder_targets_lengths_)
# sys.exit()
return {
encoder_inputs: encoder_inputs_,
encoder_inputs_length: encoder_input_lengths_,
decoder_targets: decoder_targets_,
decoder_targets_length: decoder_targets_lengths_,
}
def next_feed():
batch = next(batches)
encoder_inputs_, _ = helpers.batch(batch)
decoder_targets_, _ = helpers.batch(
[(sequence) + [EOS] for sequence in batch]
)
decoder_inputs_, _ = helpers.batch(
[[EOS] + (sequence) for sequence in batch]
)
return {
encoder_inputs: encoder_inputs_,
decoder_inputs: decoder_inputs_,
decoder_targets: decoder_targets_,
sequence_len: [len(encoder_inputs_)]*batch_size,
}
def main():
collection_uri, team_project, test_run_id, access_token = process_args()
print "Got args", collection_uri, team_project, test_run_id, access_token
worker_count = 10
q = Queue()
print "Main thread starting workers"
for i in range(worker_count):
worker = UploadWorker(q, i, collection_uri, team_project, test_run_id, access_token)
worker.daemon = True
worker.start()
print "Beginning reading of test results."
all_results = read_results(os.getcwd())
batch_size = 1000
batches = batch(all_results, batch_size)
print "Uploading results in batches of size {}".format(batch_size)
for b in batches:
q.put(b)
print "Main thread finished queueing batches"
q.join()
print "Main thread exiting"
def make_inference_inputs(self, input_seq):
inputs_, inputs_length_ = helpers.batch(input_seq)
return {
self.encoder_inputs: inputs_,
self.encoder_inputs_length: inputs_length_,
}
