def make_context_frequency_dict(context, model, context_map, device,
threshold):
model.eval()
context_data = train.batchify(context_map.context_tokenize(context), 1,
device)
data, targets = train.get_batch(280, context_data, 0)
with torch.no_grad():
hidden = model.init_hidden(bsz=1)
output, hidden = model(data, hidden)
# model returns log softmax
preds = output[-1].squeeze().exp().cpu().tolist()
hidden = train.repackage_hidden(hidden)
assert len(context_map.dictionary.idx2word) == len(preds)
probs = {
key: prob
for key, prob in zip(context_map.dictionary.idx2word, preds)
}
if int(threshold):
# threshold of number of nodes (leafs)
p_sorted = sorted(probs, key=lambda x: probs[x], reverse=True)
filtered = {k: probs[k] for k in p_sorted[:int(threshold)]}
probs = filtered
pass
else:
# threshold of probabilities
probs = {
key: prob
for key, prob in zip(context_map.dictionary.idx2word, preds)
if prob > threshold
}
pass
sort = collections.OrderedDict(
sorted(probs.items(), key=operator.itemgetter(1), reverse=False))
return sort
def test_train_one_step(self):
batch = train.get_batch(128)
optimizer = create_test_optimizer()
key = random.PRNGKey(0)
_, train_metrics = train.train_step(optimizer, batch, key)
self.assertLessEqual(train_metrics['loss'], 5)
self.assertGreaterEqual(train_metrics['accuracy'], 0)
def test_train_one_step(self):
batch = train.get_batch(128)
rng = random.PRNGKey(0)
model = train.create_model(rng)
optimizer = train.create_optimizer(model, 0.003)
optimizer, train_metrics = train.train_step(optimizer, batch)
self.assertLessEqual(train_metrics['loss'], 5)
self.assertGreaterEqual(train_metrics['accuracy'], 0)
def evaluate_lm(eval_model, criterion, data_source, bptt, TEXT):
eval_model.eval() # Turn on the evaluation mode
total_loss = 0.
ntokens = len(TEXT.vocab.stoi)
with torch.no_grad():
for i in range(0, data_source.size(0) - 1, bptt):
data, targets = train.get_batch(data_source, i, bptt)
output = eval_model(data)
output_flat = output.view(-1, ntokens)
total_loss += len(data) * criterion(output_flat, targets).item()
return total_loss / (len(data_source) - 1)
def test_train_one_step(self):
batch = train.get_batch(128)
rng = random.PRNGKey(0)
with nn.stochastic(rng):
model = train.create_model(nn.make_rng())
optimizer = train.create_optimizer(model, 0.003)
optimizer, train_metrics = train.train_step(
optimizer, batch, nn.make_rng())
self.assertLessEqual(train_metrics['loss'], 5)
self.assertGreaterEqual(train_metrics['accuracy'], 0)
def test_decode_batch(self): key = random.PRNGKey(0) state = create_test_state() batch = train.get_batch(5) train.decode_batch(state.params, batch, key)
model = Seq2seq(config=config,
w2i_target=w2i_target,
useTeacherForcing=False,
useAttention=True,
useBeamSearch=3)
print("(3) run model......")
print_every = 100
max_target_len = 20
with tf.Session(config=tf_config) as sess:
saver = tf.train.Saver()
saver.restore(sess, model_path)
source_batch, source_lens, target_batch, target_lens = get_batch(
docs_source, w2i_source, docs_target, w2i_target,
config.batch_size)
feed_dict = {
model.seq_inputs: source_batch,
model.seq_inputs_length: source_lens,
model.seq_targets: [[0] * max_target_len] * len(source_batch),
model.seq_targets_length: [max_target_len] * len(source_batch)
}
print("samples:\n")
predict_batch = sess.run(model.out, feed_dict)
for i in range(3):
print("in:", [
i2w_source[num]
for num in source_batch[i] if i2w_source[num] != "_PAD"
checkpoint_path = "checkpoint"
iter_num = 8000
train.source_w2i, train.source_i2w = train.load_vocabs(config.source_vocab_path)
train.target_w2i, train.target_i2w = train.load_vocabs(config.target_vocab_path)
config.source_vocab_size = len(train.source_w2i)
config.target_vocab_size = len(train.target_w2i)
source_test = train.read_docs_to_seqs(config.source_test_path, train.source_w2i)
target_test = train.read_docs_to_seqs(config.target_test_path, train.target_w2i)
test_pairs = [(s_seq, t_seq) for s_seq, t_seq in zip(source_test, target_test)]
source_batch, source_lengths, target_batch, target_lengths = train.get_batch(test_pairs, None)
encoder = torch.load(checkpoint_path+"/encoder.model.iter"+str(iter_num)+".pth")
decoder = torch.load(checkpoint_path+"/decoder.model.iter"+str(iter_num)+".pth")
precision, preds = train.run_epoch(source_batch, source_lengths, target_batch, target_lengths, encoder, decoder, TRAIN=False)
print precision
fout = open("predict.result","w")
for i in range(len(preds)):
fout.write(" ".join([train.source_i2w[j] for j in source_test[i]])+"\n")
fout.write(" ".join([train.target_i2w[j] for j in target_test[i]])+"\n")
fout.write(" ".join([train.target_i2w[j] for j in preds[i]])+"\n")
fout.write(str(preds[i] == target_test[i])+"\n")
def test_decode_batch(self):
key = random.PRNGKey(0)
optimizer = create_test_optimizer()
batch, masks = train.get_batch(5)
train.decode_batch(optimizer.target, batch, masks, key)
import os
from datetime import datetime
import numpy as np
import matplotlib.pyplot as plt
from models.siamese import get_siamese_model
from train import get_batch
if __name__ == "__main__":
shape = (614, 421, 3)
model = get_siamese_model(shape)
model.load_weights('saved_models/saved-model-19.h5')
files = os.listdir('./cards/')
for i, image in enumerate(get_batch(os.listdir('./cards/'), 1, shape)):
max_score = 0
test_image = image[0]
best_match = None
start = datetime.now()
for j, image2 in enumerate(get_batch(os.listdir('./cards/'), 1,
shape)):
score = model.predict([image, image2])
if max_score < score:
best_match = image2
max_score = score
if j > i + 10:
break
print(datetime.now() - start)
print(max_score)
fig = plt.figure()
plt.subplot(1, 2, 1)
