import numpy as np
from src.net import Network
import theano.tensor as T
from src.utils import get_one_hot
from src import mnist_loader
from src.model_tests import run_test
(train_images, train_labels, test_images, test_labels) = mnist_loader.load_fashion_mnist()
train_labels = get_one_hot(train_labels)
input_var = T.fmatrix('input')
y = T.fmatrix('truth')
network_dense_config = {
'mode': 'dense',
'units': [1024, 1024, 784],
'dropouts': [0.2, 0.2, 0.2],
}
dense_net = Network(
name='3_dense_test',
dimensions=[None] + list(train_images.shape[1:]),
input_var=input_var,
y=y,
config=network_dense_config,
def one_hot_encode_dataset(self, Y):
Y_one_hot = get_one_hot(Y)
return Y_one_hot
from src.utils import get_one_hot
from src import mnist_loader
from src.model_tests import run_test
(train_images, train_labels, test_images,
test_labels) = mnist_loader.load_fashion_mnist()
from sklearn.utils import shuffle
train_images, train_labels = shuffle(train_images,
train_labels,
random_state=0)
train_labels = get_one_hot(train_labels)
test_labels = get_one_hot(test_labels)
train_images = np.reshape(train_images, (train_images.shape[0], 28, 28))
test_images = np.reshape(test_images, (test_images.shape[0], 28, 28))
input_var = T.tensor4('input')
y = T.fmatrix('truth')
network_conv_config = {
'mode': 'conv',
'filters': [16, 32],
'filter_size': [[5, 5], [5, 5]],
'stride': [[1, 1], [1, 1]],
'pool': {
'mode': 'average_exc_pad',
def encode_dataset(self, X, Y):
Y_one_hot = get_one_hot(Y)
all_data = [(x.reshape(-1, 1), y.reshape(-1, 1))
for x, y in zip(X, Y_one_hot)]
return all_data
def main(model_name, model_type, feats_name, img_savepath, query_file, vocab_file, cuda):
"""Main function."""
queries = json.load(open(query_file))
vocab = json.load(open(vocab_file))
data = h5py.File(feats_name, 'r')
data_dict = dict()
for fname, feat in zip(data['filenames'], data['features']):
data_dict[fname] = feat
if model_type == 'inception':
model = inception(512, 512, 2480, batch_first=True, dropout=0.7)
elif model_type == 'vgg':
model = vgg(512, 512, 2480, batch_first=True, dropout=0.7)
elif model_type == 'squeezenet':
model = squeezenet(512, 512, 2480, batch_first=True, dropout=0.7)
else:
print("Please, specify a valid model type: inception, vgg, squeezenet"\
"instead of %s" % model_type)
return
"""Load the model weights."""
if cuda:
model = model.cuda()
model.load_state_dict(torch.load(model_name))
model.eval()
txt_trf = TextTransforms()
# pylint: disable=W0108
txt_norm = lambda x: txt_trf.normalize(x)
# First of all, add a zero vector to the features data for start/stop.
data_dict['zeros'] = np.zeros_like(data['features'][0])
zero_idx = list(data_dict.keys()).index('zeros')
answers_feats = torch.from_numpy(np.array(data_dict.values()))
answers_feats = torch.nn.functional.normalize(answers_feats, p=2, dim=1)
if cuda:
answers_feats = answers_feats.cuda()
for nq, query in enumerate(queries):
# Now, generate outfit for one image (forward and backward prediction until start/stop):
query_feats = torch.from_numpy(np.array([data_dict[q] for q in query['image_query']]))
query_feats = torch.nn.functional.normalize(query_feats, p=2, dim=1)
if cuda:
query_feats = query_feats.cuda()
first_prod = query_feats[0].unsqueeze(0) # Start with the first image
# Forward prediction
forward_seq = run_forward_lstm(model, first_prod, answers_feats, data_dict, zero_idx, cuda)
# Backward prediction
backward_seq = run_backward_lstm(model, first_prod, answers_feats, data_dict, zero_idx, cuda)
# Concatenate full sequence (forward + backward) generated by first product
first_sequence = backward_seq + [query['image_query'][0]] + forward_seq
seq_feats = torch.from_numpy(np.array([data_dict[im] for im in first_sequence]))
seq_feats = torch.nn.functional.normalize(seq_feats, p=2, dim=1)
if cuda:
seq_feats = seq_feats.cuda()
# If there are more images, substitute the nearest one by the query and recompute:
if len(query['image_query']) >= 2:
positions = [len(backward_seq)] # Position of the first query in the sequence
for i, img in enumerate(query['image_query'][1:]):
# Find NN of the next item
dists = torch.mm(query_feats[i + 1].unsqueeze(0), seq_feats.permute(1, 0))
_, idx = torch.max(dists, 1)
positions.append(idx)
start_pos = np.min(positions)
end_pos = np.max(positions)
if start_pos == positions[0]:
start_feats = query_feats[0].unsqueeze(0)
end_feats = query_feats[i + 1].unsqueeze(0)
start_item = query['image_query'][0]
end_item = query['image_query'][i + 1]
elif end_pos == positions[0]:
start_feats = query_feats[i + 1].unsqueeze(0)
end_feats = query_feats[0].unsqueeze(0)
start_item = query['image_query'][i + 1]
end_item = query['image_query'][0]
blanks = run_fill_lstm(model, start_feats, end_feats, end_pos - start_pos - 1,
answers_feats, data_dict, zero_idx, cuda)
sets = [start_item] + blanks + [end_item]
sets_feats = torch.from_numpy(np.array([data_dict[im] for im in sets]))
sets_feats = torch.nn.functional.normalize(sets_feats, p=2, dim=1)
if cuda:
sets_feats = sets_feats.cuda()
# run bi LSTM again
forward_seq = run_forward_lstm(model, sets_feats, answers_feats, data_dict, zero_idx, cuda)
backward_seq = run_backward_lstm(model, sets_feats, answers_feats, data_dict, zero_idx, cuda)
sets = backward_seq + sets + forward_seq
positions = [len(backward_seq), len(sets) - len(forward_seq) - 1]
else:
sets = backward_seq + query['image_query'] + forward_seq
if len(query['text_query']):
text_query = txt_norm(query['text_query'])
texts = torch.stack([get_one_hot(word, vocab) for word in text_query.split()])
texts = torch.autograd.Variable(texts)
if cuda:
texts = texts.cuda()
text_query_feat = model.textn(texts)
text_query_feat = torch.mean(text_query_feat.view(len(text_query_feat), -1), 0)
text_query_feat = torch.nn.functional.normalize(text_query_feat.unsqueeze(0), p=2, dim=1)
sets_text = sets[:]
for i, j in enumerate(sets):
if j not in query['image_query']:
sets_text[i] = nn_search(j, text_query_feat, data_dict, answers_feats, cuda)
create_img_outfit(sets, positions, os.path.join(img_savepath, "%d.jpg" % nq))
create_img_outfit(sets_text, positions, os.path.join(img_savepath, "%d_%s.jpg" % (nq, text_query)))