import numpy as np from src.net import Network import theano.tensor as T from src.utils import get_one_hot from src import mnist_loader from src.model_tests import run_test (train_images, train_labels, test_images, test_labels) = mnist_loader.load_fashion_mnist() train_labels = get_one_hot(train_labels) input_var = T.fmatrix('input') y = T.fmatrix('truth') network_dense_config = { 'mode': 'dense', 'units': [1024, 1024, 784], 'dropouts': [0.2, 0.2, 0.2], } dense_net = Network( name='3_dense_test', dimensions=[None] + list(train_images.shape[1:]), input_var=input_var, y=y, config=network_dense_config,
def one_hot_encode_dataset(self, Y): Y_one_hot = get_one_hot(Y) return Y_one_hot
from src.utils import get_one_hot from src import mnist_loader from src.model_tests import run_test (train_images, train_labels, test_images, test_labels) = mnist_loader.load_fashion_mnist() from sklearn.utils import shuffle train_images, train_labels = shuffle(train_images, train_labels, random_state=0) train_labels = get_one_hot(train_labels) test_labels = get_one_hot(test_labels) train_images = np.reshape(train_images, (train_images.shape[0], 28, 28)) test_images = np.reshape(test_images, (test_images.shape[0], 28, 28)) input_var = T.tensor4('input') y = T.fmatrix('truth') network_conv_config = { 'mode': 'conv', 'filters': [16, 32], 'filter_size': [[5, 5], [5, 5]], 'stride': [[1, 1], [1, 1]], 'pool': { 'mode': 'average_exc_pad',
def encode_dataset(self, X, Y): Y_one_hot = get_one_hot(Y) all_data = [(x.reshape(-1, 1), y.reshape(-1, 1)) for x, y in zip(X, Y_one_hot)] return all_data
def main(model_name, model_type, feats_name, img_savepath, query_file, vocab_file, cuda): """Main function.""" queries = json.load(open(query_file)) vocab = json.load(open(vocab_file)) data = h5py.File(feats_name, 'r') data_dict = dict() for fname, feat in zip(data['filenames'], data['features']): data_dict[fname] = feat if model_type == 'inception': model = inception(512, 512, 2480, batch_first=True, dropout=0.7) elif model_type == 'vgg': model = vgg(512, 512, 2480, batch_first=True, dropout=0.7) elif model_type == 'squeezenet': model = squeezenet(512, 512, 2480, batch_first=True, dropout=0.7) else: print("Please, specify a valid model type: inception, vgg, squeezenet"\ "instead of %s" % model_type) return """Load the model weights.""" if cuda: model = model.cuda() model.load_state_dict(torch.load(model_name)) model.eval() txt_trf = TextTransforms() # pylint: disable=W0108 txt_norm = lambda x: txt_trf.normalize(x) # First of all, add a zero vector to the features data for start/stop. data_dict['zeros'] = np.zeros_like(data['features'][0]) zero_idx = list(data_dict.keys()).index('zeros') answers_feats = torch.from_numpy(np.array(data_dict.values())) answers_feats = torch.nn.functional.normalize(answers_feats, p=2, dim=1) if cuda: answers_feats = answers_feats.cuda() for nq, query in enumerate(queries): # Now, generate outfit for one image (forward and backward prediction until start/stop): query_feats = torch.from_numpy(np.array([data_dict[q] for q in query['image_query']])) query_feats = torch.nn.functional.normalize(query_feats, p=2, dim=1) if cuda: query_feats = query_feats.cuda() first_prod = query_feats[0].unsqueeze(0) # Start with the first image # Forward prediction forward_seq = run_forward_lstm(model, first_prod, answers_feats, data_dict, zero_idx, cuda) # Backward prediction backward_seq = run_backward_lstm(model, first_prod, answers_feats, data_dict, zero_idx, cuda) # Concatenate full sequence (forward + backward) generated by first product first_sequence = backward_seq + [query['image_query'][0]] + forward_seq seq_feats = torch.from_numpy(np.array([data_dict[im] for im in first_sequence])) seq_feats = torch.nn.functional.normalize(seq_feats, p=2, dim=1) if cuda: seq_feats = seq_feats.cuda() # If there are more images, substitute the nearest one by the query and recompute: if len(query['image_query']) >= 2: positions = [len(backward_seq)] # Position of the first query in the sequence for i, img in enumerate(query['image_query'][1:]): # Find NN of the next item dists = torch.mm(query_feats[i + 1].unsqueeze(0), seq_feats.permute(1, 0)) _, idx = torch.max(dists, 1) positions.append(idx) start_pos = np.min(positions) end_pos = np.max(positions) if start_pos == positions[0]: start_feats = query_feats[0].unsqueeze(0) end_feats = query_feats[i + 1].unsqueeze(0) start_item = query['image_query'][0] end_item = query['image_query'][i + 1] elif end_pos == positions[0]: start_feats = query_feats[i + 1].unsqueeze(0) end_feats = query_feats[0].unsqueeze(0) start_item = query['image_query'][i + 1] end_item = query['image_query'][0] blanks = run_fill_lstm(model, start_feats, end_feats, end_pos - start_pos - 1, answers_feats, data_dict, zero_idx, cuda) sets = [start_item] + blanks + [end_item] sets_feats = torch.from_numpy(np.array([data_dict[im] for im in sets])) sets_feats = torch.nn.functional.normalize(sets_feats, p=2, dim=1) if cuda: sets_feats = sets_feats.cuda() # run bi LSTM again forward_seq = run_forward_lstm(model, sets_feats, answers_feats, data_dict, zero_idx, cuda) backward_seq = run_backward_lstm(model, sets_feats, answers_feats, data_dict, zero_idx, cuda) sets = backward_seq + sets + forward_seq positions = [len(backward_seq), len(sets) - len(forward_seq) - 1] else: sets = backward_seq + query['image_query'] + forward_seq if len(query['text_query']): text_query = txt_norm(query['text_query']) texts = torch.stack([get_one_hot(word, vocab) for word in text_query.split()]) texts = torch.autograd.Variable(texts) if cuda: texts = texts.cuda() text_query_feat = model.textn(texts) text_query_feat = torch.mean(text_query_feat.view(len(text_query_feat), -1), 0) text_query_feat = torch.nn.functional.normalize(text_query_feat.unsqueeze(0), p=2, dim=1) sets_text = sets[:] for i, j in enumerate(sets): if j not in query['image_query']: sets_text[i] = nn_search(j, text_query_feat, data_dict, answers_feats, cuda) create_img_outfit(sets, positions, os.path.join(img_savepath, "%d.jpg" % nq)) create_img_outfit(sets_text, positions, os.path.join(img_savepath, "%d_%s.jpg" % (nq, text_query)))