def test_normalize_rows(self): arg = np.zeros((3, 2)) self.assertTrue(np.array_equal(normalize_rows(arg), arg)) arg = np.array([[0, 3, 4], [1, 6, 4]]) expected = np.array([[0, 0.6, 0.8], [0.13736056, 0.82416338, 0.54944226]]) self.assertTrue(np.allclose(normalize_rows(arg), expected))
def compute_metrics(model, loader, k=5, mode='bilinear'): global GPU_AVAILABLE p_at_1 = 0 p_at_k = 0 ndcg = 0 for X, Y in loader: X = Variable(X) Y = [Variable(y) for y in Y] if GPU_AVAILABLE: X = X.cuda() Y = [y.cuda() for y in Y] if mode == "bilinear": outputs = model(X, Y) if GPU_AVAILABLE: outputs = [out.cpu() for out in outputs] outputs = [out.data.numpy().squeeze() for out in outputs] elif mode == "project_x": X_proj = model.project_x(X).data.numpy() X_proj = normalize_rows(X_proj) Y = [y.data.numpy() for y in Y] outputs = [ x.reshape(1, -1).dot(np.atleast_2d(y).T).squeeze() for x, y in zip(X_proj, Y) ] elif mode == "project_y": Y_proj = [model.project_y(y).data.numpy() for y in Y] Y_proj = [normalize_rows(y) for y in Y_proj] X = X.data.numpy() outputs = [ x.reshape(1, -1).dot(np.atleast_2d(y).T).squeeze() for x, y in zip(X, Y_proj) ] elif mode == "random": outputs = [np.random.random(len(y)) for y in Y] else: raise ValueError("not a valid mode") idxs = [np.argsort(out)[::-1] for out in outputs] p_at_1 += sum([np.mean(idx[:1] < 1) for idx in idxs]) p_at_k += sum([np.mean(idx[:k] < k) for idx in idxs]) ndcg += sum( [ndcg_at_k(out.tolist(), k=k, method=0) for out in outputs]) N = len(loader.dataset) return p_at_1 / N, p_at_k / N, ndcg / N
def load_vectors(self): self.db = np.load(self.vectors_path, allow_pickle=True) if self.normalize: self.db = normalize_rows(self.db) self.dim = self.db.shape[1] self.index = faiss.IndexFlatIP(self.dim) self.db = np.ascontiguousarray(self.db, dtype=np.float32) self.index.add(self.db)
def __init__(self, path, tags_file, set_): super().__init__() self._path = abspath(path) if not exists(self._path): raise OSError('{} doesn\'t exists'.format(self._path)) self._data = json.load(open(tags_file, 'r')) self._set = set_ if not self._set in data.keys(): raise ValueError('{} isn\'t a valid key'.format(set_)) # sort images for reproducibility reasons sorted_keys = sorted(list(self._data[self._set])) self._data[self._set] = [self._data[self._set][k] for k in sorted_keys] # L2 normalization of word embeddings if not 'vectors' in self._data: raise RuntimeError('there are no word embeddings for this dataset') vectors = self._data['vectors'] for tag, vec in vectors.items(): if vec is not None: vectors[tag] = normalize_rows(vec).astype(np.float32)
images_test = [images[i] for i in range(1, len(images), 2)] tags = sorted(list(tag2im.keys())) # -------------------------------------------------------------------------- print("processing input (training) embeddings ...") ebd_x = [] ebd_x_proj = [] for im in progressbar(images): feat_file = join(args.features_path, im.replace('.jpg', '.dat')) x = load(feat_file) ebd_x.append(x) x_proj = model.project_x(torch.from_numpy(x)).squeeze_(0).data.numpy() ebd_x_proj.append(x_proj) ebd_x = normalize_rows(np.array(ebd_x), 2) ebd_x_proj = normalize_rows(np.array(ebd_x_proj), 2) print("{} image features".format(len(ebd_x))) # -------------------------------------------------------------------------- print("processing output embeddings ...") ebd_y = [] ebd_y_proj = [] for tag in progressbar(tags): tag = tag.replace('*', '') if vecs[tag] is None: y = np.zeros(vec_dim, dtype=np.float32) else: y = np.array(vecs[tag], dtype=np.float32) ebd_y.append(y)
X, Y = {}, {} feat_extractor = AutoEncoder(exp=args.exp, weights_path=args.weights_path) print("loading features ... ", end='') sys.stdout.flush() for set_ in [k for k in anno.keys() if k != "tags"]: imid_list = sorted(list(anno[set_].keys())) # only for reproducibility X[set_] = None Y[set_] = [None for _ in range(len(imid_list))] if args.exp == 1: for i, imid in tqdm(enumerate(imid_list), total=len(imid_list)): # set image features fname = splitext(anno[set_][imid]["file_name"])[0] + ".dat" x = load(join(args.features_path, set_, fname)) x = normalize_rows(x.reshape(1, -1)).squeeze() x = feat_extractor.predict(kind="img", x=x) if i == 0: n_samples = len(imid_list) n_dim = x.shape[0] X[set_] = np.empty((n_samples, n_dim), dtype=np.float32) X[set_][i] = x # set word embeddings (OOV tags are set to the zero vector) tags = anno[set_][imid]["tags"] y = [[0] * vec_dim if vecs[w] is None else vecs[w] for w in tags] y = normalize_rows(np.array(y, dtype=np.float32)) y = [t.reshape(1, -1) for t in y] y = feat_extractor.predict(kind="text", y=y) Y[set_][i] = y
def load_vectors(self): self.db = np.load(self.vectors_path, allow_pickle=True) if self.normalize: self.db = normalize_rows(self.db) self.dim = self.db.shape[1]
def test_model(pool_pctg, layer_size_1, layer_size_2): tf.reset_default_graph() X_train, y_train, X_test, y_test = loadEEG() X_val = X_test[:2] y_val = y_test[:2] X_test = X_test[2:] y_test = y_test[2:] labels = np.array([0, 1, 2, 3, 4, 5]) digit_indices = [np.where(y_train == i)[0] for i in labels] tr_trip_idxs = create_triplet_idxs(X_train, digit_indices, labels) digit_indices = [np.where(y_test == i)[0] for i in labels] te_trip_idxs = create_triplet_idxs(X_test, digit_indices, labels) print 'There are ', len(tr_trip_idxs), ' training examples!' #p = np.random.permutation(len(tr_trip_idxs)) #tr_trip_idxs = tr_trip_idxs[p] # Initializing the variables # the data, shuffled and split between train and test sets #""" X_train = normalize_rows(X_train) X_test = normalize_rows(X_test) #""" D = X_train.shape[1] ts_length = X_train.shape[2] global_step = tf.Variable(0, trainable=False) starter_learning_rate = 0.001 learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 10, 0.1, staircase=True) pool_width = pool_pctg * ts_length # create training+test positive and negative pairs anchor = tf.placeholder(tf.float32, shape=([None, D, ts_length]), name='L') same = tf.placeholder(tf.float32, shape=([None, D, ts_length]), name='R') different = tf.placeholder(tf.float32, shape=([None, D, ts_length]), name='R') labels = tf.placeholder(tf.float32, shape=([None]), name='gt') dropout_f = tf.placeholder("float") bn_train = tf.placeholder(tf.bool) with tf.variable_scope("siamese") as scope: model1, filters = build_conv_net(anchor, bn_train, dropout_f, ts_length, embedding_size, pool_width, layer_size_1, layer_size_2) scope.reuse_variables() model2, _ = build_conv_net(same, bn_train, dropout_f, ts_length, embedding_size, pool_width, layer_size_1, layer_size_2) scope.reuse_variables() model3, _ = build_conv_net(different, bn_train, dropout_f, ts_length, embedding_size, pool_width, layer_size_1, layer_size_2) distance = tf.sqrt( tf.reduce_sum(tf.pow(tf.subtract(model1, model2), 2), 1, keep_dims=True)) loss = triplet_loss(model1, model2, model3) #+ regularizer(model1, model2, model3) #loss = new_new_loss(model1, model2, model3) + regularizer(model1, model2, model3) debug_val = debug_loss(model1, model2, model3) regularization = regularizer(model1, model2, model3) tr_loss = triplet_loss(model1, model2, model3) t_vars = tf.trainable_variables() d_vars = [var for var in t_vars if 'l' in var.name] batch = tf.Variable(0) optimizer = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(loss) f1 = open('X_output.txt', 'w') f2 = open('X_labels.txt', 'w') f1_t = open('X_output_test.txt', 'w') f2_t = open('X_labels_test.txt', 'w') patience_window = 10 early_stopping = False last_vals = [0 for i in range(patience_window)] skippable_batches = [] with tf.Session() as sess: tf.global_variables_initializer().run() # Training cycle for epoch in range(400): avg_loss = 0. avg_r = 0. total_batch = int( np.ceil(tr_trip_idxs.shape[0] / float(batch_size))) start_time = time.time() # Loop over all batches loss_values = [] avg_tr = 0. for i in range(total_batch): if i in skippable_batches: continue s = i * batch_size e = (i + 1) * batch_size # Fit training using batch data input1, input2, input3 = next_batch_from_idx( s, e, tr_trip_idxs, X_train) #anchor_embedding=model1.eval(feed_dict={anchor:input1,dropout_f:1.0}) #same_embedding=model1.eval(feed_dict={anchor:input2,dropout_f:1.0}) #diff_embedding=model1.eval(feed_dict={anchor:input3,dropout_f:1.0}) _, loss_value, predict, r_loss, tr_val, d = sess.run( [ optimizer, loss, distance, regularization, tr_loss, debug_val ], feed_dict={ anchor: input1, same: input2, different: input3, dropout_f: 1.0 }) print loss_value if loss_value < .001: skippable_batches.append(i) if i % 30 == 0: train_embedding = model1.eval(feed_dict={ anchor: X_train, dropout_f: 1.0 }) test_embedding = model1.eval(feed_dict={ anchor: X_test, dropout_f: 1.0 }) #val_embedding = model1.eval(feed_dict={anchor:X_val,dropout_f:1.0}) accuracy = evaluate_test_embedding( train_embedding, y_train, test_embedding, y_test) print 'ACCURACY: ', accuracy, ' EPOCH: ', epoch #pdb.set_trace() if math.isnan(loss_value): pdb.set_trace() avg_loss += loss_value loss_values.append(loss_value) avg_r += r_loss avg_tr += tr_val #print('epoch %d loss %0.2f' %(epoch,avg_loss/total_batch)) print('epoch %d time: %f loss %0.5f r_loss %0.5f tr_loss %0.5f' % (epoch, duration, avg_loss / (total_batch), avg_r / total_batch, tr_val)) duration = time.time() - start_time if epoch % 10 == 0: tr_acc = compute_accuracy(tr_trip_idxs) print "Training accuracy: ", tr_acc print( 'epoch %d time: %f loss %0.5f r_loss %0.5f tr_loss %0.5f' % (epoch, duration, avg_loss / (total_batch), avg_r / total_batch, tr_val)) train_embedding = model1.eval(feed_dict={ anchor: X_train, dropout_f: 1.0 }) test_embedding = model1.eval(feed_dict={ anchor: X_test, dropout_f: 1.0 }) #val_embedding = model1.eval(feed_dict={anchor:X_val,dropout_f:1.0}) accuracy = evaluate_test_embedding(train_embedding, y_train, test_embedding, y_test) #val_accuracy = evaluate_test_embedding(train_embedding, y_train, val_embedding, y_val) print('Accuracy given NN approach %0.2f' % (100 * accuracy)) #print('Val Accuracy given NN approach %0.2f' %(100*val_accuracy)) last_vals[(epoch / 100) % patience_window] = val_accuracy if last_vals.count(last_vals[0]) == len(last_vals): early_stopping = True """ if early_stopping: print 'Stopping early!' break """ train_embedding = model1.eval(feed_dict={ anchor: X_train, dropout_f: 1.0 }) test_embedding = model1.eval(feed_dict={ anchor: X_test, dropout_f: 1.0 }) accuracy = evaluate_test_embedding(train_embedding, y_train, test_embedding, y_test) print('Accuracy given NN approach %0.2f' % (100 * accuracy)) filter1_weights = sess.run(filters[0]) for coord, label in zip(train_embedding, y_train): f1.write(' '.join([str(a) for a in coord]) + "\n") f2.write(str(label) + "\n") for coord, label in zip(test_embedding, y_test): f1_t.write(' '.join([str(a) for a in coord]) + "\n") f2_t.write(str(label) + "\n") return accuracy
def test_model(dataset, pool_pctg=.1, layer_size=40, stride_pct=-1): tf.reset_default_graph() """ X_train, y_train, X_test, y_test = loadEEG() X_val = X_test[:2] y_val = y_test[:2] X_test = X_test[2:] y_test = y_test[2:] """ def compute_accuracy(X, triplet_idxs): n_correct = 0.0 for triplet in triplet_idxs: a = np.expand_dims(X[triplet[0]], 0) s = np.expand_dims(X[triplet[1]], 0) d = np.expand_dims(X[triplet[2]], 0) predict_same = distance.eval(feed_dict={ anchor: a, same: s, dropout_f: 1.0 }) predict_diff = distance.eval(feed_dict={ anchor: a, same: d, dropout_f: 1.0 }) if predict_same[0][0] < predict_diff[0][0]: n_correct += 1.0 return n_correct / len(triplet_idxs) dataset_list = cv_splits_for_dataset(dataset) if len(dataset_list) <= n_fold: n_fold = 0 X_train = dataset_list[n_fold].X_train y_train = dataset_list[n_fold].y_train X_test = dataset_list[n_fold].X_test y_test = dataset_list[n_fold].y_test if dataset == 'trajectories': X_train = [g.T for g in X_train] X_test = [g.T for g in X_test] n = max([ np.max([v.shape[0] for v in X_train]), np.max([v.shape[0] for v in X_test]) ]) X_train = standardize_ts_lengths(X_train, n) X_test = standardize_ts_lengths(X_test, n) y_train = np.array(y_train) y_test = np.array(y_test) X_train = normalize_rows(X_train) X_test = normalize_rows(X_test) labels = np.unique(y_train) digit_indices = [np.where(y_train == i)[0] for i in labels] tr_trip_idxs = create_triplet_idxs(X_train, digit_indices, labels) digit_indices = [np.where(y_test == i)[0] for i in labels] te_trip_idxs = create_triplet_idxs(X_test, digit_indices, labels) N = X_train.shape[0] Ntest = X_test.shape[0] D = X_train.shape[1] ts_length = X_train.shape[2] global_step = tf.Variable(0, trainable=False) starter_learning_rate = 0.001 learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 10, 0.1, staircase=True) pool_width = pool_pctg * ts_length labels = np.unique(y_train) digit_indices = [np.where(y_train == i)[0] for i in labels] tr_pairs, tr_y = create_pairs(X_train, digit_indices, labels) pos_ind = np.where(tr_y == SAME_LABEL)[0] neg_ind = np.where(tr_y == NEG_LABEL)[0] # create training+test positive and negative pairs anchor = tf.placeholder(tf.float32, shape=([None, D, ts_length]), name='L') same = tf.placeholder(tf.float32, shape=([None, D, ts_length]), name='R') different = tf.placeholder(tf.float32, shape=([None, D, ts_length]), name='R') labels = tf.placeholder(tf.float32, shape=([None]), name='gt') #digit_indices = [np.where(y_val == i)[0] for i in labels] #val_pairs, val_y = create_pairs(X_val, digit_indices, labels) digit_indices = [np.where(y_test == i)[0] for i in labels] te_pairs, te_y = create_pairs(X_test, digit_indices, labels) # Initializing the variables r = 1 N = tr_pairs.shape[0] # the data, shuffled and split between train and test sets batch_size = 24 num_pos = 30 num_neg = batch_size - num_pos global_step = tf.Variable(0, trainable=False) starter_learning_rate = 0.01 learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 10, 0.1, staircase=True) # create training+test positive and negative pairs images_L = tf.placeholder(tf.float32, shape=([None, ts_length]), name='L') images_R = tf.placeholder(tf.float32, shape=([None, ts_length]), name='R') labels = tf.placeholder(tf.float32, shape=([None, 1]), name='gt') embedding_size = 10 dropout_f = tf.placeholder("float") bn_train = tf.placeholder(tf.bool) pool_width = pool_pctg * ts_length with tf.variable_scope("siamese") as scope: model1, filters = build_conv_net(images_L, bn_train, dropout_f, ts_length, embedding_size, pool_width) scope.reuse_variables() model2, _ = build_conv_net(images_R, bn_train, dropout_f, ts_length, embedding_size, pool_width) normalize_model1 = tf.nn.l2_normalize(model1, 0) normalize_model2 = tf.nn.l2_normalize(model2, 0) cos_similarity = tf.reduce_sum(tf.multiply(normalize_model1, normalize_model1), 1, keep_dims=True) distance = tf.sqrt( tf.reduce_sum(tf.pow(tf.subtract(model1, model2), 2), 1, keep_dims=True)) #distance = 1-scipy.spatial.distance.cosine(model1, model2) #loss = contrastive_loss(labels,distance) + regularizer(model1, model2, r) #loss = c_loss(labels, model1, model2) + regularizer(model1, model2, r) loss = contrastive_loss(labels, distance) + regularizer(model1, model2, r) #ugh = c_loss(labels, model1, model2) ugh = contrastive_loss(labels, distance) #loss = contrastive_loss(labels,distance) + regularizer(model1, model2, r) regularization = regularizer(model1, model2, r) #contrastice loss t_vars = tf.trainable_variables() d_vars = [var for var in t_vars if 'l' in var.name] batch = tf.Variable(0) optimizer = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(loss) #optimizer = tf.train.RMSPropOptimizer(0.00001,momentum=0.9,epsilon=1e-6).minimize(loss) # Launch the graph f1 = open('X_output.txt', 'w') f2 = open('X_labels.txt', 'w') f1_t = open('X_output_test.txt', 'w') f2_t = open('X_labels_test.txt', 'w') #filter2_summary = tf.summary.image("Filter_2", filters[1]) patience_window = 5 last_vals = [0 for i in range(patience_window)] early_stopping = False with tf.Session() as sess: tf.global_variables_initializer().run() summary_writer = tf.summary.FileWriter('/tmp/logs', sess.graph_def) #merged = tf.summary.merge_all() # Training cycle step = 0 perf_collect = [[], []] for epoch in range(700): total_c_loss = 0 for i in range(max(int(np.ceil(N / float(batch_size))), 1)): batch_ind = np.arange(i * batch_size, min((i + 1) * batch_size, N - 1)) #pos_ind = np.arange(i*num_pos, min((i+1)*num_pos,N-1)) #neg_ind = np.arange(i*num_neg, min((i+1)*num_neg,N-1)) #pos_ind = np.random.choice( np.arange(N), num_pos) #neg_ind = np.random.choice( np.arange(N), num_neg) #batch_ind = np.concatenate((pos_ind, neg_ind)) #print('VAL ACCURACY %0.2f' % perf_collect[1][-1]) input1, input2, y = tr_pairs[batch_ind, [0]], tr_pairs[ batch_ind, 1], tr_y[batch_ind, np.newaxis] _, loss_value, predict, r_loss, c_loss = sess.run( [optimizer, loss, distance, regularization, ugh], feed_dict={ images_L: input1, images_R: input2, labels: y, dropout_f: 1.0, bn_train: True }) total_c_loss += c_loss if math.isnan(c_loss): pdb.set_trace() if epoch % 400 == 0: #tr_acc = compute_accuracy(predict,y) print('epoch %d loss %0.5f r_loss %0.5f c_loss %0.5f ' % (epoch, loss_value, r_loss, total_c_loss)) train_embedding = model1.eval(feed_dict={ images_L: X_train, dropout_f: 1.0, bn_train: False }) test_embedding = model1.eval(feed_dict={ images_L: X_test, dropout_f: 1.0, bn_train: False }) val_embedding = model1.eval(feed_dict={ images_L: X_val, dropout_f: 1.0, bn_train: False }) accuracy = evaluate_test_embedding(train_embedding, y_train, test_embedding, y_test) val_accuracy = evaluate_test_embedding(train_embedding, y_train, val_embedding, y_val) last_vals[(epoch / 100) % patience_window] = val_accuracy if last_vals.count(last_vals[0]) == len(last_vals) and i > 900: early_stopping = True print('Accuracy given NN approach %0.2f' % (100 * accuracy)) print('Val Accuracy given NN approach %0.2f' % (100 * val_accuracy)) """ if early_stopping: print 'Stopping early' break """ #print('epoch %d loss %0.2f' %(epoch,avg_loss/total_batch)) # Test model """ y = np.reshape(te_y,(te_y.shape[0],1)) feature1=model1.eval(feed_dict={images_L:te_pairs[:,0],dropout_f:1.0, bn_train:False}) feature2=model2.eval(feed_dict={images_R:te_pairs[:,1],dropout_f:1.0, bn_train:False}) te_acc = compute_accuracy_features(feature1, feature2,te_y) print('Accuracy test set %0.2f' % (100 * te_acc)) """ train_embedding = model1.eval(feed_dict={ images_L: X_train, dropout_f: 1.0, bn_train: False }) test_embedding = model1.eval(feed_dict={ images_L: X_test, dropout_f: 1.0, bn_train: False }) accuracy = evaluate_test_embedding(train_embedding, y_train, test_embedding, y_test) print('Accuracy given NN approach %0.2f' % (100 * accuracy)) return accuracy
print("loading features ... ", end='') sys.stdout.flush() for set_ in [k for k in anno.keys() if k != "tags"]: imid_list = sorted(list(anno[set_].keys())) # only for reproducibility X[set_] = None Y[set_] = [None for _ in range(len(imid_list))] for i, imid in enumerate(imid_list): # set image features fname = splitext(anno[set_][imid]["file_name"])[0] + ".dat" x = load(join(args.features_path, set_, fname)) if i == 0: n_samples = len(imid_list) n_dim = len(x) X[set_] = np.empty((n_samples, n_dim), dtype=np.float32) X[set_][i] = normalize_rows(x.reshape(1, -1)).squeeze() # set word embeddings (OOV tags are set to the zero vector) tags = anno[set_][imid]["tags"] y = [[0] * vec_dim if vecs[w] is None else vecs[w] for w in tags] Y[set_][i] = normalize_rows(np.array(y, dtype=np.float32)) print("done") # shuffle debug if args.debug: idxs = random_state.permutation(len(X["train2014"]))[:1000] X["train2014"] = X["train2014"][idxs] Y["train2014"] = [Y["train2014"][i] for i in idxs] # run