def embedding_nearest_neighbour(n_neighbors=FLAGS.n_neighbours, num_classes=FLAGS.way, num_shots=FLAGS.shot, num_tasks=FLAGS.num_tasks, num_encoding_dims=FLAGS.num_encoding_dims, test_set=FLAGS.test_set, dataset=FLAGS.dataset): print('{}-way {}-shot embedding nearest neighbour'.format( num_classes, num_shots)) if dataset != 'celeba': _, _, _, X_test, Y_test, Z_test = get_data(dataset, num_encoding_dims, test_set) task_generator = TaskGenerator(num_classes=num_classes, num_train_samples_per_class=num_shots, num_samples_per_class=num_shots + 5) partition = task_generator.get_partition_from_labels(Y_test) partitions = [partition] else: _, _, _, X_test, attributes_test, Z_test = get_data( dataset, num_encoding_dims, test_set) task_generator = TaskGenerator(num_classes=num_classes, num_train_samples_per_class=num_shots, num_samples_per_class=num_shots + 5) partitions = task_generator.get_celeba_task_pool(attributes_test) tasks = task_generator.get_tasks(num_tasks=num_tasks, partitions=partitions) accuracies = [] for i_task, task in enumerate(tasks): if (i_task + 1) % (num_tasks // 10) == 0: print('test {}, accuracy {:.5}'.format(i_task + 1, np.mean(accuracies))) ind_train_few, Y_train_few, ind_test_few, Y_test_few = task Z_train_few, Z_test_few = Z_test[ind_train_few], Z_test[ind_test_few] knn = KNeighborsClassifier(n_neighbors=n_neighbors, n_jobs=-1) knn.fit(Z_train_few, Y_train_few) accuracy = knn.score(Z_test_few, Y_test_few) accuracies.append(accuracy) print( '{}-way {}-shot embedding nearest neighbour: {:.5} with 95% CI {:.5} over {} tests' .format(num_classes, num_shots, np.mean(accuracies), 1.96 * np.std(accuracies) / np.sqrt(num_tasks), num_tasks))
def make_data_tensor(self, train=True): if train: mode = FLAGS.mt_mode num_classes = self.num_classes_train num_tasks = FLAGS.metatrain_iterations * self.batch_size num_splits = 1000 if FLAGS.num_partitions == -1: num_partitions = num_tasks else: num_partitions = FLAGS.num_partitions if FLAGS.datasource == 'celeba': assert num_classes == 2, "CelebA must have two classes" X, attributes, Z = self.X_train, self.attributes_train, self.Z_train else: X, Y, Z = self.X_train, self.Y_train, self.Z_train num_samples_per_class = self.num_samples_per_class_train num_train_samples_per_class = FLAGS.inner_update_batch_size_train print('Setting up tasks for meta-training') else: mode = FLAGS.mv_mode if mode == 'encenc': raise NotImplementedError num_tasks = FLAGS.num_eval_tasks num_splits = 100 num_partitions = num_tasks if FLAGS.datasource == 'celeba': X, attributes, Z = self.X_val, self.attributes_val, self.Z_val else: X, Y, Z = self.X_val, self.Y_val, self.Z_val num_classes = self.num_classes_val num_samples_per_class = self.num_samples_per_class_val num_train_samples_per_class = FLAGS.inner_update_batch_size_val print('Setting up tasks for meta-val') task_generator = TaskGenerator( num_classes=num_classes, num_train_samples_per_class=num_train_samples_per_class, num_samples_per_class=num_samples_per_class) partition_algorithm = FLAGS.partition_algorithm margin = FLAGS.margin print('Generating indices for {} tasks'.format(num_tasks)) if mode == 'gtgt': if FLAGS.datasource == 'celeba': partitions = task_generator.get_celeba_task_pool( attributes=attributes) else: print('Using ground truth partition to create classes') partition = task_generator.get_partition_from_labels(labels=Y) partitions = [partition] elif mode == 'encenc': if partition_algorithm == 'hyperplanes': print( 'Using {} hyperplanes-based partition(s) of encoding space to create classes, margin={}' .format(num_partitions, margin)) partitions = task_generator.get_partitions_hyperplanes( encodings=Z, num_splits=num_splits, margin=margin, num_partitions=num_partitions) elif partition_algorithm == 'kmeans': print( 'Using {} k-means based partition(s) of encoding space to create classes' .format(num_partitions)) partitions = task_generator.get_partitions_kmeans(encodings=Z, train=train) else: raise ValueError( 'Unrecognized partition-generating algorithm: either hyperplanes or kmeans' ) elif mode == 'randrand': print('Randomly sampled and labeled tasks') partitions = [] for p in tqdm(range(num_partitions)): labels = np.random.choice(FLAGS.num_clusters, size=Y.shape, replace=True) partition = task_generator.get_partition_from_labels( labels=labels) partitions.append(partition) else: raise ValueError('Unrecognized task generation scheme') print('Average number of classes per partition: {}'.format( np.mean([len(list(partition.keys())) for partition in partitions]))) if FLAGS.on_encodings: features = features_ph = tf.placeholder(Z.dtype, Z.shape) else: assert X.dtype == 'uint8' features_ph = tf.placeholder(X.dtype, X.shape) features = tf.reshape(features_ph, [-1, self.dim_input]) def gather_preprocess(task): for split in ['train', 'test']: task['{}_labels'.format(split)] = tf.one_hot( task['{}_labels'.format(split)], num_classes) if not FLAGS.on_encodings: task['{}_features'.format(split)] = tf.cast( tf.gather(features, task['{}_indices'.format(split)]), tf.float32) / 255.0 else: task['{}_features'.format(split)] = tf.gather( features, task['{}_indices'.format(split)]) return task def stack(task): features = tf.concat( (task['train_features'], task['test_features']), axis=0) labels = tf.concat((task['train_labels'], task['test_labels']), axis=0) return features, labels tasks = task_generator.get_tasks(num_tasks=num_tasks, partitions=partitions) train_ind, train_labels, test_ind, test_labels = [ task[0] for task in tasks ], [task[1] for task in tasks], [task[2] for task in tasks ], [task[3] for task in tasks] dataset = tf.data.Dataset.from_tensor_slices({ "train_indices": train_ind, "train_labels": train_labels, "test_indices": test_ind, "test_labels": test_labels }) dataset = dataset.map(map_func=gather_preprocess, num_parallel_calls=FLAGS.num_parallel_calls) dataset = dataset.map(map_func=stack, num_parallel_calls=FLAGS.num_parallel_calls) dataset = dataset.batch(batch_size=self.batch_size) dataset = dataset.prefetch(4) dataset = dataset.repeat() iterator = dataset.make_initializable_iterator() features_batch, labels_batch = iterator.get_next() if FLAGS.on_encodings: iterator.initializer.run(feed_dict={features_ph: Z}) else: iterator.initializer.run(feed_dict={features_ph: X}) return features_batch, labels_batch
def cluster_fit_color(num_classes=FLAGS.way, num_tasks=FLAGS.num_tasks, num_clusters=FLAGS.num_clusters, num_encoding_dims=FLAGS.num_encoding_dims, test_set=FLAGS.test_set, dataset=FLAGS.dataset): assert dataset == 'mnist' import keras from keras.layers import Conv2D, Flatten, Dense from keras.losses import categorical_crossentropy from keras.optimizers import Adam from sklearn.cluster import KMeans X_train, Y_train, Z_train, X_test, Y_test, Z_test = get_data( dataset, num_encoding_dims, test_set) # Z_train, Z_test = whitening(Z_train, Z_test) start = time.time() kmeans = KMeans(n_clusters=num_clusters, init='k-means++', random_state=0, precompute_distances=True, n_jobs=-1, n_init=1000, max_iter=100000).fit(Z_train) print( "Ran KMeans with n_clusters={} in {:.5} seconds, objective {}.".format( num_clusters, time.time() - start, kmeans.score(Z_train))) X_train, X_test = X_train / 255.0, X_test / 255.0 X_train, X_test = X_train.reshape((-1, 28, 28, 1)), X_test.reshape( (-1, 28, 28, 1)) cluster_labels_train = keras.utils.to_categorical(kmeans.labels_, num_clusters) cluster_labels_test = keras.utils.to_categorical(kmeans.predict(Z_test), num_clusters) model = keras.Sequential() model.add( Conv2D(filters=32, kernel_size=(3, 3), strides=(2, 2), activation='relu', padding='same', input_shape=(28, 28, 1))) model.add( Conv2D(filters=32, kernel_size=(3, 3), strides=(2, 2), activation='relu', padding='same')) model.add( Conv2D(filters=32, kernel_size=(3, 3), strides=(2, 2), activation='relu', padding='same')) model.add( Conv2D(filters=32, kernel_size=(3, 3), strides=(2, 2), activation='relu', padding='same')) model.add(Flatten()) model.add(Dense(units=num_clusters, activation='softmax')) model.summary() model.compile(loss=categorical_crossentropy, optimizer=Adam(), metrics=['accuracy']) model.fit(X_train, cluster_labels_train, batch_size=500, epochs=25, verbose=1, validation_data=(X_test, cluster_labels_test)) score = model.evaluate(X_test, cluster_labels_test, verbose=1) print('Test loss: {}\tTest accuracy: {}'.format(score[0], score[1])) model.compile(loss=categorical_crossentropy, optimizer=keras.optimizers.SGD(lr=0.01), metrics=['accuracy']) for num_shots in [1, 5, 10]: accuracies, finetuned_accuracies = [], [] num_degenerate_tasks = 0 start = time.time() task_generator = TaskGenerator(num_classes=num_classes, num_train_samples_per_class=num_shots, num_samples_per_class=num_shots + 5) partition = task_generator.get_partition_from_labels(Y_test) for i_test in range(num_tasks): if (i_test + 1) % (num_tasks // 10) == 0: print( 'test {}, accuracy {:.5}, finetuned accuracy {:.5}'.format( i_test + 1, np.mean(accuracies), np.mean(finetuned_accuracies))) task = task_generator.get_task(partition=partition) ind_train_few, Y_train_few, ind_test_few, Y_test_few = task X_train_few, X_test_few = X_test[ind_train_few], X_test[ ind_test_few] cluster_to_labels_few = defaultdict(list) Z_train_few = np.argmax(model.predict(X_train_few), axis=1) Z_test_few = np.argmax(model.predict(X_test_few), axis=1) for i in range(len(Y_train_few)): cluster_to_labels_few[Z_train_few[i]].append(Y_train_few[i]) cluster_to_label_few = defaultdict(int) for (cluster, labels) in list(cluster_to_labels_few.items()): uniques, counts = np.unique(labels, return_counts=True) cluster_to_label_few[cluster] = uniques[np.argmax(counts)] if len(cluster_to_label_few) == 0: num_degenerate_tasks += 1 continue predictions = [] for z in Z_test_few: predictions.append(cluster_to_label_few[z]) accuracies.append(accuracy_score(Y_test_few, predictions)) print('num_clusters={}, num_encoding_dims={}'.format( num_clusters, num_encoding_dims)) print( '{}-way {}-shot fit_kmeans: {:.5} with 95% CI {:.5} over {} tests'. format(num_classes, num_shots, np.mean(accuracies), 1.96 * np.std(accuracies) / np.sqrt(num_tasks), num_tasks)) print( '{}-way {}-shot fit_kmeans finetuned: {:.5} with 95% CI {:.5} over {} tests' .format(num_classes, num_shots, np.mean(finetuned_accuracies), 1.96 * np.std(finetuned_accuracies) / np.sqrt(num_tasks), num_tasks)) print( '{} few-shot classification tasks: {:.5} seconds with {} degenerate tasks.' .format(num_tasks, time.time() - start, num_degenerate_tasks))
def embedding_cluster_matching(num_classes=FLAGS.way, num_shots=FLAGS.shot, num_tasks=FLAGS.num_tasks, num_clusters=FLAGS.num_clusters, num_encoding_dims=FLAGS.num_encoding_dims, dataset=FLAGS.dataset, test_set=FLAGS.test_set): if dataset != 'celeba': _, _, Z_train, X_test, Y_test, Z_test = get_data( dataset, num_encoding_dims, test_set) else: _, _, Z_train, X_test, attributes_test, Z_test = get_data( dataset, num_encoding_dims, test_set) start = time.time() kmeans = KMeans(n_clusters=num_clusters, init='k-means++', random_state=0, precompute_distances=True, n_jobs=10, n_init=10, max_iter=3000).fit(Z_train) print( "Ran KMeans with n_clusters={} in {:.5} seconds, objective {}.".format( num_clusters, time.time() - start, kmeans.score(Z_train))) if dataset != 'celeba': task_generator = TaskGenerator(num_classes=num_classes, num_train_samples_per_class=num_shots, num_samples_per_class=num_shots + 5) partition = task_generator.get_partition_from_labels(Y_test) partitions = [partition] else: task_generator = TaskGenerator(num_classes=num_classes, num_train_samples_per_class=num_shots, num_samples_per_class=num_shots + 5) partitions = task_generator.get_celeba_task_pool(attributes_test) tasks = task_generator.get_tasks(num_tasks=num_tasks, partitions=partitions) for num_shots in [FLAGS.shot]: accuracies = [] start = time.time() num_degenerate_tasks = 0 for i_task, task in enumerate(tasks): if (i_task + 1) % (num_tasks // 10) == 0: print('test {}, accuracy {:.5}'.format(i_task + 1, np.mean(accuracies))) ind_train_few, Y_train_few, ind_test_few, Y_test_few = task Z_train_few, Z_test_few = Z_test[ind_train_few], Z_test[ ind_test_few] clusters_to_labels_few = defaultdict(list) examples_to_clusters_few = kmeans.predict(Z_train_few) for i in range(len(Y_train_few)): clusters_to_labels_few[examples_to_clusters_few[i]].append( Y_train_few[i]) for (cluster, labels) in list(clusters_to_labels_few.items()): uniques, counts = np.unique(labels, return_counts=True) clusters_to_labels_few[cluster] = [uniques[np.argmax(counts)]] # if len(np.unique(labels)) > 1: # delete degenerate clusters # del clusters_to_labels_few[cluster] if len(clusters_to_labels_few) == 0: num_degenerate_tasks += 1 continue centroid_ind_to_cluster = np.array( list(clusters_to_labels_few.keys())) centroids = kmeans.cluster_centers_[centroid_ind_to_cluster] distances = distance.cdist(Z_test_few, centroids) predicted_clusters = centroid_ind_to_cluster[np.argmin(distances, axis=1)] predictions = [] for cluster in predicted_clusters: predictions.append(clusters_to_labels_few[cluster][0]) accuracies.append(accuracy_score(Y_test_few, predictions)) print('dataset={}, encoder={}, num_encoding_dims={}, num_clusters={}'. format(dataset, FLAGS.encoder, num_clusters, num_encoding_dims)) print( '{}-way {}-shot nearest-cluster after clustering embeddings: {:.5} with 95% CI {:.5} over {} tests' .format(num_classes, num_shots, np.mean(accuracies), 1.96 * np.std(accuracies) / np.sqrt(num_tasks), num_tasks)) print( '{} few-shot classification tasks: {:.5} seconds with {} degenerate tasks.' .format(num_tasks, time.time() - start, num_degenerate_tasks))
def cluster_color_logistic_regression( C=FLAGS.inverse_reg, penalty='l2', multi_class='multinomial', n_clusters=FLAGS.num_clusters, num_classes=FLAGS.way, num_shots=FLAGS.shot, num_tasks=FLAGS.num_tasks, num_encoding_dims=FLAGS.num_encoding_dims, test_set=FLAGS.test_set, dataset=FLAGS.dataset): if dataset != 'celeba': _, _, Z_train, X_test, Y_test, Z_test = get_data( dataset, num_encoding_dims, test_set) else: _, _, Z_train, X_test, attributes_test, Z_test = get_data( dataset, num_encoding_dims, test_set) start = time.time() kmeans = KMeans(n_clusters=n_clusters, precompute_distances=True, n_jobs=-1, n_init=100).fit(Z_train) print("Ran KMeans with n_clusters={} in {:.5} seconds.".format( n_clusters, time.time() - start)) uniques, counts = np.unique(kmeans.labels_, return_counts=True) if dataset != 'celeba': task_generator = TaskGenerator(num_classes=num_classes, num_train_samples_per_class=num_shots, num_samples_per_class=num_shots + 5) partition = task_generator.get_partition_from_labels(Y_test) partitions = [partition] else: task_generator = TaskGenerator(num_classes=num_classes, num_train_samples_per_class=num_shots, num_samples_per_class=num_shots + 5) partitions = task_generator.get_celeba_task_pool(attributes_test) tasks = task_generator.get_tasks(num_tasks=num_tasks, partitions=partitions) train_accuracies, test_accuracies = [], [] start = time.time() clusters_to_indices = task_generator.get_partition_from_labels( kmeans.labels_) for i_task, task in enumerate(tasks): if (i_task + 1) % (num_tasks // 10) == 0: print('test {}, train accuracy {:.5}, test accuracy {:.5}'.format( i_task + 1, np.mean(train_accuracies), np.mean(test_accuracies))) ind_train_few, Y_train_few, ind_test_few, Y_test_few = task Z_train_few, Z_test_few = Z_test[ind_train_few], Z_test[ind_test_few] clusters_to_labels_few = defaultdict(list) indices_to_clusters_few = kmeans.predict(Z_train_few) for i in range(Z_train_few.shape[0]): clusters_to_labels_few[indices_to_clusters_few[i]].append( Y_train_few[i]) Z_train_fit, Y_train_fit = [], [] for cluster in list(clusters_to_labels_few.keys()): labels = clusters_to_labels_few[cluster] if len(np.unique(labels)) == 1: # skip degenerate clusters Z_train_fit.extend( Z_train[clusters_to_indices[cluster]] ) # propagate labels to unlabeled datapoints Y_train_fit.extend([ labels[0] for i in range(len(clusters_to_indices[cluster])) ]) Z_train_fit, Y_train_fit = np.stack(Z_train_fit, axis=0), np.stack(Y_train_fit, axis=0) Z_train_fit = np.concatenate((Z_train_fit, Z_train_few), axis=0) Y_train_fit = np.concatenate((Y_train_fit, Y_train_few), axis=0) logistic_regression = LogisticRegression(n_jobs=-1, penalty=penalty, C=C, multi_class=multi_class, solver='saga', max_iter=500) logistic_regression.fit(Z_train_fit, Y_train_fit) test_accuracies.append( logistic_regression.score(Z_test_few, Y_test_few)) train_accuracies.append( logistic_regression.score(Z_train_fit, Y_train_fit)) print('n_clusters={}, penalty={}, C={}, multi_class={}'.format( n_clusters, penalty, C, multi_class)) print( '{}-way {}-shot logistic regression after clustering: {:.5} with 95% CI {:.5} over {} tests' .format(num_classes, num_shots, np.mean(test_accuracies), 1.96 * np.std(test_accuracies) / np.sqrt(num_tasks), num_tasks)) print('Mean training accuracy: {:.5}; standard deviation: {:.5}'.format( np.mean(train_accuracies), np.std(train_accuracies))) print('{} few-shot classification tasks: {:.5} seconds.'.format( num_tasks, time.time() - start))
def embedding_logistic_regression(C=FLAGS.inverse_reg, penalty='l2', multi_class='multinomial', num_classes=FLAGS.way, num_shots=FLAGS.shot, num_tasks=FLAGS.num_tasks, num_encoding_dims=FLAGS.num_encoding_dims, test_set=FLAGS.test_set, dataset=FLAGS.dataset): print('{}-way {}-shot logistic regression'.format(num_classes, num_shots)) if dataset != 'celeba': _, _, _, X_test, Y_test, Z_test = get_data(dataset, num_encoding_dims, test_set) task_generator = TaskGenerator(num_classes=num_classes, num_train_samples_per_class=num_shots, num_samples_per_class=num_shots + 5) partition = task_generator.get_partition_from_labels(Y_test) partitions = [partition] else: _, _, _, X_test, attributes_test, Z_test = get_data( dataset, num_encoding_dims, test_set) task_generator = TaskGenerator(num_classes=num_classes, num_train_samples_per_class=num_shots, num_samples_per_class=num_shots + 5) partitions = task_generator.get_celeba_task_pool(attributes_test) tasks = task_generator.get_tasks(num_tasks=num_tasks, partitions=partitions) train_accuracies, test_accuracies = [], [] start = time.time() for i_task, task in enumerate(tasks): if (i_task + 1) % (num_tasks // 10) == 0: print('test {}, train accuracy {:.5}, test accuracy {:.5}'.format( i_task + 1, np.mean(train_accuracies), np.mean(test_accuracies))) ind_train_few, Y_train_few, ind_test_few, Y_test_few = task Z_train_few, Z_test_few = Z_test[ind_train_few], Z_test[ind_test_few] logistic_regression = LogisticRegression(n_jobs=-1, penalty=penalty, C=C, multi_class=multi_class, solver='saga', max_iter=1000) logistic_regression.fit(Z_train_few, Y_train_few) test_accuracies.append( logistic_regression.score(Z_test_few, Y_test_few)) train_accuracies.append( logistic_regression.score(Z_train_few, Y_train_few)) print('penalty={}, C={}, multi_class={}'.format(penalty, C, multi_class)) print( '{}-way {}-shot logistic regression: {:.5} with 95% CI {:.5} over {} tests' .format(num_classes, num_shots, np.mean(test_accuracies), 1.96 * np.std(test_accuracies) / np.sqrt(num_tasks), num_tasks)) print('Mean training accuracy: {:.5}; standard deviation: {:.5}'.format( np.mean(train_accuracies), np.std(train_accuracies))) print('{} few-shot classification tasks: {:.5} seconds.'.format( num_tasks, time.time() - start))
def embedding_mlp(num_classes=FLAGS.way, num_shots=FLAGS.shot, num_tasks=FLAGS.num_tasks, num_encoding_dims=FLAGS.num_encoding_dims, test_set=FLAGS.test_set, dataset=FLAGS.dataset, units=FLAGS.units, dropout=FLAGS.dropout): import keras from keras.layers import Dense, Dropout from keras.losses import categorical_crossentropy from keras.callbacks import EarlyStopping from keras import backend as K if dataset != 'celeba': _, _, _, X_test, Y_test, Z_test = get_data(dataset, num_encoding_dims, test_set) task_generator = TaskGenerator(num_classes=num_classes, num_train_samples_per_class=num_shots, num_samples_per_class=num_shots + 5) partition = task_generator.get_partition_from_labels(Y_test) partitions = [partition] else: _, _, _, X_test, attributes_test, Z_test = get_data( dataset, num_encoding_dims, test_set) task_generator = TaskGenerator(num_classes=num_classes, num_train_samples_per_class=num_shots, num_samples_per_class=num_shots + 5) partitions = task_generator.get_celeba_task_pool(attributes_test) tasks = task_generator.get_tasks(num_tasks=num_tasks, partitions=partitions) train_accuracies, test_accuracies = [], [] start = time.time() for i_task, task in enumerate(tqdm(tasks)): if (i_task + 1) % (num_tasks // 10) == 0: tqdm.write('test {}, accuracy {:.5}'.format( i_task + 1, np.mean(test_accuracies))) ind_train_few, Y_train_few, ind_test_few, Y_test_few = task Z_train_few, Z_test_few = Z_test[ind_train_few], Z_test[ind_test_few] Y_train_few, Y_test_few = keras.utils.to_categorical( Y_train_few, num_classes=num_classes), keras.utils.to_categorical( Y_test_few, num_classes=num_classes) model = keras.Sequential() model.add( Dense(units=units, activation='relu', input_dim=Z_train_few.shape[1])) model.add(Dropout(rate=dropout)) model.add(Dense(units=num_classes, activation='softmax')) model.compile(loss=categorical_crossentropy, optimizer=keras.optimizers.Adam(), metrics=['accuracy']) early_stopping = EarlyStopping(monitor='val_loss', patience=2) model.fit(Z_train_few, Y_train_few, batch_size=Z_train_few.shape[0], epochs=500, verbose=0, validation_data=(Z_test_few, Y_test_few), callbacks=[early_stopping]) train_score = model.evaluate(Z_train_few, Y_train_few, verbose=0) train_accuracies.append(train_score[1]) test_score = model.evaluate(Z_test_few, Y_test_few, verbose=0) test_accuracies.append(test_score[1]) K.clear_session() print('units={}, dropout={}'.format(units, dropout)) print( '{}-way {}-shot embedding mlp: {:.5} with 95% CI {:.5} over {} tests'. format(num_classes, num_shots, np.mean(test_accuracies), 1.96 * np.std(test_accuracies) / np.sqrt(num_tasks), num_tasks)) print('Mean training accuracy: {:.5}; standard deviation: {:.5}'.format( np.mean(train_accuracies), np.std(train_accuracies))) print('{} few-shot classification tasks: {:.5} seconds.'.format( num_tasks, time.time() - start))