def test_mlc_label(self): """test multi-label case""" dataset = self.setup_mlc_dataset() lbr = IdealLabeler(dataset) ask_id = lbr.label(np.array([12., 5., 2., 11., 14.])) np.testing.assert_array_equal(ask_id, [0, 1, 0, 0, 1]) ask_id = lbr.label(np.array([6., 2., 21., 20., 5.])) np.testing.assert_array_equal(ask_id, [0, 0, 1, 0, 1])
def test_label(self): dataset = self.setup_dataset() lbr = IdealLabeler(dataset) ask_id = lbr.label(np.array([0, 1, 2])) self.assertEqual(ask_id, 1) ask_id = lbr.label(np.array([6, 7, 8])) self.assertEqual(ask_id, 3) ask_id = lbr.label([12, 13, 14]) self.assertEqual(ask_id, 4)
def test_label(self): dataset = self.setup_dataset() lbr = IdealLabeler(dataset) ask_id = lbr.label(np.array([0, 1, 2])) self.assertEqual(ask_id, 1) ask_id = lbr.label(np.array([6, 7, 8])) self.assertEqual(ask_id, 3) ask_id = lbr.label([12, 13, 14]) self.assertEqual(ask_id, 4)
def libact_QBC(X, y, n_queries): y_train = np.array([None for _ in range(len(y))]) y_train[0], y_train[50], y_train[100] = 0, 1, 2 libact_train_dataset = Dataset(X, y_train) libact_full_dataset = Dataset(X, y) libact_learner_list = [ LogisticRegressionLibact(solver='liblinear', n_jobs=1, multi_class='ovr'), LogisticRegressionLibact(solver='liblinear', n_jobs=1, multi_class='ovr') ] libact_qs = QueryByCommittee(libact_train_dataset, models=libact_learner_list, method='lc') libact_labeler = IdealLabeler(libact_full_dataset) for libact_learner in libact_learner_list: libact_learner.train(libact_train_dataset) for _ in range(n_queries): query_idx = libact_qs.make_query() query_label = libact_labeler.label(X[query_idx]) libact_train_dataset.update(query_idx, query_label) for libact_learner in libact_learner_list: libact_learner.train(libact_train_dataset)
def libact_EER(X, y, n_queries): y_train = np.array([None for _ in range(len(y))]) y_train[0], y_train[50], y_train[100] = 0, 1, 2 libact_train_dataset = Dataset(X, y_train) libact_full_dataset = Dataset(X, y) libact_learner = LogisticRegressionLibact( solver='liblinear', n_jobs=1, multi_class='ovr') #SVM(gamma='auto', probability=True) libact_qs = EER(libact_train_dataset, model=libact_learner, loss='01') libact_labeler = IdealLabeler(libact_full_dataset) libact_learner.train(libact_train_dataset) for _ in range(n_queries): query_idx = libact_qs.make_query() query_label = libact_labeler.label(X[query_idx]) libact_train_dataset.update(query_idx, query_label) libact_learner.train(libact_train_dataset)
def main(): quota = 10 # ask human to label 10 samples n_classes = 5 E_out1, E_out2 = [], [] trn_ds, tst_ds, ds = split_train_test(n_classes) trn_ds2 = copy.deepcopy(trn_ds) qs = UncertaintySampling(trn_ds, method='lc', model=LogisticRegression()) qs2 = RandomSampling(trn_ds2) model = LogisticRegression() fig = plt.figure() ax = fig.add_subplot(2, 1, 1) ax.set_xlabel('Number of Queries') ax.set_ylabel('Error') model.train(trn_ds) E_out1 = np.append(E_out1, 1 - model.score(tst_ds)) model.train(trn_ds2) E_out2 = np.append(E_out2, 1 - model.score(tst_ds)) query_num = np.arange(0, 1) p1, = ax.plot(query_num, E_out1, 'g', label='qs Eout') p2, = ax.plot(query_num, E_out2, 'k', label='random Eout') plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), fancybox=True, shadow=True, ncol=5) plt.show(block=False) img_ax = fig.add_subplot(2, 1, 2) box = img_ax.get_position() img_ax.set_position( [box.x0, box.y0 - box.height * 0.1, box.width, box.height * 0.9]) plt.show() # Give each label its name (labels are from 0 to n_classes-1) # lbr = InteractiveLabeler(label_name=[str(lbl) for lbl in range(n_classes)]) x_ds = ds.data print(x_ds.shape) y_ds = ds.target print(y_ds.shape) lbr_ds = Dataset(x_ds, y_ds) x, _ = zip(*trn_ds.data) print(x) lbr = IdealLabeler(lbr_ds) for i in range(quota): ask_id = qs.make_query() print("asking sample from Uncertainty Sampling") # reshape the image to its width and height x, _ = zip(*trn_ds.data) lb = lbr.label(x[ask_id]) # lb = lbr.label(trn_ds.data[ask_id][0].reshape(8, 8)) trn_ds.update(ask_id, lb) model.train(trn_ds) E_out1 = np.append(E_out1, 1 - model.score(tst_ds)) ask_id = qs2.make_query() print("asking sample from Random Sample") x, _ = zip(*trn_ds2.data) lb = lbr.label(x[ask_id]) # lb = lbr.label(trn_ds2.data[ask_id][0].reshape(8, 8)) trn_ds2.update(ask_id, lb) model.train(trn_ds2) E_out2 = np.append(E_out2, 1 - model.score(tst_ds)) ax.set_xlim((0, i + 1)) ax.set_ylim((0, max(max(E_out1), max(E_out2)) + 0.2)) query_num = np.arange(0, i + 2) p1.set_xdata(query_num) p1.set_ydata(E_out1) p2.set_xdata(query_num) p2.set_ydata(E_out2) plt.show() input("Press any key to continue...")
def visualize(dataset_file, word2vec_file, word_freq_file, output_file): nfolds = 3 for i, foldid in zip( [0], [0]): #itertools.product(get_set_random_states(), range(nfolds)): #for i, foldid in itertools.product(get_set_random_states(), range(nfolds)): result = [] data = read_and_split_data2(dataset_file, random_state=i, nfold=nfolds, foldid=foldid) # convert to libact dataset docs = data['train_x'] + data['dev_x'] + data['test_x'] trn_len = len(data['train_x']) dev_len = len(data['dev_x']) test_len = len(data['test_x']) X = np.asarray([[i] for i in range(len(docs))]) X_train = np.asarray([X[i, :] for i in range(trn_len)]) X_dev = np.asarray( [X[i, :] for i in range(trn_len, trn_len + dev_len)]) X_test = np.asarray( [X[i, :] for i in range(trn_len + dev_len, len(docs))]) y = data['train_y'] + data['dev_y'] + data['test_y'] y_train = data['train_y'] y_dev = data['dev_y'] y_test = data['test_y'] n_labeled = 50 tmp = prepare_data(X_train, y_train, X_test, y_test, n_labeled, random_state=3) trn_ds, tst_ds, fully_labeled_trn_ds, n_labeled = tmp lbr = IdealLabeler(fully_labeled_trn_ds) # quota = len(y_train) - n_labeled # number of samples to query quota = 10 batchsize = 10 # Comparing UncertaintySampling strategy with RandomSampling. # model is the base learner, e.g. LogisticRegression, SVM ... etc. # train CNN Readability Classifier # train CNN Readability Classifier print('>>>>>>>>>>>UnCertainty ....') preprocessor = rlc.Preprocessor(word2vec_file, word_freq_file, topwords_as_vocab=True) cls = rlc.ReadlevelClassifier(preprocessor, useGPU=True) cls.cuda() wrapper = CNNWrapperClassifier((docs, y), cls, (data['dev_x'], data['dev_y'])) wrapper.train(trn_ds) import ipdb # ipdb.set_trace() unlabeled_entry_ids, X_pool = zip(*trn_ds.get_unlabeled_entries()) dvalue = wrapper.predict_proba(X_pool) from sklearn.manifold import TSNE X_embedded = TSNE(n_components=2).fit_transform(dvalue) labels = np.asarray([[ lbr.label(X_pool[i]) for i in range(len(unlabeled_entry_ids)) ]]).transpose() print(labels.shape, X_embedded.shape) X_embedded = np.concatenate((X_embedded, labels), axis=1) result = pd.DataFrame(data=X_embedded) with open(output_file, 'w') as f: result.to_csv(f, header=False)