def _update_models_and_counts_(X_pool, y_pool, X_test, y_test, X_train, y_train, feature_expert, instance_model, rmw_n, rmw_a, reasoning_model, docs, \ discovered_features, discovered_class0_features, discovered_class1_features, \ covered_docs, num_a_feat_chosen, instance_model_scores, reasoning_model_scores, \ num_training_samples, discovered_feature_counts, num_docs_covered): # Train instance model instance_model.fit(X_train, y_train) X_pool_csc = X_pool.tocsc() for doc_id in docs: #feature = feature_expert.most_informative_feature(X_pool[doc_id], y_pool[doc_id]) feature = feature_expert.any_informative_feature( X_pool[doc_id], y_pool[doc_id]) if feature: discovered_features.add(feature) if y_pool[doc_id] == 0: discovered_class0_features.add(feature) else: discovered_class1_features.add(feature) # Train reasoning model reasoning_model.partial_fit(X_pool[doc_id], y_pool[doc_id], feature, rmw_n, rmw_a) # train feature_model one by one # docs covered if feature: f_covered_docs = X_pool_csc[:, feature].indices covered_docs.update(f_covered_docs) # number of times a feat is chosen as a reason if feature: num_a_feat_chosen[feature] += 1 (accu, auc) = evaluate_model(instance_model, X_test, y_test) instance_model_scores['auc'].append(auc) instance_model_scores['accu'].append(accu) (accu, auc) = evaluate_model(reasoning_model, X_test, y_test) reasoning_model_scores['auc'].append(auc) reasoning_model_scores['accu'].append(accu) num_training_samples.append(X_train.shape[0]) # discovered feature counts discovered_feature_counts['class0'].append(len(discovered_class0_features)) discovered_feature_counts['class1'].append(len(discovered_class1_features)) num_docs_covered.append(len(covered_docs))
def learn(X_pool, y_pool, X_test, y_test, training_set, pool_set, feature_expert, \ selection_strategy, disagree_strat, coverage, budget, instance_model, feature_model, \ pooling_model, reasoning_model, rmw_n, rmw_a, seed=0, Debug=False, \ reasoning_strategy='random', switch=40): start = time() print '-' * 50 print 'Starting Active Learning...' instance_model_scores = {'auc': [], 'accu': []} feature_model_scores = {'auc': [], 'accu': []} pooling_model_scores = {'auc': [], 'accu': []} reasoning_model_scores = {'auc': [], 'accu': []} discovered_feature_counts = {'class0': [], 'class1': []} num_docs_covered = [] covered_docs = set() X_pool_csc = X_pool.tocsc() num_samples = len(pool_set) + len(training_set) num_feat = X_pool.shape[1] num_a_feat_chosen = np.zeros(num_feat) discovered_features = set() discovered_class0_features = set() discovered_class1_features = set() feature_expert.rg.seed(seed) if selection_strategy == 'random': doc_pick_model = RandomStrategy(seed) elif selection_strategy == 'uncertaintyIM': doc_pick_model = UNCSampling(instance_model, feature_expert, y_pool, Debug) elif selection_strategy == 'uncertaintyFM': doc_pick_model = UNCSampling(feature_model, feature_expert, y_pool, Debug) elif selection_strategy == 'uncertaintyPM': doc_pick_model = UNCSampling(pooling_model, feature_expert, y_pool, Debug) elif selection_strategy == 'uncertaintyRM': doc_pick_model = UNCSampling(reasoning_model, feature_expert, y_pool, Debug) elif selection_strategy == 'disagreement': doc_pick_model = DisagreementStrategy(instance_model, feature_model, \ feature_expert, y_pool, disagree_strat, Debug=Debug) elif selection_strategy == 'covering': doc_pick_model = CoveringStrategy(feature_expert, num_samples, y_pool, \ type='unknown', seed=seed, Debug=Debug) elif selection_strategy == 'covering_fewest': doc_pick_model = CoveringStrategy(feature_expert, num_samples, y_pool, \ type='fewest', seed=seed, Debug=Debug) elif selection_strategy == 'cheating': doc_pick_model = CheatingApproach(feature_expert, num_samples, y_pool, \ seed=seed, Debug=Debug) elif selection_strategy == 'cover_then_disagree': doc_pick_model = CoveringThenDisagreement(feature_expert, instance_model, \ feature_model, num_samples, percentage=coverage, y=y_pool, type='unknown', \ metric=disagree_strat, seed=seed, Debug=Debug) elif selection_strategy == 'cover_then_uncertaintyPM': doc_pick_model = CoverThenUncertainty(feature_expert, pooling_model, \ num_samples, percentage=coverage, y=y_pool, type='unknown', \ seed=seed, Debug=Debug) elif selection_strategy == 'cover_then_uncertaintyRM': doc_pick_model = CoverThenUncertainty(feature_expert, reasoning_model, \ num_samples, percentage=coverage, y=y_pool, type='unknown', \ seed=seed, Debug=Debug) elif selection_strategy == 'cover_then_featureCertainty': doc_pick_model = CoverThenFeatureCertainty(feature_expert, feature_model, \ num_samples, percentage=coverage, y=y_pool, type='unknown', \ seed=seed, Debug=Debug) elif selection_strategy == "optaucP": doc_pick_model = OptimizeAUC(X_test, y_test, feature_expert, \ optimize="P", seed=seed, Debug=Debug) elif selection_strategy == "optaucI": doc_pick_model = OptimizeAUC(X_test, y_test, feature_expert, \ optimize="I", seed=seed, Debug=Debug) elif selection_strategy == "optaucF": doc_pick_model = OptimizeAUC(X_test, y_test, feature_expert, \ optimize="F", seed=seed, Debug=Debug) elif selection_strategy == "optaucR": doc_pick_model = OptimizeAUC(X_test, y_test, feature_expert, \ optimize="R", seed=seed, Debug=Debug) elif selection_strategy == 'reasoning_then_featureCertainty': doc_pick_model = ReasoningThenFeatureCertainty(feature_expert, instance_model, \ feature_model, switch=switch, reasoning_strategy=reasoning_strategy, y=y_pool, type='unknown', \ seed=seed, Debug=Debug) elif selection_strategy == "unc_insuff_R": doc_pick_model = UNCForInsufficientReason(reasoning_model) elif selection_strategy == "unc_no_conflict_R": doc_pick_model = UNCWithNoConflict(reasoning_model) elif selection_strategy == "unc_prefer_no_conflict_R": doc_pick_model = UNCPreferNoConflict(reasoning_model) elif selection_strategy == "unc_prefer_conflict_R": doc_pick_model = UNCPreferConflict(reasoning_model) elif selection_strategy == "unc_three_types_R": doc_pick_model = UNCThreeTypes(reasoning_model) else: raise ValueError('Selection strategy: \'%s\' invalid!' % selection_strategy) bootstrap_size = len(training_set) training_set_empty = (bootstrap_size == 0) if not training_set_empty: X_train = X_pool[training_set] y_train = y_pool[training_set] # Train all three models using the training set data instance_model.fit(X_train, y_train) # train instance_model for doc in training_set: #feature = feature_expert.most_informative_feature(X_pool[doc], y_pool[doc]) feature = feature_expert.any_informative_feature( X_pool[doc], y_pool[doc]) if feature: feature_model.fit( feature, y_pool[doc]) # train feature_model one by one discovered_features.add(feature) if y_pool[doc] == 0: discovered_class0_features.add(feature) else: discovered_class1_features.add(feature) # Reasoning model reasoning_model.partial_fit( X_pool[doc], y_pool[doc], feature, rmw_n, rmw_a) # train feature_model one by one # docs covered if feature: f_covered_docs = X_pool_csc[:, feature].indices covered_docs.update(f_covered_docs) # number of times a feat is chosen as a reason if feature: num_a_feat_chosen[feature] += 1 if selection_strategy == 'covering' or selection_strategy == 'covering_fewest': doc_pick_model.update(X_pool, feature, doc) elif selection_strategy == 'cheating': doc_pick_model.update(X_pool, feature, y_pool[doc]) elif selection_strategy.startswith( 'cover_then') and doc_pick_model.phase == 'covering': doc_pick_model.covering.update(X_pool, feature, doc) pooling_model.fit(instance_model, feature_model, weights=[0.5, 0.5]) # train pooling_model (accu, auc) = evaluate_model(instance_model, X_test, y_test) instance_model_scores['auc'].append(auc) instance_model_scores['accu'].append(accu) (accu, auc) = evaluate_model(feature_model, X_test, y_test) feature_model_scores['auc'].append(auc) feature_model_scores['accu'].append(accu) (accu, auc) = evaluate_model(pooling_model, X_test, y_test) pooling_model_scores['auc'].append(auc) pooling_model_scores['accu'].append(accu) (accu, auc) = evaluate_model(reasoning_model, X_test, y_test) reasoning_model_scores['auc'].append(auc) reasoning_model_scores['accu'].append(accu) # discovered feature counts if isinstance(feature_model, FeatureMNBUniform): discovered_feature_counts['class0'].append( len(feature_model.class0_features)) discovered_feature_counts['class1'].append( len(feature_model.class1_features)) elif isinstance(feature_model, FeatureMNBWeighted): nz = np.sum(feature_model.feature_count_ > 0, axis=1) discovered_feature_counts['class0'].append(nz[0]) discovered_feature_counts['class1'].append(nz[1]) num_docs_covered.append(len(covered_docs)) else: if selection_strategy.startswith( 'uncertainty') or selection_strategy == 'disagreement': raise ValueError('\'%s\' requires bootstrapping!' % selection_strategy) for i in range(budget): train_set_size = len(training_set) # Choose a document based on the strategy chosen if selection_strategy.startswith('cover_then'): doc_id = doc_pick_model.choice(X_pool, i + 1, pool_set) elif selection_strategy.startswith('optauc'): doc_id = doc_pick_model.choice(X_pool, y_pool, pool_set, training_set, feature_model, reasoning_model, rmw_n, rmw_a) elif selection_strategy == 'reasoning_then_featureCertainty': doc_id = doc_pick_model.choice(X_pool, i + 1, pool_set, train_set_size) elif selection_strategy == "unc_insuff_R": doc_id = doc_pick_model.choice(X_pool, pool_set, discovered_features, max_num_feats=1) elif selection_strategy == "unc_no_conflict_R": doc_id = doc_pick_model.choice(X_pool, pool_set, discovered_class0_features, discovered_class1_features) elif selection_strategy == "unc_prefer_no_conflict_R": doc_id = doc_pick_model.choice(X_pool, pool_set, discovered_class0_features, discovered_class1_features, top_k=10) elif selection_strategy == "unc_prefer_conflict_R": doc_id = doc_pick_model.choice(X_pool, pool_set, discovered_class0_features, discovered_class1_features, top_k=10) elif selection_strategy == "unc_three_types_R": doc_id = doc_pick_model.choice(X_pool, pool_set, discovered_class0_features, discovered_class1_features, top_k=10) else: doc_id = doc_pick_model.choice(X_pool, pool_set) if doc_id == None: break # Remove the chosen document from pool and add it to the training set pool_set.remove(doc_id) training_set.append(doc_id) if i == 0 and training_set_empty: X_train = X_pool[doc_id] y_train = np.array([y_pool[doc_id]]) else: X_train = sp.vstack((X_train, X_pool[doc_id])) y_train = np.hstack((y_train, np.array([y_pool[doc_id]]))) # Ask the expert for instance label (returns the true label from the dataset) label = y_pool[doc_id] # Ask the expert for most informative feature given the label #feature = feature_expert.most_informative_feature(X_pool[doc_id], label) feature = feature_expert.any_informative_feature( X_pool[doc_id], y_pool[doc_id]) # Update the instance model instance_model.fit(X_train, y_train) # Update the feature model if feature: feature_model.fit(feature, label) discovered_features.add(feature) if y_pool[doc_id] == 0: discovered_class0_features.add(feature) else: discovered_class1_features.add(feature) reasoning_model.partial_fit(X_pool[doc_id], y_pool[doc_id], feature, rmw_n, rmw_a) # train feature_model one by one # docs covered if feature: f_covered_docs = X_pool_csc[:, feature].indices covered_docs.update(f_covered_docs) # number of times a feat is chosen as a reason if feature: num_a_feat_chosen[feature] += 1 # Update the pooling model pooling_model.fit(instance_model, feature_model, weights=[0.5, 0.5]) # print 'docs = %d, feature = %s' % (doc_id, str(feature)) if selection_strategy == 'covering' or selection_strategy == 'covering_fewest': doc_pick_model.update(X_pool, feature, doc_id) elif selection_strategy == 'cheating': doc_pick_model.update(X_pool, feature, label) elif selection_strategy.startswith( 'cover_then') and doc_pick_model.phase == 'covering': doc_pick_model.covering.update(X_pool, feature, doc_id) # print 'covering_fewest features: %d, feature model features: %d' % (len(doc_pick_model.annotated_features), len(feature_model.class0_features + feature_model.class1_features)) # Evaluate performance based on Instance Model (accu, auc) = evaluate_model(instance_model, X_test, y_test) instance_model_scores['auc'].append(auc) instance_model_scores['accu'].append(accu) # Evaluate performance on Feature Model (accu, auc) = evaluate_model(feature_model, X_test, y_test) feature_model_scores['auc'].append(auc) feature_model_scores['accu'].append(accu) # Evaluate performance on Pooled Model (accu, auc) = evaluate_model(pooling_model, X_test, y_test) pooling_model_scores['auc'].append(auc) pooling_model_scores['accu'].append(accu) # Evaluate performance of the Reasoning Model (accu, auc) = evaluate_model(reasoning_model, X_test, y_test) reasoning_model_scores['auc'].append(auc) reasoning_model_scores['accu'].append(accu) # discovered feature counts if isinstance(feature_model, FeatureMNBUniform): discovered_feature_counts['class0'].append( len(feature_model.class0_features)) discovered_feature_counts['class1'].append( len(feature_model.class1_features)) elif isinstance(feature_model, FeatureMNBWeighted): nz = np.sum(feature_model.feature_count_ > 0, axis=1) discovered_feature_counts['class0'].append(nz[0]) discovered_feature_counts['class1'].append(nz[1]) # docs covered num_docs_covered.append(len(covered_docs)) if selection_strategy.startswith('cover_then'): transition = doc_pick_model.transition else: transition = None # compute the # of training samples for plot if training_set_empty: num_training_samples = np.arange(len( instance_model_scores['accu'])) + 1 else: num_training_samples = np.arange(len( instance_model_scores['accu'])) + bootstrap_size print 'Active Learning took %2.2fs' % (time() - start) return (num_training_samples, instance_model_scores, feature_model_scores, pooling_model_scores, reasoning_model_scores, discovered_feature_counts, num_docs_covered, transition, num_a_feat_chosen)
def learn(model_type, X_pool, y_pool, X_test, y_test, training_set, pool_set, feature_expert, \ selection_strategy, budget, step_size, topk, w_o, w_r, seed=0, alpha=1, smoothing=0, poolingMNBWeights=[0.5, 0.5], poolingFM_r=100.0, lr_C=1, svm_C=1, \ zaidan_C=1, zaidan_Ccontrast=1, zaidan_nu=1, cvTrain=False, Debug=False): start = time() print '-' * 50 print 'Starting Active Learning...' _, num_feat = X_pool.shape model_scores = { 'auc': [], 'accu': [], 'wr': [], 'wo': [], 'alpha': [], 'svm_C': [], 'zaidan_C': [], 'zaidan_Ccontrast': [], 'zaidan_nu': [], 'FMrvalue': [], 'IMweight': [], 'FMweight': [] } rationales = set() rationales_c0 = set() rationales_c1 = set() number_of_docs = 0 feature_expert.rg.seed(seed) num_training_samples = [] all_features = [] # keep all the training data instance ids in docs list docs = training_set X_train = None y_train = [] sample_weight = [] for doc_id in docs: #feature = feature_expert.most_informative_feature(X_pool[doc_id], y_pool[doc_id]) feature = feature_expert.any_informative_feature( X_pool[doc_id], y_pool[doc_id]) number_of_docs = number_of_docs + 1 # append feature to all_features, even if it is None all_features.append(feature) if feature is not None: rationales.add(feature) if y_pool[doc_id] == 0: rationales_c0.add(feature) else: rationales_c1.add(feature) if cvTrain: # get optimal parameters depending on the model_type if model_type == 'mnb_LwoR': w_r, w_o = optimalMNBLwoRParameters(X_pool[training_set], y_pool[training_set], all_features) feature_counter = 0 for doc_id in docs: x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: x[0, f] = w_r * x[0, f] else: x[0, f] = w_o * x[0, f] feature_counter = feature_counter + 1 if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) elif model_type == 'mnb': w_r, w_o = optimalMNBParameters(X_pool[training_set], y_pool[training_set], all_features) feature_counter = 0 for doc_id in docs: x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: x[0, f] = w_r * x[0, f] else: x[0, f] = w_o * x[0, f] feature_counter = feature_counter + 1 if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) elif model_type == 'svm_linear': w_r, w_o, C = optimalSVMParameters(X_pool[training_set], y_pool[training_set], all_features, seed) feature_counter = 0 for doc_id in docs: x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: x[0, f] = w_r * x[0, f] else: x[0, f] = w_o * x[0, f] feature_counter = feature_counter + 1 if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) elif model_type == 'svm_linear_LwoR': w_r, w_o, C = optimalSVMLwoRParameters(X_pool[training_set], y_pool[training_set], all_features, seed) feature_counter = 0 for doc_id in docs: x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: x[0, f] = w_r * x[0, f] else: x[0, f] = w_o * x[0, f] feature_counter = feature_counter + 1 if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) if model_type == 'poolingMNB': classpriors = np.zeros(2) classpriors[1] = (np.sum(y_pool[docs]) * 1.) / (len(docs) * 1.) classpriors[0] = 1. - classpriors[1] alpha, poolingFM_r, poolingMNBWeights = optimalPoolingMNBParameters( X_pool[training_set], y_pool[training_set], all_features, smoothing, num_feat) feature_model = FeatureMNBUniform(rationales_c0, rationales_c1, num_feat, smoothing, classpriors, poolingFM_r) feature_counter = 0 for doc_id in docs: if all_features[feature_counter]: # updates feature model with features one at a time feature_model.fit(all_features[feature_counter], y_pool[doc_id]) feature_counter = feature_counter + 1 x = sp.csr_matrix(X_pool[doc_id], dtype=float) if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) if model_type == 'Zaidan': zaidan_C, zaidan_Ccontrast, zaidan_nu = optimalZaidanParameters( X_pool[training_set], y_pool[training_set], all_features, seed) feature_counter = 0 for doc_id in docs: x = sp.csr_matrix(X_pool[doc_id], dtype=np.float64) if all_features[feature_counter] is not None: x_pseudo = (X_pool[doc_id]).todense() # create pseudoinstances based on rationales provided; one pseudoinstance is created for each rationale. x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: test = x[0, f] x_pseudo[0, f] = x[0, f] / zaidan_nu else: x_pseudo[0, f] = 0.0 x_pseudo = sp.csr_matrix(x_pseudo, dtype=np.float64) if not y_train: X_train = x if all_features[feature_counter] is not None: X_train = sp.vstack((X_train, x_pseudo)) else: X_train = sp.vstack((X_train, x)) if all_features[feature_counter] is not None: X_train = sp.vstack((X_train, x_pseudo)) y_train.append(y_pool[doc_id]) if all_features[feature_counter] is not None: # append y label again for the pseudoinstance created y_train.append(y_pool[doc_id]) sample_weight.append(zaidan_C) if all_features[feature_counter] is not None: # append instance weight=zaidan_Ccontrast for the pseudoinstance created sample_weight.append(zaidan_Ccontrast) feature_counter = feature_counter + 1 # Train the model if model_type == 'lrl2': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l2', random_state=random_state) elif model_type == 'lrl1': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l1', random_state=random_state) elif model_type == 'mnb': model = MultinomialNB(alpha=alpha) elif model_type == 'mnb_LwoR': model = MultinomialNB(alpha=alpha) elif model_type == 'svm_linear_LwoR': random_state = np.random.RandomState(seed=seed) model = LinearSVC(C=C, random_state=random_state) elif model_type == 'svm_linear': random_state = np.random.RandomState(seed=seed) model = LinearSVC(C=C, random_state=random_state) elif model_type == 'poolingMNB': instance_model = MultinomialNB(alpha=alpha) model = PoolingMNB() elif model_type == 'Zaidan': random_state = np.random.RandomState(seed=seed) model = svm.SVC(kernel='linear', C=1.0, random_state=random_state) if model_type == 'poolingMNB': instance_model.fit(X_train, y_train) model.fit(instance_model, feature_model, weights=poolingMNBWeights) # train pooling_model elif model_type == 'Zaidan': model.fit(X_train, np.array(y_train), sample_weight=sample_weight) else: model.fit(X_train, np.array(y_train)) (accu, auc) = evaluate_model(model, X_test, y_test) model_scores['auc'].append(auc) model_scores['accu'].append(accu) if model_type == 'poolingMNB': model_scores['alpha'].append(alpha) model_scores['FMrvalue'].append(poolingFM_r) model_scores['IMweight'].append(poolingMNBWeights[0]) model_scores['FMweight'].append(poolingMNBWeights[1]) else: model_scores['FMrvalue'].append(0.0) model_scores['IMweight'].append(0.0) model_scores['FMweight'].append(0.0) if model_type == 'Zaidan': model_scores['zaidan_C'].append(zaidan_C) model_scores['zaidan_Ccontrast'].append(zaidan_Ccontrast) model_scores['zaidan_nu'].append(zaidan_nu) else: model_scores['zaidan_C'].append(0.0) model_scores['zaidan_Ccontrast'].append(0.0) model_scores['zaidan_nu'].append(0.0) if model_type == 'mnb' or model_type == 'mnb_LwoR': model_scores['alpha'].append(alpha) else: model_scores['alpha'].append(0.0) if model_type == 'svm_linear' or model_type == 'svm_linear_LwoR': model_scores['svm_C'].append(C) else: model_scores['svm_C'].append(0.0) if model_type == 'svm_linear' or model_type == 'svm_linear_LwoR' or model_type == 'mnb' or model_type == 'mnb_LwoR': model_scores['wr'].append(w_r) model_scores['wo'].append(w_o) else: model_scores['wr'].append(0.0) model_scores['wo'].append(0.0) num_training_samples.append(number_of_docs) feature_expert.rg.seed(seed) if selection_strategy == 'random': doc_pick_model = RandomStrategy(seed) elif selection_strategy == 'unc': doc_pick_model = UNCSampling() elif selection_strategy == "pnc": doc_pick_model = UNCPreferNoConflict() elif selection_strategy == "pnr": doc_pick_model = UNCPreferNoRationale() elif selection_strategy == "pr": doc_pick_model = UNCPreferRationale() elif selection_strategy == "pc": doc_pick_model = UNCPreferConflict() elif selection_strategy == "tt": doc_pick_model = UNCThreeTypes() elif selection_strategy == "pipe": doc_pick_model = Pipe([UNCSampling(), UNCPreferConflict()], [10, 30]) else: raise ValueError('Selection strategy: \'%s\' invalid!' % selection_strategy) k = step_size #while X_train.shape[0] < budget: while number_of_docs < budget: # Choose a document based on the strategy chosen if selection_strategy == "pnc": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "pnr": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "pr": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "pc": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "tt": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "pipe": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) else: doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k) if doc_ids is None or len(doc_ids) == 0: break for doc_id in doc_ids: #feature = feature_expert.most_informative_feature(X_pool[doc_id], y_pool[doc_id]) feature = feature_expert.any_informative_feature( X_pool[doc_id], y_pool[doc_id]) all_features.append(feature) number_of_docs = number_of_docs + 1 if feature is not None: rationales.add(feature) if y_pool[doc_id] == 0: rationales_c0.add(feature) else: rationales_c1.add(feature) # Remove the chosen document from pool and add it to the training set pool_set.remove(doc_id) training_set.append(long(doc_id)) if cvTrain: # get optimal parameters depending on the model_type X_train = None y_train = [] sample_weight = [] if model_type == 'mnb_LwoR': if np.mod(number_of_docs, 20) == 10: w_r, w_o = optimalMNBLwoRParameters( X_pool[training_set], y_pool[training_set], all_features) feature_counter = 0 for doc_id in training_set: x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: x[0, f] = w_r * x[0, f] else: x[0, f] = w_o * x[0, f] feature_counter = feature_counter + 1 if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) elif model_type == 'mnb': if np.mod(number_of_docs, 20) == 10: w_r, w_o = optimalMNBParameters(X_pool[training_set], y_pool[training_set], all_features) feature_counter = 0 for doc_id in training_set: x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: x[0, f] = w_r * x[0, f] else: x[0, f] = w_o * x[0, f] feature_counter = feature_counter + 1 if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) elif model_type == 'svm_linear_LwoR': if np.mod(number_of_docs, 20) == 10: w_r, w_o, C = optimalSVMLwoRParameters( X_pool[training_set], y_pool[training_set], all_features, seed) feature_counter = 0 for doc_id in training_set: x = sp.csr_matrix(X_pool[doc_id], dtype=np.float64) x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: x[0, f] = w_r * x[0, f] else: x[0, f] = w_o * x[0, f] feature_counter = feature_counter + 1 if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) elif model_type == 'svm_linear': if np.mod(number_of_docs, 20) == 10: w_r, w_o, C = optimalSVMParameters(X_pool[training_set], y_pool[training_set], all_features, seed) feature_counter = 0 for doc_id in training_set: x = sp.csr_matrix(X_pool[doc_id], dtype=np.float64) x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: x[0, f] = w_r * x[0, f] else: x[0, f] = w_o * x[0, f] feature_counter = feature_counter + 1 if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) if model_type == 'poolingMNB': classpriors = np.zeros(2) classpriors[1] = (np.sum(y_pool[docs]) * 1.) / (len(docs) * 1.) classpriors[0] = 1. - classpriors[1] if np.mod(number_of_docs, 20) == 10: alpha, poolingFM_r, poolingMNBWeights = optimalPoolingMNBParameters( X_pool[training_set], y_pool[training_set], all_features, smoothing, num_feat) feature_model = FeatureMNBUniform(rationales_c0, rationales_c1, num_feat, smoothing, classpriors, poolingFM_r) feature_counter = 0 for doc_id in training_set: if all_features[feature_counter]: # updates feature model with features one at a time feature_model.fit(all_features[feature_counter], y_pool[doc_id]) feature_counter = feature_counter + 1 x = sp.csr_matrix(X_pool[doc_id], dtype=float) if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) if model_type == 'Zaidan': if np.mod(number_of_docs, 20) == 10: zaidan_C, zaidan_Ccontrast, zaidan_nu = optimalZaidanParameters( X_pool[training_set], y_pool[training_set], all_features, seed) feature_counter = 0 for doc_id in training_set: x = sp.csr_matrix(X_pool[doc_id], dtype=np.float64) if all_features[feature_counter] is not None: x_pseudo = (X_pool[doc_id]).todense() # create pseudoinstances based on rationales provided; one pseudoinstance is created for each rationale. x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: test = x[0, f] x_pseudo[0, f] = x[0, f] / zaidan_nu else: x_pseudo[0, f] = 0.0 x_pseudo = sp.csr_matrix(x_pseudo, dtype=np.float64) if not y_train: X_train = x if all_features[feature_counter] is not None: X_train = sp.vstack((X_train, x_pseudo)) else: X_train = sp.vstack((X_train, x)) if all_features[feature_counter] is not None: X_train = sp.vstack((X_train, x_pseudo)) y_train.append(y_pool[doc_id]) if all_features[feature_counter] is not None: # append y label again for the pseudoinstance created y_train.append(y_pool[doc_id]) sample_weight.append(zaidan_C) if all_features[feature_counter] is not None: # append instance weight=zaidan_Ccontrast for the pseudoinstance created sample_weight.append(zaidan_Ccontrast) feature_counter = feature_counter + 1 # Train the model if model_type == 'lrl2': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l2', random_state=random_state) elif model_type == 'lrl1': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l1', random_state=random_state) elif model_type == 'mnb': model = MultinomialNB(alpha=alpha) elif model_type == 'mnb_LwoR': model = MultinomialNB(alpha=alpha) elif model_type == 'svm_linear_LwoR': random_state = np.random.RandomState(seed=seed) model = LinearSVC(C=C, random_state=random_state) elif model_type == 'svm_linear': random_state = np.random.RandomState(seed=seed) model = LinearSVC(C=C, random_state=random_state) elif model_type == 'poolingMNB': instance_model = MultinomialNB(alpha=alpha) model = PoolingMNB() elif model_type == 'Zaidan': random_state = np.random.RandomState(seed=seed) model = svm.SVC(kernel='linear', C=svm_C, random_state=random_state) if model_type == 'poolingMNB': instance_model.fit(X_train, y_train) model.fit(instance_model, feature_model, weights=poolingMNBWeights) # train pooling_model elif model_type == 'Zaidan': model.fit(X_train, np.array(y_train), sample_weight=sample_weight) else: model.fit(X_train, np.array(y_train)) (accu, auc) = evaluate_model(model, X_test, y_test) model_scores['auc'].append(auc) model_scores['accu'].append(accu) if model_type == 'poolingMNB': model_scores['alpha'].append(alpha) model_scores['FMrvalue'].append(poolingFM_r) model_scores['IMweight'].append(poolingMNBWeights[0]) model_scores['FMweight'].append(poolingMNBWeights[1]) else: model_scores['FMrvalue'].append(0.0) model_scores['IMweight'].append(0.0) model_scores['FMweight'].append(0.0) if model_type == 'Zaidan': model_scores['zaidan_C'].append(zaidan_C) model_scores['zaidan_Ccontrast'].append(zaidan_Ccontrast) model_scores['zaidan_nu'].append(zaidan_nu) else: model_scores['zaidan_C'].append(0.0) model_scores['zaidan_Ccontrast'].append(0.0) model_scores['zaidan_nu'].append(0.0) if model_type == 'mnb' or model_type == 'mnb_LwoR': model_scores['alpha'].append(alpha) else: model_scores['alpha'].append(0.0) if model_type == 'svm_linear' or model_type == 'svm_linear_LwoR': model_scores['svm_C'].append(C) else: model_scores['svm_C'].append(0.0) if model_type == 'svm_linear' or model_type == 'svm_linear_LwoR' or model_type == 'mnb' or model_type == 'mnb_LwoR': model_scores['wr'].append(w_r) model_scores['wo'].append(w_o) else: model_scores['wr'].append(0.0) model_scores['wo'].append(0.0) num_training_samples.append(number_of_docs) print 'Active Learning took %2.2fs' % (time() - start) return (np.array(num_training_samples), model_scores)
def learn(model_type, X_pool, y_pool, X_test, y_test, training_set, pool_set, feature_expert, \ selection_strategy, budget, step_size, topk, w_o, w_r, seed=0, alpha=1, poolingMNBWeights=[0.5, 0.5], Meville_etal_r=100.0, lr_C=1, svm_C=1, \ Zaidan_etal_C=1, Zaidan_etal_Ccontrast=1, Zaidan_etal_mu=1, Debug=False): start = time() print('-' * 50) print('Starting Active Learning...') _, num_feat = X_pool.shape model_scores = {'auc': [], 'accu': []} rationales = set() rationales_c0 = set() rationales_c1 = set() feature_expert.rg.seed(seed) num_training_samples = [] number_of_docs = 0 docs = training_set X_train = None y_train = [] sample_weight = list() if model_type == 'Melville_etal': # create feature model classpriors = np.zeros(2) classpriors[1] = (np.sum(y_pool[docs]) * 1.) / (len(docs) * 1.) classpriors[0] = 1. - classpriors[1] feature_model = FeatureMNBUniform(rationales_c0, rationales_c1, num_feat, classpriors, Meville_etal_r) for doc_id in docs: number_of_docs = number_of_docs + 1 feature = feature_expert.any_informative_feature( X_pool[doc_id], y_pool[doc_id]) if model_type == 'Melville_etal': if feature: feature_model.fit(feature, y_pool[doc_id]) rationales.add(feature) if y_pool[doc_id] == 0: rationales_c0.add(feature) else: rationales_c1.add(feature) if model_type == 'Zaidan_etal': x = sp.csr_matrix(X_pool[doc_id], dtype=np.float64) if feature is not None: x_pseudo = (X_pool[doc_id]).todense() # create pseudoinstances based on rationales provided; one pseudoinstance is created per rationale. x_feats = x[0].indices for f in x_feats: if f == feature: test = x[0, f] x_pseudo[0, f] = x[0, f] / Zaidan_etal_mu else: x_pseudo[0, f] = 0.0 x_pseudo = sp.csr_matrix(x_pseudo, dtype=np.float64) else: x = sp.csr_matrix(X_pool[doc_id], dtype=float) if "Melville_etal" not in model_type: x_feats = x[0].indices for f in x_feats: if f == feature: x[0, f] = w_r * x[0, f] else: x[0, f] = w_o * x[0, f] if model_type == 'Zaidan_etal': if not y_train: X_train = x if feature is not None: X_train = sp.vstack((X_train, x_pseudo)) else: X_train = sp.vstack((X_train, x)) if feature is not None: X_train = sp.vstack((X_train, x_pseudo)) y_train.append(y_pool[doc_id]) if feature is not None: # append y label again for the pseudoinstance created y_train.append(y_pool[doc_id]) sample_weight.append(Zaidan_etal_C) if feature is not None: # append instance weight=Zaidan_etal_Ccontrast for the pseudoinstance created sample_weight.append(Zaidan_etal_Ccontrast) else: if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) # Train the model if model_type == 'lrl2': random_state = RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l2', random_state=random_state) elif model_type == 'lrl1': random_state = RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l1', random_state=random_state) elif model_type == 'mnb': model = MultinomialNB(alpha=alpha) elif model_type == 'svm_linear': random_state = RandomState(seed=seed) model = LinearSVC(C=svm_C, random_state=random_state) elif model_type == 'Melville_etal': instance_model = MultinomialNB(alpha=alpha) model = PoolingMNB() elif model_type == 'Zaidan_etal': random_state = RandomState(seed=seed) model = svm.SVC(kernel='linear', C=svm_C, random_state=random_state) if model_type == 'Melville_etal': #feature_model.fit(feature, y_pool[doc_id]) instance_model.fit(X_train, y_train) model.fit(instance_model, feature_model, weights=poolingMNBWeights) # train pooling_model elif model_type == 'Zaidan_etal': model.fit(X_train, np.array(y_train), sample_weight=sample_weight) else: model.fit(X_train, np.array(y_train)) (accu, auc) = evaluate_model(model, X_test, y_test) model_scores['auc'].append(auc) model_scores['accu'].append(accu) num_training_samples.append(number_of_docs) feature_expert.rg.seed(seed) if selection_strategy == 'RND': doc_pick_model = RandomStrategy(seed) elif selection_strategy == 'UNC': doc_pick_model = UNCSampling() elif selection_strategy == 'UNC_PNC': doc_pick_model = UNCPreferNoConflict() elif selection_strategy == 'UNC_PC': doc_pick_model = UNCPreferConflict() else: raise ValueError('Selection strategy: \'%s\' invalid!' % selection_strategy) k = step_size while X_train.shape[0] < budget: # Choose a document based on the strategy chosen if selection_strategy == 'UNC_PNC': doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == 'UNC_PC': doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) else: doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k) if doc_ids is None or len(doc_ids) == 0: break for doc_id in doc_ids: # Remove the chosen document from pool and add it to the training set pool_set.remove(doc_id) training_set.append(doc_id) #feature = feature_expert.most_informative_feature(X_pool[doc_id], y_pool[doc_id]) feature = feature_expert.any_informative_feature( X_pool[doc_id], y_pool[doc_id]) if model_type == 'Melville_etal': if feature: feature_model.fit(feature, y_pool[doc_id]) number_of_docs = number_of_docs + 1 rationales.add(feature) if y_pool[doc_id] == 0: rationales_c0.add(feature) else: rationales_c1.add(feature) if model_type == 'Zaidan_etal': x = sp.csr_matrix(X_pool[doc_id], dtype=np.float64) if feature is not None: x_pseudo = (X_pool[doc_id]).todense() # create pseudoinstances based on rationales provided; one pseudoinstance is created for each rationale. x_feats = x[0].indices for f in x_feats: if f == feature: test = x[0, f] x_pseudo[0, f] = x[0, f] / Zaidan_etal_mu else: x_pseudo[0, f] = 0.0 x_pseudo = sp.csr_matrix(x_pseudo, dtype=np.float64) else: x = sp.csr_matrix(X_pool[doc_id], dtype=float) if "Melville_etal" not in model_type: x_feats = x[0].indices for f in x_feats: if f == feature: x[0, f] = w_r * x[0, f] else: x[0, f] = w_o * x[0, f] if model_type == 'Zaidan_etal': X_train = sp.vstack((X_train, x)) if feature is not None: X_train = sp.vstack((X_train, x_pseudo)) y_train.append(y_pool[doc_id]) if feature is not None: # append y label again for the pseudoinstance created y_train.append(y_pool[doc_id]) sample_weight.append(Zaidan_etal_C) if feature is not None: # append instance weight=Zaidan_etal_Ccontrast for the pseudoinstance created sample_weight.append(Zaidan_etal_Ccontrast) else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) # Train the model if model_type == 'lrl2': random_state = RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l2', random_state=random_state) elif model_type == 'lrl1': random_state = RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l1', random_state=random_state) elif model_type == 'mnb': model = MultinomialNB(alpha=alpha) elif model_type == 'svm_linear': random_state = RandomState(seed=seed) model = LinearSVC(C=svm_C, random_state=random_state) elif model_type == 'Melville_etal': instance_model = MultinomialNB(alpha=alpha) model = PoolingMNB() elif model_type == 'Zaidan_etal': random_state = RandomState(seed=seed) model = svm.SVC(kernel='linear', C=svm_C, random_state=random_state) if model_type == 'Melville_etal': instance_model.fit(X_train, y_train) model.fit(instance_model, feature_model, weights=poolingMNBWeights) # train pooling_model elif model_type == 'Zaidan_etal': model.fit(X_train, np.array(y_train), sample_weight=sample_weight) else: model.fit(X_train, np.array(y_train)) (accu, auc) = evaluate_model(model, X_test, y_test) model_scores['auc'].append(auc) model_scores['accu'].append(accu) num_training_samples.append(number_of_docs) print('Active Learning took %2.2fs' % (time() - start)) return (np.array(num_training_samples), model_scores)
def learn(model_type, X_pool, y_pool, X_test, y_test, training_set, pool_set, feature_expert, \ selection_strategy, budget, step_size, topk, w_o, w_r, seed=0, lr_C=1, svm_C=1, svm_gamma=0, Debug=False): start = time() print '-' * 50 print 'Starting Active Learning...' model_scores = {'auc':[], 'accu':[]} rationales = set() rationales_c0 = set() rationales_c1 = set() feature_expert.rg.seed(seed) num_training_samples = [] docs = training_set X_train = None y_train = [] for doc_id in docs: #feature = feature_expert.most_informative_feature(X_pool[doc_id], y_pool[doc_id]) feature = feature_expert.any_informative_feature(X_pool[doc_id], y_pool[doc_id]) rationales.add(feature) if y_pool[doc_id] == 0: rationales_c0.add(feature) else: rationales_c1.add(feature) x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == feature: x[0,f] = w_r*x[0,f] else: x[0,f] = w_o*x[0,f] if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) # Train the model if model_type=='lrl2': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l2', random_state=random_state) elif model_type=='mnb': model = MultinomialNB(alpha=1) elif model_type=='svm_linear': model = svm.SVC(kernel='linear', C=svm_C, probability=True) #model = svm.SVC(kernel='linear', probability=True) elif model_type=='svm_rbf': model = svm.SVC(kernel='rbf', gamma=svm_gamma, C=svm_C, probability=True) #model = svm.SVC(kernel='rbf', probability=True) elif model_type=='svm_poly': model = svm.SVC(kernel='rbf', gamma=svm_gamma, C=svm_C, probability=True) #model = svm.SVC(kernel='poly', probability=True) elif model_type=='adaptive_lr': random_state = np.random.RandomState(seed=seed) #model = LogisticRegression(C=C, penalty='l2', random_state=random_state) #model.fit(X_train, np.array(y_train)) model = LogisticRegressionAdaptive() elif model_type=='adaptive_svm': random_state = np.random.RandomState(seed=seed) model = AdaptiveSVM() elif model_type=='SGD': model = SGDClassifier(loss="log",penalty='l2', n_iter=100) model.fit(X_train, np.array(y_train)) (accu, auc) = evaluate_model(model, X_test, y_test) model_scores['auc'].append(auc) model_scores['accu'].append(accu) num_training_samples.append(X_train.shape[0]) feature_expert.rg.seed(seed) if selection_strategy == 'random': doc_pick_model = RandomStrategy(seed) elif selection_strategy == 'uncertainty': doc_pick_model = UNCSampling(model, feature_expert, y_pool, Debug) elif selection_strategy == "optauc": doc_pick_model = OptimizeAUC(X_test, y_test, feature_expert, \ optimize="R", seed=seed, Debug=Debug) elif selection_strategy == "unc_prefer_no_conflict": doc_pick_model = UNCPreferNoConflict(model) elif selection_strategy == "unc_prefer_conflict": doc_pick_model = UNCPreferConflict(model) elif selection_strategy == "unc_three_types": doc_pick_model = UNCThreeTypes(model) else: raise ValueError('Selection strategy: \'%s\' invalid!' % selection_strategy) k = step_size while X_train.shape[0] < budget: # Choose a document based on the strategy chosen if selection_strategy == "unc_prefer_no_conflict": doc_ids = doc_pick_model.choices(X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "unc_prefer_conflict": doc_ids = doc_pick_model.choices(X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "unc_three_types": doc_ids = doc_pick_model.choices(X_pool, pool_set, k, rationales_c0, rationales_c1, topk) else: doc_ids = doc_pick_model.choices(X_pool, pool_set, k) if doc_ids is None or len(doc_ids) == 0: break for doc_id in doc_ids: # Remove the chosen document from pool and add it to the training set pool_set.remove(doc_id) training_set.append(doc_id) #feature = feature_expert.most_informative_feature(X_pool[doc_id], y_pool[doc_id]) feature = feature_expert.any_informative_feature(X_pool[doc_id], y_pool[doc_id]) rationales.add(feature) if y_pool[doc_id] == 0: rationales_c0.add(feature) else: rationales_c1.add(feature) x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == feature: x[0,f] = w_r*x[0,f] else: x[0,f] = w_o*x[0,f] X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) # Train the model if model_type=='lrl2': random_state2 = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l2', random_state=random_state) elif model_type=='mnb': model = MultinomialNB(alpha=1) elif model_type=='svm_linear': model = svm.SVC(kernel='linear', C=svm_C, probability=True) #model = svm.SVC(kernel='linear', probability=True) elif model_type=='svm_rbf': model = svm.SVC(kernel='rbf', gamma=svm_gamma, C=svm_C, probability=True) #model = svm.SVC(kernel='rbf', probability=True) elif model_type=='svm_poly': model = svm.SVC(kernel='rbf', gamma=svm_gamma, C=svm_C, probability=True) #model = svm.SVC(kernel='poly', probability=True) elif model_type=='adaptive_lr': #random_state = np.random.RandomState(seed=seed) model = LogisticRegressionAdaptive() elif model_type=='adaptive_svm': random_state = np.random.RandomState(seed=seed) model = AdaptiveSVM() elif model_type=='SGD': model = SGDClassifier(loss="log",penalty='l2', n_iter=100) # fit the model and evaluate model.fit(X_train, np.array(y_train)) (accu, auc) = evaluate_model(model, X_test, y_test) model_scores['auc'].append(auc) model_scores['accu'].append(accu) num_training_samples.append(X_train.shape[0]) print 'Active Learning took %2.2fs' % (time() - start) return (np.array(num_training_samples), model_scores)
def learn(model_type, X_pool, y_pool, X_test, y_test, training_set, pool_set, feature_expert, \ selection_strategy, budget, step_size, topk, w_o, w_r, seed=0, alpha=1, poolingMNBWeights=[0.5, 0.5], Meville_etal_r=100.0, lr_C=1, svm_C=1, \ Zaidan_etal_C=1, Zaidan_etal_Ccontrast=1, Zaidan_etal_mu=1, Debug=False): start = time() print '-' * 50 print 'Starting Active Learning...' _, num_feat = X_pool.shape model_scores = {'auc':[], 'accu':[]} rationales = set() rationales_c0 = set() rationales_c1 = set() feature_expert.rg.seed(seed) num_training_samples = [] number_of_docs = 0 docs = training_set X_train = None y_train = [] if model_type=='Melville_etal': # create feature model classpriors=np.zeros(2) classpriors[1] = (np.sum(y_pool[docs])*1.)/(len(docs)*1.) classpriors[0] = 1. - classpriors[1] feature_model = FeatureMNBUniform(rationales_c0, rationales_c1, num_feat, classpriors, Meville_etal_r) for doc_id in docs: number_of_docs=number_of_docs+1 feature = feature_expert.any_informative_feature(X_pool[doc_id], y_pool[doc_id]) if model_type == 'Melville_etal': if feature: feature_model.fit(feature, y_pool[doc_id]) rationales.add(feature) if y_pool[doc_id] == 0: rationales_c0.add(feature) else: rationales_c1.add(feature) if model_type == 'Zaidan_etal': x = sp.csr_matrix(X_pool[doc_id], dtype=np.float64) if feature is not None: x_pseudo = (X_pool[doc_id]).todense() # create pseudoinstances based on rationales provided; one pseudoinstance is created per rationale. x_feats = x[0].indices for f in x_feats: if f == feature: test= x[0,f] x_pseudo[0,f] = x[0,f]/Zaidan_etal_mu else: x_pseudo[0,f] = 0.0 x_pseudo=sp.csr_matrix(x_pseudo, dtype=np.float64) else: x = sp.csr_matrix(X_pool[doc_id], dtype=float) if "Melville_etal" not in model_type: x_feats = x[0].indices for f in x_feats: if f == feature: x[0,f] = w_r*x[0,f] else: x[0,f] = w_o*x[0,f] if model_type=='Zaidan_etal': if not y_train: X_train = x if feature is not None: X_train = sp.vstack((X_train, x_pseudo)) else: X_train = sp.vstack((X_train, x)) if feature is not None: X_train = sp.vstack((X_train, x_pseudo)) y_train.append(y_pool[doc_id]) if feature is not None: # append y label again for the pseudoinstance created y_train.append(y_pool[doc_id]) sample_weight.append(Zaidan_etal_C) if feature is not None: # append instance weight=Zaidan_etal_Ccontrast for the pseudoinstance created sample_weight.append(Zaidan_etal_Ccontrast) else: if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) # Train the model if model_type=='lrl2': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l2', random_state=random_state) elif model_type=='lrl1': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l1', random_state=random_state) elif model_type=='mnb': model = MultinomialNB(alpha=alpha) elif model_type=='svm_linear': random_state = np.random.RandomState(seed=seed) model = LinearSVC(C=svm_C, random_state=random_state) elif model_type=='Melville_etal': instance_model=MultinomialNB(alpha=alpha) model = PoolingMNB() elif model_type=='Zaidan_etal': random_state = np.random.RandomState(seed=seed) model = svm.SVC(kernel='linear', C=svm_C, random_state=random_state) if model_type=='Melville_etal': #feature_model.fit(feature, y_pool[doc_id]) instance_model.fit(X_train, y_train) model.fit(instance_model, feature_model, weights=poolingMNBWeights) # train pooling_model elif model_type=='Zaidan_etal': model.fit(X_train, np.array(y_train), sample_weight=sample_weight) else: model.fit(X_train, np.array(y_train)) (accu, auc) = evaluate_model(model, X_test, y_test) model_scores['auc'].append(auc) model_scores['accu'].append(accu) num_training_samples.append(number_of_docs) feature_expert.rg.seed(seed) if selection_strategy == 'RND': doc_pick_model = RandomStrategy(seed) elif selection_strategy == 'UNC': doc_pick_model = UNCSampling() elif selection_strategy == 'UNC_PNC': doc_pick_model = UNCPreferNoConflict() elif selection_strategy == 'UNC_PC': doc_pick_model = UNCPreferConflict() else: raise ValueError('Selection strategy: \'%s\' invalid!' % selection_strategy) k = step_size while X_train.shape[0] < budget: # Choose a document based on the strategy chosen if selection_strategy == 'UNC_PNC': doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == 'UNC_PC': doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) else: doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k) if doc_ids is None or len(doc_ids) == 0: break for doc_id in doc_ids: # Remove the chosen document from pool and add it to the training set pool_set.remove(doc_id) training_set.append(doc_id) #feature = feature_expert.most_informative_feature(X_pool[doc_id], y_pool[doc_id]) feature = feature_expert.any_informative_feature(X_pool[doc_id], y_pool[doc_id]) if model_type=='Melville_etal': if feature: feature_model.fit(feature, y_pool[doc_id]) number_of_docs=number_of_docs+1 rationales.add(feature) if y_pool[doc_id] == 0: rationales_c0.add(feature) else: rationales_c1.add(feature) if model_type=='Zaidan_etal': x = sp.csr_matrix(X_pool[doc_id], dtype=np.float64) if feature is not None: x_pseudo = (X_pool[doc_id]).todense() # create pseudoinstances based on rationales provided; one pseudoinstance is created for each rationale. x_feats = x[0].indices for f in x_feats: if f == feature: test= x[0,f] x_pseudo[0,f] = x[0,f]/Zaidan_etal_mu else: x_pseudo[0,f] = 0.0 x_pseudo=sp.csr_matrix(x_pseudo, dtype=np.float64) else: x = sp.csr_matrix(X_pool[doc_id], dtype=float) if "Melville_etal" not in model_type: x_feats = x[0].indices for f in x_feats: if f == feature: x[0,f] = w_r*x[0,f] else: x[0,f] = w_o*x[0,f] if model_type=='Zaidan_etal': X_train = sp.vstack((X_train, x)) if feature is not None: X_train = sp.vstack((X_train, x_pseudo)) y_train.append(y_pool[doc_id]) if feature is not None: # append y label again for the pseudoinstance created y_train.append(y_pool[doc_id]) sample_weight.append(Zaidan_etal_C) if feature is not None: # append instance weight=Zaidan_etal_Ccontrast for the pseudoinstance created sample_weight.append(Zaidan_etal_Ccontrast) else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) # Train the model if model_type=='lrl2': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l2', random_state=random_state) elif model_type=='lrl1': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l1', random_state=random_state) elif model_type=='mnb': model = MultinomialNB(alpha=alpha) elif model_type=='svm_linear': random_state = np.random.RandomState(seed=seed) model = LinearSVC(C=svm_C, random_state=random_state) elif model_type=='Melville_etal': instance_model=MultinomialNB(alpha=alpha) model = PoolingMNB() elif model_type=='Zaidan_etal': random_state = np.random.RandomState(seed=seed) model = svm.SVC(kernel='linear', C=svm_C, random_state=random_state) if model_type=='Melville_etal': instance_model.fit(X_train, y_train) model.fit(instance_model, feature_model, weights=poolingMNBWeights) # train pooling_model elif model_type=='Zaidan_etal': model.fit(X_train, np.array(y_train), sample_weight=sample_weight) else: model.fit(X_train, np.array(y_train)) (accu, auc) = evaluate_model(model, X_test, y_test) model_scores['auc'].append(auc) model_scores['accu'].append(accu) num_training_samples.append(number_of_docs) print 'Active Learning took %2.2fs' % (time() - start) return (np.array(num_training_samples), model_scores)
def learn(model_type, X_pool, y_pool, X_test, y_test, training_set, pool_set, feature_expert, \ selection_strategy, budget, step_size, topk, w_o, w_r, seed=0, alpha=1, smoothing=0, poolingMNBWeights=[0.5, 0.5], poolingFM_r=100.0, lr_C=1, svm_C=1, \ zaidan_C=1, zaidan_Ccontrast=1, zaidan_nu=1, cvTrain=False, Debug=False): start = time() print '-' * 50 print 'Starting Active Learning...' _, num_feat = X_pool.shape model_scores = {'auc':[], 'accu':[], 'wr':[], 'wo':[], 'alpha':[], 'svm_C':[], 'zaidan_C':[], 'zaidan_Ccontrast':[], 'zaidan_nu':[], 'FMrvalue':[], 'IMweight':[], 'FMweight':[]} rationales = set() rationales_c0 = set() rationales_c1 = set() number_of_docs = 0 feature_expert.rg.seed(seed) num_training_samples = [] all_features=[] # keep all the training data instance ids in docs list docs = training_set X_train = None y_train = [] sample_weight = [] for doc_id in docs: #feature = feature_expert.most_informative_feature(X_pool[doc_id], y_pool[doc_id]) feature = feature_expert.any_informative_feature(X_pool[doc_id], y_pool[doc_id]) number_of_docs=number_of_docs+1 # append feature to all_features, even if it is None all_features.append(feature) if feature is not None: rationales.add(feature) if y_pool[doc_id] == 0: rationales_c0.add(feature) else: rationales_c1.add(feature) if cvTrain: # get optimal parameters depending on the model_type if model_type=='mnb_LwoR': w_r, w_o=optimalMNBLwoRParameters(X_pool[training_set], y_pool[training_set], all_features) feature_counter=0 for doc_id in docs: x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: x[0,f] = w_r*x[0,f] else: x[0,f] = w_o*x[0,f] feature_counter=feature_counter+1 if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) elif model_type=='mnb': w_r, w_o=optimalMNBParameters(X_pool[training_set], y_pool[training_set], all_features) feature_counter=0 for doc_id in docs: x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: x[0,f] = w_r*x[0,f] else: x[0,f] = w_o*x[0,f] feature_counter=feature_counter+1 if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) elif model_type=='svm_linear': w_r, w_o, C= optimalSVMParameters(X_pool[training_set], y_pool[training_set], all_features, seed) feature_counter=0 for doc_id in docs: x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: x[0,f] = w_r*x[0,f] else: x[0,f] = w_o*x[0,f] feature_counter=feature_counter+1 if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) elif model_type=='svm_linear_LwoR': w_r, w_o, C= optimalSVMLwoRParameters(X_pool[training_set], y_pool[training_set], all_features, seed) feature_counter=0 for doc_id in docs: x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: x[0,f] = w_r*x[0,f] else: x[0,f] = w_o*x[0,f] feature_counter=feature_counter+1 if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) if model_type=='poolingMNB': classpriors=np.zeros(2) classpriors[1]=(np.sum(y_pool[docs])*1.)/(len(docs)*1.) classpriors[0]= 1. - classpriors[1] alpha, poolingFM_r, poolingMNBWeights = optimalPoolingMNBParameters(X_pool[training_set], y_pool[training_set], all_features, smoothing, num_feat) feature_model=FeatureMNBUniform(rationales_c0, rationales_c1, num_feat, smoothing, classpriors, poolingFM_r) feature_counter=0 for doc_id in docs: if all_features[feature_counter]: # updates feature model with features one at a time feature_model.fit(all_features[feature_counter], y_pool[doc_id]) feature_counter=feature_counter+1 x = sp.csr_matrix(X_pool[doc_id], dtype=float) if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) if model_type=='Zaidan': zaidan_C, zaidan_Ccontrast, zaidan_nu = optimalZaidanParameters(X_pool[training_set], y_pool[training_set], all_features, seed) feature_counter=0 for doc_id in docs: x = sp.csr_matrix(X_pool[doc_id], dtype=np.float64) if all_features[feature_counter] is not None: x_pseudo = (X_pool[doc_id]).todense() # create pseudoinstances based on rationales provided; one pseudoinstance is created for each rationale. x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: test= x[0,f] x_pseudo[0,f] = x[0,f]/zaidan_nu else: x_pseudo[0,f] = 0.0 x_pseudo=sp.csr_matrix(x_pseudo, dtype=np.float64) if not y_train: X_train = x if all_features[feature_counter] is not None: X_train = sp.vstack((X_train, x_pseudo)) else: X_train = sp.vstack((X_train, x)) if all_features[feature_counter] is not None: X_train = sp.vstack((X_train, x_pseudo)) y_train.append(y_pool[doc_id]) if all_features[feature_counter] is not None: # append y label again for the pseudoinstance created y_train.append(y_pool[doc_id]) sample_weight.append(zaidan_C) if all_features[feature_counter] is not None: # append instance weight=zaidan_Ccontrast for the pseudoinstance created sample_weight.append(zaidan_Ccontrast) feature_counter = feature_counter+1 # Train the model if model_type=='lrl2': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l2', random_state=random_state) elif model_type=='lrl1': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l1', random_state=random_state) elif model_type=='mnb': model = MultinomialNB(alpha=alpha) elif model_type=='mnb_LwoR': model = MultinomialNB(alpha=alpha) elif model_type=='svm_linear_LwoR': random_state = np.random.RandomState(seed=seed) model = LinearSVC(C=C, random_state=random_state) elif model_type=='svm_linear': random_state = np.random.RandomState(seed=seed) model = LinearSVC(C=C, random_state=random_state) elif model_type=='poolingMNB': instance_model=MultinomialNB(alpha=alpha) model = PoolingMNB() elif model_type=='Zaidan': random_state = np.random.RandomState(seed=seed) model = svm.SVC(kernel='linear', C=1.0, random_state=random_state) if model_type=='poolingMNB': instance_model.fit(X_train, y_train) model.fit(instance_model, feature_model, weights=poolingMNBWeights) # train pooling_model elif model_type=='Zaidan': model.fit(X_train, np.array(y_train), sample_weight=sample_weight) else: model.fit(X_train, np.array(y_train)) (accu, auc) = evaluate_model(model, X_test, y_test) model_scores['auc'].append(auc) model_scores['accu'].append(accu) if model_type=='poolingMNB': model_scores['alpha'].append(alpha) model_scores['FMrvalue'].append(poolingFM_r) model_scores['IMweight'].append(poolingMNBWeights[0]) model_scores['FMweight'].append(poolingMNBWeights[1]) else: model_scores['FMrvalue'].append(0.0) model_scores['IMweight'].append(0.0) model_scores['FMweight'].append(0.0) if model_type=='Zaidan': model_scores['zaidan_C'].append(zaidan_C) model_scores['zaidan_Ccontrast'].append(zaidan_Ccontrast) model_scores['zaidan_nu'].append(zaidan_nu) else: model_scores['zaidan_C'].append(0.0) model_scores['zaidan_Ccontrast'].append(0.0) model_scores['zaidan_nu'].append(0.0) if model_type=='mnb' or model_type=='mnb_LwoR': model_scores['alpha'].append(alpha) else: model_scores['alpha'].append(0.0) if model_type=='svm_linear' or model_type=='svm_linear_LwoR': model_scores['svm_C'].append(C) else: model_scores['svm_C'].append(0.0) if model_type=='svm_linear' or model_type=='svm_linear_LwoR' or model_type=='mnb' or model_type=='mnb_LwoR': model_scores['wr'].append(w_r) model_scores['wo'].append(w_o) else: model_scores['wr'].append(0.0) model_scores['wo'].append(0.0) num_training_samples.append(number_of_docs) feature_expert.rg.seed(seed) if selection_strategy == 'random': doc_pick_model = RandomStrategy(seed) elif selection_strategy == 'unc': doc_pick_model = UNCSampling() elif selection_strategy == "pnc": doc_pick_model = UNCPreferNoConflict() elif selection_strategy == "pnr": doc_pick_model = UNCPreferNoRationale() elif selection_strategy == "pr": doc_pick_model = UNCPreferRationale() elif selection_strategy == "pc": doc_pick_model = UNCPreferConflict() elif selection_strategy == "tt": doc_pick_model = UNCThreeTypes() elif selection_strategy == "pipe": doc_pick_model = Pipe([UNCSampling(), UNCPreferConflict()], [10, 30]) else: raise ValueError('Selection strategy: \'%s\' invalid!' % selection_strategy) k = step_size #while X_train.shape[0] < budget: while number_of_docs < budget: # Choose a document based on the strategy chosen if selection_strategy == "pnc": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "pnr": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "pr": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "pc": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "tt": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "pipe": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) else: doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k) if doc_ids is None or len(doc_ids) == 0: break for doc_id in doc_ids: #feature = feature_expert.most_informative_feature(X_pool[doc_id], y_pool[doc_id]) feature = feature_expert.any_informative_feature(X_pool[doc_id], y_pool[doc_id]) all_features.append(feature) number_of_docs=number_of_docs + 1 if feature is not None: rationales.add(feature) if y_pool[doc_id] == 0: rationales_c0.add(feature) else: rationales_c1.add(feature) # Remove the chosen document from pool and add it to the training set pool_set.remove(doc_id) training_set.append(long(doc_id)) if cvTrain: # get optimal parameters depending on the model_type X_train = None y_train = [] sample_weight = [] if model_type=='mnb_LwoR': if np.mod(number_of_docs,20)==10: w_r, w_o=optimalMNBLwoRParameters(X_pool[training_set], y_pool[training_set], all_features) feature_counter=0 for doc_id in training_set: x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: x[0,f] = w_r*x[0,f] else: x[0,f] = w_o*x[0,f] feature_counter=feature_counter+1 if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) elif model_type=='mnb': if np.mod(number_of_docs,20)==10: w_r, w_o=optimalMNBParameters(X_pool[training_set], y_pool[training_set], all_features) feature_counter=0 for doc_id in training_set: x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: x[0,f] = w_r*x[0,f] else: x[0,f] = w_o*x[0,f] feature_counter=feature_counter+1 if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) elif model_type=='svm_linear_LwoR': if np.mod(number_of_docs,20)==10: w_r, w_o, C= optimalSVMLwoRParameters(X_pool[training_set], y_pool[training_set], all_features, seed) feature_counter=0 for doc_id in training_set: x = sp.csr_matrix(X_pool[doc_id], dtype=np.float64) x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: x[0,f] = w_r*x[0,f] else: x[0,f] = w_o*x[0,f] feature_counter=feature_counter+1 if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) elif model_type=='svm_linear': if np.mod(number_of_docs,20)==10: w_r, w_o, C= optimalSVMParameters(X_pool[training_set], y_pool[training_set], all_features, seed) feature_counter=0 for doc_id in training_set: x = sp.csr_matrix(X_pool[doc_id], dtype=np.float64) x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: x[0,f] = w_r*x[0,f] else: x[0,f] = w_o*x[0,f] feature_counter=feature_counter+1 if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) if model_type=='poolingMNB': classpriors=np.zeros(2) classpriors[1]=(np.sum(y_pool[docs])*1.)/(len(docs)*1.) classpriors[0]= 1. - classpriors[1] if np.mod(number_of_docs,20)==10: alpha, poolingFM_r, poolingMNBWeights = optimalPoolingMNBParameters(X_pool[training_set], y_pool[training_set], all_features, smoothing, num_feat) feature_model=FeatureMNBUniform(rationales_c0, rationales_c1, num_feat, smoothing, classpriors, poolingFM_r) feature_counter=0 for doc_id in training_set: if all_features[feature_counter]: # updates feature model with features one at a time feature_model.fit(all_features[feature_counter], y_pool[doc_id]) feature_counter=feature_counter+1 x = sp.csr_matrix(X_pool[doc_id], dtype=float) if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) if model_type=='Zaidan': if np.mod(number_of_docs,20)==10: zaidan_C, zaidan_Ccontrast, zaidan_nu = optimalZaidanParameters(X_pool[training_set], y_pool[training_set], all_features, seed) feature_counter=0 for doc_id in training_set: x = sp.csr_matrix(X_pool[doc_id], dtype=np.float64) if all_features[feature_counter] is not None: x_pseudo = (X_pool[doc_id]).todense() # create pseudoinstances based on rationales provided; one pseudoinstance is created for each rationale. x_feats = x[0].indices for f in x_feats: if f == all_features[feature_counter]: test= x[0,f] x_pseudo[0,f] = x[0,f]/zaidan_nu else: x_pseudo[0,f] = 0.0 x_pseudo=sp.csr_matrix(x_pseudo, dtype=np.float64) if not y_train: X_train = x if all_features[feature_counter] is not None: X_train = sp.vstack((X_train, x_pseudo)) else: X_train = sp.vstack((X_train, x)) if all_features[feature_counter] is not None: X_train = sp.vstack((X_train, x_pseudo)) y_train.append(y_pool[doc_id]) if all_features[feature_counter] is not None: # append y label again for the pseudoinstance created y_train.append(y_pool[doc_id]) sample_weight.append(zaidan_C) if all_features[feature_counter] is not None: # append instance weight=zaidan_Ccontrast for the pseudoinstance created sample_weight.append(zaidan_Ccontrast) feature_counter = feature_counter+1 # Train the model if model_type=='lrl2': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l2', random_state=random_state) elif model_type=='lrl1': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l1', random_state=random_state) elif model_type=='mnb': model = MultinomialNB(alpha=alpha) elif model_type=='mnb_LwoR': model = MultinomialNB(alpha=alpha) elif model_type=='svm_linear_LwoR': random_state = np.random.RandomState(seed=seed) model = LinearSVC(C=C, random_state=random_state) elif model_type=='svm_linear': random_state = np.random.RandomState(seed=seed) model = LinearSVC(C=C, random_state=random_state) elif model_type=='poolingMNB': instance_model=MultinomialNB(alpha=alpha) model = PoolingMNB() elif model_type=='Zaidan': random_state = np.random.RandomState(seed=seed) model = svm.SVC(kernel='linear', C=svm_C, random_state=random_state) if model_type=='poolingMNB': instance_model.fit(X_train, y_train) model.fit(instance_model, feature_model, weights=poolingMNBWeights) # train pooling_model elif model_type=='Zaidan': model.fit(X_train, np.array(y_train), sample_weight=sample_weight) else: model.fit(X_train, np.array(y_train)) (accu, auc) = evaluate_model(model, X_test, y_test) model_scores['auc'].append(auc) model_scores['accu'].append(accu) if model_type=='poolingMNB': model_scores['alpha'].append(alpha) model_scores['FMrvalue'].append(poolingFM_r) model_scores['IMweight'].append(poolingMNBWeights[0]) model_scores['FMweight'].append(poolingMNBWeights[1]) else: model_scores['FMrvalue'].append(0.0) model_scores['IMweight'].append(0.0) model_scores['FMweight'].append(0.0) if model_type=='Zaidan': model_scores['zaidan_C'].append(zaidan_C) model_scores['zaidan_Ccontrast'].append(zaidan_Ccontrast) model_scores['zaidan_nu'].append(zaidan_nu) else: model_scores['zaidan_C'].append(0.0) model_scores['zaidan_Ccontrast'].append(0.0) model_scores['zaidan_nu'].append(0.0) if model_type=='mnb' or model_type=='mnb_LwoR': model_scores['alpha'].append(alpha) else: model_scores['alpha'].append(0.0) if model_type=='svm_linear' or model_type=='svm_linear_LwoR': model_scores['svm_C'].append(C) else: model_scores['svm_C'].append(0.0) if model_type=='svm_linear' or model_type=='svm_linear_LwoR' or model_type=='mnb' or model_type=='mnb_LwoR': model_scores['wr'].append(w_r) model_scores['wo'].append(w_o) else: model_scores['wr'].append(0.0) model_scores['wo'].append(0.0) num_training_samples.append(number_of_docs) print 'Active Learning took %2.2fs' % (time() - start) return (np.array(num_training_samples), model_scores)
def learn(model_type, X_pool, y_pool, X_test, y_test, training_set, pool_set, feature_expert, \ selection_strategy, budget, step_size, topk, w_o, w_r, seed=0, lr_C=1, svm_C=1, Debug=False): start = time() print '-' * 50 print 'Starting Active Learning...' model_scores = {'auc':[], 'accu':[]} rationales = set() rationales_c0 = set() rationales_c1 = set() feature_expert.rg.seed(seed) num_training_samples = [] docs = training_set X_train = None y_train = [] for doc_id in docs: #feature = feature_expert.most_informative_feature(X_pool[doc_id], y_pool[doc_id]) feature = feature_expert.any_informative_feature(X_pool[doc_id], y_pool[doc_id]) rationales.add(feature) if y_pool[doc_id] == 0: rationales_c0.add(feature) else: rationales_c1.add(feature) x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == feature: x[0,f] = w_r*x[0,f] else: x[0,f] = w_o*x[0,f] if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) # Train the model if model_type=='lrl2': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l2', random_state=random_state) elif model_type=='lrl1': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l1', random_state=random_state) elif model_type=='mnb': model = MultinomialNB(alpha=1) elif model_type=='svm_linear': model = LinearSVC(C=svm_C) model.fit(X_train, np.array(y_train)) (accu, auc) = evaluate_model(model, X_test, y_test) model_scores['auc'].append(auc) model_scores['accu'].append(accu) num_training_samples.append(X_train.shape[0]) feature_expert.rg.seed(seed) if selection_strategy == 'random': doc_pick_model = RandomStrategy(seed) elif selection_strategy == 'unc': doc_pick_model = UNCSampling() elif selection_strategy == "pnc": doc_pick_model = UNCPreferNoConflict() elif selection_strategy == "pnr": doc_pick_model = UNCPreferNoRationale() elif selection_strategy == "pr": doc_pick_model = UNCPreferRationale() elif selection_strategy == "pc": doc_pick_model = UNCPreferConflict() elif selection_strategy == "tt": doc_pick_model = UNCThreeTypes() elif selection_strategy == "pipe": doc_pick_model = Pipe([UNCSampling(), UNCPreferConflict()], [10, 30]) else: raise ValueError('Selection strategy: \'%s\' invalid!' % selection_strategy) k = step_size while X_train.shape[0] < budget: # Choose a document based on the strategy chosen if selection_strategy == "pnc": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "pnr": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "pr": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "pc": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "tt": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "pipe": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) else: doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k) if doc_ids is None or len(doc_ids) == 0: break for doc_id in doc_ids: # Remove the chosen document from pool and add it to the training set pool_set.remove(doc_id) training_set.append(doc_id) #feature = feature_expert.most_informative_feature(X_pool[doc_id], y_pool[doc_id]) feature = feature_expert.any_informative_feature(X_pool[doc_id], y_pool[doc_id]) rationales.add(feature) if y_pool[doc_id] == 0: rationales_c0.add(feature) else: rationales_c1.add(feature) x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == feature: x[0,f] = w_r*x[0,f] else: x[0,f] = w_o*x[0,f] X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) # Train the model if model_type=='lrl2': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l2', random_state=random_state) elif model_type=='lrl1': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l1', random_state=random_state) elif model_type=='mnb': model = MultinomialNB(alpha=1) elif model_type=='svm_linear': model = LinearSVC(C=svm_C) # fit the model and evaluate model.fit(X_train, np.array(y_train)) (accu, auc) = evaluate_model(model, X_test, y_test) model_scores['auc'].append(auc) model_scores['accu'].append(accu) num_training_samples.append(X_train.shape[0]) print 'Active Learning took %2.2fs' % (time() - start) return (np.array(num_training_samples), model_scores)
def learn(model_type, X_pool, y_pool, X_test, y_test, training_set, pool_set, feature_expert, \ selection_strategy, budget, step_size, topk, w_o, w_r, seed=0, lr_C=1, svm_C=1, svm_gamma=0, zaidan_C=0.01, zaidan_Ccontrast=1.0, zaidan_nu=1, Debug=False): start = time() print '-' * 50 print 'Starting Active Learning...' model_scores = {'auc': [], 'accu': []} rationales = set() rationales_c0 = set() rationales_c1 = set() feature_expert.rg.seed(seed) num_training_samples = [] docs = training_set X_train = None y_train = [] sample_weight = [] number_of_docs = 0 for doc_id in docs: #feature = feature_expert.most_informative_feature(X_pool[doc_id], y_pool[doc_id]) feature = feature_expert.any_informative_feature( X_pool[doc_id], y_pool[doc_id]) number_of_docs = number_of_docs + 1 if feature is not None: rationales.add(feature) if y_pool[doc_id] == 0: rationales_c0.add(feature) else: rationales_c1.add(feature) x = sp.csr_matrix(X_pool[doc_id], dtype=float) if feature is not None: x_pseudo = (X_pool[doc_id]).todense() # create pseudoinstances based on rationales provided; one pseudoinstance is created for each rationale. x_feats = x[0].indices for f in x_feats: if f == feature: x_pseudo[0, f] = x[0, f] else: x_pseudo[0, f] = 0.0 x_pseudo = sp.csr_matrix(x_pseudo) if not y_train: X_train = x if feature is not None: X_train = sp.vstack((X_train, x_pseudo)) else: X_train = sp.vstack((X_train, x)) if feature is not None: X_train = sp.vstack((X_train, x_pseudo)) y_train.append(y_pool[doc_id]) if feature is not None: # append y label again for the pseudoinstance created y_train.append(y_pool[doc_id]) sample_weight.append(zaidan_C) if feature is not None: # append instance weight=zaidan_Ccontrast for the pseudoinstance created sample_weight.append(zaidan_Ccontrast) # Train the model if model_type == 'lrl2': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l2', random_state=random_state) elif model_type == 'mnb': model = MultinomialNB(alpha=1) elif model_type == 'svm_linear': model = svm.SVC(kernel='linear', C=svm_C, probability=True) #model = svm.SVC(kernel='linear', probability=True) elif model_type == 'svm_rbf': model = svm.SVC(kernel='rbf', gamma=svm_gamma, C=svm_C, probability=True) #model = svm.SVC(kernel='rbf', probability=True) elif model_type == 'svm_poly': model = svm.SVC(kernel='rbf', gamma=svm_gamma, C=svm_C, probability=True) #model = svm.SVC(kernel='poly', probability=True) elif model_type == 'adaptive_lr': random_state = np.random.RandomState(seed=seed) #model = LogisticRegression(C=C, penalty='l2', random_state=random_state) #model.fit(X_train, np.array(y_train)) model = LogisticRegressionAdaptive() elif model_type == 'adaptive_svm': random_state = np.random.RandomState(seed=seed) model = AdaptiveSVM() elif model_type == 'SGD': model = SGDClassifier(loss="log", penalty='l2', n_iter=100) # fit model using sample weight model.fit(X_train, np.array(y_train), sample_weight=sample_weight) (accu, auc) = evaluate_model(model, X_test, y_test) model_scores['auc'].append(auc) model_scores['accu'].append(accu) num_training_samples.append(number_of_docs) feature_expert.rg.seed(seed) if selection_strategy == 'random': doc_pick_model = RandomStrategy(seed) elif selection_strategy == 'uncertainty': doc_pick_model = UNCSampling(model, feature_expert, y_pool, Debug) elif selection_strategy == "optauc": doc_pick_model = OptimizeAUC(X_test, y_test, feature_expert, \ optimize="R", seed=seed, Debug=Debug) elif selection_strategy == "unc_prefer_no_conflict": doc_pick_model = UNCPreferNoConflict(model) elif selection_strategy == "unc_prefer_conflict": doc_pick_model = UNCPreferConflict(model) elif selection_strategy == "unc_three_types": doc_pick_model = UNCThreeTypes(model) else: raise ValueError('Selection strategy: \'%s\' invalid!' % selection_strategy) k = step_size while number_of_docs < budget: # Choose a document based on the strategy chosen if selection_strategy == "unc_prefer_no_conflict": doc_ids = doc_pick_model.choices(X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "unc_prefer_conflict": doc_ids = doc_pick_model.choices(X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "unc_three_types": doc_ids = doc_pick_model.choices(X_pool, pool_set, k, rationales_c0, rationales_c1, topk) else: doc_ids = doc_pick_model.choices(X_pool, pool_set, k) if doc_ids is None or len(doc_ids) == 0: break for doc_id in doc_ids: # Remove the chosen document from pool and add it to the training set pool_set.remove(doc_id) training_set.append(doc_id) number_of_docs = number_of_docs + 1 #feature = feature_expert.most_informative_feature(X_pool[doc_id], y_pool[doc_id]) feature = feature_expert.any_informative_feature( X_pool[doc_id], y_pool[doc_id]) if feature is not None: rationales.add(feature) if y_pool[doc_id] == 0: rationales_c0.add(feature) else: rationales_c1.add(feature) x = sp.csr_matrix(X_pool[doc_id], dtype=float) if feature is not None: x_pseudo = (X_pool[doc_id]).todense() # create pseudoinstances based on rationales provided; one pseudoinstance is created for each rationale. x_feats = x[0].indices for f in x_feats: if f == feature: x_pseudo[0, f] = x[0, f] else: x_pseudo[0, f] = 0.0 x_pseudo = sp.csr_matrix(x_pseudo) if not y_train: X_train = x if feature is not None: X_train = sp.vstack((X_train, x_pseudo)) else: X_train = sp.vstack((X_train, x)) if feature is not None: X_train = sp.vstack((X_train, x_pseudo)) y_train.append(y_pool[doc_id]) if feature is not None: # append y label again for the pseudoinstance created y_train.append(y_pool[doc_id]) sample_weight.append(zaidan_C) if feature is not None: # append instance weight=zaidan_Ccontrast for the pseudoinstance created sample_weight.append(zaidan_Ccontrast) # Train the model if model_type == 'lrl2': random_state2 = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l2', random_state=random_state) elif model_type == 'mnb': model = MultinomialNB(alpha=1) elif model_type == 'svm_linear': model = svm.SVC(kernel='linear', C=svm_C, probability=True) #model = svm.SVC(kernel='linear', probability=True) elif model_type == 'svm_rbf': model = svm.SVC(kernel='rbf', gamma=svm_gamma, C=svm_C, probability=True) #model = svm.SVC(kernel='rbf', probability=True) elif model_type == 'svm_poly': model = svm.SVC(kernel='rbf', gamma=svm_gamma, C=svm_C, probability=True) #model = svm.SVC(kernel='poly', probability=True) elif model_type == 'adaptive_lr': #random_state = np.random.RandomState(seed=seed) model = LogisticRegressionAdaptive() elif model_type == 'adaptive_svm': random_state = np.random.RandomState(seed=seed) model = AdaptiveSVM() elif model_type == 'SGD': model = SGDClassifier(loss="log", penalty='l2', n_iter=100) # fit model using sample weight model.fit(X_train, np.array(y_train), sample_weight=sample_weight) (accu, auc) = evaluate_model(model, X_test, y_test) model_scores['auc'].append(auc) model_scores['accu'].append(accu) num_training_samples.append(number_of_docs) print 'Active Learning took %2.2fs' % (time() - start) return (np.array(num_training_samples), model_scores)
def learn(X_pool, y_pool, X_test, y_test, training_set, pool_set, feature_expert, \ selection_strategy, disagree_strat, coverage, budget, instance_model, feature_model, \ pooling_model, reasoning_model, rmw_n, rmw_a, seed=0, Debug=False, \ reasoning_strategy='random', switch=40): start = time() print '-' * 50 print 'Starting Active Learning...' instance_model_scores = {'auc':[], 'accu':[]} feature_model_scores = {'auc':[], 'accu':[]} pooling_model_scores = {'auc':[], 'accu':[]} reasoning_model_scores = {'auc':[], 'accu':[]} discovered_feature_counts = {'class0':[], 'class1': []} num_docs_covered = [] covered_docs = set() X_pool_csc = X_pool.tocsc() num_samples = len(pool_set) + len(training_set) num_feat = X_pool.shape[1] num_a_feat_chosen = np.zeros(num_feat) discovered_features = set() discovered_class0_features = set() discovered_class1_features = set() feature_expert.rg.seed(seed) if selection_strategy == 'random': doc_pick_model = RandomStrategy(seed) elif selection_strategy == 'uncertaintyIM': doc_pick_model = UNCSampling(instance_model, feature_expert, y_pool, Debug) elif selection_strategy == 'uncertaintyFM': doc_pick_model = UNCSampling(feature_model, feature_expert, y_pool, Debug) elif selection_strategy == 'uncertaintyPM': doc_pick_model = UNCSampling(pooling_model, feature_expert, y_pool, Debug) elif selection_strategy == 'uncertaintyRM': doc_pick_model = UNCSampling(reasoning_model, feature_expert, y_pool, Debug) elif selection_strategy == 'disagreement': doc_pick_model = DisagreementStrategy(instance_model, feature_model, \ feature_expert, y_pool, disagree_strat, Debug=Debug) elif selection_strategy == 'covering': doc_pick_model = CoveringStrategy(feature_expert, num_samples, y_pool, \ type='unknown', seed=seed, Debug=Debug) elif selection_strategy == 'covering_fewest': doc_pick_model = CoveringStrategy(feature_expert, num_samples, y_pool, \ type='fewest', seed=seed, Debug=Debug) elif selection_strategy == 'cheating': doc_pick_model = CheatingApproach(feature_expert, num_samples, y_pool, \ seed=seed, Debug=Debug) elif selection_strategy == 'cover_then_disagree': doc_pick_model = CoveringThenDisagreement(feature_expert, instance_model, \ feature_model, num_samples, percentage=coverage, y=y_pool, type='unknown', \ metric=disagree_strat, seed=seed, Debug=Debug) elif selection_strategy == 'cover_then_uncertaintyPM': doc_pick_model = CoverThenUncertainty(feature_expert, pooling_model, \ num_samples, percentage=coverage, y=y_pool, type='unknown', \ seed=seed, Debug=Debug) elif selection_strategy == 'cover_then_uncertaintyRM': doc_pick_model = CoverThenUncertainty(feature_expert, reasoning_model, \ num_samples, percentage=coverage, y=y_pool, type='unknown', \ seed=seed, Debug=Debug) elif selection_strategy == 'cover_then_featureCertainty': doc_pick_model = CoverThenFeatureCertainty(feature_expert, feature_model, \ num_samples, percentage=coverage, y=y_pool, type='unknown', \ seed=seed, Debug=Debug) elif selection_strategy == "optaucP": doc_pick_model = OptimizeAUC(X_test, y_test, feature_expert, \ optimize="P", seed=seed, Debug=Debug) elif selection_strategy == "optaucI": doc_pick_model = OptimizeAUC(X_test, y_test, feature_expert, \ optimize="I", seed=seed, Debug=Debug) elif selection_strategy == "optaucF": doc_pick_model = OptimizeAUC(X_test, y_test, feature_expert, \ optimize="F", seed=seed, Debug=Debug) elif selection_strategy == "optaucR": doc_pick_model = OptimizeAUC(X_test, y_test, feature_expert, \ optimize="R", seed=seed, Debug=Debug) elif selection_strategy == 'reasoning_then_featureCertainty': doc_pick_model = ReasoningThenFeatureCertainty(feature_expert, instance_model, \ feature_model, switch=switch, reasoning_strategy=reasoning_strategy, y=y_pool, type='unknown', \ seed=seed, Debug=Debug) elif selection_strategy == "unc_insuff_R": doc_pick_model = UNCForInsufficientReason(reasoning_model) elif selection_strategy == "unc_no_conflict_R": doc_pick_model = UNCWithNoConflict(reasoning_model) elif selection_strategy == "unc_prefer_no_conflict_R": doc_pick_model = UNCPreferNoConflict(reasoning_model) elif selection_strategy == "unc_prefer_conflict_R": doc_pick_model = UNCPreferConflict(reasoning_model) elif selection_strategy == "unc_three_types_R": doc_pick_model = UNCThreeTypes(reasoning_model) else: raise ValueError('Selection strategy: \'%s\' invalid!' % selection_strategy) bootstrap_size = len(training_set) training_set_empty = (bootstrap_size == 0) if not training_set_empty: X_train = X_pool[training_set] y_train = y_pool[training_set] # Train all three models using the training set data instance_model.fit(X_train, y_train) # train instance_model for doc in training_set: #feature = feature_expert.most_informative_feature(X_pool[doc], y_pool[doc]) feature = feature_expert.any_informative_feature(X_pool[doc], y_pool[doc]) if feature: feature_model.fit(feature, y_pool[doc]) # train feature_model one by one discovered_features.add(feature) if y_pool[doc] == 0: discovered_class0_features.add(feature) else: discovered_class1_features.add(feature) # Reasoning model reasoning_model.partial_fit(X_pool[doc], y_pool[doc], feature, rmw_n, rmw_a) # train feature_model one by one # docs covered if feature: f_covered_docs = X_pool_csc[:, feature].indices covered_docs.update(f_covered_docs) # number of times a feat is chosen as a reason if feature: num_a_feat_chosen[feature] += 1 if selection_strategy == 'covering' or selection_strategy == 'covering_fewest': doc_pick_model.update(X_pool, feature, doc) elif selection_strategy == 'cheating': doc_pick_model.update(X_pool, feature, y_pool[doc]) elif selection_strategy.startswith('cover_then') and doc_pick_model.phase == 'covering': doc_pick_model.covering.update(X_pool, feature, doc) pooling_model.fit(instance_model, feature_model, weights=[0.5, 0.5]) # train pooling_model (accu, auc) = evaluate_model(instance_model, X_test, y_test) instance_model_scores['auc'].append(auc) instance_model_scores['accu'].append(accu) (accu, auc) = evaluate_model(feature_model, X_test, y_test) feature_model_scores['auc'].append(auc) feature_model_scores['accu'].append(accu) (accu, auc) = evaluate_model(pooling_model, X_test, y_test) pooling_model_scores['auc'].append(auc) pooling_model_scores['accu'].append(accu) (accu, auc) = evaluate_model(reasoning_model, X_test, y_test) reasoning_model_scores['auc'].append(auc) reasoning_model_scores['accu'].append(accu) # discovered feature counts if isinstance(feature_model, FeatureMNBUniform): discovered_feature_counts['class0'].append(len(feature_model.class0_features)) discovered_feature_counts['class1'].append(len(feature_model.class1_features)) elif isinstance(feature_model, FeatureMNBWeighted): nz = np.sum(feature_model.feature_count_>0, axis=1) discovered_feature_counts['class0'].append(nz[0]) discovered_feature_counts['class1'].append(nz[1]) num_docs_covered.append(len(covered_docs)) else: if selection_strategy.startswith('uncertainty') or selection_strategy == 'disagreement': raise ValueError('\'%s\' requires bootstrapping!' % selection_strategy) for i in range(budget): train_set_size=len(training_set) # Choose a document based on the strategy chosen if selection_strategy.startswith('cover_then'): doc_id = doc_pick_model.choice(X_pool, i+1, pool_set) elif selection_strategy.startswith('optauc'): doc_id = doc_pick_model.choice(X_pool, y_pool, pool_set, training_set, feature_model, reasoning_model, rmw_n, rmw_a) elif selection_strategy == 'reasoning_then_featureCertainty': doc_id = doc_pick_model.choice(X_pool, i+1, pool_set, train_set_size) elif selection_strategy == "unc_insuff_R": doc_id = doc_pick_model.choice(X_pool, pool_set, discovered_features, max_num_feats=1) elif selection_strategy == "unc_no_conflict_R": doc_id = doc_pick_model.choice(X_pool, pool_set, discovered_class0_features, discovered_class1_features) elif selection_strategy == "unc_prefer_no_conflict_R": doc_id = doc_pick_model.choice(X_pool, pool_set, discovered_class0_features, discovered_class1_features, top_k=10) elif selection_strategy == "unc_prefer_conflict_R": doc_id = doc_pick_model.choice(X_pool, pool_set, discovered_class0_features, discovered_class1_features, top_k=10) elif selection_strategy == "unc_three_types_R": doc_id = doc_pick_model.choice(X_pool, pool_set, discovered_class0_features, discovered_class1_features, top_k=10) else: doc_id = doc_pick_model.choice(X_pool, pool_set) if doc_id == None: break # Remove the chosen document from pool and add it to the training set pool_set.remove(doc_id) training_set.append(doc_id) if i == 0 and training_set_empty: X_train = X_pool[doc_id] y_train = np.array([y_pool[doc_id]]) else: X_train = sp.vstack((X_train, X_pool[doc_id])) y_train = np.hstack((y_train, np.array([y_pool[doc_id]]))) # Ask the expert for instance label (returns the true label from the dataset) label = y_pool[doc_id] # Ask the expert for most informative feature given the label #feature = feature_expert.most_informative_feature(X_pool[doc_id], label) feature = feature_expert.any_informative_feature(X_pool[doc_id], y_pool[doc_id]) # Update the instance model instance_model.fit(X_train, y_train) # Update the feature model if feature: feature_model.fit(feature, label) discovered_features.add(feature) if y_pool[doc_id] == 0: discovered_class0_features.add(feature) else: discovered_class1_features.add(feature) reasoning_model.partial_fit(X_pool[doc_id], y_pool[doc_id], feature, rmw_n, rmw_a) # train feature_model one by one # docs covered if feature: f_covered_docs = X_pool_csc[:, feature].indices covered_docs.update(f_covered_docs) # number of times a feat is chosen as a reason if feature: num_a_feat_chosen[feature] += 1 # Update the pooling model pooling_model.fit(instance_model, feature_model, weights=[0.5, 0.5]) # print 'docs = %d, feature = %s' % (doc_id, str(feature)) if selection_strategy == 'covering' or selection_strategy == 'covering_fewest': doc_pick_model.update(X_pool, feature, doc_id) elif selection_strategy == 'cheating': doc_pick_model.update(X_pool, feature, label) elif selection_strategy.startswith('cover_then') and doc_pick_model.phase == 'covering': doc_pick_model.covering.update(X_pool, feature, doc_id) # print 'covering_fewest features: %d, feature model features: %d' % (len(doc_pick_model.annotated_features), len(feature_model.class0_features + feature_model.class1_features)) # Evaluate performance based on Instance Model (accu, auc) = evaluate_model(instance_model, X_test, y_test) instance_model_scores['auc'].append(auc) instance_model_scores['accu'].append(accu) # Evaluate performance on Feature Model (accu, auc) = evaluate_model(feature_model, X_test, y_test) feature_model_scores['auc'].append(auc) feature_model_scores['accu'].append(accu) # Evaluate performance on Pooled Model (accu, auc) = evaluate_model(pooling_model, X_test, y_test) pooling_model_scores['auc'].append(auc) pooling_model_scores['accu'].append(accu) # Evaluate performance of the Reasoning Model (accu, auc) = evaluate_model(reasoning_model, X_test, y_test) reasoning_model_scores['auc'].append(auc) reasoning_model_scores['accu'].append(accu) # discovered feature counts if isinstance(feature_model, FeatureMNBUniform): discovered_feature_counts['class0'].append(len(feature_model.class0_features)) discovered_feature_counts['class1'].append(len(feature_model.class1_features)) elif isinstance(feature_model, FeatureMNBWeighted): nz = np.sum(feature_model.feature_count_>0, axis=1) discovered_feature_counts['class0'].append(nz[0]) discovered_feature_counts['class1'].append(nz[1]) # docs covered num_docs_covered.append(len(covered_docs)) if selection_strategy.startswith('cover_then'): transition = doc_pick_model.transition else: transition = None # compute the # of training samples for plot if training_set_empty: num_training_samples = np.arange(len(instance_model_scores['accu'])) + 1 else: num_training_samples = np.arange(len(instance_model_scores['accu'])) + bootstrap_size print 'Active Learning took %2.2fs' % (time() - start) return (num_training_samples, instance_model_scores, feature_model_scores, pooling_model_scores,reasoning_model_scores, discovered_feature_counts, num_docs_covered, transition, num_a_feat_chosen)
def learn(model_type, X_pool, y_pool, X_test, y_test, training_set, pool_set, feature_expert, \ selection_strategy, budget, step_size, topk, w_o, w_r, seed=0, lr_C=1, svm_C=1, Debug=False): start = time() print '-' * 50 print 'Starting Active Learning...' model_scores = {'auc': [], 'accu': []} rationales = set() rationales_c0 = set() rationales_c1 = set() feature_expert.rg.seed(seed) num_training_samples = [] docs = training_set X_train = None y_train = [] for doc_id in docs: #feature = feature_expert.most_informative_feature(X_pool[doc_id], y_pool[doc_id]) feature = feature_expert.any_informative_feature( X_pool[doc_id], y_pool[doc_id]) rationales.add(feature) if y_pool[doc_id] == 0: rationales_c0.add(feature) else: rationales_c1.add(feature) x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == feature: x[0, f] = w_r * x[0, f] else: x[0, f] = w_o * x[0, f] if not y_train: X_train = x else: X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) # Train the model if model_type == 'lrl2': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l2', random_state=random_state) elif model_type == 'lrl1': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l1', random_state=random_state) elif model_type == 'mnb': model = MultinomialNB(alpha=1) elif model_type == 'svm_linear': model = LinearSVC(C=svm_C) model.fit(X_train, np.array(y_train)) (accu, auc) = evaluate_model(model, X_test, y_test) model_scores['auc'].append(auc) model_scores['accu'].append(accu) num_training_samples.append(X_train.shape[0]) feature_expert.rg.seed(seed) if selection_strategy == 'random': doc_pick_model = RandomStrategy(seed) elif selection_strategy == 'unc': doc_pick_model = UNCSampling() elif selection_strategy == "pnc": doc_pick_model = UNCPreferNoConflict() elif selection_strategy == "pnr": doc_pick_model = UNCPreferNoRationale() elif selection_strategy == "pr": doc_pick_model = UNCPreferRationale() elif selection_strategy == "pc": doc_pick_model = UNCPreferConflict() elif selection_strategy == "tt": doc_pick_model = UNCThreeTypes() elif selection_strategy == "pipe": doc_pick_model = Pipe([UNCSampling(), UNCPreferConflict()], [10, 30]) else: raise ValueError('Selection strategy: \'%s\' invalid!' % selection_strategy) k = step_size while X_train.shape[0] < budget: # Choose a document based on the strategy chosen if selection_strategy == "pnc": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "pnr": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "pr": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "pc": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "tt": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) elif selection_strategy == "pipe": doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k, rationales_c0, rationales_c1, topk) else: doc_ids = doc_pick_model.choices(model, X_pool, pool_set, k) if doc_ids is None or len(doc_ids) == 0: break for doc_id in doc_ids: # Remove the chosen document from pool and add it to the training set pool_set.remove(doc_id) training_set.append(doc_id) #feature = feature_expert.most_informative_feature(X_pool[doc_id], y_pool[doc_id]) feature = feature_expert.any_informative_feature( X_pool[doc_id], y_pool[doc_id]) rationales.add(feature) if y_pool[doc_id] == 0: rationales_c0.add(feature) else: rationales_c1.add(feature) x = sp.csr_matrix(X_pool[doc_id], dtype=float) x_feats = x[0].indices for f in x_feats: if f == feature: x[0, f] = w_r * x[0, f] else: x[0, f] = w_o * x[0, f] X_train = sp.vstack((X_train, x)) y_train.append(y_pool[doc_id]) # Train the model if model_type == 'lrl2': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l2', random_state=random_state) elif model_type == 'lrl1': random_state = np.random.RandomState(seed=seed) model = LogisticRegression(C=lr_C, penalty='l1', random_state=random_state) elif model_type == 'mnb': model = MultinomialNB(alpha=1) elif model_type == 'svm_linear': model = LinearSVC(C=svm_C) # fit the model and evaluate model.fit(X_train, np.array(y_train)) (accu, auc) = evaluate_model(model, X_test, y_test) model_scores['auc'].append(auc) model_scores['accu'].append(accu) num_training_samples.append(X_train.shape[0]) print 'Active Learning took %2.2fs' % (time() - start) return (np.array(num_training_samples), model_scores)