def _test_classification_report(n_classes=2): classifiers = ClassifiersFactory() classifiers.add_classifier('gb', GradientBoostingClassifier(n_estimators=10)) classifiers.add_classifier('rf', RandomForestClassifier()) classifiers.add_classifier('ada', AdaBoostClassifier(n_estimators=10)) X, y = generate_classification_sample(1000, 5, n_classes=n_classes) classifiers.fit(X, y) X, y = generate_classification_sample(1000, 5, n_classes=n_classes) test_lds = LabeledDataStorage(X, y, sample_weight=None) report = classifiers.test_on_lds(test_lds) val = numpy.mean(X['column0']) labels_dict = None if n_classes > 2: labels_dict = {} for i in range(n_classes): labels_dict[i] = str(i) _classification_mask_report(report, "column0 > %f" % val, X, labels_dict) _classification_mask_report(report, lambda x: numpy.array(x['column0']) < val, X, labels_dict) _classification_mask_report(report, None, X, labels_dict)
def test_factory(): factory = ClassifiersFactory() try: from rep.estimators.tmva import TMVAClassifier factory.add_classifier('tmva', TMVAClassifier()) except ImportError: pass factory.add_classifier('rf', RandomForestClassifier(n_estimators=10)) factory.add_classifier('ada', AdaBoostClassifier(n_estimators=20)) X, y, sample_weight = generate_classification_data() assert factory == factory.fit(X, y, sample_weight=sample_weight, features=list(X.columns), parallel_profile='threads-4') for cl in factory.values(): assert list(cl.features) == list(X.columns) proba = factory.predict_proba(X, parallel_profile='threads-4') labels = factory.predict(X, parallel_profile='threads-4') for key, val in labels.items(): score = accuracy_score(y, val) print(key, score) assert score > 0.7, key for key, val in proba.items(): assert numpy.allclose(val.sum(axis=1), 1), 'probabilities do not sum to 1' assert numpy.all(val >= 0.), 'negative probabilities' auc_score = roc_auc_score(y, val[:, 1]) print(auc_score) assert auc_score > 0.8 for key, iterator in factory.staged_predict_proba(X).items(): assert key != 'tmva', 'tmva does not support staged pp' for p in iterator: assert p.shape == (len(X), 2) # checking that last iteration coincides with previous assert numpy.all(p == proba[key]) # testing picklability dump_string = cPickle.dumps(factory) clf_loaded = cPickle.loads(dump_string) assert type(factory) == type(clf_loaded) probs1 = factory.predict_proba(X) probs2 = clf_loaded.predict_proba(X) for key, val in probs1.items(): assert numpy.all(val == probs2[key]), 'something strange was loaded' report = ClassificationReport({'rf': factory['rf']}, LabeledDataStorage(X, y, sample_weight)) report.feature_importance_shuffling(roc_auc_score_mod).plot(new_plot=True, figsize=(18, 3)) report = factory.test_on_lds(LabeledDataStorage(X, y, sample_weight)) report = factory.test_on(X, y, sample_weight=sample_weight) val = numpy.mean(X['column0']) yield check_report_with_mask, report, "column0 > %f" % (val / 2.), X yield check_report_with_mask, report, lambda x: numpy.array(x['column0']) < val * 2., X yield check_report_with_mask, report, None, X
def _check_classification_report(n_classes=2): classifiers = ClassifiersFactory() classifiers.add_classifier('gb', GradientBoostingClassifier(n_estimators=10)) classifiers.add_classifier('rf', RandomForestClassifier()) classifiers.add_classifier('ada', AdaBoostClassifier(n_estimators=10)) X, y = generate_classification_sample(1000, 5, n_classes=n_classes) classifiers.fit(X, y) X, y = generate_classification_sample(1000, 5, n_classes=n_classes) test_lds = LabeledDataStorage(X, y, sample_weight=None) report = classifiers.test_on_lds(test_lds) val = numpy.mean(X['column0']) labels_dict = None if n_classes > 2: labels_dict = {i: str(i) for i in range(n_classes)} _classification_mask_report(report, "column0 > %f" % val, X, labels_dict) _classification_mask_report(report, lambda x: numpy.array(x['column0']) < val, X, labels_dict) _classification_mask_report(report, None, X, labels_dict) check_classification_learning_curve_masks(report, n_classes=n_classes)
class Mayou(Classifier): """Classifier for raredecay analysis""" __DEFAULT_CLF_CFG = dict( xgb=dict( n_estimators=450, eta=0.1, subsample=0.9, bagging=None ), rdf=dict( n_estimators=1600, # 1600 max_features='auto', # only 1 feature seems to be pretty good... max_depth=200, min_samples_split=250, min_samples_leaf=150, min_weight_fraction_leaf=0., max_leaf_nodes=None, bootstrap=False, oob_score=False, n_jobs=7, class_weight=None, bagging=None ), nn=dict( layers=[100, 100], hidden_activation='logistic', output_activation='linear', input_noise=0, # [0,1,2,3,4,5,10,20], hidden_noise=0, input_dropout=0, hidden_dropout=0.05, decode_from=1, weight_l1=0.01, weight_l2=0.03, scaler='standard', trainers=[{'optimize': 'adagrad', 'patience': 2, 'momentum': 0.5, 'nesterov': True, 'learning_rate': 0.2, 'min_improvement': 0.01}], bagging=None ), gb=dict( learning_rate=0.05, n_estimators=500, max_depth=4, min_samples_split=600, min_samples_leaf=1, min_weight_fraction_leaf=0., subsample=0.8, max_features=None, max_leaf_nodes=None, bagging=None ), ) __DEFAULT_BAG_CFG = dict( n_estimators=20, max_samples=0.9, max_features=1.0, ) def __init__(self, base_estimators=None, bagging_base=None, stacking='xgb', features_stack=None, bagging_stack=None, hunting=False, transform=True, transform_pred=True): """blablabla Parameters ---------- base_estimators : dict('clf': classifier OR keyword-parameters) Contains all the level-0 classifiers. The key is the name of the classifier and the value is either a **prefitted** classifier or a dictionary containing the keyword arguments to instantiate such a classifier. If no pre-trained classifier is provided, the key-value has to be one of the following: - **'xgb'** creates an XGBoost classifier - **'rdf'** creates a Random Forest classifier - **'erf'** creates a Forest consisting of Extra Randomized Trees - **'nn'** creates an artificial Neural Network from TheaNets - **'ada'** creates an AdaBoost instance with Decision Trees as basis - **'gb'** creates a Gradient Boosted classifier with Decision Trees as basis """ if base_estimators is None: OrderedDict(self.__DEFAULT_CLF_CFG) else: self._base_estimators = base_estimators if isinstance(stacking, str): self._clf_1 = {stacking: None} elif isinstance(stacking, dict): self._clf_1 = stacking elif stacking in (False, None): stacking = False else: self._clf_1 = {'clf_stacking': stacking} # stacking is a classifier self._transform_data = transform self._bagging = bagging_base self._hunting = hunting self._clf_1_bagging = bagging_stack self._features_stack = features_stack self._clf_0 = {} self._factory = ClassifiersFactory() self._base_scaler = None self._pred_scaler = None def get_params(self, deep=True): out = dict( base_estimators=None, bagging_base=None, stacking='xgb', features_stack=None, bagging_stack=None, hunting=False ) return out def _transform(self, X, fit=False): if self._transform_data: columns = copy.deepcopy(X.keys()) index = copy.deepcopy(X.index) if fit: self._base_scaler = StandardScaler(copy=True) self._base_scaler.fit_transform(X) else: X = self._base_scaler.transform(X) X = pd.DataFrame(X, index=index, columns=columns) return X def _transform_pred(self, X, fit=False): if self._transform_pred: columns = copy.deepcopy(X.keys()) index = copy.deepcopy(X.index) if fit: self._pred_scaler = StandardScaler(copy=True) # don't change data! self._pred_scaler.fit_transform(X) else: X = self._pred_scaler.transform(X) X = pd.DataFrame(X, index=index, columns=columns) return X def _make_clf(self, clf, bagging=None): """Creates a classifier from a dict or returns the clf""" if isinstance(clf, dict): key, val = clf.popitem() try: val = self.__DEFAULT_CLF_CFG.get(key) if val is None else val except KeyError: logger.error(str(val) + " not an implemented classifier.") raise temp_bagging = val.pop('bagging', bagging) bagging = temp_bagging if bagging is None else bagging if key == 'rdf': config_clf = dict(val) # possible multi-threading arguments clf = SklearnClassifier(RandomForestClassifier(**config_clf)) elif key == 'erf': config_clf = dict(val) # possible multi-threading arguments clf = SklearnClassifier(ExtraTreesClassifier(**config_clf)) elif key == 'nn': config_clf = dict(val) # possible multi-threading arguments clf = TheanetsClassifier(**config_clf) elif key == 'ada': config_clf = dict(val) # possible multi-threading arguments clf = SklearnClassifier(AdaBoostClassifier(**config_clf)) elif key == 'gb': config_clf = dict(val) # possible multi-threading arguments clf = SklearnClassifier(GradientBoostingClassifier(**config_clf)) elif key == 'xgb': config_clf = dict(val) # possible multi-threading arguments clf = XGBoostClassifier(**config_clf) elif hasattr(clf, 'fit'): bagging = False # return the classifier # bagging over the instantiated estimators if isinstance(bagging, int) and bagging >= 1: bagging = dict(self.__DEFAULT_BAG_CFG, n_estimators=bagging) if isinstance(bagging, dict): # TODO: implement multi-thread: bagging.update({'base_estimator': clf}) clf = SklearnClassifier(BaggingClassifier(**bagging)) else: raise ValueError(str(clf) + " not valid as a classifier.") clf = {key: clf} return clf def _factory_fit(self, X, y, sample_weight): # create classifiers from initial dictionary if self._base_estimators != {}: for key, val in self._base_estimators.items(): clf = self._make_clf({key: val}, bagging=self._bagging) self._clf_0.update(clf) self._base_estimators = {} # add base estimators to factory for key, val in self._clf_0.iteritems(): self._factory.add_classifier(key, val) # parallel on factory level -> good mixture of clfs (one uses lot of RAM, one cpu...) parallel_profile = 'threads-' + str(min([len(self._factory.items()), globals_.free_cpus()])) # fit all classifiers print "start fitting factory" self._factory.fit(X, y, sample_weight, parallel_profile=parallel_profile) return self def _factory_predict(self, X): columns = copy.deepcopy(X.keys()) index = copy.deepcopy(X.index) # parallel on factory level -> good mixture of clfs (one uses lot of RAM, one cpu...) parallel_profile = 'threads-' + str(min([len(self._factory.items()), globals_.free_cpus()])) # predict, return a dictionary predictions = self._factory.predict(X, parallel_profile=parallel_profile) # slice the arrays of predictions in the dict right for key, val in predictions.items(): predictions[key] = val[:, 1] return pd.DataFrame(predictions, index=index, columns=columns) # @profile def _factory_predict_proba(self, X): index = X.index # parallel on factory level -> good mixture of clfs (one uses lot of RAM, one cpu...) parallel_profile = 'threads-' + str(min([len(self._factory.items()), globals_.free_cpus()])) print parallel_profile parallel_profile = None # predict, return a dictionary predictions = self._factory.predict_proba(X, parallel_profile=parallel_profile) # slice the arrays of predictions in the dict right for key, val in predictions.items(): predictions[key] = val[:, 1] return pd.DataFrame(predictions, index=index) def _get_X_stack(self, X, fit_scaler=False): # get the predictions of the base estimators lvl_0_proba = pd.DataFrame(self._factory_predict_proba(X), index=X.index, columns=self._factory.keys()) lvl_0_proba = self._transform_pred(lvl_0_proba, fit=fit_scaler) # add data features to stacking data if self._features_stack is not None: if self._features_stack == 'all': self._features_stack = self.features elif not set(self._features_stack).issubset(self.features): raise RuntimeError("Stacked features not in features of the data fitted to") X_data = pd.DataFrame(X, columns=self._features_stack) lvl_0_proba = pd.concat([lvl_0_proba, X_data], axis=1, copy=False) return lvl_0_proba def _clf_1_fit(self, X, y, sample_weight): X_stack = self._get_X_stack(X, fit_scaler=True) if self._clf_1 not in (False, None): if self._clf_1.values()[0] is None or isinstance(self._clf_1.values()[0], dict): self._clf_1 = self._make_clf(self._clf_1, bagging=self._clf_1_bagging) self._clf = copy.deepcopy(self._clf_1.values()[0]) self._clf.fit(X_stack, y, sample_weight) def _set_features(self, X): """Set the 'features' attribute for the classifier""" if isinstance(X, pd.DataFrame): self.features = X.columns.values else: self.features = ["Feature_" + str(i) for i in range(X.shape[1])] def fit(self, X, y, sample_weight=None): # initiate properties self._set_features(X) self.classes_ = range(len(set(y))) X = self._transform(X, fit=True) # fit the base estimators self._factory_fit(X, y, sample_weight) # fit the stacking classifier self._clf_1_fit(X, y, sample_weight) return self def predict(self, X): # TODO: inside get_X_stack: lvl_0_proba = self._factory_predict_proba(X) X = self._transform(X) X_stack = self._get_X_stack(X) return self._clf.predict(X_stack) def predict_proba(self, X): X = self._transform(X) X_stack = self._get_X_stack(X) return self._clf.predict_proba(X_stack) def test_on(self, X, y, sample_weight=None): lds = LabeledDataStorage(X, y, sample_weight) return self.test_on_lds(lds) def test_on_lds(self, lds): lds.data = self._transform(lds.data) return ClassificationReport({'Mayou clf': self}, lds) def staged_predict_proba(self, X): X = self._transform(X) X_stack = self._get_X_stack(X) try: temp_proba = self._clf.staged_predict_proba(X_stack) # TODO: change error catching except: print "error occured in mayou, staged predict proba not supported" return temp_proba def stacker_test_on_lds(self, lds): """Return report for the stacker only""" lds.data = self._get_X_stack(self._transform(lds.data)) return ClassificationReport({'Mayou stacker': self._clf}, lds) def stacker_test_on(self, X, y, sample_weight=None): """Return report for the stacker only""" lds = LabeledDataStorage(X, y, sample_weight) return self.stacker_test_on_lds(lds)