def test_getMetricWithMeasure(self): metric = Metric('random forest') metric.addValue('accuracy', 0.75) metric.addValue('recall', 0.9) metric.addValue('precision', 0.2) metric.addValue('acc', 0.98) metric.addValue('test_value', 500) actual_result = metric.getMetricWithMeasure() expected_result = {'accuracy': 0.75, 'recall': 0.9, 'precision': 0.2, 'acc': 0.98, 'test_value': 500} self.assertEqual( actual_result, expected_result ) actual_result = metric.getMetricWithMeasure('acc') expected_result = {'acc': 0.98} self.assertEqual( actual_result, expected_result ) actual_result = metric.getMetricWithMeasure(['recall', 'acc', 'test_value']) expected_result = {'recall': 0.9, 'acc': 0.98, 'test_value': 500} self.assertEqual( actual_result, expected_result ) try: actual_result = metric.getMetricWithMeasure( 480 ) fail(self) except ValueError as ve: self.assertEqual( str(ve), 'Metric.getMetricWithMeasure must be given either a string of metric or array of strings of metrics desired.' )
def test_eval(self): iris = load_iris() X_train, y_train = iris.data[:120], iris.target[:120] X_test, y_test = iris.data[120:], iris.target[120:] rf = Model('rf', RandomForestClassifier, {'random_state':0}) rf.run(X_train, y_train) metrics = rf.eval(X_test, y_test, metrics=['acc']) rf_metric = Metric('rf') rf_metric.addValue('acc', 0.7666666666666667) expected_metrics = rf_metric.getValues() self.assertEqual( metrics, expected_metrics )
def eval(self, X, y, metrics=[], fold_num=None): if len(X) == len(y): # If we aren't running cross validation, then clear out the metrics if not fold_num: self.metrics_manager.clearMetrics() for model_name in self.models_dict.keys(): model = self.models_dict[model_name] if fold_num and type(fold_num) == int: metric_object = Metric(model_name, fold=fold_num) else: metric_object = Metric(model_name) # If a ROC curve is requested, graph it! if 'roc' in metrics or 'ROC' in metrics: fpr, tpr, roc_auc = model.calcROC(X, y, self.type_of_experiment) self.metrics_manager.addToROCCurve(model_name, fpr, tpr, roc_auc) model_metrics = model.eval(X, y, metrics) for measure_name in model_metrics.keys(): metric_object.addValue(measure_name, model_metrics[measure_name]) self.metrics_manager.addMetric(metric_object) if not fold_num: return self.metrics_manager.getMetrics() else: if self.name: raise ValueError( 'Data and target provided to \'{}\' must be same length:\n\tlen of data: {}\n\tlen of target: {}' .format(self.name, str(len(X)), str(len(y)))) else: raise ValueError( 'Provided data and target must be same length:\n\tlen of data: {}\n\tlen of target: {}' .format(str(len(X)), str(len(y))))
def test_getMetrics(self): metrics_manager = MetricsManager() metric1 = Metric('metric1') metric1.addValue('acc', 0.48) metric1.addValue('recall', 0.3) metric1.addValue('precision', 0.7) metric2 = Metric('metric2') metric2.addValue('acc', 0.98) metric2.addValue('precision', 0.42) metric3 = Metric('metric3') metric3.addValue('recall', 0.34) metric3.addValue('precision', 0) metric4 = Metric('metric4') metric4.addValue('acc', 0.8) metric4.addValue('recall', 0.35) metric5 = Metric('metric5') metric5.addValue('acc', 0.14) metric5.addValue('recall', 0.03) metric5.addValue('precision', 0.71) metric6 = Metric('metric6') metric6.addValue('acc', 0.92) metric6.addValue('recall', 1.0) metric6.addValue('precision', 0.63) metrics_manager.addMetric(metric1) metrics_manager.addMetric(metric2) metrics_manager.addMetric(metric3) metrics_manager.addMetric(metric4) metrics_manager.addMetric(metric5) metrics_manager.addMetric(metric6) actual_result_metrics = metrics_manager.getMetrics() expected_result_metrics = [metric1, metric2, metric3, metric4, metric5, metric6] self.assertListEqual( actual_result_metrics, expected_result_metrics ) metrics_manager = MetricsManager() metric1 = Metric('metric1') metric1.addValue('acc', 0.48) metric1.addValue('recall', 0.3) metric1.addValue('precision', 0.7) metric2 = Metric('metric2') metric2.addValue('acc', 0.98) metric2.addValue('precision', 0.42) metric3 = Metric('metric3') metric3.addValue('recall', 0.34) metric3.addValue('precision', 0) metric4 = Metric('metric4') metric4.addValue('acc', 0.8) metric4.addValue('recall', 0.35) metric5 = Metric('metric5') metric5.addValue('acc', 0.14) metric5.addValue('recall', 0.03) metric5.addValue('precision', 0.71) metric6 = Metric('metric6') metric6.addValue('acc', 0.92) metric6.addValue('recall', 1.0) metric6.addValue('precision', 0.63) metrics_manager.addMetric(metric1) metrics_manager.addMetric(metric2) metrics_manager.addMetric(metric3) metrics_manager.addMetric(metric4) metrics_manager.addMetric(metric5) metrics_manager.addMetric(metric6) # Metric 3 will be different! metric3 = Metric('metric3') metric3.addValue('recall', 0.68) metric3.addValue('precision', 0.74) actual_result_metrics = metrics_manager.getMetrics() expected_result_metrics = [metric1, metric2, metric3, metric4, metric5, metric6] self.assertNotEqual( actual_result_metrics, expected_result_metrics )
def test_equals(self): metric1 = Metric('some name') metric1.addValue('acc', 0.98) metric1.addValue('recall', 0.72) metric2 = Metric('some other name') metric2.addValue('acc', 0.98) metric2.addValue('recall', 0.72) self.assertEqual( metric1, metric2 ) metric1 = Metric('some name') metric1.addValue('acc', 0.98) metric1.addValue('recall', 0.72) metric2 = Metric('some other name') metric2.addValue('acc', 0.20) metric2.addValue('recall', 0.5) self.assertNotEqual( metric1, metric2 )
def test_addValue(self): metric = Metric('random forest') metric.addValue('accuracy', 0.75) self.assertEqual( metric.getValue('accuracy'), 0.75 ) metric.addValue('recall', 0.9) metric.addValue('precision', 0.2) self.assertEqual( metric.getValue('accuracy'), 0.75 ) self.assertEqual( metric.getValue('recall'), 0.9 ) self.assertEqual( metric.getValue('precision'), 0.2 ) metric.addValue('accuracy', 0.99) metric.addValue('accuracy', 0.3) metric.addValue('accuracy', 0.25672837482) self.assertEqual( metric.getValue('accuracy'), 0.25672837482 ) try: metric.addValue('accuracy', ['hello!']) fail(self) except ValueError as ve: self.assertEqual( str(ve), 'Metric.addValue must have \'m_type\' as string and \'value\' as integer or floating point number instead of type(m_type) => <class \'str\'> and type(value) => <class \'list\'>') try: metric.addValue(['accuracy'], 0.9) fail(self) except ValueError as ve: self.assertEqual( str(ve), 'Metric.addValue must have \'m_type\' as string and \'value\' as integer or floating point number instead of type(m_type) => <class \'list\'> and type(value) => <class \'float\'>')
def test_eval(self): iris = load_iris() X_train, y_train = iris.data[:120], iris.target[:120] X_test, y_test = iris.data[120:], iris.target[120:] experiment = Experiment('exp1', models=['rf', 'dt'], exp_type='classification') experiment.train(X_train, y_train) metrics = experiment.eval(X_test, y_test, metrics=['acc']) rf_metric = Metric('rf') rf_metric.addValue('acc', 0.7666666666666667) dt_metric = Metric('dt') dt_metric.addValue('acc', 0.8) expected_metrics = [rf_metric, dt_metric] self.assertEqual( metrics, expected_metrics ) metrics = experiment.eval(X_test, y_test, metrics='acc') expected_metrics = [rf_metric, dt_metric] self.assertEqual( metrics, expected_metrics ) # 10-fold cross validation test X = iris.data y = iris.target experiment = Experiment('exp1', models=['rf', 'dt'], exp_type='classification') experiment.train(X, y, cv=True, n_folds=10, shuffle=True, metrics=['acc', 'rec', 'prec']) experiment.setRandomState(0) rf_metric = Metric('rf') rf_metric.addValue('acc_avg', 0.9400000000000001) rf_metric.addValue('acc_std', 0.04666666666666665) rf_metric.addValue('rec_avg', 0.9400000000000001) rf_metric.addValue('rec_std', 0.04666666666666665) rf_metric.addValue('prec_avg', 0.9531746031746031) rf_metric.addValue('prec_std', 0.03412698412698411) dt_metric = Metric('dt') dt_metric.addValue('acc_avg', 0.9400000000000001) dt_metric.addValue('acc_std', 0.05537749241945382) dt_metric.addValue('rec_avg', 0.9400000000000001) dt_metric.addValue('rec_std', 0.05537749241945382) dt_metric.addValue('prec_avg', 0.9541666666666668) dt_metric.addValue('prec_std', 0.036244412085277455) expected_metrics = {'rf': rf_metric, 'dt': dt_metric} metrics = experiment.getSummarizedMetrics() self.assertEqual( metrics, expected_metrics ) try: experiment.train(X, y, cv=True, n_folds=0, shuffle=True, metrics=['acc', 'rec', 'prec']) fail(self) except ValueError as ve: self.assertEqual( str(ve), 'Number of folds in cross validation (n_folds) must be larger than zero!' ) try: experiment.train(X, y, cv=True, n_folds=-1, shuffle=True, metrics=['acc', 'rec', 'prec']) fail(self) except ValueError as ve: self.assertEqual( str(ve), 'Number of folds in cross validation (n_folds) must be larger than zero!' )