def test_getMetricWithMeasure(self):
metric = Metric('random forest')
metric.addValue('accuracy', 0.75)
metric.addValue('recall', 0.9)
metric.addValue('precision', 0.2)
metric.addValue('acc', 0.98)
metric.addValue('test_value', 500)
actual_result = metric.getMetricWithMeasure()
expected_result = {'accuracy': 0.75, 'recall': 0.9, 'precision': 0.2, 'acc': 0.98, 'test_value': 500}
self.assertEqual( actual_result, expected_result )
actual_result = metric.getMetricWithMeasure('acc')
expected_result = {'acc': 0.98}
self.assertEqual( actual_result, expected_result )
actual_result = metric.getMetricWithMeasure(['recall', 'acc', 'test_value'])
expected_result = {'recall': 0.9, 'acc': 0.98, 'test_value': 500}
self.assertEqual( actual_result, expected_result )
try:
actual_result = metric.getMetricWithMeasure( 480 )
fail(self)
except ValueError as ve:
self.assertEqual( str(ve), 'Metric.getMetricWithMeasure must be given either a string of metric or array of strings of metrics desired.' )
def test_eval(self):
iris = load_iris()
X_train, y_train = iris.data[:120], iris.target[:120]
X_test, y_test = iris.data[120:], iris.target[120:]
rf = Model('rf', RandomForestClassifier, {'random_state':0})
rf.run(X_train, y_train)
metrics = rf.eval(X_test, y_test, metrics=['acc'])
rf_metric = Metric('rf')
rf_metric.addValue('acc', 0.7666666666666667)
expected_metrics = rf_metric.getValues()
self.assertEqual( metrics, expected_metrics )
def eval(self, X, y, metrics=[], fold_num=None):
if len(X) == len(y):
# If we aren't running cross validation, then clear out the metrics
if not fold_num:
self.metrics_manager.clearMetrics()
for model_name in self.models_dict.keys():
model = self.models_dict[model_name]
if fold_num and type(fold_num) == int:
metric_object = Metric(model_name, fold=fold_num)
else:
metric_object = Metric(model_name)
# If a ROC curve is requested, graph it!
if 'roc' in metrics or 'ROC' in metrics:
fpr, tpr, roc_auc = model.calcROC(X, y,
self.type_of_experiment)
self.metrics_manager.addToROCCurve(model_name, fpr, tpr,
roc_auc)
model_metrics = model.eval(X, y, metrics)
for measure_name in model_metrics.keys():
metric_object.addValue(measure_name,
model_metrics[measure_name])
self.metrics_manager.addMetric(metric_object)
if not fold_num:
return self.metrics_manager.getMetrics()
else:
if self.name:
raise ValueError(
'Data and target provided to \'{}\' must be same length:\n\tlen of data: {}\n\tlen of target: {}'
.format(self.name, str(len(X)), str(len(y))))
else:
raise ValueError(
'Provided data and target must be same length:\n\tlen of data: {}\n\tlen of target: {}'
.format(str(len(X)), str(len(y))))
def test_getMetrics(self):
metrics_manager = MetricsManager()
metric1 = Metric('metric1')
metric1.addValue('acc', 0.48)
metric1.addValue('recall', 0.3)
metric1.addValue('precision', 0.7)
metric2 = Metric('metric2')
metric2.addValue('acc', 0.98)
metric2.addValue('precision', 0.42)
metric3 = Metric('metric3')
metric3.addValue('recall', 0.34)
metric3.addValue('precision', 0)
metric4 = Metric('metric4')
metric4.addValue('acc', 0.8)
metric4.addValue('recall', 0.35)
metric5 = Metric('metric5')
metric5.addValue('acc', 0.14)
metric5.addValue('recall', 0.03)
metric5.addValue('precision', 0.71)
metric6 = Metric('metric6')
metric6.addValue('acc', 0.92)
metric6.addValue('recall', 1.0)
metric6.addValue('precision', 0.63)
metrics_manager.addMetric(metric1)
metrics_manager.addMetric(metric2)
metrics_manager.addMetric(metric3)
metrics_manager.addMetric(metric4)
metrics_manager.addMetric(metric5)
metrics_manager.addMetric(metric6)
actual_result_metrics = metrics_manager.getMetrics()
expected_result_metrics = [metric1, metric2, metric3, metric4, metric5, metric6]
self.assertListEqual( actual_result_metrics, expected_result_metrics )
metrics_manager = MetricsManager()
metric1 = Metric('metric1')
metric1.addValue('acc', 0.48)
metric1.addValue('recall', 0.3)
metric1.addValue('precision', 0.7)
metric2 = Metric('metric2')
metric2.addValue('acc', 0.98)
metric2.addValue('precision', 0.42)
metric3 = Metric('metric3')
metric3.addValue('recall', 0.34)
metric3.addValue('precision', 0)
metric4 = Metric('metric4')
metric4.addValue('acc', 0.8)
metric4.addValue('recall', 0.35)
metric5 = Metric('metric5')
metric5.addValue('acc', 0.14)
metric5.addValue('recall', 0.03)
metric5.addValue('precision', 0.71)
metric6 = Metric('metric6')
metric6.addValue('acc', 0.92)
metric6.addValue('recall', 1.0)
metric6.addValue('precision', 0.63)
metrics_manager.addMetric(metric1)
metrics_manager.addMetric(metric2)
metrics_manager.addMetric(metric3)
metrics_manager.addMetric(metric4)
metrics_manager.addMetric(metric5)
metrics_manager.addMetric(metric6)
# Metric 3 will be different!
metric3 = Metric('metric3')
metric3.addValue('recall', 0.68)
metric3.addValue('precision', 0.74)
actual_result_metrics = metrics_manager.getMetrics()
expected_result_metrics = [metric1, metric2, metric3, metric4, metric5, metric6]
self.assertNotEqual( actual_result_metrics, expected_result_metrics )
def test_equals(self):
metric1 = Metric('some name')
metric1.addValue('acc', 0.98)
metric1.addValue('recall', 0.72)
metric2 = Metric('some other name')
metric2.addValue('acc', 0.98)
metric2.addValue('recall', 0.72)
self.assertEqual( metric1, metric2 )
metric1 = Metric('some name')
metric1.addValue('acc', 0.98)
metric1.addValue('recall', 0.72)
metric2 = Metric('some other name')
metric2.addValue('acc', 0.20)
metric2.addValue('recall', 0.5)
self.assertNotEqual( metric1, metric2 )
def test_addValue(self):
metric = Metric('random forest')
metric.addValue('accuracy', 0.75)
self.assertEqual( metric.getValue('accuracy'), 0.75 )
metric.addValue('recall', 0.9)
metric.addValue('precision', 0.2)
self.assertEqual( metric.getValue('accuracy'), 0.75 )
self.assertEqual( metric.getValue('recall'), 0.9 )
self.assertEqual( metric.getValue('precision'), 0.2 )
metric.addValue('accuracy', 0.99)
metric.addValue('accuracy', 0.3)
metric.addValue('accuracy', 0.25672837482)
self.assertEqual( metric.getValue('accuracy'), 0.25672837482 )
try:
metric.addValue('accuracy', ['hello!'])
fail(self)
except ValueError as ve:
self.assertEqual( str(ve), 'Metric.addValue must have \'m_type\' as string and \'value\' as integer or floating point number instead of type(m_type) => <class \'str\'> and type(value) => <class \'list\'>')
try:
metric.addValue(['accuracy'], 0.9)
fail(self)
except ValueError as ve:
self.assertEqual( str(ve), 'Metric.addValue must have \'m_type\' as string and \'value\' as integer or floating point number instead of type(m_type) => <class \'list\'> and type(value) => <class \'float\'>')
def test_eval(self):
iris = load_iris()
X_train, y_train = iris.data[:120], iris.target[:120]
X_test, y_test = iris.data[120:], iris.target[120:]
experiment = Experiment('exp1', models=['rf', 'dt'], exp_type='classification')
experiment.train(X_train, y_train)
metrics = experiment.eval(X_test, y_test, metrics=['acc'])
rf_metric = Metric('rf')
rf_metric.addValue('acc', 0.7666666666666667)
dt_metric = Metric('dt')
dt_metric.addValue('acc', 0.8)
expected_metrics = [rf_metric, dt_metric]
self.assertEqual( metrics, expected_metrics )
metrics = experiment.eval(X_test, y_test, metrics='acc')
expected_metrics = [rf_metric, dt_metric]
self.assertEqual( metrics, expected_metrics )
# 10-fold cross validation test
X = iris.data
y = iris.target
experiment = Experiment('exp1', models=['rf', 'dt'], exp_type='classification')
experiment.train(X, y, cv=True, n_folds=10, shuffle=True, metrics=['acc', 'rec', 'prec'])
experiment.setRandomState(0)
rf_metric = Metric('rf')
rf_metric.addValue('acc_avg', 0.9400000000000001)
rf_metric.addValue('acc_std', 0.04666666666666665)
rf_metric.addValue('rec_avg', 0.9400000000000001)
rf_metric.addValue('rec_std', 0.04666666666666665)
rf_metric.addValue('prec_avg', 0.9531746031746031)
rf_metric.addValue('prec_std', 0.03412698412698411)
dt_metric = Metric('dt')
dt_metric.addValue('acc_avg', 0.9400000000000001)
dt_metric.addValue('acc_std', 0.05537749241945382)
dt_metric.addValue('rec_avg', 0.9400000000000001)
dt_metric.addValue('rec_std', 0.05537749241945382)
dt_metric.addValue('prec_avg', 0.9541666666666668)
dt_metric.addValue('prec_std', 0.036244412085277455)
expected_metrics = {'rf': rf_metric, 'dt': dt_metric}
metrics = experiment.getSummarizedMetrics()
self.assertEqual( metrics, expected_metrics )
try:
experiment.train(X, y, cv=True, n_folds=0, shuffle=True, metrics=['acc', 'rec', 'prec'])
fail(self)
except ValueError as ve:
self.assertEqual( str(ve), 'Number of folds in cross validation (n_folds) must be larger than zero!' )
try:
experiment.train(X, y, cv=True, n_folds=-1, shuffle=True, metrics=['acc', 'rec', 'prec'])
fail(self)
except ValueError as ve:
self.assertEqual( str(ve), 'Number of folds in cross validation (n_folds) must be larger than zero!' )
