def test_create_result_report(self):
metrics = [multiclass_recall, multiclass_precision, multiclass_f1]
r = ResultReport(approach=Approach.load(self.approach.id),
metrics=metrics)
df = r.as_dataframe()
for m in [f.__name__ for f in metrics]:
self.assertTrue(m in df.columns)
def test_create_runpool(self):
# Force reload runs from database
self.approach = Approach.load(self.approach.id)
runpool = RunPool(self.approach.runs)
for run in runpool.iteruns():
self.assertTrue(isinstance(run, Run))
def test_iterate_all_runs_runpool(self):
self.approach = Approach.load(self.approach.id)
runpool = RunPool(self.approach.runs)
i = 0
for run in runpool.iteruns():
self.assertTrue(isinstance(run, Run))
run.status = "finished"
i += 1
self.assertEqual(i, runpool.iter)
def status(approach_id):
if not _is_running_in_project():
print("You must use driftai CLI inside an driftai project directory")
return
print("Loading approach data...")
stat = Approach.load(approach_id).status
if not stat["done"]:
print("Approach {} is still running".format(approach_id))
print(stat["progress_bar"] + " Done runs: " + str(stat["done_runs"]) + " Total runs: " + str(stat["total_runs"]))
else:
print("There are no left runs for Approach {approach_id}!".format(approach_id))
def test_get_subdataset_runs(self):
runnable = import_from("test.lr.logistic_regression",
"LogisticRegressionApproach")
runs = RunGenerator.from_runnable_approach(runnable())
for run in runs:
run.save()
runs = Approach.load(self.approach.id).runs
self.assertTrue(len(runs) > 0)
for run in runs:
self.assertIsInstance(run, Run)
return runs
def evaluate(approach_id, metric):
if not _is_running_in_project():
print("You must use driftai CLI inside an driftai project directory")
return
if not Approach.collection().exists(approach_id):
print("Approach with id {} does not exist.".format(approach_id))
return
approach = Approach.load(approach_id)
r = ResultReport(approach=approach, metrics=[str_to_metric_fn[m] for m in metric])
r.as_dataframe()\
.to_csv(approach_id + "_evaluation.csv", index=False)
def test_generate_runs_from_subdataset(self):
# Trick to load runnable approach
LogisticRegressionApproach = import_from("test.lr.logistic_regression",
"LogisticRegressionApproach")
ra = LogisticRegressionApproach()
# Generate the runs
run_gens = RunGenerator.from_runnable_approach(ra)
# Write runs to database
ra.approach.runs = run_gens
ra.approach.update()
# Reload approach to test if runs were correctly stored
approach = Approach.load(ra.approach.id)
self.assertEqual(len(approach.runs), len(run_gens))
def test_using_sklearn_metrics(self):
from sklearn.metrics import classification_report
r = ResultReport(approach=Approach.load(self.approach.id),
metrics=[classification_report])
df = r.as_dataframe()
self.assertIsNotNone(df.classification_report[0])
def test_create_result_report(self):
metrics = ["recall", "precision", "f1"]
r = ResultReport(approach=Approach.load(self.approach.id),
metrics=[recall, precision, f1])
df = r.as_dataframe()
self.assertTrue(all(m in df.columns for m in metrics))