def test_evaluation_list_per_fold(self): openml.config.server = self.production_server size = 1000 task_ids = [6] uploader_ids = [1] flow_ids = [6969] evaluations = openml.evaluations.list_evaluations( "predictive_accuracy", size=size, offset=0, task=task_ids, flow=flow_ids, uploader=uploader_ids, per_fold=True) self.assertEqual(len(evaluations), size) for run_id in evaluations.keys(): self.assertIsNone(evaluations[run_id].value) self.assertIsNotNone(evaluations[run_id].values) # potentially we could also test array values, but these might be # added in the future evaluations = openml.evaluations.list_evaluations( "predictive_accuracy", size=size, offset=0, task=task_ids, flow=flow_ids, uploader=uploader_ids, per_fold=False) for run_id in evaluations.keys(): self.assertIsNotNone(evaluations[run_id].value) self.assertIsNone(evaluations[run_id].values)
def test_evaluation_list_filter_task(self): openml.config.server = self.production_server task_id = 7312 evaluations = openml.evaluations.list_evaluations("predictive_accuracy", task=[task_id]) self.assertGreater(len(evaluations), 100) for run_id in evaluations.keys(): self.assertEquals(evaluations[run_id].task_id, task_id)
def test_evaluation_list_filter_run(self): openml.config.server = self.production_server run_id = 1 evaluations = openml.evaluations.list_evaluations("predictive_accuracy", id=[run_id]) self.assertEquals(len(evaluations), 1) for run_id in evaluations.keys(): self.assertEquals(evaluations[run_id].run_id, run_id)
def test_evaluation_list_filter_flow(self): openml.config.server = self.production_server flow_id = 100 evaluations = openml.evaluations.list_evaluations("predictive_accuracy", flow=[flow_id]) self.assertGreater(len(evaluations), 2) for run_id in evaluations.keys(): self.assertEquals(evaluations[run_id].flow_id, flow_id)
def test_evaluation_list_filter_uploader_ID_10(self): openml.config.server = self.production_server setup_id = 10 evaluations = openml.evaluations.list_evaluations("predictive_accuracy", setup=[setup_id]) self.assertGreater(len(evaluations), 100) for run_id in evaluations.keys(): self.assertEquals(evaluations[run_id].setup_id, setup_id)
def test_evaluation_list_filter_uploader_ID_10(self): openml.config.server = self.production_server setup_id = 10 evaluations = openml.evaluations.list_evaluations( "predictive_accuracy", size=60, setups=[setup_id]) self.assertGreater(len(evaluations), 50) for run_id in evaluations.keys(): self.assertEqual(evaluations[run_id].setup_id, setup_id) # default behaviour of this method: return aggregated results (not # per fold) self.assertIsNotNone(evaluations[run_id].value) self.assertIsNone(evaluations[run_id].values)
def test_evaluation_list_filter_task(self): openml.config.server = self.production_server task_id = 7312 evaluations = openml.evaluations.list_evaluations("predictive_accuracy", task=[task_id]) self.assertGreater(len(evaluations), 100) for run_id in evaluations.keys(): self.assertEquals(evaluations[run_id].task_id, task_id) # default behaviour of this method: return aggregated results (not # per fold) self.assertIsNotNone(evaluations[run_id].value) self.assertIsNone(evaluations[run_id].values)
def test_evaluation_list_filter_uploader_ID_10(self): openml.config.server = self.production_server setup_id = 10 evaluations = openml.evaluations.list_evaluations("predictive_accuracy", setup=[setup_id]) self.assertGreater(len(evaluations), 50) for run_id in evaluations.keys(): self.assertEquals(evaluations[run_id].setup_id, setup_id) # default behaviour of this method: return aggregated results (not # per fold) self.assertIsNotNone(evaluations[run_id].value) self.assertIsNone(evaluations[run_id].values)
def test_evaluation_list_filter_run(self): openml.config.server = self.production_server run_id = 12 evaluations = openml.evaluations.list_evaluations( "predictive_accuracy", size=2, runs=[run_id]) self.assertEqual(len(evaluations), 1) for run_id in evaluations.keys(): self.assertEqual(evaluations[run_id].run_id, run_id) # default behaviour of this method: return aggregated results (not # per fold) self.assertIsNotNone(evaluations[run_id].value) self.assertIsNone(evaluations[run_id].values)
def test_evaluation_list_per_fold(self): openml.config.server = self.production_server size = 1000 task_ids = [6] uploader_ids = [1] flow_ids = [6969] evaluations = openml.evaluations.list_evaluations( "predictive_accuracy", size=size, offset=0, task=task_ids, flow=flow_ids, uploader=uploader_ids, per_fold=True) self.assertEquals(len(evaluations), size) for run_id in evaluations.keys(): self.assertIsNone(evaluations[run_id].value) self.assertIsNotNone(evaluations[run_id].values) # potentially we could also test array values, but these might be # added in the future evaluations = openml.evaluations.list_evaluations( "predictive_accuracy", size=size, offset=0, task=task_ids, flow=flow_ids, uploader=uploader_ids, per_fold=False) for run_id in evaluations.keys(): self.assertIsNotNone(evaluations[run_id].value) self.assertIsNone(evaluations[run_id].values)