def test_evaluate_dataset_incomplete(self, create_job):
"""Check that if the validation of a dataset fails it is not added."""
# We just assume that the validation fails because there is no
# low-level data for the recordings in this dataset
with self.assertRaises(dataset_eval.IncompleteDatasetException):
dataset_eval.evaluate_dataset(self.test_dataset_id, False, 'local')
create_job.assert_not_called()
def test_evaluate_dataset_incomplete(self, create_job):
"""Check that if the validation of a dataset fails it is not added."""
# We just assume that the validation fails because there is no
# low-level data for the recordings in this dataset
with self.assertRaises(dataset_eval.IncompleteDatasetException):
dataset_eval.evaluate_dataset(self.test_dataset_id, False, 'local')
create_job.assert_not_called()
def test_view_json(self):
resp = self.client.get(url_for("datasets.view_json", id=self.test_uuid))
self.assert404(resp)
dataset_id = dataset.create_from_dict(self.test_data, author_id=self.test_user_id)
resp = self.client.get(url_for("datasets.view_json", id=dataset_id))
self.assert200(resp)
dataset_eval.evaluate_dataset(dataset_id, False, dataset_eval.EVAL_LOCAL)
self.temporary_login(self.test_user_id)
def test_evaluate_dataset(self, validate_dataset_contents):
"""Test that a dataset can be submitted for evaluation if it is complete"""
dataset_eval.evaluate_dataset(self.test_dataset_id, False, 'local')
self.assertTrue(dataset_eval.job_exists(self.test_dataset_id))
# Evaluate a second time it will raise
with self.assertRaises(dataset_eval.JobExistsException):
dataset_eval.evaluate_dataset(self.test_dataset_id, False, 'local')
# We validate after checking if the dataset exists, so this will only
# be called once
validate_dataset_contents.assert_called_once()
def test_view_json(self):
resp = self.client.get(url_for("datasets.view_json",
id=self.test_uuid))
self.assert404(resp)
dataset_id = dataset.create_from_dict(self.test_data,
author_id=self.test_user_id)
resp = self.client.get(url_for("datasets.view_json", id=dataset_id))
self.assert200(resp)
dataset_eval.evaluate_dataset(dataset_id, False)
self.temporary_login(self.test_user_id)
def test_evaluate_dataset(self, validate_dataset_contents):
"""Test that a dataset can be submitted for evaluation if it is complete"""
dataset_eval.evaluate_dataset(self.test_dataset_id, False, 'local')
self.assertTrue(dataset_eval.job_exists(self.test_dataset_id))
# Evaluate a second time it will raise
with self.assertRaises(dataset_eval.JobExistsException):
dataset_eval.evaluate_dataset(self.test_dataset_id, False, 'local')
# We validate after checking if the dataset exists, so this will only
# be called once
validate_dataset_contents.assert_called_once()
def test_eval_job_delete(self):
resp = self.client.delete(
url_for("datasets.eval_job",
dataset_id=self.test_uuid,
job_id=self.test_uuid))
self.assert404(resp)
dataset_id = dataset.create_from_dict(self.test_data,
author_id=self.test_user_id)
resp = self.client.delete(
url_for("datasets.eval_job",
dataset_id=dataset_id,
job_id=self.test_uuid))
self.assert404(resp)
job_id = dataset_eval.evaluate_dataset(dataset_id, False,
dataset_eval.EVAL_LOCAL)
resp = self.client.delete(
url_for("datasets.eval_job", dataset_id=dataset_id, job_id=job_id))
self.assert401(resp)
self.assertEqual(
resp.json, {
'success': False,
'error': 'You are not allowed to delete this evaluation job.'
})
# As an author
self.temporary_login(self.test_user_id)
resp = self.client.delete(
url_for("datasets.eval_job", dataset_id=dataset_id, job_id=job_id))
self.assert200(resp)
self.assertIsNone(dataset_eval.get_job(job_id))
def test_view_json(self):
resp = self.client.get(
url_for("datasets.view_json", dataset_id=self.test_uuid))
self.assert404(resp)
dataset_id = dataset.create_from_dict(self.test_data,
author_id=self.test_user_id)
resp = self.client.get(
url_for("datasets.view_json", dataset_id=dataset_id))
self.assert200(resp)
dataset_eval.evaluate_dataset(
dataset_id, False, dataset_eval.EVAL_LOCAL,
dataset_eval.DEFAULT_PARAMETER_C,
dataset_eval.DEFAULT_PARAMETER_GAMMA,
dataset_eval.DEFAULT_PARAMETER_PREPROCESSING)
self.temporary_login(self.test_user_id)
def test_eval_job_delete(self):
resp = self.client.delete("/datasets/%s/%s" % (self.test_uuid, self.test_uuid))
self.assert404(resp)
dataset_id = dataset.create_from_dict(self.test_data, author_id=self.test_user_id)
resp = self.client.delete("/datasets/%s/%s" % (dataset_id, self.test_uuid))
self.assert404(resp)
job_id = dataset_eval.evaluate_dataset(dataset_id, False, dataset_eval.EVAL_LOCAL)
resp = self.client.delete("/datasets/%s/%s" % (dataset_id, job_id))
self.assert401(resp)
# As an author
self.temporary_login(self.test_user_id)
resp = self.client.delete("/datasets/%s/%s" % (dataset_id, job_id))
self.assert200(resp)
self.assertIsNone(dataset_eval.get_job(job_id))
def test_eval_job_delete(self):
resp = self.client.delete(url_for("datasets.eval_job", dataset_id=self.test_uuid, job_id=self.test_uuid))
self.assert404(resp)
dataset_id = dataset.create_from_dict(self.test_data, author_id=self.test_user_id)
resp = self.client.delete(url_for("datasets.eval_job", dataset_id=dataset_id, job_id=self.test_uuid))
self.assert404(resp)
job_id = dataset_eval.evaluate_dataset(dataset_id, False, dataset_eval.EVAL_LOCAL)
resp = self.client.delete(url_for("datasets.eval_job", dataset_id=dataset_id, job_id=job_id))
self.assert401(resp)
self.assertEqual(resp.json, {'success': False, 'error': 'You are not allowed to delete this evaluation job.'})
# As an author
self.temporary_login(self.test_user_id)
resp = self.client.delete(url_for("datasets.eval_job", dataset_id=dataset_id, job_id=job_id))
self.assert200(resp)
self.assertIsNone(dataset_eval.get_job(job_id))
def test_eval_job_delete(self):
resp = self.client.delete("/datasets/%s/%s" %
(self.test_uuid, self.test_uuid))
self.assert404(resp)
dataset_id = dataset.create_from_dict(self.test_data,
author_id=self.test_user_id)
resp = self.client.delete("/datasets/%s/%s" %
(dataset_id, self.test_uuid))
self.assert404(resp)
job_id = dataset_eval.evaluate_dataset(dataset_id, False)
resp = self.client.delete("/datasets/%s/%s" % (dataset_id, job_id))
self.assert401(resp)
# As an author
self.temporary_login(self.test_user_id)
resp = self.client.delete("/datasets/%s/%s" % (dataset_id, job_id))
self.assert200(resp)
self.assertIsNone(dataset_eval.get_job(job_id))
