def test_get_job(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(type(job), dict)
self.assertIsNone(dataset_eval.get_job("f47ac10b-58cc-4372-a567-0e02b2c3d479"))
def test_get_next_pending_job(self):
job1_id = dataset_eval._create_job(self.conn,
self.test_dataset_id,
True,
dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3],
gamma_value=[4, 5, 6],
preprocessing_values=["basic"],
filter_type=None)
job1 = dataset_eval.get_job(job1_id)
job2_id = dataset_eval._create_job(self.conn,
self.test_dataset_id,
True,
dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3],
gamma_value=[4, 5, 6],
preprocessing_values=["basic"],
filter_type=None)
job2 = dataset_eval.get_job(job2_id)
next_pending = dataset_eval.get_next_pending_job()
self.assertEqual(job1, next_pending)
dataset_eval.set_job_status(
job_id=job1_id,
status=dataset_eval.STATUS_FAILED,
)
next_pending = dataset_eval.get_next_pending_job()
self.assertEqual(job2, next_pending)
def test_eval_job_location(self):
job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE, None)
job1 = dataset_eval.get_job(job1_id)
self.assertEqual(job1["eval_location"], dataset_eval.EVAL_REMOTE)
job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, None)
job2 = dataset_eval.get_job(job2_id)
self.assertEqual(job2["eval_location"], dataset_eval.EVAL_LOCAL)
def test_delete_job(self):
with self.assertRaises(dataset_eval.JobNotFoundException):
dataset_eval.delete_job(self.test_uuid)
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True)
self.assertIsNotNone(dataset_eval.get_job(job_id))
dataset_eval.delete_job(job_id)
self.assertIsNone(dataset_eval.get_job(job_id))
def test_eval_job_location(self):
job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE, None)
job1 = dataset_eval.get_job(job1_id)
self.assertEqual(job1["eval_location"], dataset_eval.EVAL_REMOTE)
job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, None)
job2 = dataset_eval.get_job(job2_id)
self.assertEqual(job2["eval_location"], dataset_eval.EVAL_LOCAL)
def test_get_next_pending_job_remote(self):
# If we have a remote pending job with the most recent timestamp, skip it
job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE)
job1 = dataset_eval.get_job(job1_id)
job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
job2 = dataset_eval.get_job(job2_id)
next_pending = dataset_eval.get_next_pending_job()
self.assertEqual(job2, next_pending)
def test_get_next_pending_job_remote(self):
# If we have a remote pending job with the most recent timestamp, skip it
job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE)
job1 = dataset_eval.get_job(job1_id)
job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
job2 = dataset_eval.get_job(job2_id)
next_pending = dataset_eval.get_next_pending_job()
self.assertEqual(job2, next_pending)
def test_set_job_status(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id)
job = dataset_eval.get_job(job_id)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
dataset_eval.set_job_status(
job_id=job_id,
status=dataset_eval.STATUS_FAILED,
)
job = dataset_eval.get_job(job_id)
self.assertEqual(job["status"], dataset_eval.STATUS_FAILED)
def test_set_job_status(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
job = dataset_eval.get_job(job_id)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
dataset_eval.set_job_status(
job_id=job_id,
status=dataset_eval.STATUS_FAILED,
)
job = dataset_eval.get_job(job_id)
self.assertEqual(job["status"], dataset_eval.STATUS_FAILED)
def test_delete_job(self):
with self.assertRaises(dataset_eval.JobNotFoundException):
dataset_eval.delete_job(self.test_uuid)
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
snapshots = dataset.get_snapshots_for_dataset(self.test_dataset_id)
self.assertEqual(len(snapshots), 1)
self.assertIsNotNone(dataset_eval.get_job(job_id))
dataset_eval.delete_job(job_id)
snapshots = dataset.get_snapshots_for_dataset(self.test_dataset_id)
self.assertEqual(len(snapshots), 0)
self.assertIsNone(dataset_eval.get_job(job_id))
def test_delete_job(self):
with self.assertRaises(dataset_eval.JobNotFoundException):
dataset_eval.delete_job(self.test_uuid)
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
snapshots = dataset.get_snapshots_for_dataset(self.test_dataset_id)
self.assertEqual(len(snapshots), 1)
self.assertIsNotNone(dataset_eval.get_job(job_id))
dataset_eval.delete_job(job_id)
snapshots = dataset.get_snapshots_for_dataset(self.test_dataset_id)
self.assertEqual(len(snapshots), 0)
self.assertIsNone(dataset_eval.get_job(job_id))
def test_eval_job_location(self):
job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
job1 = dataset_eval.get_job(job1_id)
self.assertEqual(job1["eval_location"], dataset_eval.EVAL_REMOTE)
job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
job2 = dataset_eval.get_job(job2_id)
self.assertEqual(job2["eval_location"], dataset_eval.EVAL_LOCAL)
def test_set_job_status(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
job = dataset_eval.get_job(job_id)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
dataset_eval.set_job_status(
job_id=job_id,
status=dataset_eval.STATUS_FAILED,
)
job = dataset_eval.get_job(job_id)
self.assertEqual(job["status"], dataset_eval.STATUS_FAILED)
def test_delete_job(self):
with self.assertRaises(dataset_eval.JobNotFoundException):
dataset_eval.delete_job(self.test_uuid)
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
snapshots = dataset.get_snapshots_for_dataset(self.test_dataset_id)
self.assertEqual(len(snapshots), 1)
self.assertIsNotNone(dataset_eval.get_job(job_id))
dataset_eval.delete_job(job_id)
snapshots = dataset.get_snapshots_for_dataset(self.test_dataset_id)
self.assertEqual(len(snapshots), 0)
self.assertIsNone(dataset_eval.get_job(job_id))
def test_get_next_pending_job_remote(self):
# If we have a remote pending job with the most recent timestamp, skip it
job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
job1 = dataset_eval.get_job(job1_id)
job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
job2 = dataset_eval.get_job(job2_id)
next_pending = dataset_eval.get_next_pending_job()
self.assertEqual(job2, next_pending)
def test_get_next_pending_job(self):
job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id)
job1 = dataset_eval.get_job(job1_id)
job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id)
job2 = dataset_eval.get_job(job2_id)
next_pending = dataset_eval.get_next_pending_job()
self.assertEqual(job1, next_pending)
dataset_eval.set_job_status(
job_id=job1_id,
status=dataset_eval.STATUS_FAILED,
)
next_pending = dataset_eval.get_next_pending_job()
self.assertEqual(job2, next_pending)
def test_get_job(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
random_id = "f47ac10b-58cc-4372-a567-0e02b2c3d479"
# just in case
self.assertNotEqual(random_id, job_id)
self.assertIsNone(dataset_eval.get_job(random_id))
def test_get_job(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
random_id = "f47ac10b-58cc-4372-a567-0e02b2c3d479"
# just in case
self.assertNotEqual(random_id, job_id)
self.assertIsNone(dataset_eval.get_job(random_id))
def test_eval_job_delete(self):
resp = self.client.delete(
url_for("datasets.eval_job",
dataset_id=self.test_uuid,
job_id=self.test_uuid))
self.assert404(resp)
dataset_id = dataset.create_from_dict(self.test_data,
author_id=self.test_user_id)
resp = self.client.delete(
url_for("datasets.eval_job",
dataset_id=dataset_id,
job_id=self.test_uuid))
self.assert404(resp)
job_id = dataset_eval.evaluate_dataset(dataset_id, False,
dataset_eval.EVAL_LOCAL)
resp = self.client.delete(
url_for("datasets.eval_job", dataset_id=dataset_id, job_id=job_id))
self.assert401(resp)
self.assertEqual(
resp.json, {
'success': False,
'error': 'You are not allowed to delete this evaluation job.'
})
# As an author
self.temporary_login(self.test_user_id)
resp = self.client.delete(
url_for("datasets.eval_job", dataset_id=dataset_id, job_id=job_id))
self.assert200(resp)
self.assertIsNone(dataset_eval.get_job(job_id))
def test_create_job_artistfilter(self):
# Artist filtering as an option
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, False, dataset_eval.EVAL_LOCAL, dataset_eval.FILTER_ARTIST)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
self.assertEqual(job["options"]["filter_type"], "artist")
def test_create_job_normalize(self):
# dataset normalization
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
self.assertEqual(job["options"]["normalize"], True)
def test_create_job_normalize(self):
# dataset normalization
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
self.assertEqual(job["options"]["normalize"], True)
def test_create_job_artistfilter(self):
# Artist filtering as an option
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, False, dataset_eval.EVAL_LOCAL, dataset_eval.FILTER_ARTIST)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
self.assertEqual(job["options"]["filter_type"], "artist")
def test_get_job(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
random_id = "f47ac10b-58cc-4372-a567-0e02b2c3d479"
# just in case
self.assertNotEqual(random_id, job_id)
self.assertIsNone(dataset_eval.get_job(random_id))
def test_create_job_normalize(self):
# dataset normalization
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
self.assertEqual(job["options"]["normalize"], True)
def test_create_job_artistfilter(self):
# Artist filtering as an option
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, False, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=dataset_eval.FILTER_ARTIST)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
self.assertEqual(job["options"]["filter_type"], "artist")
def test_create_job_svm_params(self):
# C, gamma, and preprocessing values
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=dataset_eval.FILTER_ARTIST)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
self.assertEqual(job["options"]["c_values"], [1, 2, 3])
self.assertEqual(job["options"]["gamma_values"], [4, 5, 6])
self.assertEqual(job["options"]["preprocessing_values"], ["basic"])
def test_set_job_result(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
result = {
u"accuracy": 1,
u"parameters": {},
u"confusion_matrix": {},
}
dataset_eval.set_job_result(
job_id=job_id,
result=json.dumps(result),
)
job = dataset_eval.get_job(job_id)
self.assertEqual(job["result"], result)
def test_set_job_result(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id)
result = {
u"accuracy": 1,
u"parameters": {},
u"confusion_matrix": {},
}
dataset_eval.set_job_result(
job_id=job_id,
result=json.dumps(result),
)
job = dataset_eval.get_job(job_id)
self.assertEqual(job["result"], result)
def test_set_job_result(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
result = {
u"accuracy": 1,
u"parameters": {},
u"confusion_matrix": {},
}
dataset_eval.set_job_result(
job_id=job_id,
result=json.dumps(result),
)
job = dataset_eval.get_job(job_id)
self.assertEqual(job["result"], result)
def test_eval_job_delete(self):
resp = self.client.delete("/datasets/%s/%s" % (self.test_uuid, self.test_uuid))
self.assert404(resp)
dataset_id = dataset.create_from_dict(self.test_data, author_id=self.test_user_id)
resp = self.client.delete("/datasets/%s/%s" % (dataset_id, self.test_uuid))
self.assert404(resp)
job_id = dataset_eval.evaluate_dataset(dataset_id, False, dataset_eval.EVAL_LOCAL)
resp = self.client.delete("/datasets/%s/%s" % (dataset_id, job_id))
self.assert401(resp)
# As an author
self.temporary_login(self.test_user_id)
resp = self.client.delete("/datasets/%s/%s" % (dataset_id, job_id))
self.assert200(resp)
self.assertIsNone(dataset_eval.get_job(job_id))
def test_eval_job_delete(self):
resp = self.client.delete(url_for("datasets.eval_job", dataset_id=self.test_uuid, job_id=self.test_uuid))
self.assert404(resp)
dataset_id = dataset.create_from_dict(self.test_data, author_id=self.test_user_id)
resp = self.client.delete(url_for("datasets.eval_job", dataset_id=dataset_id, job_id=self.test_uuid))
self.assert404(resp)
job_id = dataset_eval.evaluate_dataset(dataset_id, False, dataset_eval.EVAL_LOCAL)
resp = self.client.delete(url_for("datasets.eval_job", dataset_id=dataset_id, job_id=job_id))
self.assert401(resp)
self.assertEqual(resp.json, {'success': False, 'error': 'You are not allowed to delete this evaluation job.'})
# As an author
self.temporary_login(self.test_user_id)
resp = self.client.delete(url_for("datasets.eval_job", dataset_id=dataset_id, job_id=job_id))
self.assert200(resp)
self.assertIsNone(dataset_eval.get_job(job_id))
def test_eval_job_delete(self):
resp = self.client.delete("/datasets/%s/%s" %
(self.test_uuid, self.test_uuid))
self.assert404(resp)
dataset_id = dataset.create_from_dict(self.test_data,
author_id=self.test_user_id)
resp = self.client.delete("/datasets/%s/%s" %
(dataset_id, self.test_uuid))
self.assert404(resp)
job_id = dataset_eval.evaluate_dataset(dataset_id, False)
resp = self.client.delete("/datasets/%s/%s" % (dataset_id, job_id))
self.assert401(resp)
# As an author
self.temporary_login(self.test_user_id)
resp = self.client.delete("/datasets/%s/%s" % (dataset_id, job_id))
self.assert200(resp)
self.assertIsNone(dataset_eval.get_job(job_id))
def test_create_job(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
