def test_job_exists(self):
self.assertFalse(dataset_eval.job_exists(self.test_dataset_id))
dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
self.assertTrue(dataset_eval.job_exists(self.test_dataset_id))
def test_get_next_pending_job(self):
job1_id = dataset_eval._create_job(self.conn,
self.test_dataset_id,
True,
dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3],
gamma_value=[4, 5, 6],
preprocessing_values=["basic"],
filter_type=None)
job1 = dataset_eval.get_job(job1_id)
job2_id = dataset_eval._create_job(self.conn,
self.test_dataset_id,
True,
dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3],
gamma_value=[4, 5, 6],
preprocessing_values=["basic"],
filter_type=None)
job2 = dataset_eval.get_job(job2_id)
next_pending = dataset_eval.get_next_pending_job()
self.assertEqual(job1, next_pending)
dataset_eval.set_job_status(
job_id=job1_id,
status=dataset_eval.STATUS_FAILED,
)
next_pending = dataset_eval.get_next_pending_job()
self.assertEqual(job2, next_pending)
def test_eval_job_location(self):
job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE, None)
job1 = dataset_eval.get_job(job1_id)
self.assertEqual(job1["eval_location"], dataset_eval.EVAL_REMOTE)
job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, None)
job2 = dataset_eval.get_job(job2_id)
self.assertEqual(job2["eval_location"], dataset_eval.EVAL_LOCAL)
def test_eval_job_location(self):
job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE, None)
job1 = dataset_eval.get_job(job1_id)
self.assertEqual(job1["eval_location"], dataset_eval.EVAL_REMOTE)
job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, None)
job2 = dataset_eval.get_job(job2_id)
self.assertEqual(job2["eval_location"], dataset_eval.EVAL_LOCAL)
def test_get_next_pending_job_remote(self):
# If we have a remote pending job with the most recent timestamp, skip it
job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE)
job1 = dataset_eval.get_job(job1_id)
job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
job2 = dataset_eval.get_job(job2_id)
next_pending = dataset_eval.get_next_pending_job()
self.assertEqual(job2, next_pending)
def test_get_next_pending_job_remote(self):
# If we have a remote pending job with the most recent timestamp, skip it
job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE)
job1 = dataset_eval.get_job(job1_id)
job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
job2 = dataset_eval.get_job(job2_id)
next_pending = dataset_eval.get_next_pending_job()
self.assertEqual(job2, next_pending)
def test_create_job_badlocation(self):
# an invalid eval_location
with self.assertRaises(ValueError):
dataset_eval._create_job(self.conn,
self.test_dataset_id,
True,
"not_a_location",
c_value=[1, 2, 3],
gamma_value=[4, 5, 6],
preprocessing_values=["basic"],
filter_type=None)
def test_create_job_badfilter(self):
# An unknown filter type
with self.assertRaises(ValueError):
dataset_eval._create_job(self.conn,
self.test_dataset_id,
True,
dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3],
gamma_value=[4, 5, 6],
preprocessing_values=["basic"],
filter_type="test")
def test_eval_job_location(self):
job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
job1 = dataset_eval.get_job(job1_id)
self.assertEqual(job1["eval_location"], dataset_eval.EVAL_REMOTE)
job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
job2 = dataset_eval.get_job(job2_id)
self.assertEqual(job2["eval_location"], dataset_eval.EVAL_LOCAL)
def test_get_next_pending_job_remote(self):
# If we have a remote pending job with the most recent timestamp, skip it
job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
job1 = dataset_eval.get_job(job1_id)
job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
job2 = dataset_eval.get_job(job2_id)
next_pending = dataset_eval.get_next_pending_job()
self.assertEqual(job2, next_pending)
def test_get_next_pending_job(self):
job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id)
job1 = dataset_eval.get_job(job1_id)
job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id)
job2 = dataset_eval.get_job(job2_id)
next_pending = dataset_eval.get_next_pending_job()
self.assertEqual(job1, next_pending)
dataset_eval.set_job_status(
job_id=job1_id,
status=dataset_eval.STATUS_FAILED,
)
next_pending = dataset_eval.get_next_pending_job()
self.assertEqual(job2, next_pending)
def test_get_pending_jobs_for_user_done(self):
""" Check that a remote eval job with a done status doesn't show """
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE, None)
db.dataset_eval.set_job_status(job_id, db.dataset_eval.STATUS_DONE)
response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id)
self.assertEqual(response, [])
def test_get_job(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(type(job), dict)
self.assertIsNone(dataset_eval.get_job("f47ac10b-58cc-4372-a567-0e02b2c3d479"))
def test_get_pending_jobs_for_user_local(self):
""" Check that a local eval dataset for this user doesn't show """
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, None)
job_details = db.dataset_eval.get_job(job_id)
response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id)
self.assertEqual(response, [])
def test_get_job(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
random_id = "f47ac10b-58cc-4372-a567-0e02b2c3d479"
# just in case
self.assertNotEqual(random_id, job_id)
self.assertIsNone(dataset_eval.get_job(random_id))
def test_get_job(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
random_id = "f47ac10b-58cc-4372-a567-0e02b2c3d479"
# just in case
self.assertNotEqual(random_id, job_id)
self.assertIsNone(dataset_eval.get_job(random_id))
def test_get_pending_jobs_for_user_local(self):
""" Check that a local eval dataset for this user doesn't show """
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, None)
job_details = db.dataset_eval.get_job(job_id)
response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id)
self.assertEqual(response, [])
def test_get_pending_jobs_for_user_done(self):
""" Check that a remote eval job with a done status doesn't show """
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE, None)
db.dataset_eval.set_job_status(job_id, db.dataset_eval.STATUS_DONE)
response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id)
self.assertEqual(response, [])
def test_delete_job(self):
with self.assertRaises(dataset_eval.JobNotFoundException):
dataset_eval.delete_job(self.test_uuid)
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True)
self.assertIsNotNone(dataset_eval.get_job(job_id))
dataset_eval.delete_job(job_id)
self.assertIsNone(dataset_eval.get_job(job_id))
def test_create_job_artistfilter(self):
# Artist filtering as an option
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, False, dataset_eval.EVAL_LOCAL, dataset_eval.FILTER_ARTIST)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
self.assertEqual(job["options"]["filter_type"], "artist")
def test_create_job_artistfilter(self):
# Artist filtering as an option
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, False, dataset_eval.EVAL_LOCAL, dataset_eval.FILTER_ARTIST)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
self.assertEqual(job["options"]["filter_type"], "artist")
def test_create_job_normalize(self):
# dataset normalization
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
self.assertEqual(job["options"]["normalize"], True)
def test_create_job_normalize(self):
# dataset normalization
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
self.assertEqual(job["options"]["normalize"], True)
def test_get_job(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
random_id = "f47ac10b-58cc-4372-a567-0e02b2c3d479"
# just in case
self.assertNotEqual(random_id, job_id)
self.assertIsNone(dataset_eval.get_job(random_id))
def test_get_pending_jobs_for_user_local(self):
""" Check that a local eval dataset for this user doesn't show """
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
job_details = db.dataset_eval.get_job(job_id)
response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id)
self.assertEqual(response, [])
def test_get_pending_jobs_for_user_other_user(self):
""" Check that a remote eval job for another user doesn't show """
another_user_id = user.create("another user")
another_dataset_id = dataset.create_from_dict(self.test_data, author_id=another_user_id)
job_id = dataset_eval._create_job(self.conn, another_dataset_id, True, dataset_eval.EVAL_REMOTE, None)
response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id)
self.assertEqual(response, [])
def test_get_pending_jobs_for_user_done(self):
""" Check that a remote eval job with a done status doesn't show """
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
db.dataset_eval.set_job_status(job_id, db.dataset_eval.STATUS_DONE)
response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id)
self.assertEqual(response, [])
def test_get_pending_jobs_for_user_other_user(self):
""" Check that a remote eval job for another user doesn't show """
another_user_id = user.create("another user")
another_dataset_id = dataset.create_from_dict(self.test_data, author_id=another_user_id)
job_id = dataset_eval._create_job(self.conn, another_dataset_id, True, dataset_eval.EVAL_REMOTE, None)
response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id)
self.assertEqual(response, [])
def test_create_job_normalize(self):
# dataset normalization
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
self.assertEqual(job["options"]["normalize"], True)
def test_create_job_artistfilter(self):
# Artist filtering as an option
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, False, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=dataset_eval.FILTER_ARTIST)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
self.assertEqual(job["options"]["filter_type"], "artist")
def test_set_job_status(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id)
job = dataset_eval.get_job(job_id)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
dataset_eval.set_job_status(
job_id=job_id,
status=dataset_eval.STATUS_FAILED,
)
job = dataset_eval.get_job(job_id)
self.assertEqual(job["status"], dataset_eval.STATUS_FAILED)
def test_get_pending_jobs_for_user_other_user(self):
""" Check that a remote eval job for another user doesn't show """
another_user_id = user.create("another user")
another_dataset_id = dataset.create_from_dict(self.test_data, author_id=another_user_id)
job_id = dataset_eval._create_job(self.conn, another_dataset_id, True, dataset_eval.EVAL_REMOTE,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id)
self.assertEqual(response, [])
def test_set_job_status(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
job = dataset_eval.get_job(job_id)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
dataset_eval.set_job_status(
job_id=job_id,
status=dataset_eval.STATUS_FAILED,
)
job = dataset_eval.get_job(job_id)
self.assertEqual(job["status"], dataset_eval.STATUS_FAILED)
def test_delete_job(self):
with self.assertRaises(dataset_eval.JobNotFoundException):
dataset_eval.delete_job(self.test_uuid)
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
snapshots = dataset.get_snapshots_for_dataset(self.test_dataset_id)
self.assertEqual(len(snapshots), 1)
self.assertIsNotNone(dataset_eval.get_job(job_id))
dataset_eval.delete_job(job_id)
snapshots = dataset.get_snapshots_for_dataset(self.test_dataset_id)
self.assertEqual(len(snapshots), 0)
self.assertIsNone(dataset_eval.get_job(job_id))
def test_delete_job(self):
with self.assertRaises(dataset_eval.JobNotFoundException):
dataset_eval.delete_job(self.test_uuid)
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
snapshots = dataset.get_snapshots_for_dataset(self.test_dataset_id)
self.assertEqual(len(snapshots), 1)
self.assertIsNotNone(dataset_eval.get_job(job_id))
dataset_eval.delete_job(job_id)
snapshots = dataset.get_snapshots_for_dataset(self.test_dataset_id)
self.assertEqual(len(snapshots), 0)
self.assertIsNone(dataset_eval.get_job(job_id))
def test_create_job_svm_params(self):
# C, gamma, and preprocessing values
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=dataset_eval.FILTER_ARTIST)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
self.assertEqual(job["options"]["c_values"], [1, 2, 3])
self.assertEqual(job["options"]["gamma_values"], [4, 5, 6])
self.assertEqual(job["options"]["preprocessing_values"], ["basic"])
def test_set_job_status(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
job = dataset_eval.get_job(job_id)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
dataset_eval.set_job_status(
job_id=job_id,
status=dataset_eval.STATUS_FAILED,
)
job = dataset_eval.get_job(job_id)
self.assertEqual(job["status"], dataset_eval.STATUS_FAILED)
def test_get_remote_pending_jobs_for_user(self):
""" Check that we get remote pending jobs for a user """
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE, None)
job_details = db.dataset_eval.get_job(job_id)
response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id)
expected_response = [{
'job_id' : str(job_id),
'dataset_name' : self.test_data['name'],
'created' : job_details['created']
}]
self.assertEqual(response, expected_response)
def test_get_remote_pending_jobs_for_user(self):
""" Check that we get remote pending jobs for a user """
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE, None)
job_details = db.dataset_eval.get_job(job_id)
response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id)
expected_response = [{
'job_id' : str(job_id),
'dataset_name' : self.test_data['name'],
'created' : job_details['created']
}]
self.assertEqual(response, expected_response)
def test_delete_job(self):
with self.assertRaises(dataset_eval.JobNotFoundException):
dataset_eval.delete_job(self.test_uuid)
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL,
c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"],
filter_type=None)
snapshots = dataset.get_snapshots_for_dataset(self.test_dataset_id)
self.assertEqual(len(snapshots), 1)
self.assertIsNotNone(dataset_eval.get_job(job_id))
dataset_eval.delete_job(job_id)
snapshots = dataset.get_snapshots_for_dataset(self.test_dataset_id)
self.assertEqual(len(snapshots), 0)
self.assertIsNone(dataset_eval.get_job(job_id))
def test_set_job_result(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
result = {
u"accuracy": 1,
u"parameters": {},
u"confusion_matrix": {},
}
dataset_eval.set_job_result(
job_id=job_id,
result=json.dumps(result),
)
job = dataset_eval.get_job(job_id)
self.assertEqual(job["result"], result)
def test_set_job_result(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id)
result = {
u"accuracy": 1,
u"parameters": {},
u"confusion_matrix": {},
}
dataset_eval.set_job_result(
job_id=job_id,
result=json.dumps(result),
)
job = dataset_eval.get_job(job_id)
self.assertEqual(job["result"], result)
def test_create_job_badlocation(self):
# an invalid eval_location
with self.assertRaises(ValueError):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, "not_a_location")
def test_create_job_badfilter(self):
# An unknown filter type
with self.assertRaises(ValueError):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, "test")
def test_job_exists(self):
self.assertFalse(dataset_eval.job_exists(self.test_dataset_id))
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL)
self.assertTrue(dataset_eval.job_exists(self.test_dataset_id))
def test_create_job(self):
job_id = dataset_eval._create_job(self.conn, self.test_dataset_id)
job = dataset_eval.get_job(job_id)
self.assertIsNotNone(job)
self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)
