def test_job_exists(self): self.assertFalse(dataset_eval.job_exists(self.test_dataset_id)) dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type=None) self.assertTrue(dataset_eval.job_exists(self.test_dataset_id))
def test_get_next_pending_job(self): job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type=None) job1 = dataset_eval.get_job(job1_id) job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type=None) job2 = dataset_eval.get_job(job2_id) next_pending = dataset_eval.get_next_pending_job() self.assertEqual(job1, next_pending) dataset_eval.set_job_status( job_id=job1_id, status=dataset_eval.STATUS_FAILED, ) next_pending = dataset_eval.get_next_pending_job() self.assertEqual(job2, next_pending)
def test_eval_job_location(self): job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE, None) job1 = dataset_eval.get_job(job1_id) self.assertEqual(job1["eval_location"], dataset_eval.EVAL_REMOTE) job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, None) job2 = dataset_eval.get_job(job2_id) self.assertEqual(job2["eval_location"], dataset_eval.EVAL_LOCAL)
def test_get_next_pending_job_remote(self): # If we have a remote pending job with the most recent timestamp, skip it job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE) job1 = dataset_eval.get_job(job1_id) job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL) job2 = dataset_eval.get_job(job2_id) next_pending = dataset_eval.get_next_pending_job() self.assertEqual(job2, next_pending)
def test_create_job_badlocation(self): # an invalid eval_location with self.assertRaises(ValueError): dataset_eval._create_job(self.conn, self.test_dataset_id, True, "not_a_location", c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type=None)
def test_create_job_badfilter(self): # An unknown filter type with self.assertRaises(ValueError): dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type="test")
def test_eval_job_location(self): job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE, c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type=None) job1 = dataset_eval.get_job(job1_id) self.assertEqual(job1["eval_location"], dataset_eval.EVAL_REMOTE) job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type=None) job2 = dataset_eval.get_job(job2_id) self.assertEqual(job2["eval_location"], dataset_eval.EVAL_LOCAL)
def test_get_next_pending_job_remote(self): # If we have a remote pending job with the most recent timestamp, skip it job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE, c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type=None) job1 = dataset_eval.get_job(job1_id) job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type=None) job2 = dataset_eval.get_job(job2_id) next_pending = dataset_eval.get_next_pending_job() self.assertEqual(job2, next_pending)
def test_get_next_pending_job(self): job1_id = dataset_eval._create_job(self.conn, self.test_dataset_id) job1 = dataset_eval.get_job(job1_id) job2_id = dataset_eval._create_job(self.conn, self.test_dataset_id) job2 = dataset_eval.get_job(job2_id) next_pending = dataset_eval.get_next_pending_job() self.assertEqual(job1, next_pending) dataset_eval.set_job_status( job_id=job1_id, status=dataset_eval.STATUS_FAILED, ) next_pending = dataset_eval.get_next_pending_job() self.assertEqual(job2, next_pending)
def test_get_pending_jobs_for_user_done(self): """ Check that a remote eval job with a done status doesn't show """ job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE, None) db.dataset_eval.set_job_status(job_id, db.dataset_eval.STATUS_DONE) response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id) self.assertEqual(response, [])
def test_get_job(self): job_id = dataset_eval._create_job(self.conn, self.test_dataset_id) job = dataset_eval.get_job(job_id) self.assertIsNotNone(job) self.assertEqual(type(job), dict) self.assertIsNone(dataset_eval.get_job("f47ac10b-58cc-4372-a567-0e02b2c3d479"))
def test_get_pending_jobs_for_user_local(self): """ Check that a local eval dataset for this user doesn't show """ job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, None) job_details = db.dataset_eval.get_job(job_id) response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id) self.assertEqual(response, [])
def test_get_job(self): job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL) random_id = "f47ac10b-58cc-4372-a567-0e02b2c3d479" # just in case self.assertNotEqual(random_id, job_id) self.assertIsNone(dataset_eval.get_job(random_id))
def test_delete_job(self): with self.assertRaises(dataset_eval.JobNotFoundException): dataset_eval.delete_job(self.test_uuid) job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True) self.assertIsNotNone(dataset_eval.get_job(job_id)) dataset_eval.delete_job(job_id) self.assertIsNone(dataset_eval.get_job(job_id))
def test_create_job_artistfilter(self): # Artist filtering as an option job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, False, dataset_eval.EVAL_LOCAL, dataset_eval.FILTER_ARTIST) job = dataset_eval.get_job(job_id) self.assertIsNotNone(job) self.assertEqual(job["status"], dataset_eval.STATUS_PENDING) self.assertEqual(job["options"]["filter_type"], "artist")
def test_create_job_normalize(self): # dataset normalization job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL) job = dataset_eval.get_job(job_id) self.assertIsNotNone(job) self.assertEqual(job["status"], dataset_eval.STATUS_PENDING) self.assertEqual(job["options"]["normalize"], True)
def test_get_job(self): job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type=None) random_id = "f47ac10b-58cc-4372-a567-0e02b2c3d479" # just in case self.assertNotEqual(random_id, job_id) self.assertIsNone(dataset_eval.get_job(random_id))
def test_get_pending_jobs_for_user_local(self): """ Check that a local eval dataset for this user doesn't show """ job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type=None) job_details = db.dataset_eval.get_job(job_id) response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id) self.assertEqual(response, [])
def test_get_pending_jobs_for_user_other_user(self): """ Check that a remote eval job for another user doesn't show """ another_user_id = user.create("another user") another_dataset_id = dataset.create_from_dict(self.test_data, author_id=another_user_id) job_id = dataset_eval._create_job(self.conn, another_dataset_id, True, dataset_eval.EVAL_REMOTE, None) response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id) self.assertEqual(response, [])
def test_get_pending_jobs_for_user_done(self): """ Check that a remote eval job with a done status doesn't show """ job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE, c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type=None) db.dataset_eval.set_job_status(job_id, db.dataset_eval.STATUS_DONE) response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id) self.assertEqual(response, [])
def test_create_job_normalize(self): # dataset normalization job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type=None) job = dataset_eval.get_job(job_id) self.assertIsNotNone(job) self.assertEqual(job["status"], dataset_eval.STATUS_PENDING) self.assertEqual(job["options"]["normalize"], True)
def test_create_job_artistfilter(self): # Artist filtering as an option job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, False, dataset_eval.EVAL_LOCAL, c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type=dataset_eval.FILTER_ARTIST) job = dataset_eval.get_job(job_id) self.assertIsNotNone(job) self.assertEqual(job["status"], dataset_eval.STATUS_PENDING) self.assertEqual(job["options"]["filter_type"], "artist")
def test_set_job_status(self): job_id = dataset_eval._create_job(self.conn, self.test_dataset_id) job = dataset_eval.get_job(job_id) self.assertEqual(job["status"], dataset_eval.STATUS_PENDING) dataset_eval.set_job_status( job_id=job_id, status=dataset_eval.STATUS_FAILED, ) job = dataset_eval.get_job(job_id) self.assertEqual(job["status"], dataset_eval.STATUS_FAILED)
def test_get_pending_jobs_for_user_other_user(self): """ Check that a remote eval job for another user doesn't show """ another_user_id = user.create("another user") another_dataset_id = dataset.create_from_dict(self.test_data, author_id=another_user_id) job_id = dataset_eval._create_job(self.conn, another_dataset_id, True, dataset_eval.EVAL_REMOTE, c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type=None) response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id) self.assertEqual(response, [])
def test_set_job_status(self): job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL) job = dataset_eval.get_job(job_id) self.assertEqual(job["status"], dataset_eval.STATUS_PENDING) dataset_eval.set_job_status( job_id=job_id, status=dataset_eval.STATUS_FAILED, ) job = dataset_eval.get_job(job_id) self.assertEqual(job["status"], dataset_eval.STATUS_FAILED)
def test_delete_job(self): with self.assertRaises(dataset_eval.JobNotFoundException): dataset_eval.delete_job(self.test_uuid) job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL) snapshots = dataset.get_snapshots_for_dataset(self.test_dataset_id) self.assertEqual(len(snapshots), 1) self.assertIsNotNone(dataset_eval.get_job(job_id)) dataset_eval.delete_job(job_id) snapshots = dataset.get_snapshots_for_dataset(self.test_dataset_id) self.assertEqual(len(snapshots), 0) self.assertIsNone(dataset_eval.get_job(job_id))
def test_create_job_svm_params(self): # C, gamma, and preprocessing values job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type=dataset_eval.FILTER_ARTIST) job = dataset_eval.get_job(job_id) self.assertIsNotNone(job) self.assertEqual(job["status"], dataset_eval.STATUS_PENDING) self.assertEqual(job["options"]["c_values"], [1, 2, 3]) self.assertEqual(job["options"]["gamma_values"], [4, 5, 6]) self.assertEqual(job["options"]["preprocessing_values"], ["basic"])
def test_set_job_status(self): job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type=None) job = dataset_eval.get_job(job_id) self.assertEqual(job["status"], dataset_eval.STATUS_PENDING) dataset_eval.set_job_status( job_id=job_id, status=dataset_eval.STATUS_FAILED, ) job = dataset_eval.get_job(job_id) self.assertEqual(job["status"], dataset_eval.STATUS_FAILED)
def test_get_remote_pending_jobs_for_user(self): """ Check that we get remote pending jobs for a user """ job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_REMOTE, None) job_details = db.dataset_eval.get_job(job_id) response = dataset_eval.get_remote_pending_jobs_for_user(self.test_user_id) expected_response = [{ 'job_id' : str(job_id), 'dataset_name' : self.test_data['name'], 'created' : job_details['created'] }] self.assertEqual(response, expected_response)
def test_delete_job(self): with self.assertRaises(dataset_eval.JobNotFoundException): dataset_eval.delete_job(self.test_uuid) job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, c_value=[1, 2, 3], gamma_value=[4, 5, 6], preprocessing_values=["basic"], filter_type=None) snapshots = dataset.get_snapshots_for_dataset(self.test_dataset_id) self.assertEqual(len(snapshots), 1) self.assertIsNotNone(dataset_eval.get_job(job_id)) dataset_eval.delete_job(job_id) snapshots = dataset.get_snapshots_for_dataset(self.test_dataset_id) self.assertEqual(len(snapshots), 0) self.assertIsNone(dataset_eval.get_job(job_id))
def test_set_job_result(self): job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL) result = { u"accuracy": 1, u"parameters": {}, u"confusion_matrix": {}, } dataset_eval.set_job_result( job_id=job_id, result=json.dumps(result), ) job = dataset_eval.get_job(job_id) self.assertEqual(job["result"], result)
def test_set_job_result(self): job_id = dataset_eval._create_job(self.conn, self.test_dataset_id) result = { u"accuracy": 1, u"parameters": {}, u"confusion_matrix": {}, } dataset_eval.set_job_result( job_id=job_id, result=json.dumps(result), ) job = dataset_eval.get_job(job_id) self.assertEqual(job["result"], result)
def test_create_job_badlocation(self): # an invalid eval_location with self.assertRaises(ValueError): job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, "not_a_location")
def test_create_job_badfilter(self): # An unknown filter type with self.assertRaises(ValueError): job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL, "test")
def test_job_exists(self): self.assertFalse(dataset_eval.job_exists(self.test_dataset_id)) job_id = dataset_eval._create_job(self.conn, self.test_dataset_id, True, dataset_eval.EVAL_LOCAL) self.assertTrue(dataset_eval.job_exists(self.test_dataset_id))
def test_create_job(self): job_id = dataset_eval._create_job(self.conn, self.test_dataset_id) job = dataset_eval.get_job(job_id) self.assertIsNotNone(job) self.assertEqual(job["status"], dataset_eval.STATUS_PENDING)