def test_add1(self):
self.opts["delay"] = 2000
job = Job.create(db.session, **self.opts)
self.assertEquals(job.submittime, job.createtime + 2000 * 1000)
# negative delay equals to 0 seconds
self.opts["delay"] = -2000
job = Job.create(db.session, **self.opts)
self.assertEquals(job.submittime, job.createtime)
def test_list(self):
job = Job.create(db.session, **self.jobOpts)
self.opts["job_uid"] = job.uid
for x in range(5):
Timetable.create(db.session, **self.opts)
arr = Timetable.list(db.session, None)
self.assertEquals(len(arr), 5)
def test_get(self):
job = Job.create(db.session, **self.opts)
res = Job.get(db.session, job.uid)
self.assertEquals(res.json(), job.json())
# fetch job with non-existent job id
res = Job.get(db.session, "")
self.assertEquals(res, None)
def test_execCommandOrder(self):
job = Job.create(db.session, **self.opts)
cmd = job.execCommand(sparkContext, ["foo=bar"], {"spark.sql.shuffle.partitions": 200})
anon = [x for x in cmd if x.startswith("--") or x == "spark-submit" or x.endswith(".jar")]
self.assertEqual(anon, ["spark-submit", "--name", "--master", "--conf", "--conf", "--conf",
"--conf", "--conf", "--conf", "--class", "/tmp/file.jar"])
self.assertEqual(len(cmd), 24)
self.assertEqual(cmd[20:], ["a", "b", "c", "foo=bar"])
def test_execCommand(self):
job = Job.create(db.session, **self.opts)
cmd = job.execCommand(sparkContext)
self.assertEqual(sorted(cmd), sorted(["spark-submit", "--name", "test-job", "--master",
"spark://sandbox:7077", "--conf", "%s=%s" % (SPARK_OCTOHAVEN_JOB_ID, job.uid),
"--conf", "spark.executor.memory=4g", "--conf", "spark.driver.memory=4g",
"--conf", "spark.file.overwrite=true", "--conf", "spark.shuffle.spill=true",
"--class", "com.test.Main", "/tmp/file.jar", "a", "b", "c"]))
def test_stats(self):
job = Job.create(db.session, **self.jobOpts)
self.opts["job_uid"] = job.uid
timetable = Timetable.create(db.session, **self.opts)
latestStats = None
for x in range(10):
spawnedJob = Job.create(db.session, **self.jobOpts)
latestStats = TimetableStats(timetable_uid=timetable.uid, job_uid=spawnedJob.uid,
createtime=x)
db.session.add(latestStats)
db.session.commit()
# retrieve stats and compare with iterator
stats = timetable.json()["stats"]
self.assertEquals(stats["jobs"], 10)
self.assertEquals(stats["last_time"], latestStats.createtime)
self.assertEquals(stats["last_job_uid"], latestStats.job_uid)
def test_list(self):
i = 0L
for x in range(10):
Job.create(db.session, **self.opts)
arr = Job.list(db.session, None)
self.assertEquals(len(arr), 10)
times = [x.createtime for x in arr]
self.assertEquals(times, sorted(times, reverse=True))
# test selecting status
arr = Job.list(db.session, Job.READY, limit=1)
self.assertEquals(len(arr), 1)
arr = Job.list(db.session, Job.READY, limit=5)
self.assertEquals(len(arr), 5)
arr = Job.list(db.session, Job.READY, limit=0)
self.assertEquals(len(arr), 10)
arr = Job.list(db.session, Job.READY, limit=-1)
self.assertEquals(len(arr), 10)
def test_get(self):
job = Job.create(db.session, **self.jobOpts)
self.opts["job_uid"] = job.uid
timetable = Timetable.create(db.session, **self.opts)
res = Timetable.get(db.session, timetable.uid)
self.assertEquals(res.json(), timetable.json())
# test non-existent key
res = Timetable.get(db.session, "")
self.assertEquals(res, None)
def test_cancel(self):
job = Job.create(db.session, **self.jobOpts)
self.opts["job_uid"] = job.uid
timetable = Timetable.create(db.session, **self.opts)
Timetable.cancel(db.session, timetable)
timetable.status = Timetable.CANCELLED
# try cancelling already cancelled timetable
with self.assertRaises(StandardError):
Timetable.cancel(db.session, timetable)
def test_run(self):
job = Job.create(db.session, **self.opts)
Job.run(db.session, job)
self.assertEqual(job.status, Job.RUNNING)
# check that start time is close to current time
self.assertTrue(job.starttime > utils.currentTimeMillis() - 2000)
# should fail to run already running job
with self.assertRaises(StandardError):
Job.run(db.session, job)
def test_pause(self):
job = Job.create(db.session, **self.jobOpts)
self.opts["job_uid"] = job.uid
timetable = Timetable.create(db.session, **self.opts)
Timetable.pause(db.session, timetable)
timetable.status = Timetable.PAUSED
# try pausing already paused timetable
with self.assertRaises(StandardError):
Timetable.pause(db.session, timetable)
def test_resume(self):
job = Job.create(db.session, **self.jobOpts)
self.opts["job_uid"] = job.uid
timetable = Timetable.create(db.session, **self.opts)
timetable.status = Timetable.PAUSED
Timetable.resume(db.session, timetable)
timetable.status = Timetable.ACTIVE
# try resuming already active timetable
with self.assertRaises(StandardError):
Timetable.resume(db.session, timetable)
def test_json(self):
job = Job.create(db.session, **self.jobOpts)
self.opts["job_uid"] = job.uid
timetable = Timetable.create(db.session, **self.opts)
res = Timetable.get(db.session, timetable.uid).json()
self.assertEquals(res["name"], timetable.name)
self.assertEquals(res["status"], timetable.status)
self.assertEquals(res["createtime"], timetable.createtime)
self.assertEquals(res["canceltime"], timetable.canceltime)
self.assertEquals(res["cron"], timetable.cronExpression().json())
self.assertEquals(res["job"], timetable.job.json())
def test_json(self):
job = Job.create(db.session, **self.opts)
obj = job.json()
self.assertEquals(obj["name"], job.name)
self.assertEquals(obj["status"], job.status)
self.assertEquals(obj["createtime"], job.createtime)
self.assertEquals(obj["submittime"], job.submittime)
self.assertEquals(obj["entrypoint"], job.entrypoint)
self.assertEquals(obj["jar"], job.jar)
self.assertEquals(obj["options"], job.getSparkOptions())
self.assertEquals(obj["jobconf"], job.getJobConf())
def test_listRunning(self):
arr = [("ready", Job.READY), ("running", Job.RUNNING), ("finished", Job.FINISHED),
("running", Job.RUNNING)]
for name, status in arr:
job = Job.create(db.session, **self.opts)
job.name = name
job.status = status
db.session.commit()
jobs = Job.listRunning(db.session)
self.assertEqual(len(jobs), 2)
self.assertEqual([x.status for x in jobs], [Job.RUNNING, Job.RUNNING])
def test_finish(self):
job = Job.create(db.session, **self.opts)
# should not be able to finish not running job
with self.assertRaises(StandardError):
Job.finish(db.session, job)
# Launch job and finish it
Job.run(db.session, job)
Job.finish(db.session, job)
self.assertEqual(job.status, Job.FINISHED)
self.assertTrue(job.finishtime > utils.currentTimeMillis() - 2000)
# should fail to finish already finished job
with self.assertRaises(StandardError):
Job.finish(db.session, job)
def test_registerNewJob(self):
job = Job.create(db.session, **self.jobOpts)
self.opts["job_uid"] = job.uid
timetable = Timetable.create(db.session, **self.opts)
spawnedJob = None
for x in range(7):
spawnedJob = Timetable.registerNewJob(db.session, timetable)
time.sleep(0.05)
# retrieve stats and compare with iterator
stats = timetable.json()["stats"]
self.assertEquals(stats["jobs"], 7)
self.assertEquals(stats["last_job_uid"], spawnedJob.uid)
def test_close(self):
job = Job.create(db.session, **self.opts)
self.assertEquals(job.status, Job.READY)
Job.close(db.session, job)
self.assertEquals(job.status, Job.CLOSED)
# try closing already closed job
with self.assertRaises(StandardError):
Job.close(db.session, job)
with self.assertRaises(StandardError):
job.status = Job.RUNNING
Job.close(db.session, job)
with self.assertRaises(StandardError):
job.status = Job.FINISHED
Job.close(db.session, job)
def test_add(self):
with self.assertRaises(StandardError):
Timetable.create(db.session, **{})
with self.assertRaises(StandardError):
Timetable.create(db.session, **{"name": "test"})
with self.assertRaises(StandardError):
Timetable.create(db.session, **{"name": "test", "cron": "* * * * * *"})
# test correct input, though it has more keys than required
# should fail since there is no such job exists
job = Job.create(db.session, **self.jobOpts)
self.opts["job_uid"] = job.uid
timetable = Timetable.create(db.session, **self.opts)
self.assertTrue(timetable.createtime > utils.currentTimeMillis() - 5000)
self.assertEquals(timetable.job.json(), job.json())
def test_listRunnable(self):
arr = [("ready", Job.READY), ("running", Job.RUNNING), ("finished", Job.FINISHED),
("running", Job.RUNNING), ("delayed", Job.DELAYED)]
for name, status in arr:
job = Job.create(db.session, **self.opts)
job.name = name
job.status = status
if status is Job.DELAYED:
job.submittime = job.submittime - 10000
job.priority = job.submittime / 1000L
db.session.commit()
jobs = Job.listRunnable(db.session, 5, utils.currentTimeMillis())
self.assertEqual(len(jobs), 2)
self.assertEqual([x.name for x in jobs], ["delayed", "ready"])
self.assertEqual([x.status for x in jobs], [Job.DELAYED, Job.READY])
def test_jobCopy(self):
job = Job.create(db.session, **self.opts)
copy = job.jobCopy(name="a", status=Job.READY, priority=1, createtime=2L, submittime=2L)
self.assertEquals(copy.uid, None)
self.assertEquals(copy.name, "a")
self.assertEquals(copy.status, Job.READY)
self.assertEquals(copy.priority, 1)
self.assertEquals(copy.createtime, 2L)
self.assertEquals(copy.submittime, 2L)
self.assertEquals(copy.sparkappid, None)
self.assertEquals(copy.starttime, None)
self.assertEquals(copy.finishtime, None)
# these properties should be the same
self.assertEquals(copy.options, job.options)
self.assertEquals(copy.jobconf, job.jobconf)
self.assertEquals(copy.entrypoint, job.entrypoint)
self.assertEquals(copy.jar, job.jar)
def test_add(self):
job = Job.create(db.session, **self.opts)
arr = Job.list(db.session, None)
self.assertTrue(len(arr), 1)
