def test_add1(self): self.opts["delay"] = 2000 job = Job.create(db.session, **self.opts) self.assertEquals(job.submittime, job.createtime + 2000 * 1000) # negative delay equals to 0 seconds self.opts["delay"] = -2000 job = Job.create(db.session, **self.opts) self.assertEquals(job.submittime, job.createtime)
def test_list(self): job = Job.create(db.session, **self.jobOpts) self.opts["job_uid"] = job.uid for x in range(5): Timetable.create(db.session, **self.opts) arr = Timetable.list(db.session, None) self.assertEquals(len(arr), 5)
def test_get(self): job = Job.create(db.session, **self.opts) res = Job.get(db.session, job.uid) self.assertEquals(res.json(), job.json()) # fetch job with non-existent job id res = Job.get(db.session, "") self.assertEquals(res, None)
def test_execCommandOrder(self): job = Job.create(db.session, **self.opts) cmd = job.execCommand(sparkContext, ["foo=bar"], {"spark.sql.shuffle.partitions": 200}) anon = [x for x in cmd if x.startswith("--") or x == "spark-submit" or x.endswith(".jar")] self.assertEqual(anon, ["spark-submit", "--name", "--master", "--conf", "--conf", "--conf", "--conf", "--conf", "--conf", "--class", "/tmp/file.jar"]) self.assertEqual(len(cmd), 24) self.assertEqual(cmd[20:], ["a", "b", "c", "foo=bar"])
def test_execCommand(self): job = Job.create(db.session, **self.opts) cmd = job.execCommand(sparkContext) self.assertEqual(sorted(cmd), sorted(["spark-submit", "--name", "test-job", "--master", "spark://sandbox:7077", "--conf", "%s=%s" % (SPARK_OCTOHAVEN_JOB_ID, job.uid), "--conf", "spark.executor.memory=4g", "--conf", "spark.driver.memory=4g", "--conf", "spark.file.overwrite=true", "--conf", "spark.shuffle.spill=true", "--class", "com.test.Main", "/tmp/file.jar", "a", "b", "c"]))
def test_stats(self): job = Job.create(db.session, **self.jobOpts) self.opts["job_uid"] = job.uid timetable = Timetable.create(db.session, **self.opts) latestStats = None for x in range(10): spawnedJob = Job.create(db.session, **self.jobOpts) latestStats = TimetableStats(timetable_uid=timetable.uid, job_uid=spawnedJob.uid, createtime=x) db.session.add(latestStats) db.session.commit() # retrieve stats and compare with iterator stats = timetable.json()["stats"] self.assertEquals(stats["jobs"], 10) self.assertEquals(stats["last_time"], latestStats.createtime) self.assertEquals(stats["last_job_uid"], latestStats.job_uid)
def test_list(self): i = 0L for x in range(10): Job.create(db.session, **self.opts) arr = Job.list(db.session, None) self.assertEquals(len(arr), 10) times = [x.createtime for x in arr] self.assertEquals(times, sorted(times, reverse=True)) # test selecting status arr = Job.list(db.session, Job.READY, limit=1) self.assertEquals(len(arr), 1) arr = Job.list(db.session, Job.READY, limit=5) self.assertEquals(len(arr), 5) arr = Job.list(db.session, Job.READY, limit=0) self.assertEquals(len(arr), 10) arr = Job.list(db.session, Job.READY, limit=-1) self.assertEquals(len(arr), 10)
def test_get(self): job = Job.create(db.session, **self.jobOpts) self.opts["job_uid"] = job.uid timetable = Timetable.create(db.session, **self.opts) res = Timetable.get(db.session, timetable.uid) self.assertEquals(res.json(), timetable.json()) # test non-existent key res = Timetable.get(db.session, "") self.assertEquals(res, None)
def test_cancel(self): job = Job.create(db.session, **self.jobOpts) self.opts["job_uid"] = job.uid timetable = Timetable.create(db.session, **self.opts) Timetable.cancel(db.session, timetable) timetable.status = Timetable.CANCELLED # try cancelling already cancelled timetable with self.assertRaises(StandardError): Timetable.cancel(db.session, timetable)
def test_run(self): job = Job.create(db.session, **self.opts) Job.run(db.session, job) self.assertEqual(job.status, Job.RUNNING) # check that start time is close to current time self.assertTrue(job.starttime > utils.currentTimeMillis() - 2000) # should fail to run already running job with self.assertRaises(StandardError): Job.run(db.session, job)
def test_pause(self): job = Job.create(db.session, **self.jobOpts) self.opts["job_uid"] = job.uid timetable = Timetable.create(db.session, **self.opts) Timetable.pause(db.session, timetable) timetable.status = Timetable.PAUSED # try pausing already paused timetable with self.assertRaises(StandardError): Timetable.pause(db.session, timetable)
def test_resume(self): job = Job.create(db.session, **self.jobOpts) self.opts["job_uid"] = job.uid timetable = Timetable.create(db.session, **self.opts) timetable.status = Timetable.PAUSED Timetable.resume(db.session, timetable) timetable.status = Timetable.ACTIVE # try resuming already active timetable with self.assertRaises(StandardError): Timetable.resume(db.session, timetable)
def test_json(self): job = Job.create(db.session, **self.jobOpts) self.opts["job_uid"] = job.uid timetable = Timetable.create(db.session, **self.opts) res = Timetable.get(db.session, timetable.uid).json() self.assertEquals(res["name"], timetable.name) self.assertEquals(res["status"], timetable.status) self.assertEquals(res["createtime"], timetable.createtime) self.assertEquals(res["canceltime"], timetable.canceltime) self.assertEquals(res["cron"], timetable.cronExpression().json()) self.assertEquals(res["job"], timetable.job.json())
def test_json(self): job = Job.create(db.session, **self.opts) obj = job.json() self.assertEquals(obj["name"], job.name) self.assertEquals(obj["status"], job.status) self.assertEquals(obj["createtime"], job.createtime) self.assertEquals(obj["submittime"], job.submittime) self.assertEquals(obj["entrypoint"], job.entrypoint) self.assertEquals(obj["jar"], job.jar) self.assertEquals(obj["options"], job.getSparkOptions()) self.assertEquals(obj["jobconf"], job.getJobConf())
def test_listRunning(self): arr = [("ready", Job.READY), ("running", Job.RUNNING), ("finished", Job.FINISHED), ("running", Job.RUNNING)] for name, status in arr: job = Job.create(db.session, **self.opts) job.name = name job.status = status db.session.commit() jobs = Job.listRunning(db.session) self.assertEqual(len(jobs), 2) self.assertEqual([x.status for x in jobs], [Job.RUNNING, Job.RUNNING])
def test_finish(self): job = Job.create(db.session, **self.opts) # should not be able to finish not running job with self.assertRaises(StandardError): Job.finish(db.session, job) # Launch job and finish it Job.run(db.session, job) Job.finish(db.session, job) self.assertEqual(job.status, Job.FINISHED) self.assertTrue(job.finishtime > utils.currentTimeMillis() - 2000) # should fail to finish already finished job with self.assertRaises(StandardError): Job.finish(db.session, job)
def test_registerNewJob(self): job = Job.create(db.session, **self.jobOpts) self.opts["job_uid"] = job.uid timetable = Timetable.create(db.session, **self.opts) spawnedJob = None for x in range(7): spawnedJob = Timetable.registerNewJob(db.session, timetable) time.sleep(0.05) # retrieve stats and compare with iterator stats = timetable.json()["stats"] self.assertEquals(stats["jobs"], 7) self.assertEquals(stats["last_job_uid"], spawnedJob.uid)
def test_close(self): job = Job.create(db.session, **self.opts) self.assertEquals(job.status, Job.READY) Job.close(db.session, job) self.assertEquals(job.status, Job.CLOSED) # try closing already closed job with self.assertRaises(StandardError): Job.close(db.session, job) with self.assertRaises(StandardError): job.status = Job.RUNNING Job.close(db.session, job) with self.assertRaises(StandardError): job.status = Job.FINISHED Job.close(db.session, job)
def test_add(self): with self.assertRaises(StandardError): Timetable.create(db.session, **{}) with self.assertRaises(StandardError): Timetable.create(db.session, **{"name": "test"}) with self.assertRaises(StandardError): Timetable.create(db.session, **{"name": "test", "cron": "* * * * * *"}) # test correct input, though it has more keys than required # should fail since there is no such job exists job = Job.create(db.session, **self.jobOpts) self.opts["job_uid"] = job.uid timetable = Timetable.create(db.session, **self.opts) self.assertTrue(timetable.createtime > utils.currentTimeMillis() - 5000) self.assertEquals(timetable.job.json(), job.json())
def test_listRunnable(self): arr = [("ready", Job.READY), ("running", Job.RUNNING), ("finished", Job.FINISHED), ("running", Job.RUNNING), ("delayed", Job.DELAYED)] for name, status in arr: job = Job.create(db.session, **self.opts) job.name = name job.status = status if status is Job.DELAYED: job.submittime = job.submittime - 10000 job.priority = job.submittime / 1000L db.session.commit() jobs = Job.listRunnable(db.session, 5, utils.currentTimeMillis()) self.assertEqual(len(jobs), 2) self.assertEqual([x.name for x in jobs], ["delayed", "ready"]) self.assertEqual([x.status for x in jobs], [Job.DELAYED, Job.READY])
def test_jobCopy(self): job = Job.create(db.session, **self.opts) copy = job.jobCopy(name="a", status=Job.READY, priority=1, createtime=2L, submittime=2L) self.assertEquals(copy.uid, None) self.assertEquals(copy.name, "a") self.assertEquals(copy.status, Job.READY) self.assertEquals(copy.priority, 1) self.assertEquals(copy.createtime, 2L) self.assertEquals(copy.submittime, 2L) self.assertEquals(copy.sparkappid, None) self.assertEquals(copy.starttime, None) self.assertEquals(copy.finishtime, None) # these properties should be the same self.assertEquals(copy.options, job.options) self.assertEquals(copy.jobconf, job.jobconf) self.assertEquals(copy.entrypoint, job.entrypoint) self.assertEquals(copy.jar, job.jar)
def test_add(self): job = Job.create(db.session, **self.opts) arr = Job.list(db.session, None) self.assertTrue(len(arr), 1)