result = resp.read().decode(encoding='UTF-8', errors='strict') if resp.code == 500: return json.dumps({"error": result}) else: return '{"data": %s}' % result except: logging.exception("SparkSQL job execution error") return json.dumps({'error': str(sys.exc_info()[1])}) finally: conn.close() executors = { 'default': ThreadPoolExecutor(20), 'processpool': ProcessPoolExecutor(5) } job_defaults = { 'coalesce': False, 'max_instances': 300 } scheduler = BackgroundScheduler(executors=executors, job_defaults=job_defaults) for job in session.execute('SELECT * FROM spark_sql_jobs'): logging.info("Add SparkSQL job[%s] hour[%s] sql[%s]" % (job['job_id'], job['hour'], job['sql'][0:50])) scheduler.scheduled_job('cron', [job['job_id'], job['sql'], job['ttl']], id=job['job_id'], day=job['day'], hour=job['hour'], second='0', minute='0')(exec_sparkjob) scheduler.start()
return json.dumps({"error": result}) else: return '{"data": %s}' % result except: logging.exception("SparkSQL job execution error") return json.dumps({'error': str(sys.exc_info()[1])}) finally: conn.close() executors = { 'default': ThreadPoolExecutor(20), 'processpool': ProcessPoolExecutor(5) } job_defaults = {'coalesce': False, 'max_instances': 300} scheduler = BackgroundScheduler(executors=executors, job_defaults=job_defaults) for job in session.execute('SELECT * FROM spark_sql_jobs'): logging.info("Add SparkSQL job[%s] hour[%s] sql[%s]" % (job['job_id'], job['hour'], job['sql'][0:50])) scheduler.scheduled_job('cron', [job['job_id'], job['sql'], job['ttl']], id=job['job_id'], day=job['day'], hour=job['hour'], second='0', minute='0')(exec_sparkjob) scheduler.start()
class ExScheduler(object): def __init__(self): self.__scheduler = BackgroundScheduler() self.__jobstatuslist = {} def start_scheduler(self, **options): self.__scheduler.configure(**options) self.__scheduler.start() def shutdownscheduler(self): self.__scheduler.shutdown(wait=False) def getjob(self, jobid): return self.__scheduler.get_job(jobid) def scheduledjob(self, *args, **kw): return self.__scheduler.scheduled_job(*args, **kw) def addjob(self, *args, **kw): job = self.__scheduler.add_job(*args, **kw) self.__scheduler.wakeup() return job def removejob(self, job_id, jobstore=None): self.__scheduler.remove_job(job_id=job_id, jobstore=jobstore) def pausejob(self, job_id, jobstore=None): self.__scheduler.pause_job(job_id=job_id, jobstore=jobstore) self.setjobstatus(job_id, JobStatus.PAUSED) def resumejob(self, job_id, jobstore=None): self.__scheduler.resume_job(job_id=job_id, jobstore=jobstore) self.setjobstatus(job_id, JobStatus.SCHEDULING) def modifyjob(self, job_id, jobstore=None, **kw): job = self.__scheduler.modify_job(job_id=job_id, jobstore=jobstore, **kw) return job def getjoblist(self): joblist = self.__scheduler.get_jobs() return map(self._getjobinfo, joblist) def addlistener(self, callback, mask=EVENT_ALL): self.__scheduler.add_listener(callback, mask) def setjobstatus(self, jobid, jobstatus): self.__jobstatuslist[jobid] = jobstatus def jobstatusinitial(self, job_id, jobstore=None): job = self.__scheduler.get_job(job_id=job_id, jobstore=jobstore) if job_id not in self.__jobstatuslist: status = (JobStatus.SCHEDULING if job.next_run_time else JobStatus.PAUSED) if hasattr( job, 'next_run_time') else JobStatus.PENDING self.setjobstatus(job_id, status) def _getjobinfo(self, job): self.jobstatusinitial(job.id) return { "id": str(job.id), "name": str(job.name), "kwargs": job.kwargs, "trigger": trigger_str_to_dict(str(job.trigger)), "next_run_time": datetime_repr(job.next_run_time) if self.__jobstatuslist[str(job.id)] == JobStatus.SCHEDULING or self.__jobstatuslist[str(job.id)] == JobStatus.RUNNING else "--", "status": self.__jobstatuslist[str(job.id)] }