result = resp.read().decode(encoding='UTF-8', errors='strict')
if resp.code == 500:
return json.dumps({"error": result})
else:
return '{"data": %s}' % result
except:
logging.exception("SparkSQL job execution error")
return json.dumps({'error': str(sys.exc_info()[1])})
finally:
conn.close()
executors = {
'default': ThreadPoolExecutor(20),
'processpool': ProcessPoolExecutor(5)
}
job_defaults = {
'coalesce': False,
'max_instances': 300
}
scheduler = BackgroundScheduler(executors=executors, job_defaults=job_defaults)
for job in session.execute('SELECT * FROM spark_sql_jobs'):
logging.info("Add SparkSQL job[%s] hour[%s] sql[%s]" % (job['job_id'], job['hour'], job['sql'][0:50]))
scheduler.scheduled_job('cron', [job['job_id'], job['sql'], job['ttl']], id=job['job_id'], day=job['day'], hour=job['hour'], second='0', minute='0')(exec_sparkjob)
scheduler.start()
return json.dumps({"error": result})
else:
return '{"data": %s}' % result
except:
logging.exception("SparkSQL job execution error")
return json.dumps({'error': str(sys.exc_info()[1])})
finally:
conn.close()
executors = {
'default': ThreadPoolExecutor(20),
'processpool': ProcessPoolExecutor(5)
}
job_defaults = {'coalesce': False, 'max_instances': 300}
scheduler = BackgroundScheduler(executors=executors, job_defaults=job_defaults)
for job in session.execute('SELECT * FROM spark_sql_jobs'):
logging.info("Add SparkSQL job[%s] hour[%s] sql[%s]" %
(job['job_id'], job['hour'], job['sql'][0:50]))
scheduler.scheduled_job('cron', [job['job_id'], job['sql'], job['ttl']],
id=job['job_id'],
day=job['day'],
hour=job['hour'],
second='0',
minute='0')(exec_sparkjob)
scheduler.start()
class ExScheduler(object):
def __init__(self):
self.__scheduler = BackgroundScheduler()
self.__jobstatuslist = {}
def start_scheduler(self, **options):
self.__scheduler.configure(**options)
self.__scheduler.start()
def shutdownscheduler(self):
self.__scheduler.shutdown(wait=False)
def getjob(self, jobid):
return self.__scheduler.get_job(jobid)
def scheduledjob(self, *args, **kw):
return self.__scheduler.scheduled_job(*args, **kw)
def addjob(self, *args, **kw):
job = self.__scheduler.add_job(*args, **kw)
self.__scheduler.wakeup()
return job
def removejob(self, job_id, jobstore=None):
self.__scheduler.remove_job(job_id=job_id, jobstore=jobstore)
def pausejob(self, job_id, jobstore=None):
self.__scheduler.pause_job(job_id=job_id, jobstore=jobstore)
self.setjobstatus(job_id, JobStatus.PAUSED)
def resumejob(self, job_id, jobstore=None):
self.__scheduler.resume_job(job_id=job_id, jobstore=jobstore)
self.setjobstatus(job_id, JobStatus.SCHEDULING)
def modifyjob(self, job_id, jobstore=None, **kw):
job = self.__scheduler.modify_job(job_id=job_id,
jobstore=jobstore,
**kw)
return job
def getjoblist(self):
joblist = self.__scheduler.get_jobs()
return map(self._getjobinfo, joblist)
def addlistener(self, callback, mask=EVENT_ALL):
self.__scheduler.add_listener(callback, mask)
def setjobstatus(self, jobid, jobstatus):
self.__jobstatuslist[jobid] = jobstatus
def jobstatusinitial(self, job_id, jobstore=None):
job = self.__scheduler.get_job(job_id=job_id, jobstore=jobstore)
if job_id not in self.__jobstatuslist:
status = (JobStatus.SCHEDULING
if job.next_run_time else JobStatus.PAUSED) if hasattr(
job, 'next_run_time') else JobStatus.PENDING
self.setjobstatus(job_id, status)
def _getjobinfo(self, job):
self.jobstatusinitial(job.id)
return {
"id":
str(job.id),
"name":
str(job.name),
"kwargs":
job.kwargs,
"trigger":
trigger_str_to_dict(str(job.trigger)),
"next_run_time":
datetime_repr(job.next_run_time)
if self.__jobstatuslist[str(job.id)] == JobStatus.SCHEDULING or
self.__jobstatuslist[str(job.id)] == JobStatus.RUNNING else "--",
"status":
self.__jobstatuslist[str(job.id)]
}
