def create_sparkjob(job_id, sql, ttl, **cron_args):
if scheduler.get_job(job_id) is not None:
raise Exception('repeated job_id[%s]' % job_id)
ttl = 60 * 60 * 24 * int(ttl)
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
session.execute("INSERT INTO spark_sql_jobs (job_id, day, hour, sql, status, ttl, created_at)"
" VALUES (%s, %s, %s, %s, %s, %s, %s)",
[job_id, cron_args['day'], cron_args['hour'], sql, '1', ttl, now])
logging.info("Add job:%s" % job_id)
scheduler.scheduled_job('cron', [job_id, sql, ttl], id=job_id, **cron_args)(exec_sparkjob)
def get(self, *args, **kwargs):
if self.request.headers['Accept'].__contains__('application/json'):
job_id = self.get_argument('job_id', default='')
if job_id is not '':
job_id = " where job_id = '%s'" % job_id
jobs = session.execute("select * from spark_sql_jobs %s " % job_id)
self.return_json(jobs)
else:
jobs = session.execute("select * from spark_sql_jobs")
self.render('/jobs', data=jobs, user=self.current_user)
def get(self, *args, **kwargs):
if self.request.headers['Accept'].__contains__('application/json'):
job_id = self.get_argument('job_id', default='')
if job_id is not '':
job_id = " where job_id = '%s'" % job_id
jobs = session.execute("select * from spark_sql_jobs %s " % job_id)
self.return_json(jobs)
else:
jobs = session.execute("select * from spark_sql_jobs")
self.render('/jobs', data=jobs, user=self.current_user)
def create_sparkjob(job_id, sql, ttl, **cron_args):
if scheduler.get_job(job_id) is not None:
raise Exception('repeated job_id[%s]' % job_id)
ttl = 60 * 60 * 24 * int(ttl)
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
session.execute(
"INSERT INTO spark_sql_jobs (job_id, day, hour, sql, status, ttl, created_at)"
" VALUES (%s, %s, %s, %s, %s, %s, %s)",
[job_id, cron_args['day'], cron_args['hour'], sql, '1', ttl, now])
logging.info("Add job:%s" % job_id)
scheduler.scheduled_job('cron', [job_id, sql, ttl], id=job_id,
**cron_args)(exec_sparkjob)
def post(self):
name = self.get_argument("name")
user = session.execute("SELECT * FROM users WHERE name = %s", name)[0]
if not user:
logging.error("user(name=%s) not exist", name)
self.return_json({"cause": 'user.not.exist'}, 400)
# self.redirect("/user_not_exist")
return
if user['status'] != 1:
logging.error("user(name=%s) status is not normal")
self.return_json({"cause": "user.status.abnormal"}, 500)
return
if user['password'] != self.get_argument("password"):
logging.error("user(name=%s) password mismatch", name)
self.return_json({"cause": "password.mismatch"}, 500)
return
self.set_secure_cookie("cookie_user_name", str(user['name']), expires_days=None)
self.redirect(self.get_argument("next", "/"))
def remove_sparkjob(job_id):
scheduler.remove_job(job_id)
session.execute("DELETE from spark_sql_jobs where job_id = '%s'" % job_id)
def start_sparkjob(job_id):
rows = session.execute("SELECT * FROM spark_sql_jobs where job_id=%s",
job_id)
if len(rows) > 0:
exec_sparkjob(rows[0].job_id, rows[0].sql, rows[0].ttl)
return json.dumps({"error": result})
else:
return '{"data": %s}' % result
except:
logging.exception("SparkSQL job execution error")
return json.dumps({'error': str(sys.exc_info()[1])})
finally:
conn.close()
executors = {
'default': ThreadPoolExecutor(20),
'processpool': ProcessPoolExecutor(5)
}
job_defaults = {'coalesce': False, 'max_instances': 300}
scheduler = BackgroundScheduler(executors=executors, job_defaults=job_defaults)
for job in session.execute('SELECT * FROM spark_sql_jobs'):
logging.info("Add SparkSQL job[%s] hour[%s] sql[%s]" %
(job['job_id'], job['hour'], job['sql'][0:50]))
scheduler.scheduled_job('cron', [job['job_id'], job['sql'], job['ttl']],
id=job['job_id'],
day=job['day'],
hour=job['hour'],
second='0',
minute='0')(exec_sparkjob)
scheduler.start()
def get(self, *args, **kwargs):
jobs = session.execute("select * from spark_sql_jobs")
self.render('/jobs', data=jobs, user=self.current_user)
def remove_sparkjob(job_id):
scheduler.remove_job(job_id)
session.execute("DELETE from spark_sql_jobs where job_id = '%s'" % job_id)
def start_sparkjob(job_id):
rows = session.execute("SELECT * FROM spark_sql_jobs where job_id=%s", job_id)
if len(rows) > 0:
exec_sparkjob(rows[0].job_id, rows[0].sql, rows[0].ttl)
result = resp.read().decode(encoding='UTF-8', errors='strict')
if resp.code == 500:
return json.dumps({"error": result})
else:
return '{"data": %s}' % result
except:
logging.exception("SparkSQL job execution error")
return json.dumps({'error': str(sys.exc_info()[1])})
finally:
conn.close()
executors = {
'default': ThreadPoolExecutor(20),
'processpool': ProcessPoolExecutor(5)
}
job_defaults = {
'coalesce': False,
'max_instances': 300
}
scheduler = BackgroundScheduler(executors=executors, job_defaults=job_defaults)
for job in session.execute('SELECT * FROM spark_sql_jobs'):
logging.info("Add SparkSQL job[%s] hour[%s] sql[%s]" % (job['job_id'], job['hour'], job['sql'][0:50]))
scheduler.scheduled_job('cron', [job['job_id'], job['sql'], job['ttl']], id=job['job_id'], day=job['day'], hour=job['hour'], second='0', minute='0')(exec_sparkjob)
scheduler.start()
def get(self, *args, **kwargs):
jobs = session.execute("select * from spark_sql_jobs")
self.render('/jobs', data=jobs, user=self.current_user)
def get_current_user(self):
user_name = self.get_secure_cookie("cookie_user_name")
if not user_name:
return None
return session.execute("SELECT * FROM users WHERE name = %s",
user_name)[0]