def post(self):
global _etl_page
_name = request.json.get('server_name')
context = dict(form=setForm(_etl_page),
result=True,
etl_page=_etl_page,
etl_href="etl_mod_post",
etl_do=getStatus('do'),
etl_info=getStatus('info'))
context.update(setMenus())
if _etl_page == 0:
etl = EtlManager(EtlDir)
elif _etl_page == 1:
etl = EtlManager(EtlMod)
elif _etl_page == 2:
etl = EtlManager(EtlTask)
deleteTask(_name)
elif _etl_page == 3:
etl = EtlManager(EtlJob)
deleteJob(_name)
_str = {'name': _name}
etl.delete_by(**_str)
if _etl_page == 0:
_template_name = 'etltools/dirlist.html'
_page, _info = getList(EtlDir)
context['dirs'] = _info
context['pagination'] = _page
elif _etl_page == 1:
_page, _info = getList(EtlMod)
_template_name = 'etltools/modlist.html'
context['mods'] = _info
context['pagination'] = _page
elif _etl_page == 2:
_page, _info = getList(EtlTask)
_template_name = 'etltools/tasklist.html'
context['tasks'] = _info
context['pagination'] = _page
elif _etl_page == 3:
_page, _info = getList(EtlJob)
_template_name = 'etltools/joblist.html'
context['jobs'] = _info
context['pagination'] = _page
context['etl_info'] = getStatus('info')
return jsonify(body=render_template(_template_name, **context))
def getAll(model):
"""
获取数据模板的所有数据记录
:param model:
:return:
"""
etl = EtlManager(model)
_recs = etl.list()
if _recs == None:
return None
_posts = []
for _rec in _recs:
_post = {}
_post['name'] = _rec.name
if model == EtlDir:
_post['desc'] = _rec.desc
_post['uuid'] = _rec.dir
_post['created_at'] = _rec.created_at
elif model == EtlServer:
_post['desc'] = _rec.desc
_post['url'] = _rec.url
_post['status'] = _rec.status
_post['created_at'] = _rec.created_at
elif model == EtlMod:
_post['desc'] = _rec.desc
_post['r_name'] = _rec.r_name
_post['uuid'] = _rec.filename
_post['cmd'] = _rec.cmd
_post['version'] = _rec.version
_post['created_at'] = _rec.created_at
elif model == EtlTask:
_post['mod'] = _rec.mod
_post['server'] = _rec.server
_post['created_at'] = _rec.created_at
_post['status'] = _rec.status
elif model == EtlJob:
_post['mod'] = _rec.mod
_post['server'] = _rec.server
_post['schedule'] = _rec.schedule
_post['created_at'] = _rec.created_at
_post['status'] = _rec.status
_posts.append(_post)
return _posts
def setJobStatus(job_name, status):
etl = EtlManager(EtlJob)
_rec = etl.get(job_name)
if _rec is not None:
val = {'status': status}
etl = EtlManager(EtlJob)
etl.update_by_name(job_name, **val)
def setTaskStatus(task_name, status):
etl = EtlManager(EtlTask)
_rec = etl.get(task_name)
if _rec is not None:
val = {'status': status}
etl = EtlManager(EtlTask)
etl.update_by_name(task_name, **val)
def updateJobStatus(job_name, status):
"""
根据返回状态修改数据模型中的status
0～20：步进5（约20秒）；21～40：步进3（约35秒）；41～60：步进2（约50秒）；61～80：步进1（约100秒）；81～99：步进1（约100秒）；99：不变
:return:
"""
etl = EtlManager(EtlJob)
_rec = etl.get(job_name)
if _rec is not None:
_sts = _rec['status']
if status != _sts:
if status == 'RUNNING':
if _sts == 'DONE' or _sts == 'ERROR':
_sts = '100'
elif _sts == 'SCHEDULE':
_sts = '0'
_st = int(_sts)
if _st < 21:
_st += 5
elif _st < 41:
_st += 3
elif _st < 99:
_st += 1
val = {'status': str(_st)}
elif status == 'DONE':
val = {'status': 'DONE'}
elif status == 'SCHEDULE':
val = {'status': 'SCHEDULE'}
else:
val = {'status': 'ERROR'}
etl = EtlManager(EtlJob)
etl.update(job_name, **val)
def init_etl_model():
etl = EtlManager(EtlDir)
param = {
'desc': u'用于管理ETL脚本文件。',
}
etl.create('ETL脚本', **param)
param = {
'desc': u'用于管理Spark运行文件。',
}
etl.create('Spark脚本', **param)
def post(self):
global _etl_page, _task_log
_url = rest_get_url('etl-task/')
if _url != '':
_taskname = request.json.get('task-name')
'''
2015.8.31 TODO: 目前尚未考虑多台ETL服务器的情况
_server = request.json.get('server')
'''
_mod = request.json.get('mod')
if _etl_page == 2:
etl = EtlManager(EtlMod)
_rec = etl.get(_mod)
if _rec is not None:
_path = '/home/shenwei/nebula/static/etl_mod/' + _rec.filename
""" 调用 REST API 激活任务
接口参数：
{ 'src-dir': 模块文件全路径，例如：'/home/shenwei/data/myphone.json.ktr',
'task-name': 任务名称，例如'my-tester'}
"""
_api_data = {
'src-dir': _path,
'task-name': _taskname,
'cmd': _rec.cmd
}
restAPI(_url, _api_data)
setTaskStatus(_taskname, '1')
else:
setTaskStatus(_taskname, 'ERROR')
context = dict(form=setForm(_etl_page),
result=True,
etl_page=_etl_page,
etl_href="etl_mod_post",
etl_do=getStatus('do'),
etl_info=getStatus('info'))
context.update(setMenus())
if _etl_page == 0:
_page, _info = getList(EtlServer)
_template_name = 'etltools/dirlist.html'
context['servers'] = _info
elif _etl_page == 1:
_page, _info = getList(EtlMod)
_template_name = 'etltools/modlist.html'
context['mods'] = _info
elif _etl_page == 2:
_page, _info = getList(EtlTask)
_template_name = 'etltools/tasklist.html'
context['tasks'] = _info
_task_log = []
context['text'] = _task_log
elif _etl_page == 3:
_page, _info = getList(EtlJob)
_template_name = 'etltools/joblist.html'
context['jobs'] = _info
context['pagination'] = _page
context['etl_info'] = getStatus('info')
return jsonify(body=render_template(_template_name, **context))
def submitForm():
"""
处理SUBMIT事件
:return:
转向的HTML内容
"""
global _status, _etl_page
context = dict(
form=setForm(_etl_page),
result=False,
etl_page=_etl_page,
etl_href="etl_mod_post",
etl_do=getStatus('do'),
etl_png="../static/etl.jpg",
etl_info=getStatus('info'),
)
context.update(setMenus())
if _etl_page == 0:
"""资源目录
"""
form = DirForm()
context['form'] = form
elif _etl_page == 1:
"""资源
"""
form = UpLoadForm()
context['form'] = form
elif _etl_page == 2:
"""任务
"""
form = TaskForm()
context['form'] = form
elif _etl_page == 3:
"""作业
"""
form = JobForm()
context['form'] = form
if form.validate_on_submit():
if _etl_page == 0:
name = form.name.data
desc = form.desc.data
param = {
'desc': desc,
'dir': getUUID(),
}
etl = EtlManager(EtlDir)
_rec = etl.get(name)
if _rec == None:
etl.create(name, **param)
"""!!!创建目录
"""
setStatus('info', u"")
context['etl_info'] = getStatus('info')
setStatus('do', 0)
context['etl_do'] = getStatus('do')
else:
setStatus('info', u"错误：该资源目录名称已经存在！")
context['etl_info'] = getStatus('info')
elif _etl_page == 1:
name = form.name.data
desc = form.desc.data
cmd = form.cmd.data
version = form.version.data
_filename = form.filename.data.filename
param = {
'desc': desc,
'r_name': _filename,
'filename': getUUID(),
'cmd': cmd,
'version': version,
}
context['filename'] = _filename
etl = EtlManager(EtlMod)
_rec = etl.get(name)
if _rec == None:
_path = '/home/shenwei/nebula/static/etl_mod/' + param[
'filename']
form.filename.data.save(_path)
context['result'] = _filename
setStatus('filename', _filename)
context['etl_page'] = _etl_page
context['etl_do'] = getStatus('do')
context['filename'] = _filename
etl.create(name, **param)
setStatus('info', u"")
context['etl_info'] = getStatus('info')
setStatus('do', 0)
context['form'] = UpLoadForm()
context['etl_do'] = getStatus('do')
else:
setStatus('info', u"错误：该模块名称已经存在！")
context['etl_info'] = getStatus('info')
setStatus('info', u"")
elif _etl_page == 2:
name = form.name.data
mod = form.mod.data
server = form.server.data
param = {
'mod': mod,
'server': server,
'status': '0' # 新任务的工作进度为 0
}
""" 查看该名称已使用？
"""
etl = EtlManager(EtlTask)
_rec = etl.get(name)
if _rec == None:
""" 新任务创建完成，需把任务请求发给指定的ETL服务器执行。
1）用scp拷贝模型到指定ETL服务器的模型目录；
2）调用该ETL服务器的REST API发起任务请求。
"""
context['etl_page'] = _etl_page
etl.create(name, **param)
setStatus('info', u"")
context['etl_info'] = getStatus('info')
setStatus('do', 0)
context['form'] = TaskForm()
context['etl_do'] = getStatus('do')
else:
setStatus('info', u"错误：该模块名称已经存在！")
context['etl_info'] = getStatus('info')
setStatus('info', u"")
elif _etl_page == 3:
name = form.name.data
mod = form.mod.data
server = form.server.data
schedule = form.schedule.data
param = {
'mod': mod,
'server': server,
'schedule': schedule,
'status': '0', # 新作业的工作进度为 0
}
etl = EtlManager(EtlJob)
_rec = etl.get(name)
if _rec == None:
"""新作业创建完成，需把任务请求发给指定的ETL服务器执行。
1）用scp拷贝模型到指定ETL服务器的模型目录；
2）调用该ETL服务器的REST API发起任务请求。
"""
context['etl_page'] = _etl_page
etl.create(name, **param)
setStatus('info', u"")
context['etl_info'] = getStatus('info')
setStatus('do', 0)
context['form'] = JobForm()
context['etl_do'] = getStatus('do')
else:
setStatus('info', u"错误：该模块名称已经存在！")
context['etl_info'] = getStatus('info')
setStatus('info', u"")
else:
setStatus('do', 0)
context['etl_do'] = getStatus('do')
if _etl_page == 0:
_page, _info = getList(EtlDir)
context['pagination'] = _page
context['dirs'] = _info
elif _etl_page == 1:
_page, _info = getList(EtlMod)
context['pagination'] = _page
context['mods'] = _info
elif _etl_page == 2:
_page, _info = getList(EtlTask)
context['pagination'] = _page
context['tasks'] = _info
elif _etl_page == 3:
_page, _info = getList(EtlJob)
context['pagination'] = _page
context['jobs'] = _info
return render_template('etltools/etlbase.html', **context)