def get(self):
context = dict()
context.update(setMenus())
context.update({
'url': CONF.webapp.rest_api_register_web,
})
return render_template(self.template_name, **context)
def post(self):
global _etl_page, _job_log
_name = request.json.get('job_name')
context = dict(form=setForm(_etl_page),
result=True,
etl_page=_etl_page,
etl_href="etl_mod_post",
etl_do=getStatus('do'),
etl_info=getStatus('info'))
context.update(setMenus())
_page, _info = getList(EtlTask)
_template_name = 'etltools/text.html'
context['tasks'] = _info
context['pagination'] = _page
context['etl_info'] = getStatus('info')
""" 获取 ETL任务 的日志
"""
_url = rest_get_url('etl-job-log/')
_job_log = rest_get_job_log(_url, _name)['text']
context['text'] = _job_log
return jsonify(body=render_template(_template_name, **context))
def get(self):
context = dict(
form=MusicFileForm(),
result=False,
)
context.update(setMenus())
return render_template(self._template_name, **context)
def get_context_data(self, **kwargs):
context = super(DashboardView, self).get_context_data(**kwargs)
context.update(dict(resources=self._get_statistics(), ))
context.update(setMenus())
return context
def get_context_data(self, **kwargs):
context = super(EMPTYView, self).get_context_data(**kwargs)
empty_id = request.args.get("resource_id", None)
context.update({
"resource_id": empty_id,
})
context.update(setMenus())
return context
def post(self):
form = UpLoadForm()
context = dict(
form=form,
)
context.update(setMenus())
return jsonify(body=render_template(self._template_name, **context))
def post(self):
context = dict(
form=None,
result=False,
)
context.update(setMenus())
return render_template(self._template_name, **context)
def submitForm():
"""
处理SUBMIT事件
:return:
转向的HTML内容
"""
global q
context = dict(
form=UpLoadForm()
)
context.update({
"icon": "icon-folder-open",
"title": u"数据导入：",
})
context.update(setMenus())
form=UpLoadForm()
if form.validate_on_submit():
desc = form.desc.data
version = form.version.data
_filename = form.filename.data.filename
_uuid = getUUID()
_path = '/home/shenwei/nebula/static/etl_mod/%s'% _uuid
form.filename.data.save(_path)
f_hash = getFileHash(_path)
param = {
'desc': desc,
'r_name': _filename,
'filename': _uuid,
'version': version,
'f_hash': f_hash,
}
context['filename'] = _filename
context['result'] = _filename
upfile = UpFileManager(UpFileMod)
""" 防止数据被重复录入：
1）查看是否已经存在 该文件的 hash值
2）若不存在，则说明：该文件未被录入；否则，表示该文件数据已录入
"""
_hash = upfile.get(f_hash)
if not _hash:
upfile.create(_filename, **param)
try:
q.send(_path,block=False)
context.update({'info': "已提交处理！"})
except:
context.update({'info': "文件处理出错：满了！"})
return render_template('webapp/datain.html', **context)
def get(self):
"""
用于 列表翻页 的刷新
:return:
"""
context = dict(
form=UpLoadForm(),
)
context.update(setMenus())
return render_template('webapp/datain.html', **context)
def post(self):
global _etl_page
_name = request.json.get('server_name')
context = dict(form=setForm(_etl_page),
result=True,
etl_page=_etl_page,
etl_href="etl_mod_post",
etl_do=getStatus('do'),
etl_info=getStatus('info'))
context.update(setMenus())
if _etl_page == 0:
etl = EtlManager(EtlDir)
elif _etl_page == 1:
etl = EtlManager(EtlMod)
elif _etl_page == 2:
etl = EtlManager(EtlTask)
deleteTask(_name)
elif _etl_page == 3:
etl = EtlManager(EtlJob)
deleteJob(_name)
_str = {'name': _name}
etl.delete_by(**_str)
if _etl_page == 0:
_template_name = 'etltools/dirlist.html'
_page, _info = getList(EtlDir)
context['dirs'] = _info
context['pagination'] = _page
elif _etl_page == 1:
_page, _info = getList(EtlMod)
_template_name = 'etltools/modlist.html'
context['mods'] = _info
context['pagination'] = _page
elif _etl_page == 2:
_page, _info = getList(EtlTask)
_template_name = 'etltools/tasklist.html'
context['tasks'] = _info
context['pagination'] = _page
elif _etl_page == 3:
_page, _info = getList(EtlJob)
_template_name = 'etltools/joblist.html'
context['jobs'] = _info
context['pagination'] = _page
context['etl_info'] = getStatus('info')
return jsonify(body=render_template(_template_name, **context))
def get_context_data(self, **kwargs):
context = super(CDHListView, self).get_context_data(**kwargs)
cdh_id = request.args.get("resource_id", None)
form = UpLoadForm()
context.update({
"resource_id": cdh_id,
"icon": "icon-folder-open",
"title": u"数据导入：",
"form": form,
})
context.update(setMenus())
return context
def get(self):
"""
用于 列表翻页 的刷新
:return:
"""
global _page_idx, _page
try:
_page_idx[_page] = int(request.args.get('page', 1))
except ValueError:
_page_idx[_page] = 1
finally:
context = dict(
result=False,
_page=_page,
etl_href="etl_mod_post",
etl_png="../static/etl.jpg",
)
context.update(setMenus())
if _page == 0:
_resp_page, _info = getList(DataElement)
template_name = 'datamng/dataelementlist.html'
elif _page == 1:
_resp_page, _info = getList(MemberMod)
template_name = 'datamng/memberlist.html'
elif _page == 2:
_resp_page, _info = getList(ProjectMod)
template_name = 'datamng/projectlist.html'
elif _page == 3:
_resp_page, _info = getList(TaskMod)
template_name = 'datamng/tasklist.html'
elif _page == 4:
_resp_page, _info = getList(ProductMod)
template_name = 'datamng/productlist.html'
elif _page == 5:
_resp_page, _info = getList(EnginerringMod)
template_name = 'datamng/enginerringlist.html'
elif _page == 6:
_resp_page, _info = getList(DeliveryMod)
template_name = 'datamng/deliverylist.html'
elif _page == 7:
_resp_page, _info = getList(UpFileMod)
template_name = 'datamng/upfilelist.html'
context['pagination'] = _resp_page
context['dirs'] = _info
return render_template('datamng/datamngbase.html', **context)
def get(self):
global _status, _etl_page
try:
_page_idx[_etl_page] = int(request.args.get('page', 1))
except ValueError:
_page_idx[_etl_page] = 1
finally:
context = dict(
result=False,
form=setForm(_etl_page),
etl_page=_etl_page,
etl_href="etl_mod_post",
etl_body=True,
etl_do=getStatus('do'),
filename=getStatus('filename'),
etl_info=getStatus('info'),
etl_png="../static/etl.jpg",
)
context.update(setMenus())
""" 获取UI的当前页面，并获取该页面对应数据模型的记录数据（供列表显示）
"""
_page = request.args.get('etl_app')
if _page == 'etl_mod_post':
_etl_page = 1
_page, _info = getList(EtlMod)
context['pagination'] = _page
context['tasks'] = _info
elif _page == 'etl_pen_post':
_etl_page = 2
_page, _info = getList(EtlTask)
context['pagination'] = _page
context['tasks'] = _info
elif _page == 'etl_scheduler_post':
_etl_page = 3
_page, _info = getList(EtlJob)
context['pagination'] = _page
context['jobs'] = _info
else:
_etl_page = 0
_page, _info = getList(EtlDir)
context['pagination'] = _page
context['dirs'] = _info
setStatus('do', 0)
context['etl_page'] = _etl_page
return render_template(self._template_name, **context)
def post(self):
"""
请求来自于：1）页面切换；2）定时器
:return:
"""
_value = int(request.json.get('value'))
context = dict(
form=UpLoadForm(),
)
context.update(setMenus())
template_name = 'webapp/form/register.html'
return jsonify(body=render_template(template_name, **context))
def get(self):
"""
用于 列表翻页 的刷新
:return:
"""
global _status, _page_idx, _task_log, _etl_page
try:
_page_idx[_etl_page] = int(request.args.get('page', 1))
except ValueError:
_page_idx[_etl_page] = 1
finally:
context = dict(form=setForm(_etl_page),
result=False,
etl_page=_etl_page,
etl_href="etl_mod_post",
etl_do=getStatus('do'),
etl_png="../static/etl.jpg",
etl_info=getStatus('info'))
context.update(setMenus())
if _etl_page == 0:
_page, _info = getList(EtlDir)
context['pagination'] = _page
context['dirs'] = _info
elif _etl_page == 1:
_page, _info = getList(EtlMod)
context['pagination'] = _page
context['mods'] = _info
elif _etl_page == 2:
_page, _info = getList(EtlTask)
context['pagination'] = _page
context['tasks'] = _info
context['text'] = _task_log
elif _etl_page == 3:
_page, _info = getList(EtlJob)
context['pagination'] = _page
context['jobs'] = _info
context['text'] = _job_log
return render_template('etltools/etlbase.html', **context)
def post(self):
"""
请求来自于：页面切换
:return:
"""
global _page
_value = int(request.json.get('value'))
if _value <= 7:
_page = _value
context = dict(
result=False,
_page=_page,
etl_href="etl_mod_post",
etl_png="../static/etl.jpg",
)
context.update(setMenus())
if _value == 0:
_resp_page, _info = getList(DataElement)
template_name = 'datamng/dataelementlist.html'
elif _value == 1:
_resp_page, _info = getList(MemberMod)
template_name = 'datamng/memberlist.html'
elif _value == 2:
_resp_page, _info = getList(ProjectMod)
template_name = 'datamng/projectlist.html'
elif _value == 3:
_resp_page, _info = getList(TaskMod)
template_name = 'datamng/tasklist.html'
elif _value == 4:
_resp_page, _info = getList(ProductMod)
template_name = 'datamng/productlist.html'
elif _value == 5:
_resp_page, _info = getList(EnginerringMod)
template_name = 'datamng/enginerringlist.html'
elif _value == 6:
_resp_page, _info = getList(DeliveryMod)
template_name = 'datamng/deliverylist.html'
elif _value == 7:
_resp_page, _info = getList(UpFileMod)
template_name = 'datamng/upfilelist.html'
context['dirs'] = _info
context['pagination'] = _resp_page
return jsonify(body=render_template(template_name, **context))
def post(self):
global _etl_page, _status
_etl_page = int(request.json.get('value'))
form = setForm(_etl_page)
context = dict(result=False,
form=form,
etl_page=_etl_page,
etl_href="etl_mod_post",
etl_do=getStatus('do'),
etl_png="../static/etl.jpg",
etl_info=getStatus('info'))
context.update(setMenus())
return jsonify(body=render_template(self._template_name, **context))
def post(self):
form = MusicFileForm()
context = dict(form=form, )
context.update(setMenus())
if form.validate_on_submit():
_filename = form.filename.data.filename
_ext = _filename.split('.')[-1]
_path = os.path.join(
os.path.realpath('.') + '/static/', "_temp_music.%s" % _ext)
form.filename.data.save(_path)
_ret = json.loads(self.doCLI(_path))
if _ret["status"] == 0:
context['result'] = True
context['file_name'] = _filename
context['music_type'] = _ret['result']
context['image_file'] = "%s.png" % _ret['result']
return render_template(self._template_name, **context)
def post(self):
global _page
_name = request.json.get('keyid')
context = dict(
result=True,
_page=_page,
etl_href="etl_mod_post",
)
context.update(setMenus())
etl = DataManager(DataElement)
_str = {'keyid': _name}
etl.delete_by(**_str)
_template_name = 'datamng/dataelementlist.html'
_resp_page, _info = getList(DataElement)
context['dirs'] = _info
context['pagination'] = _resp_page
return jsonify(body=render_template(_template_name, **context))
def post(self):
global _etl_page, _task_log
_url = rest_get_url('etl-task/')
if _url != '':
_taskname = request.json.get('task-name')
'''
2015.8.31 TODO: 目前尚未考虑多台ETL服务器的情况
_server = request.json.get('server')
'''
_mod = request.json.get('mod')
if _etl_page == 2:
etl = EtlManager(EtlMod)
_rec = etl.get(_mod)
if _rec is not None:
_path = '/home/shenwei/nebula/static/etl_mod/' + _rec.filename
""" 调用 REST API 激活任务
接口参数：
{ 'src-dir': 模块文件全路径，例如：'/home/shenwei/data/myphone.json.ktr',
'task-name': 任务名称，例如'my-tester'}
"""
_api_data = {
'src-dir': _path,
'task-name': _taskname,
'cmd': _rec.cmd
}
restAPI(_url, _api_data)
setTaskStatus(_taskname, '1')
else:
setTaskStatus(_taskname, 'ERROR')
context = dict(form=setForm(_etl_page),
result=True,
etl_page=_etl_page,
etl_href="etl_mod_post",
etl_do=getStatus('do'),
etl_info=getStatus('info'))
context.update(setMenus())
if _etl_page == 0:
_page, _info = getList(EtlServer)
_template_name = 'etltools/dirlist.html'
context['servers'] = _info
elif _etl_page == 1:
_page, _info = getList(EtlMod)
_template_name = 'etltools/modlist.html'
context['mods'] = _info
elif _etl_page == 2:
_page, _info = getList(EtlTask)
_template_name = 'etltools/tasklist.html'
context['tasks'] = _info
_task_log = []
context['text'] = _task_log
elif _etl_page == 3:
_page, _info = getList(EtlJob)
_template_name = 'etltools/joblist.html'
context['jobs'] = _info
context['pagination'] = _page
context['etl_info'] = getStatus('info')
return jsonify(body=render_template(_template_name, **context))
def post(self):
"""
请求来自于：1）页面切换；2）定时器
:return:
"""
global _status, _task_log, _job_log, _etl_page
try:
_value = int(request.json.get('value'))
if _value <= 3:
_etl_page = _value
if _etl_page == 2:
""" 收集 ETL任务 的执行状态
"""
_url = rest_get_url('etl-task-status/')
if _url != '':
_sts = rest_get_task_status(_url)
for _st in _sts:
""" 根据返回的任务状态［组］修改数据模型中的 status 值
为体现良好的UI体验，任务进度采用log函数效果（前端快，后端缓慢）
0～20：步进5（约20秒）；21～40：步进3（约35秒）；41～60：步进2（约50秒）；61～80：步进1（约100秒）；81～99：步进1（约100秒）；99：不变
"""
updateTaskStatus(_st, _sts[_st])
elif _etl_page == 3:
""" 收集 ETL作业 的执行状态
"""
_url = rest_get_url('etl-job-status/')
if _url != '':
_sts = rest_get_task_status(_url)
for _st in _sts:
""" 根据返回的任务状态［组］修改数据模型中的 status 值
为体现良好的UI体验，任务进度采用log函数效果（前端快，后端缓慢）
0～20：步进5（约20秒）；21～40：步进3（约35秒）；41～60：步进2（约50秒）；61～80：步进1（约100秒）；81～99：步进1（约100秒）；99：不变
"""
updateJobStatus(_st, _sts[_st])
except Exception:
return submitForm()
context = dict(form=setForm(_etl_page),
result=False,
etl_page=_etl_page,
etl_href="etl_mod_post",
etl_do=getStatus('do'),
etl_png="../static/etl.jpg",
etl_info=getStatus('info'))
context.update(setMenus())
if _etl_page == 1:
if getStatus('do') == 1:
template_name = 'etltools/form/register.html'
else:
template_name = 'etltools/modlist.html'
_page, _info = getList(EtlMod)
context['pagination'] = _page
context['mods'] = _info
elif _etl_page == 2:
if getStatus('do') == 1:
template_name = 'etltools/form/register.html'
else:
template_name = 'etltools/tasklist.html'
_page, _info = getList(EtlTask)
context['pagination'] = _page
context['tasks'] = _info
context['text'] = _task_log
elif _etl_page == 3:
if getStatus('do') == 1:
template_name = 'etltools/form/register.html'
context['form'] = JobForm()
else:
template_name = 'etltools/joblist.html'
_page, _info = getList(EtlJob)
context['pagination'] = _page
context['jobs'] = _info
context['text'] = _job_log
else:
if getStatus('do') == 1:
template_name = 'etltools/form/register.html'
else:
template_name = 'etltools/dirlist.html'
_page, _info = getList(EtlDir)
context['pagination'] = _page
context['dirs'] = _info
context['path'] = u'根目录'
return jsonify(body=render_template(template_name, **context))
def submitForm():
"""
处理SUBMIT事件
:return:
转向的HTML内容
"""
global _status, _etl_page
context = dict(
form=setForm(_etl_page),
result=False,
etl_page=_etl_page,
etl_href="etl_mod_post",
etl_do=getStatus('do'),
etl_png="../static/etl.jpg",
etl_info=getStatus('info'),
)
context.update(setMenus())
if _etl_page == 0:
"""资源目录
"""
form = DirForm()
context['form'] = form
elif _etl_page == 1:
"""资源
"""
form = UpLoadForm()
context['form'] = form
elif _etl_page == 2:
"""任务
"""
form = TaskForm()
context['form'] = form
elif _etl_page == 3:
"""作业
"""
form = JobForm()
context['form'] = form
if form.validate_on_submit():
if _etl_page == 0:
name = form.name.data
desc = form.desc.data
param = {
'desc': desc,
'dir': getUUID(),
}
etl = EtlManager(EtlDir)
_rec = etl.get(name)
if _rec == None:
etl.create(name, **param)
"""!!!创建目录
"""
setStatus('info', u"")
context['etl_info'] = getStatus('info')
setStatus('do', 0)
context['etl_do'] = getStatus('do')
else:
setStatus('info', u"错误：该资源目录名称已经存在！")
context['etl_info'] = getStatus('info')
elif _etl_page == 1:
name = form.name.data
desc = form.desc.data
cmd = form.cmd.data
version = form.version.data
_filename = form.filename.data.filename
param = {
'desc': desc,
'r_name': _filename,
'filename': getUUID(),
'cmd': cmd,
'version': version,
}
context['filename'] = _filename
etl = EtlManager(EtlMod)
_rec = etl.get(name)
if _rec == None:
_path = '/home/shenwei/nebula/static/etl_mod/' + param[
'filename']
form.filename.data.save(_path)
context['result'] = _filename
setStatus('filename', _filename)
context['etl_page'] = _etl_page
context['etl_do'] = getStatus('do')
context['filename'] = _filename
etl.create(name, **param)
setStatus('info', u"")
context['etl_info'] = getStatus('info')
setStatus('do', 0)
context['form'] = UpLoadForm()
context['etl_do'] = getStatus('do')
else:
setStatus('info', u"错误：该模块名称已经存在！")
context['etl_info'] = getStatus('info')
setStatus('info', u"")
elif _etl_page == 2:
name = form.name.data
mod = form.mod.data
server = form.server.data
param = {
'mod': mod,
'server': server,
'status': '0' # 新任务的工作进度为 0
}
""" 查看该名称已使用？
"""
etl = EtlManager(EtlTask)
_rec = etl.get(name)
if _rec == None:
""" 新任务创建完成，需把任务请求发给指定的ETL服务器执行。
1）用scp拷贝模型到指定ETL服务器的模型目录；
2）调用该ETL服务器的REST API发起任务请求。
"""
context['etl_page'] = _etl_page
etl.create(name, **param)
setStatus('info', u"")
context['etl_info'] = getStatus('info')
setStatus('do', 0)
context['form'] = TaskForm()
context['etl_do'] = getStatus('do')
else:
setStatus('info', u"错误：该模块名称已经存在！")
context['etl_info'] = getStatus('info')
setStatus('info', u"")
elif _etl_page == 3:
name = form.name.data
mod = form.mod.data
server = form.server.data
schedule = form.schedule.data
param = {
'mod': mod,
'server': server,
'schedule': schedule,
'status': '0', # 新作业的工作进度为 0
}
etl = EtlManager(EtlJob)
_rec = etl.get(name)
if _rec == None:
"""新作业创建完成，需把任务请求发给指定的ETL服务器执行。
1）用scp拷贝模型到指定ETL服务器的模型目录；
2）调用该ETL服务器的REST API发起任务请求。
"""
context['etl_page'] = _etl_page
etl.create(name, **param)
setStatus('info', u"")
context['etl_info'] = getStatus('info')
setStatus('do', 0)
context['form'] = JobForm()
context['etl_do'] = getStatus('do')
else:
setStatus('info', u"错误：该模块名称已经存在！")
context['etl_info'] = getStatus('info')
setStatus('info', u"")
else:
setStatus('do', 0)
context['etl_do'] = getStatus('do')
if _etl_page == 0:
_page, _info = getList(EtlDir)
context['pagination'] = _page
context['dirs'] = _info
elif _etl_page == 1:
_page, _info = getList(EtlMod)
context['pagination'] = _page
context['mods'] = _info
elif _etl_page == 2:
_page, _info = getList(EtlTask)
context['pagination'] = _page
context['tasks'] = _info
elif _etl_page == 3:
_page, _info = getList(EtlJob)
context['pagination'] = _page
context['jobs'] = _info
return render_template('etltools/etlbase.html', **context)
def get(self):
global _page
try:
_page_idx[_page] = int(request.args.get('page', 1))
except ValueError:
_page_idx[_page] = 1
finally:
context = dict(
result=False,
_page=_page,
etl_href="etl_mod_post",
etl_body=True,
etl_png="../static/etl.jpg",
)
context.update(setMenus())
""" 获取UI的当前页面，并获取该页面对应数据模型的记录数据（供列表显示）
"""
__page = request.args.get('etl_app')
if __page == 'data_element_list':
_page = 0
_resp_page, _info = getList(DataElement)
context['pagination'] = _resp_page
context['dirs'] = _info
elif __page == 'member_list':
_page = 0
_resp_page, _info = getList(MemberMod)
context['pagination'] = _resp_page
context['dirs'] = _info
elif __page == 'project_list':
_page = 0
_resp_page, _info = getList(ProductManager)
context['pagination'] = _resp_page
context['dirs'] = _info
elif __page == 'product_list':
_page = 0
_resp_page, _info = getList(ProductMod)
context['pagination'] = _resp_page
context['dirs'] = _info
elif __page == 'task_list':
_page = 0
_resp_page, _info = getList(TaskMod)
context['pagination'] = _resp_page
context['dirs'] = _info
elif __page == 'enginerring_list':
_page = 0
_resp_page, _info = getList(EnginerringMod)
context['pagination'] = _resp_page
context['dirs'] = _info
elif __page == 'delivery_list':
_page = 0
_resp_page, _info = getList(DeliveryMod)
context['pagination'] = _resp_page
context['dirs'] = _info
elif __page == 'upfile_list':
_page = 0
_resp_page, _info = getList(UpFileMod)
context['pagination'] = _resp_page
context['dirs'] = _info
else:
_page = 0
_resp_page, _info = getList(DataElement)
context['pagination'] = _resp_page
context['dirs'] = _info
context['_page'] = _page
return render_template(self._template_name, **context)