def post(self): global _etl_page _name = request.json.get('server_name') context = dict(form=setForm(_etl_page), result=True, etl_page=_etl_page, etl_href="etl_mod_post", etl_do=getStatus('do'), etl_info=getStatus('info')) context.update(setMenus()) if _etl_page == 0: etl = EtlManager(EtlDir) elif _etl_page == 1: etl = EtlManager(EtlMod) elif _etl_page == 2: etl = EtlManager(EtlTask) deleteTask(_name) elif _etl_page == 3: etl = EtlManager(EtlJob) deleteJob(_name) _str = {'name': _name} etl.delete_by(**_str) if _etl_page == 0: _template_name = 'etltools/dirlist.html' _page, _info = getList(EtlDir) context['dirs'] = _info context['pagination'] = _page elif _etl_page == 1: _page, _info = getList(EtlMod) _template_name = 'etltools/modlist.html' context['mods'] = _info context['pagination'] = _page elif _etl_page == 2: _page, _info = getList(EtlTask) _template_name = 'etltools/tasklist.html' context['tasks'] = _info context['pagination'] = _page elif _etl_page == 3: _page, _info = getList(EtlJob) _template_name = 'etltools/joblist.html' context['jobs'] = _info context['pagination'] = _page context['etl_info'] = getStatus('info') return jsonify(body=render_template(_template_name, **context))
def getAll(model): """ 获取数据模板的所有数据记录 :param model: :return: """ etl = EtlManager(model) _recs = etl.list() if _recs == None: return None _posts = [] for _rec in _recs: _post = {} _post['name'] = _rec.name if model == EtlDir: _post['desc'] = _rec.desc _post['uuid'] = _rec.dir _post['created_at'] = _rec.created_at elif model == EtlServer: _post['desc'] = _rec.desc _post['url'] = _rec.url _post['status'] = _rec.status _post['created_at'] = _rec.created_at elif model == EtlMod: _post['desc'] = _rec.desc _post['r_name'] = _rec.r_name _post['uuid'] = _rec.filename _post['cmd'] = _rec.cmd _post['version'] = _rec.version _post['created_at'] = _rec.created_at elif model == EtlTask: _post['mod'] = _rec.mod _post['server'] = _rec.server _post['created_at'] = _rec.created_at _post['status'] = _rec.status elif model == EtlJob: _post['mod'] = _rec.mod _post['server'] = _rec.server _post['schedule'] = _rec.schedule _post['created_at'] = _rec.created_at _post['status'] = _rec.status _posts.append(_post) return _posts
def setJobStatus(job_name, status): etl = EtlManager(EtlJob) _rec = etl.get(job_name) if _rec is not None: val = {'status': status} etl = EtlManager(EtlJob) etl.update_by_name(job_name, **val)
def setTaskStatus(task_name, status): etl = EtlManager(EtlTask) _rec = etl.get(task_name) if _rec is not None: val = {'status': status} etl = EtlManager(EtlTask) etl.update_by_name(task_name, **val)
def updateJobStatus(job_name, status): """ 根据返回状态修改数据模型中的status 0~20:步进5(约20秒);21~40:步进3(约35秒);41~60:步进2(约50秒);61~80:步进1(约100秒);81~99:步进1(约100秒);99:不变 :return: """ etl = EtlManager(EtlJob) _rec = etl.get(job_name) if _rec is not None: _sts = _rec['status'] if status != _sts: if status == 'RUNNING': if _sts == 'DONE' or _sts == 'ERROR': _sts = '100' elif _sts == 'SCHEDULE': _sts = '0' _st = int(_sts) if _st < 21: _st += 5 elif _st < 41: _st += 3 elif _st < 99: _st += 1 val = {'status': str(_st)} elif status == 'DONE': val = {'status': 'DONE'} elif status == 'SCHEDULE': val = {'status': 'SCHEDULE'} else: val = {'status': 'ERROR'} etl = EtlManager(EtlJob) etl.update(job_name, **val)
def init_etl_model(): etl = EtlManager(EtlDir) param = { 'desc': u'用于管理ETL脚本文件。', } etl.create('ETL脚本', **param) param = { 'desc': u'用于管理Spark运行文件。', } etl.create('Spark脚本', **param)
def post(self): global _etl_page, _task_log _url = rest_get_url('etl-task/') if _url != '': _taskname = request.json.get('task-name') ''' 2015.8.31 TODO: 目前尚未考虑多台ETL服务器的情况 _server = request.json.get('server') ''' _mod = request.json.get('mod') if _etl_page == 2: etl = EtlManager(EtlMod) _rec = etl.get(_mod) if _rec is not None: _path = '/home/shenwei/nebula/static/etl_mod/' + _rec.filename """ 调用 REST API 激活任务 接口参数: { 'src-dir': 模块文件全路径,例如:'/home/shenwei/data/myphone.json.ktr', 'task-name': 任务名称,例如'my-tester'} """ _api_data = { 'src-dir': _path, 'task-name': _taskname, 'cmd': _rec.cmd } restAPI(_url, _api_data) setTaskStatus(_taskname, '1') else: setTaskStatus(_taskname, 'ERROR') context = dict(form=setForm(_etl_page), result=True, etl_page=_etl_page, etl_href="etl_mod_post", etl_do=getStatus('do'), etl_info=getStatus('info')) context.update(setMenus()) if _etl_page == 0: _page, _info = getList(EtlServer) _template_name = 'etltools/dirlist.html' context['servers'] = _info elif _etl_page == 1: _page, _info = getList(EtlMod) _template_name = 'etltools/modlist.html' context['mods'] = _info elif _etl_page == 2: _page, _info = getList(EtlTask) _template_name = 'etltools/tasklist.html' context['tasks'] = _info _task_log = [] context['text'] = _task_log elif _etl_page == 3: _page, _info = getList(EtlJob) _template_name = 'etltools/joblist.html' context['jobs'] = _info context['pagination'] = _page context['etl_info'] = getStatus('info') return jsonify(body=render_template(_template_name, **context))
def submitForm(): """ 处理SUBMIT事件 :return: 转向的HTML内容 """ global _status, _etl_page context = dict( form=setForm(_etl_page), result=False, etl_page=_etl_page, etl_href="etl_mod_post", etl_do=getStatus('do'), etl_png="../static/etl.jpg", etl_info=getStatus('info'), ) context.update(setMenus()) if _etl_page == 0: """资源目录 """ form = DirForm() context['form'] = form elif _etl_page == 1: """资源 """ form = UpLoadForm() context['form'] = form elif _etl_page == 2: """任务 """ form = TaskForm() context['form'] = form elif _etl_page == 3: """作业 """ form = JobForm() context['form'] = form if form.validate_on_submit(): if _etl_page == 0: name = form.name.data desc = form.desc.data param = { 'desc': desc, 'dir': getUUID(), } etl = EtlManager(EtlDir) _rec = etl.get(name) if _rec == None: etl.create(name, **param) """!!!创建目录 """ setStatus('info', u"") context['etl_info'] = getStatus('info') setStatus('do', 0) context['etl_do'] = getStatus('do') else: setStatus('info', u"错误:该资源目录名称已经存在!") context['etl_info'] = getStatus('info') elif _etl_page == 1: name = form.name.data desc = form.desc.data cmd = form.cmd.data version = form.version.data _filename = form.filename.data.filename param = { 'desc': desc, 'r_name': _filename, 'filename': getUUID(), 'cmd': cmd, 'version': version, } context['filename'] = _filename etl = EtlManager(EtlMod) _rec = etl.get(name) if _rec == None: _path = '/home/shenwei/nebula/static/etl_mod/' + param[ 'filename'] form.filename.data.save(_path) context['result'] = _filename setStatus('filename', _filename) context['etl_page'] = _etl_page context['etl_do'] = getStatus('do') context['filename'] = _filename etl.create(name, **param) setStatus('info', u"") context['etl_info'] = getStatus('info') setStatus('do', 0) context['form'] = UpLoadForm() context['etl_do'] = getStatus('do') else: setStatus('info', u"错误:该模块名称已经存在!") context['etl_info'] = getStatus('info') setStatus('info', u"") elif _etl_page == 2: name = form.name.data mod = form.mod.data server = form.server.data param = { 'mod': mod, 'server': server, 'status': '0' # 新任务的工作进度为 0 } """ 查看该名称已使用? """ etl = EtlManager(EtlTask) _rec = etl.get(name) if _rec == None: """ 新任务创建完成,需把任务请求发给指定的ETL服务器执行。 1)用scp拷贝模型到指定ETL服务器的模型目录; 2)调用该ETL服务器的REST API发起任务请求。 """ context['etl_page'] = _etl_page etl.create(name, **param) setStatus('info', u"") context['etl_info'] = getStatus('info') setStatus('do', 0) context['form'] = TaskForm() context['etl_do'] = getStatus('do') else: setStatus('info', u"错误:该模块名称已经存在!") context['etl_info'] = getStatus('info') setStatus('info', u"") elif _etl_page == 3: name = form.name.data mod = form.mod.data server = form.server.data schedule = form.schedule.data param = { 'mod': mod, 'server': server, 'schedule': schedule, 'status': '0', # 新作业的工作进度为 0 } etl = EtlManager(EtlJob) _rec = etl.get(name) if _rec == None: """新作业创建完成,需把任务请求发给指定的ETL服务器执行。 1)用scp拷贝模型到指定ETL服务器的模型目录; 2)调用该ETL服务器的REST API发起任务请求。 """ context['etl_page'] = _etl_page etl.create(name, **param) setStatus('info', u"") context['etl_info'] = getStatus('info') setStatus('do', 0) context['form'] = JobForm() context['etl_do'] = getStatus('do') else: setStatus('info', u"错误:该模块名称已经存在!") context['etl_info'] = getStatus('info') setStatus('info', u"") else: setStatus('do', 0) context['etl_do'] = getStatus('do') if _etl_page == 0: _page, _info = getList(EtlDir) context['pagination'] = _page context['dirs'] = _info elif _etl_page == 1: _page, _info = getList(EtlMod) context['pagination'] = _page context['mods'] = _info elif _etl_page == 2: _page, _info = getList(EtlTask) context['pagination'] = _page context['tasks'] = _info elif _etl_page == 3: _page, _info = getList(EtlJob) context['pagination'] = _page context['jobs'] = _info return render_template('etltools/etlbase.html', **context)