def update_dispatch_detail(dispatch_id, interface_id, dispatch_name, dispatch_desc, minute, hour, day, month, week, old_status, new_status, user_id): """修改调度详情""" try: # 修改调度表 DispatchModel.update_dispatch_detail(db.etl_db, dispatch_id, interface_id, dispatch_name, dispatch_desc, minute, hour, day, month, week, new_status, user_id) # 修改调度状态 run_id = 'scheduler_%s' % dispatch_id # 仍删除 if (old_status == 0 and new_status == 0): pass # 新增 or 先新增后暂停 elif (old_status == 0 and new_status == 1 or old_status == 0 and new_status == 2): scheduler_handler.add_job(run_id, dispatch_id, minute, hour, day, month, week) # 修改 else: scheduler_handler.modify_job(run_id, dispatch_id, minute, hour, day, month, week) # 先新增后暂停 if (old_status == 0 and new_status == 2): scheduler_handler.pause_job(run_id) # 暂停 elif (old_status == 1 and new_status == 2): scheduler_handler.pause_job(run_id) # 删除 elif (old_status == 1 and new_status == 0 or old_status == 2 and new_status == 0): scheduler_handler.remove_job(run_id) # 恢复 elif (old_status == 2 and new_status == 1): scheduler_handler.resume_job(run_id) return Response(dispatch_id=dispatch_id) except pymysql.err.IntegrityError as e: log.error('请勿重复为接口添加调度 [ERROR: %s]' % e, exc_info=True) abort(400, **make_result(status=400, msg='请勿重复为接口添加调度'))
def action_dispatch(dispatch_id, action, user_id): """暂停/恢复调度任务""" for item in dispatch_id: try: run_id = 'scheduler_%s' % item # 暂停调度任务 if action == 1: DispatchModel.update_dispatch_status( db.etl_db, item, 2, user_id) SchedulerHandler.pause_job(run_id) # 恢复调度任务 elif action == 2: DispatchModel.update_dispatch_status( db.etl_db, item, 1, user_id) SchedulerHandler.resume_job(run_id) except Exception as e: log.error('暂停/恢复调度异常 [ERROR: %s]' % e, exc_info=True) abort(400, **make_result(status=400, msg='暂停/恢复调度任务异常')) return Response(dispatch_id=dispatch_id)
def get_ftp_list(ftp_name, ftp_type, ftp_host, is_deleted, page, limit): """获取FTP配置列表""" condition = [] if ftp_name: condition.append('ftp_name LIKE "%%%%%s%%%%"' % ftp_name) if ftp_host: condition.append('ftp_host LIKE "%%%%%s%%%%"' % ftp_host) if ftp_type: condition.append('ftp_type = %s' % ftp_type) if is_deleted == 1: condition.append('is_deleted = 0') elif is_deleted == 2: condition.append('is_deleted = 1') condition = 'WHERE ' + ' AND '.join(condition) if condition else '' result = FtpModel.get_ftp_list(db.etl_db, condition, page, limit) total = FtpModel.get_ftp_list_count(db.etl_db, condition) return Response(result=result, total=total)
def put(job_id): """修改任务""" payload = get_payload() params = Response(job_id=job_id, interface_id=int(payload.get('interface_id', 0)), job_name=payload.get('job_name', ''), job_desc=payload.get('job_desc', ''), job_index=payload.get('job_index', ''), server_id=int(payload.get('server_id', 0)), server_dir=payload.get('server_dir', ''), server_script=payload.get('server_script', ''), return_code=int(payload.get('return_code', 0)), old_prep=payload.get('old_prep', ''), job_prep=payload.get('job_prep', ''), old_params=payload.get('old_params', ''), job_params=payload.get('job_params', ''), is_deleted=int(payload.get('is_deleted', 0))) log.info('修改任务[params: %s]' % str(params)) return params
def action_ftp_event(ftp_event_id, action, user_id): """暂停/恢复调度事件""" for item in ftp_event_id: try: run_id = 'ftp_event_%s' % item # 暂停调度任务 if action == 1: FtpEventModel.update_ftp_event_status( db.etl_db, item, 2, user_id) EventHandler.pause_job(run_id) # 恢复调度任务 elif action == 2: FtpEventModel.update_ftp_event_status( db.etl_db, item, 1, user_id) EventHandler.resume_job(run_id) except Exception as e: log.error('暂停/恢复调度异常 [ERROR: %s]' % e, exc_info=True) abort(400, **make_result(status=400, msg='暂停/恢复调度任务异常')) return Response(ftp_event_id=ftp_event_id)
def get_alert_list(alert_channel, conf_name, is_deleted, page, limit): """获取预警配置列表""" condition = [] if conf_name: condition.append('conf_name LIKE "%%%%%s%%%%"' % conf_name) if is_deleted == 1: condition.append('is_deleted = 0') elif is_deleted == 2: condition.append('is_deleted = 1') if alert_channel == 1: condition.append('alert_channel = 1') elif alert_channel == 2: condition.append('alert_channel = 2') condition = 'WHERE ' + ' AND '.join(condition) if condition else '' result = AlertModel.get_alert_conf_list(db.etl_db, condition, page, limit) total = AlertModel.get_alert_conf_count(db.etl_db, condition) return Response(result=result, total=total)
def get_execute_job_log_history(job_id, exec_type, start_time, end_time, run_status, page, limit): """获取任务历史日志列表""" condition = [] if exec_type: condition.append('c.exec_type = %s' % exec_type) if start_time: condition.append('b.insert_time >= %s' % start_time) if end_time: condition.append('b.insert_time <= %s' % end_time) if run_status: condition.append('b.`status` = "%s"' % run_status) condition = ' AND ' + ' AND '.join(condition) if condition else '' result = ExecuteModel.get_execute_job_history(db.etl_db, job_id, condition, page, limit) total = ExecuteModel.get_execute_job_history_count( db.etl_db, job_id, condition) return Response(result=result, total=total)
def verify_add_ftp_detail(ftp_name, ftp_desc, ftp_type, ftp_host, ftp_port, ftp_user, ftp_passwd, user_id): """新增FTP配置详情""" if not ftp_name: abort(400, **make_result(status=400, msg='FTP名称不存在')) if ftp_type < 1 or ftp_type > 2: abort(400, **make_result(status=400, msg='FTP类型错误')) if not ftp_host: abort(400, **make_result(status=400, msg='FTP域名不存在')) if not ftp_port: abort(400, **make_result(status=400, msg='FTP端口不存在')) return Response(ftp_name=ftp_name, ftp_desc=ftp_desc, ftp_type=ftp_type, ftp_host=ftp_host, ftp_port=ftp_port, ftp_user=ftp_user, ftp_passwd=ftp_passwd, user_id=user_id)
def verify_add_param(param_type, param_name, param_index, source_id, param_value, param_desc, user_id): """新增参数""" if param_type < 0 or param_type > 1: abort(400, **make_result(status=400, msg='参数类型错误')) if not param_name: abort(400, **make_result(status=400, msg='参数名称不得为空')) if param_type == 1 and source_id == 0: abort(400, **make_result(status=400, msg='数据源不得为空')) if not param_index: abort(400, **make_result(status=400, msg='参数目录不得为空')) elif re.findall(',', param_index): abort(400, **make_result(status=400, msg='参数目录中不得出现逗号字符","')) if not param_value: abort(400, **make_result(status=400, msg='参数值不得为空')) if param_type != 2 and param_value.startswith('$'): abort(400, **make_result(status=400, msg='非上下文参数[参数值]不能以$开头')) if param_type == 2 and not param_value.startswith('$'): abort(400, **make_result(status=400, msg='上下文参数[参数值]只能以$开头')) return Response(param_type=param_type, param_name=param_name, param_index=param_index, source_id=source_id, param_value=param_value, param_desc=param_desc, user_id=user_id)
def delete_interface(interface_id, user_id): """删除接口""" # 查询是否在调度内 if InterfaceModel.get_schedule_detail(db.etl_db, interface_id): abort(400, **make_result(status=400, msg='调度任务运行中, 请停止调度任务后删除')) # 查询是否有任务依赖 ids = InterfaceModel.get_job_prep_by_interface(db.etl_db, interface_id) job_ids = [] out_ids = [] for items in ids: if items['job_id']: job_ids.append(items['job_id']) if items['out_id']: out_ids.append(items['out_id']) if set(out_ids) - set(job_ids): abort(400, **make_result(status=400, msg='接口中任务作为其他任务依赖, 请停止依赖任务后删除')) # 删除接口 InterfaceModel.delete_interface(db.etl_db, interface_id, user_id) return Response(interface_id=interface_id)
def verify_add_dispatch_alert(dispatch_id, alert_s, alert_f, conf_id_s, conf_id_f, user_id, send_mail_s, send_mail_f): """新增调度预警""" if not dispatch_id: abort(400, **make_result(status=400, msg='调度id不得为空')) if not alert_s and not alert_f: abort(400, **make_result(status=400, msg='请勿保存空配置')) if alert_s and not conf_id_s: abort(400, **make_result(status=400, msg='预警配置(成功), 配置参数不得为空')) if alert_f and not conf_id_f: abort(400, **make_result(status=400, msg='预警配置(失败), 配置参数不得为空')) return Response(dispatch_id=dispatch_id, alert_s=alert_s, alert_f=alert_f, conf_id_s=conf_id_s, conf_id_f=conf_id_f, user_id=user_id, send_mail_s=send_mail_s, send_mail_f=send_mail_f)
def put(ftp_event_id): """修改文件事件详情""" payload = get_payload() params = Response( ftp_event_id=ftp_event_id, event_name=payload.get('event_name', ''), event_desc=payload.get('event_desc', ''), ftp_id=int(payload.get('ftp_id', 0)), data_path=payload.get('data_path', ''), file_name=payload.get('file_name', ''), interface_id=payload.get('interface_id', ''), start_time=int(payload.get('start_time', 0)), end_time=int(payload.get('end_time', 24)), interval_value=int(payload.get('interval_value', 0)), date_time=payload.get('date_time', ''), old_status=int(payload.get('old_status', 0)), new_status=int(payload.get('new_status', 0)) ) log.info('修改文件事件详情[params: %s]' % str(params)) return params
def verify_add_ftp_event_detail(event_name, event_desc, ftp_id, data_path, file_name, interface_id, start_time, end_time, date_time, interval_value, user_id): """新增文件事件详情""" if not event_name: abort(400, **make_result(status=400, msg='事件名称不存在')) if not ftp_id: abort(400, **make_result(status=400, msg='FTP配置id不存在')) if not data_path: abort(400, **make_result(status=400, msg='文件路径不存在')) if not file_name: abort(400, **make_result(status=400, msg='文件名称不存在')) if not interface_id: abort(400, **make_result(status=400, msg='任务流id不存在')) if not interval_value: abort(400, **make_result(status=400, msg='间隔值不得为空')) if interval_value < 1 or interval_value > 60: abort(400, **make_result(status=400, msg='间隔值应在1-60分钟之间')) if not date_time: now = time.strftime('%Y-%m-%d', time.localtime()) date_time = date_add(now, -1) if start_time < 0 or start_time > 23: abort(400, **make_result(status=400, msg='开始时间错误')) if end_time < 1 or end_time > 24: abort(400, **make_result(status=400, msg='结束时间错误')) if end_time <= start_time: abort(400, **make_result(status=400, msg='结束时间不得小于等于开始时间')) if not user_id: abort(400, **make_result(status=400, msg='用户不存在')) return Response(event_name=event_name, event_desc=event_desc, ftp_id=ftp_id, data_path=data_path, file_name=file_name, interface_id=interface_id, interval_value=interval_value, start_time=start_time, end_time=end_time, date_time=date_time, user_id=user_id)
def verify_update_alert_conf_detail(conf_id, alert_channel, conf_name, param_config, param_host, param_port, param_pass, is_deleted, user_id): """修改预警配置""" if not conf_id: abort(400, **make_result(status=400, msg='预警配置id不存在')) if alert_channel < 1 or alert_channel > 2: abort(400, **make_result(status=400, msg='预警渠道参数错误')) if not conf_name: abort(400, **make_result(status=400, msg='配置名称参数错误')) if not param_config: abort(400, **make_result(status=400, msg='参数配置不得为空')) return Response(conf_id=conf_id, alert_channel=alert_channel, conf_name=conf_name, param_config=param_config, param_host=param_host, param_port=param_port, param_pass=param_pass, is_deleted=is_deleted, user_id=user_id)
def get_datasource_list(source_name, source_type, source_host, is_deleted, page, limit): """获取数据源列表""" condition = [] if source_name: condition.append('source_name LIKE "%%%%%s%%%%"' % source_name) if source_host: condition.append('source_host LIKE "%%%%%s%%%%"' % source_host) if source_type: condition.append('source_type = %s' % source_type) if is_deleted == 1: condition.append('is_deleted = 0') elif is_deleted == 2: condition.append('is_deleted = 1') condition = 'WHERE ' + ' AND '.join(condition) if condition else '' result = DataSourceModel.get_datasource_list(db.etl_db, condition, page, limit) total = DataSourceModel.get_datasource_list_count(db.etl_db, condition) return Response(result=result, total=total)
def get_job_list(job_name, job_index, interface_id, is_deleted, page, limit): """获取任务列表""" condition = [] if job_name: condition.append('job_name LIKE "%%%%%s%%%%"' % job_name) if job_index: condition.append('job_index IN (%s)' % ','.join('"%s"' % item for item in job_index)) if interface_id: condition.append('interface_id = %s' % interface_id) if is_deleted == 1: condition.append('is_deleted = 0') elif is_deleted == 2: condition.append('is_deleted = 1') condition = 'WHERE ' + ' AND '.join(condition) if condition else '' result = JobModel.get_job_list(db.etl_db, condition, page, limit) total = JobModel.get_job_count(db.etl_db, condition) return Response(result=result, total=total)
def get_job_list(job_name, start_time, end_time, interface_id, is_deleted, page, limit): """获取任务列表""" condition = [] if job_name: condition.append('job_name LIKE "%%%%%s%%%%"' % job_name) if start_time: condition.append('insert_time >= %s' % start_time) if end_time: condition.append('insert_time <= %s' % end_time) if interface_id: condition.append('interface_id = %s' % interface_id) if is_deleted == 1: condition.append('is_deleted = 0') elif is_deleted == 2: condition.append('is_deleted = 1') condition = 'WHERE ' + ' AND '.join(condition) if condition else '' result = JobModel.get_job_list(db.etl_db, condition, page, limit) total = JobModel.get_job_count(db.etl_db, condition) return Response(result=result, total=total)
def verify_add_job_id(job_name, interface_id, job_desc, server_id, job_prep, server_dir, server_script, user_id): """新增任务""" if not job_name: abort(400, **make_result(status=400, msg='任务名称不存在')) if not interface_id: abort(400, **make_result(status=400, msg='接口id不存在')) if not server_id: abort(400, **make_result(status=400, msg='执行服务器id不存在')) if not server_script: abort(400, **make_result(status=400, msg='脚本命令不存在')) if not user_id: abort(400, **make_result(status=400, msg='用户不存在')) return Response(job_name=job_name, interface_id=interface_id, job_desc=job_desc, server_id=server_id, job_prep=job_prep, server_dir=server_dir, server_script=server_script, user_id=user_id)
def verify_update_job_id(job_id, interface_id, job_name, job_desc, job_index, server_id, server_dir, server_script, return_code, old_prep, job_prep, user_id, old_params, job_params, is_deleted): """修改任务""" if not job_id: abort(400, **make_result(status=400, msg='任务id不存在')) if not interface_id: abort(400, **make_result(status=400, msg='任务流id不存在')) if not job_name: abort(400, **make_result(status=400, msg='任务名称不存在')) if not job_index: abort(400, **make_result(status=400, msg='任务目录不存在')) elif re.findall(',', job_index): abort(400, **make_result(status=400, msg='任务目录中不得出现逗号字符","')) if not user_id: abort(400, **make_result(status=400, msg='用户不存在')) if not server_id: abort(400, **make_result(status=400, msg='执行服务器id不存在')) if is_deleted < 0 or is_deleted > 1: abort(400, **make_result(status=400, msg='是否删除参数错误')) if return_code < 0: abort(400, **make_result(status=400, msg='返回状态码应大于等于0')) return Response(job_id=job_id, interface_id=interface_id, job_name=job_name, job_desc=job_desc, job_index=job_index, server_id=server_id, server_dir=server_dir, server_script=server_script, return_code=return_code, old_prep=old_prep, job_prep=job_prep, old_params=old_params, job_params=job_params, user_id=user_id, is_deleted=is_deleted)
def add_dispatch_alert(dispatch_id, alert_s, alert_f, conf_id_s, conf_id_f, user_id, send_mail_s, send_mail_f): """新增调度预警""" data = [] if alert_s: data.append({ 'dispatch_id': dispatch_id, 'config_id': conf_id_s, 'alert_type': 1, 'user_id': user_id, 'send_mail': send_mail_s }) if alert_f: data.append({ 'dispatch_id': dispatch_id, 'config_id': conf_id_f, 'alert_type': 2, 'user_id': user_id, 'send_mail': send_mail_f }) # 批处理 DispatchAlertModel.add_dispatch_alert(db.etl_db, data) return Response(dispatch_id=dispatch_id)
def add_ftp_event_detail(event_name, event_desc, ftp_id, data_path, file_name, interface_id, start_time, end_time, date_time, interval_value, user_id): """新增文件事件详情""" # 新增事件详情 ftp_event_id = FtpEventModel.add_ftp_event_detail( db.etl_db, event_name, event_desc, ftp_id, data_path, file_name, interval_value, start_time, end_time, date_time, user_id) # 新增调度 run_id = 'ftp_event_%s' % ftp_event_id minute = '*/%s' % interval_value hour = '%d-%d' % (start_time, end_time) EventHandler.add_event(run_id, ftp_event_id, minute, hour) # 新增任务流依赖 interface_id = interface_id.split(',') insert_data = [] for item in interface_id: insert_data.append({ 'ftp_event_id': ftp_event_id, 'interface_id': item }) FtpEventModel.add_file_event_interface(db.etl_db, insert_data) return Response(ftp_event_id=ftp_event_id)
def add_job_detail(job_name, interface_id, job_desc, job_index, server_id, server_dir, job_prep, job_params, server_script, user_id, return_code): """新增任务详情""" # 任务名称查重 if JobModel.get_job_detail_by_name(db.etl_db, job_name): abort(400, **make_result(status=400, msg='任务名称重复, 已存在数据库中')) # 新增任务详情 job_id = JobModel.add_job_detail(db.etl_db, job_name, interface_id, job_desc, job_index, server_id, server_dir, server_script, return_code, user_id) # 新增任务依赖 job_prep = job_prep.split(',') if job_prep else [] data = [] for prep_id in job_prep: data.append({ 'job_id': job_id, 'prep_id': prep_id, 'user_id': user_id, 'insert_time': int(time.time()), 'update_time': int(time.time()) }) JobModel.add_job_prep(db.etl_db, data) if data else None # 新增任务参数 job_params = job_params.split(',') if job_params else [] data = [] for param_id in job_params: data.append({ 'job_id': job_id, 'param_id': param_id, 'insert_time': int(time.time()), 'update_time': int(time.time()), 'user_id': user_id }) JobModel.add_job_param(db.etl_db, data) if data else None return Response(job_id=job_id)
def delete_interface(interface_id, user_id): """删除任务流""" # 查询是否在调度内 if InterfaceModel.get_schedule_detail(db.etl_db, interface_id): abort(400, **make_result(status=400, msg='调度任务运行中, 请停止调度任务后删除')) # 任务流前后置依赖 parent = InterfaceModel.get_interface_parent(db.etl_db, interface_id) child = InterfaceModel.get_interface_child(db.etl_db, interface_id) if parent or child: abort(400, **make_result(status=400, msg='任务流存在前/后置依赖, 不可删除')) # # 查询是否有任务依赖 # ids = InterfaceModel.get_job_prep_by_interface(db.etl_db, interface_id) # job_ids = [] # out_ids = [] # for items in ids: # if items['job_id']: # job_ids.append(items['job_id']) # if items['out_id']: # out_ids.append(items['out_id']) # if set(out_ids) - set(job_ids): # abort(400, **make_result(status=400, msg='任务流中任务作为其他任务依赖, 请停止依赖任务后删除')) # 删除任务流 InterfaceModel.delete_interface(db.etl_db, interface_id, user_id) return Response(interface_id=interface_id)
def verify_update_interface_detail(interface_id, interface_name, interface_desc, interface_index, old_parent, parent_interface, run_time, retry, user_id, is_deleted): """修改任务流详情""" if not interface_id: abort(400, **make_result(status=400, msg='任务流id不存在')) if not interface_name: abort(400, **make_result(status=400, msg='任务流名称不存在')) if not interface_index: abort(400, **make_result(status=400, msg='任务流目录不存在')) elif re.findall(',', interface_index): abort(400, **make_result(status=400, msg='任务流目录中不得出现逗号字符","')) if run_time and not re.findall(r'\d{4}-\d{2}-\d{2}', run_time): abort(400, **make_result(status=400, msg='数据日期格式错误')) if retry < 0 or retry > 10: abort(400, **make_result(status=400, msg='重试次数请限制在0-10之内')) if is_deleted < 0 or is_deleted > 1: abort(400, **make_result(status=400, msg='是否删除参数错误')) # 原任务流前置 old_parent = [int(item) for item in old_parent.split(',')] if old_parent else [] # 任务流前置 parent_interface = [int(item) for item in parent_interface.split(',')] if parent_interface else [] return Response(interface_id=interface_id, interface_name=interface_name, interface_desc=interface_desc, interface_index=interface_index, run_time=run_time, old_parent=old_parent, parent_interface=parent_interface, retry=retry, user_id=user_id, is_deleted=is_deleted)
def test_ftp_link(ftp_id, ftp_type, ftp_host, ftp_port, ftp_user, ftp_passwd): """测试FTP连接""" if ftp_id: detail = FtpModel.get_ftp_detail(db.etl_db, ftp_id) if isinstance(detail['ftp_passwd'], bytes): detail['ftp_passwd'] = detail['ftp_passwd'].decode( 'utf-8', 'ignore') try: if detail['ftp_type'] == 1: ftp = FtpLink(detail['ftp_host'], detail['ftp_port'], detail['ftp_user'], detail['ftp_passwd']) ftp.close() FtpModel.update_ftp_status(db.etl_db, ftp_id, 0) elif detail['ftp_type'] == 2: ftp = SftpLink(detail['ftp_host'], detail['ftp_port'], detail['ftp_user'], detail['ftp_passwd']) ftp.close() FtpModel.update_ftp_status(db.etl_db, ftp_id, 0) else: FtpModel.update_ftp_status(db.etl_db, ftp_id, 1) return Response(status=400, msg='FTP服务器类型未知') except: FtpModel.update_ftp_status(db.etl_db, ftp_id, 1) return Response(status=400, msg='FTP连接异常') return Response(status=200, msg='FTP连接成功') else: try: if ftp_type == 1: ftp = FtpLink(ftp_host, ftp_port, ftp_user, ftp_passwd) ftp.close() elif ftp_type == 2: ftp = SftpLink(ftp_host, ftp_port, ftp_user, ftp_passwd) ftp.close() else: return Response(status=400, msg='FTP服务器类型未知') except: return Response(status=400, msg='FTP连接异常') return Response(status=200, msg='FTP连接成功')
def get_datasource_list_all(): """获取所有数据源""" result = DataSourceModel.get_datasource_list_all(db.etl_db) return Response(result=result)
def verify_get_ftp_detail(ftp_id): """获取FTP配置详情""" if not ftp_id: abort(400, **make_result(status=400, msg='FTP配置id不存在')) return Response(ftp_id=ftp_id)
def verify_delete_ftp(ftp_id, user_id): """删除FTP配置""" if not ftp_id: abort(400, **make_result(status=400, msg='FTP配置id不存在')) return Response(ftp_id=ftp_id, user_id=user_id)
def verify_run_dispatch(dispatch_id): """立即执行调度任务""" if not dispatch_id: abort(400, **make_result(status=400, msg='调度id不存在')) return Response(dispatch_id=dispatch_id)
def verify_delete_dispatch_detail(dispatch_id, user_id): """删除调度详情""" if not dispatch_id: abort(400, **make_result(status=400, msg='调度id不存在')) return Response(dispatch_id=dispatch_id, user_id=user_id)