async def save_to_db(data): # async def save_to_db(task_unique_id, service_path, method, params_json): """ 保存任务到DB :param data: list data[][task_unique_id] data[][service_path] data[][method] data[][params_json] data[][key1] data[][key2] data[][key3] :return: """ if isinstance(data, dict): data = [data] model = AsyncModelBase() sql_list = [] for item in data: key = 'task_unique_id, service_path, method, key1, key2, key3, params, create_time' val = '%s, %s, %s, %s, %s, %s, %s, %s' duplicate = ['params = %s', 'create_time = %s'] value = (item['task_unique_id'], item['service_path'], item['method'], item.get('key1', ''), item.get('key2', ''), item.get('key3', ''), item['params_json'], DateUtils.time_now(), item['params_json'], DateUtils.time_now()) sql_list.append({ model.sql_constants.SQL_TYPE: model.sql_constants.INSERT, model.sql_constants.TABLE_NAME: 'tbl_cfg_task', model.sql_constants.DICT_DATA: { model.sql_constants.KEY: key, model.sql_constants.VAL: val, model.sql_constants.DUPLICATE_KEY_UPDATE: duplicate }, model.sql_constants.VALUE_TUPLE: value }) # result = await model.do_sqls(sql_list) if result is None: for item in data: logger.info( 'Task Add [%s] to DB failed, path [%s], method [%s], params: %s', item['task_unique_id'], item['service_path'], item['method'], item['params_json']) else: for item in data: logger.info( 'Task Add [%s] to DB success, path [%s], method [%s], params: %s', item['task_unique_id'], item['service_path'], item['method'], item['params_json']) return result
def run(): """ 开始执行 :return: """ if warning_to_email is None: print 'warning mail error, check setting.conf [monitor]' return last_len = 0 loop_num = 0 conn_failed_num = 0 while True: try: conn = redis.get_conn() list_len = conn.llen(list_cache_key) except Exception, e: conn_failed_num += 1 if conn_failed_num >= 6: response = send_warning_info( 'monitor', json.dumps({ 'exception message': e.message, 'time': DateUtils.time_now() })) if response is True: conn_failed_num = 0 list_len = 0 logger.error('monitor redis get_conn failed, e: %s', e.message) if list_len > 0 and list_len >= last_len: loop_num += 1 # 队列长度保持不变或增加 if loop_num >= 6: response = send_warning_info( 'task', json.dumps({ 'list_len': list_len, 'time': DateUtils.time_now() })) if response is True: loop_num = 0 else: logger.info('task len add') else: last_len = list_len loop_num = 0 # print 'task status is normal' time.sleep(10)
async def update_to_db(task_unique_id, status): """ 更新任务状态 :param task_unique_id: :param status: :return: """ model = AsyncModelBase() fields = [ 'status = %s', 'execute_time = %s' ] condition = 'task_unique_id = %s' value = (status, DateUtils.time_now(), task_unique_id) result = await model.update('tbl_cfg_task', { model.sql_constants.FIELDS: fields, model.sql_constants.CONDITION: condition }, value) if result is None: logger.info('Task Update [%s] failed, status: %s', task_unique_id, status) else: logger.info('Task Update [%s] to DB success, status: %s', task_unique_id, status) return result
async def save_to_db(task_unique_id, service_path, method, params_json): """ 保存任务到DB :param task_unique_id: :param service_path: :param method: :param params_json: :return: """ model = AsyncModelBase() key = 'task_unique_id, service_path, method, params, create_time' val = '%s, %s, %s, %s, %s' duplicate = [ 'create_time = %s' ] value = (task_unique_id, service_path, method, params_json, DateUtils.time_now(), DateUtils.time_now()) result = await model.insert('tbl_cfg_task', { model.sql_constants.KEY: key, model.sql_constants.VAL: val, model.sql_constants.DUPLICATE_KEY_UPDATE: duplicate }, value) if not result: logger.info('Task Add [%s] to DB failed, path [%s], method [%s], params: %s', task_unique_id, service_path, method, params_json) else: logger.info('Task Add [%s] to DB success, path [%s], method [%s], params: %s', task_unique_id, service_path, method, params_json) return result
async def add_job(job_list): """ 添加任务至任务队列 :param job_list: :return: """ logger.info('添加任务至队列') current_time = DateUtils.time_now() if job_list: for job in job_list: if isinstance(job['params'], str): try: job['params'] = json.loads(job['params']) except Exception as e: logger.exception('JSON ERROR', e) del job['create_time'] try: cache_key = ServiceBase.schedule.JOB_KEY + job[ 'start_time'].replace(' ', '').replace('-', '').replace( ':', '') if current_time >= job['start_time']: # 当前时间大于任务执行时间, 则立刻执行任务, 然后删除当前的key await do_job(job, is_normal=False) await redis.delete(cache_key) else: # 将任务添加进对应的队列组, 采用set所以当添加重复元素时,重复元素会被忽略 await redis.sadd(cache_key, json.dumps(job, cls=CJsonEncoder)) length = await redis.scard(cache_key) await redis.hset(ServiceBase.schedule.SCHEDULE_KEY, cache_key, length) except Exception as e: logger.exception('ADD JOB ERROR', e) await task.save_task_error(job, e)
def run(): """ 开始执行 :return: """ if warning_to_email is None: print 'warning mail error, check setting.conf [monitor]' return last_len = 0 loop_num = 0 conn_failed_num = 0 while True: try: conn = redis.get_conn() list_len = conn.llen(list_cache_key) except Exception, e: conn_failed_num += 1 if conn_failed_num >= 6: response = send_warning_info('monitor', json.dumps({ 'exception message': e.message, 'time': DateUtils.time_now() })) if response is True: conn_failed_num = 0 list_len = 0 logger.error('monitor redis get_conn failed, e: %s', e.message) if list_len > 0 and list_len >= last_len: loop_num += 1 # 队列长度保持不变或增加 if loop_num >= 6: response = send_warning_info('task', json.dumps({ 'list_len': list_len, 'time': DateUtils.time_now() })) if response is True: loop_num = 0 else: logger.info('task len add') else: last_len = list_len loop_num = 0 # print 'task status is normal' time.sleep(10)
async def cal_next_start_time(job, is_normal=True): """ 计算下一次定时任务发生的时间 :param job: :param is_normal :return: """ # 无限循环执行, 必定带正则表达式,否则直接报错 # 解析正则表达式, 计算出下一次需要执行的时间点 if not is_normal: current_time = int(DateUtils.timestamps_now()) else: current_time = DateUtils.str_to_time(job['start_time']) left_time = cron_utils.analyze(current_time + 1, job['cron']) start_time = DateUtils.format_time(current_time + left_time) # 计算距离start_time最近的RUN_TIME秒 current_date = start_time[:16] + ':00' current_count = 1 while current_date < start_time: # 避免死循环 if current_count >= 1000: break current_count += 1 current_date = DateUtils.add_second(current_date, seconds=run_time) start_time = current_date job['start_time'] = start_time cache_key = ServiceBase.schedule.JOB_KEY + job['start_time'].replace( ' ', '').replace('-', '').replace(':', '') now_date = DateUtils.time_now() # 如果下一次的执行时间小于当前时间,则跳至下一个执行的时间节点 if job['start_time'] < now_date: logger.info('任务下一次执行时间小于当前时间') current_date = now_date[:16] + ':00' while current_date < now_date: current_date = DateUtils.add_second(current_date, seconds=2 * run_time) job['start_time'] = current_date cache_key = ServiceBase.schedule.JOB_KEY + job['start_time'].replace( ' ', '').replace('-', '').replace(':', '') model = importlib.import_module('task.schedule.model') model = model.Model() await model.update_job(job) await redis.sadd(cache_key, json.dumps(job, cls=CJsonEncoder)) length = await redis.scard(cache_key) await redis.hset(ServiceBase.schedule.SCHEDULE_KEY, cache_key, length)
async def check_queue(cls): """ 检查任务队列 每10秒检查一次,如果任务队列的长度比前一次检查的长度长,且前一次检查长度不为0,那么发送报警邮件 :return: """ try: task_queue_len = await redis.llen(cls.task_queue) except Exception: raise if task_queue_len > 0: if (cls.task_queue_length > 0) and (task_queue_len >= cls.task_queue_length): if cls.task_queue_report_num >= 3: report_html = """ <tr> <td style="font-size:30px;">任务队列未能即时消费,当前任务数量:{},前一次检查任务数量:{},请注意.</td> </tr> <tr> <td>可能原因:</td> </tr> <tr> <td>1.任务消费能力不足。 如果是,程序将在3次检查之后自动增加消费者进行处理。之后如果仍然不能停止报警,见2</td> </tr> <tr> <td>2.任务进程死亡。检查任务死亡原因,并重启服务</td> </tr> <tr> <td>Task Monitor report at: {}</td> </tr> """.format(task_queue_len, cls.task_queue_length, DateUtils.time_now()) await Report.report("", cls.mail_template(report_html)) if cls.task_queue_report_num >= 6: await task.add("", "", None, False, { 'loop_num': 1, }) else: cls.task_queue_report_num = cls.task_queue_report_num + 1 else: cls.task_queue_report_num = 0 cls.task_queue_length = task_queue_len