async def do_schedule():
"""
扫描下一个时间点可以执行的定时任务
:return:
"""
current_datetime = datetime.datetime.now()
current_second = current_datetime.second
quotient = int(current_second / run_time)
remainder = int(current_second % run_time)
if remainder > 0:
quotient = quotient + 1
next_second = quotient * run_time
if next_second == 60:
next_datetime = current_datetime + datetime.timedelta(minutes=1)
next_second = 0
else:
next_datetime = current_datetime
next_timestamp = time.mktime(next_datetime.timetuple())
next_time = DateUtils.format_time(next_timestamp,
time_format=key_YYMMDDHHMM)
# 当前时间点位于0秒或者30秒，则读取job队列并执行
cache_key = ServiceBase.schedule.JOB_KEY + next_time + format_second(
next_second)
job_list = await redis.smembers(cache_key)
if job_list:
# 开协程执行任务
await do_job_list(job_list)
await redis.delete(cache_key)
await redis.hdel(ServiceBase.schedule.SCHEDULE_KEY, cache_key)
async def async_add_job(service_path='',
method='',
params={},
start_time='',
cron='',
job_id='',
group_name='default',
repeat_count=0):
"""
添加定时任务
:param service_path: 需要执行的service路径
:param method: 需要执行的方法
:param params: 需要传入的参数
:param start_time: (2018-06-20 16:30:00)
:param cron: 这里采用五位表达式，从左到右依次表示秒、分、时、天、月
可以使用具体数字或者区间
20 表示[20]
1-3 表示[1, 2, 3]
1,4,6,7 表示[1, 4, 6, 7]
* 表示所有,
1/5 从第1个开始，每五个执行一次
:param job_id: 任务编号，每个任务的编号都要求唯一
:param group_name:
:param repeat_count: 如果要求无限次执行, 则该值需要传入-1, 同一个任务有限次多次执行的情况暂不考虑,
如果业务上有需要, 希望你用多个任务来处理这件事
:return:
"""
if cron:
current_time = int(DateUtils.timestamps_now())
left_time = cron_utils.analyze(current_time + 1, cron)
start_time = DateUtils.format_time(current_time + left_time)
# 计算距离start_time最近的RUN_TIME秒
current_date = start_time[:16] + ':00'
current_count = 1
while current_date < start_time:
# 避免死循环
if current_count >= 1000:
break
current_count += 1
current_date = DateUtils.add_second(current_date, seconds=RUN_TIME)
start_time = current_date
job_params = {
'job_id': job_id,
'group_name': group_name,
'start_time': start_time,
'limit_time': 0,
'repeat_count': repeat_count,
'cron': cron,
'path': service_path,
'method': method,
'params': params
}
result = await save_job(job_params)
return result
async def cal_next_start_time(job, is_normal=True):
"""
计算下一次定时任务发生的时间
:param job:
:param is_normal
:return:
"""
# 无限循环执行, 必定带正则表达式，否则直接报错
# 解析正则表达式, 计算出下一次需要执行的时间点
if not is_normal:
current_time = int(DateUtils.timestamps_now())
else:
current_time = DateUtils.str_to_time(job['start_time'])
left_time = cron_utils.analyze(current_time + 1, job['cron'])
start_time = DateUtils.format_time(current_time + left_time)
# 计算距离start_time最近的RUN_TIME秒
current_date = start_time[:16] + ':00'
current_count = 1
while current_date < start_time:
# 避免死循环
if current_count >= 1000:
break
current_count += 1
current_date = DateUtils.add_second(current_date, seconds=run_time)
start_time = current_date
job['start_time'] = start_time
cache_key = ServiceBase.schedule.JOB_KEY + job['start_time'].replace(
' ', '').replace('-', '').replace(':', '')
now_date = DateUtils.time_now()
# 如果下一次的执行时间小于当前时间，则跳至下一个执行的时间节点
if job['start_time'] < now_date:
logger.info('任务下一次执行时间小于当前时间')
current_date = now_date[:16] + ':00'
while current_date < now_date:
current_date = DateUtils.add_second(current_date,
seconds=2 * run_time)
job['start_time'] = current_date
cache_key = ServiceBase.schedule.JOB_KEY + job['start_time'].replace(
' ', '').replace('-', '').replace(':', '')
model = importlib.import_module('task.schedule.model')
model = model.Model()
await model.update_job(job)
await redis.sadd(cache_key, json.dumps(job, cls=CJsonEncoder))
length = await redis.scard(cache_key)
await redis.hset(ServiceBase.schedule.SCHEDULE_KEY, cache_key, length)
def start(self):
"""
从数据库中读取数据，初始化定时任务
:return:
：update: wsy 2017/8/11
"""
if self.job_stores:
conn = self.pool.connection()
cursor = conn.cursor(cursorclass=MySQLdb.cursors.DictCursor)
# 查找出数据库中现存的所有任务
sql = 'SELECT * FROM tbl_cfg_schedule_job ORDER BY start_time DESC' # where start_time > now() or (repeat_count = -1)
cursor.execute(sql)
job_list = cursor.fetchall()
for job in job_list:
try:
job_params = pickle.loads(job['job_data'])
if job_params['cron']:
current_timestamp = int(time.time())
job_params['left_time'] = self.cron_utils.analyze(
current_timestamp, job_params['cron'])
job_params['start_time'] = DateUtils.format_time(
current_timestamp + job_params['left_time'])
if job_params['type'] == 'one':
"""
单次任务
"""
start_timestamp = int(
time.mktime(
time.strptime(job['start_time'],
'%Y-%m-%d %H:%M:%S')))
current_timestamp = int(time.time())
if current_timestamp > start_timestamp:
left_time = 0
print('启动时间小于当前时间,立刻执行')
else:
left_time = start_timestamp - current_timestamp
job_params['left_time'] = left_time
elif job_params['type'] == 'many':
pass
elif job_params['type'] == 'circle':
pass
thread = threading.Thread(target=self.func,
args=[job_params])
thread.start()
self.job_id_list[job_params['job_id']] = 1
print '定时任务启动'
print job_params
except Exception, e:
print Exception, ':', e
def start(self):
"""
从数据库中读取数据，初始化定时任务
:return:
：update: wsy 2017/8/11
"""
if self.job_stores:
conn = self.pool.connection()
cursor = conn.cursor(cursorclass=MySQLdb.cursors.DictCursor)
# 查找出数据库中现存的所有任务
sql = 'SELECT * FROM tbl_cfg_schedule_job ORDER BY start_time DESC' # where start_time > now() or (repeat_count = -1)
cursor.execute(sql)
job_list = cursor.fetchall()
for job in job_list:
try:
job_params = pickle.loads(job['job_data'])
if job_params['cron']:
current_timestamp = int(time.time())
job_params['left_time'] = self.cron_utils.analyze(current_timestamp, job_params['cron'])
job_params['start_time'] = DateUtils.format_time(current_timestamp + job_params['left_time'])
if job_params['type'] == 'one':
"""
单次任务
"""
start_timestamp = int(time.mktime(time.strptime(job['start_time'], '%Y-%m-%d %H:%M:%S')))
current_timestamp = int(time.time())
if current_timestamp > start_timestamp:
left_time = 0
print ('启动时间小于当前时间,立刻执行')
else:
left_time = start_timestamp - current_timestamp
job_params['left_time'] = left_time
elif job_params['type'] == 'many':
pass
elif job_params['type'] == 'circle':
pass
thread = threading.Thread(target=self.func, args=[job_params])
thread.start()
self.job_id_list[job_params['job_id']] = 1
print '定时任务启动'
print job_params
except Exception, e:
print Exception, ':', e
def add_job(self,
service_path,
method,
params={},
job_id='',
group='default',
start_time='',
limit_time=0,
repeat_count=0,
cron=''):
"""
添加定时任务
:param job:
:return:
"""
if job_id in self.job_id_list:
print Exception('job_id {%s} 已存在' % job_id)
return
if self.job_stores and not job_id:
print Exception('如果想序列化至数据源，则job_id非空')
return
try:
left_time = 0
if cron:
current_timestamp = int(time.time())
# 如果传入cron表达式，则以cron表达式为主
left_time = self.cron_utils.analyze(current_timestamp, cron)
start_time = DateUtils.format_time(current_timestamp +
left_time)
if start_time and not limit_time and not repeat_count:
"""
单次任务
"""
start_timestamp = int(
time.mktime(time.strptime(start_time,
'%Y-%m-%d %H:%M:%S')))
current_timestamp = int(time.time())
if current_timestamp > start_timestamp:
print '启动时间小于当前时间, 立刻启动'
left_time = 0
else:
left_time = start_timestamp - current_timestamp
type = 'one'
elif limit_time >= 0 and repeat_count > 0:
"""
多次任务，但不是无限循环
"""
type = 'many'
elif repeat_count < 0:
"""
无限循环
"""
type = 'circle'
job_params = {
'params': params,
'service_path': service_path,
'method': method,
'left_time': left_time,
'limit_time': limit_time,
'repeat_count': repeat_count,
'type': type,
'job_id': job_id,
'group_name': group,
'start_time': start_time,
'cron': cron
}
thread = threading.Thread(target=self.func, args=[job_params])
thread.start()
self.job_id_list[job_id] = 1
# 将任务存储至数据库
if self.job_stores:
self.save_job(job_params)
print '定时任务启动'
print job_params
return thread
except Exception, e:
print Exception("启动定时任务失败")
return
def add_job(self, service_path, method, params={}, job_id='',
group='default', start_time='', limit_time=0, repeat_count=0, cron=''):
"""
添加定时任务
:param job:
:return:
"""
if job_id in self.job_id_list:
print Exception('job_id {%s} 已存在' % job_id)
return
if self.job_stores and not job_id:
print Exception('如果想序列化至数据源，则job_id非空')
return
try:
left_time = 0
if cron:
current_timestamp = int(time.time())
# 如果传入cron表达式，则以cron表达式为主
left_time = self.cron_utils.analyze(current_timestamp, cron)
start_time = DateUtils.format_time(current_timestamp + left_time)
if start_time and not limit_time and not repeat_count:
"""
单次任务
"""
start_timestamp = int(time.mktime(time.strptime(start_time, '%Y-%m-%d %H:%M:%S')))
current_timestamp = int(time.time())
if current_timestamp > start_timestamp:
print '启动时间小于当前时间, 立刻启动'
left_time = 0
else:
left_time = start_timestamp - current_timestamp
type = 'one'
elif limit_time >= 0 and repeat_count > 0:
"""
多次任务，但不是无限循环
"""
type = 'many'
elif repeat_count < 0:
"""
无限循环
"""
type = 'circle'
job_params = {
'params': params,
'service_path': service_path,
'method': method,
'left_time': left_time,
'limit_time': limit_time,
'repeat_count': repeat_count,
'type': type,
'job_id': job_id,
'group_name': group,
'start_time': start_time,
'cron': cron
}
thread = threading.Thread(target=self.func, args=[job_params])
thread.start()
self.job_id_list[job_id] = 1
# 将任务存储至数据库
if self.job_stores:
self.save_job(job_params)
print '定时任务启动'
print job_params
return thread
except Exception, e:
print Exception("启动定时任务失败")
return
return compare_time, index, index + 1, str_list, 1
else:
return compare_time, index, 0, str_list, 1
else:
for str_item in str_list:
if int(str_item) > compare_time:
index = str_list.index(str_item)
if len(str_list) > index + 1:
return int(str_item), index, index + 1, str_list, 1
else:
return int(str_item), index, 0, str_list, 1
return int(str_list[0]), 0, -1, str_list, 1
if __name__ == '__main__':
cron_utils = CronUtils()
current_time = int(time.time())
# cron_str = '0 0 1 * *'
cron_str = '0 0 10,17 * *'
for i in range(20):
left_time = cron_utils.analyze(current_time, cron_str)
if left_time == 0:
left_time += 1
print DateUtils.format_time(current_time), left_time
current_time += left_time
else:
current_time += left_time
print DateUtils.format_time(current_time), left_time
current_time += 1
