def run():
time_now = functions.get_time_now()
cur.execute(
"select video_id, cid from data_blibli_download where add_time=%s and is_download=False;",
(add_time, ))
task_cfgs = cur.fetchall()
for task_cfg in task_cfgs:
video_id = task_cfg[0]
cid = task_cfg[1]
cur.execute(
"update data_blibli_download set redis_task_id=%s where video_id=%s;",
(time_now, video_id))
conn.commit()
REDIS_OBJ.lpush(redis_key,
json.dumps({
'video_id': video_id,
'cid': cid
}))
def run():
time_now = functions.get_time_now()
cur.execute(
"select video_id, video_url from weibo_video_info_download where is_download=False;"
)
task_cfgs = cur.fetchall()
for task_cfg in task_cfgs:
video_id = task_cfg[0]
url = task_cfg[1]
cur.execute(
"update weibo_video_info_download set redis_task_id=%s where video_id=%s;",
(time_now, video_id))
conn.commit()
REDIS_OBJ.lpush(redis_key,
json.dumps({
'video_id': video_id,
'url': url
}))
log(global_data.log_path, 'info', end_time, global_data.data_id_list)
log(global_data.log_path, 'info', end_time, '下载失败任务列表如下：')
log(global_data.log_path, 'info', end_time, global_data.data_id_fail_list)
log(global_data.log_path, 'info', end_time, '下载成功任务列表如下：')
log(global_data.log_path, 'info', end_time, global_data.data_id_success_list)
def run(t_num=5):
while True:
task_list = functions.load_task(t_num, redis_key)
if task_list:
for task in task_list:
task_id = task['video_id']
global_data.data_id_list.append(task_id)
DLD_FILE(task_list)
else:
break
if __name__ == '__main__':
start_time = functions.get_time_now()
log_name = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))
log_name = ''.join([site_name, log_name, '.txt'])
global_data.log_path = os.path.join(os.getcwd(), 'logs', log_name)
log(global_data.log_path, 'info', start_time, '开始下载任务')
run(t_num=5)
end_time = functions.get_time_now()
print_status()