def run(): time_now = functions.get_time_now() cur.execute( "select video_id, cid from data_blibli_download where add_time=%s and is_download=False;", (add_time, )) task_cfgs = cur.fetchall() for task_cfg in task_cfgs: video_id = task_cfg[0] cid = task_cfg[1] cur.execute( "update data_blibli_download set redis_task_id=%s where video_id=%s;", (time_now, video_id)) conn.commit() REDIS_OBJ.lpush(redis_key, json.dumps({ 'video_id': video_id, 'cid': cid }))
def run(): time_now = functions.get_time_now() cur.execute( "select video_id, video_url from weibo_video_info_download where is_download=False;" ) task_cfgs = cur.fetchall() for task_cfg in task_cfgs: video_id = task_cfg[0] url = task_cfg[1] cur.execute( "update weibo_video_info_download set redis_task_id=%s where video_id=%s;", (time_now, video_id)) conn.commit() REDIS_OBJ.lpush(redis_key, json.dumps({ 'video_id': video_id, 'url': url }))
log(global_data.log_path, 'info', end_time, global_data.data_id_list) log(global_data.log_path, 'info', end_time, '下载失败任务列表如下:') log(global_data.log_path, 'info', end_time, global_data.data_id_fail_list) log(global_data.log_path, 'info', end_time, '下载成功任务列表如下:') log(global_data.log_path, 'info', end_time, global_data.data_id_success_list) def run(t_num=5): while True: task_list = functions.load_task(t_num, redis_key) if task_list: for task in task_list: task_id = task['video_id'] global_data.data_id_list.append(task_id) DLD_FILE(task_list) else: break if __name__ == '__main__': start_time = functions.get_time_now() log_name = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time())) log_name = ''.join([site_name, log_name, '.txt']) global_data.log_path = os.path.join(os.getcwd(), 'logs', log_name) log(global_data.log_path, 'info', start_time, '开始下载任务') run(t_num=5) end_time = functions.get_time_now() print_status()