def error(msg):
"""Error message logger"""
msg = _get_time() + " [Error] " + str(msg)
if IS_SHOW_ERROR:
output.print_msg(msg, False)
if ERROR_LOG_PATH != "":
with thread_lock:
tool.write_file(msg, ERROR_LOG_PATH)
def step(msg):
"""Step message logger"""
msg = _get_time() + " " + str(msg)
if IS_SHOW_STEP:
output.print_msg(msg, False)
if STEP_LOG_PATH != "":
with thread_lock:
tool.write_file(msg, STEP_LOG_PATH)
def trace(msg):
"""Trace(Debugger) message logger"""
msg = _get_time() + " " + str(msg)
if IS_SHOW_TRACE:
output.print_msg(msg, False)
if TRACE_LOG_PATH != "":
with thread_lock:
tool.write_file(msg, TRACE_LOG_PATH)
def main():
account_list_from_storage = get_account_from_storage()
if len(account_list_from_storage) > 0:
account_list_from_save_data = get_account_from_save_data()
for account_id in account_list_from_storage:
if account_id not in account_list_from_save_data:
account_list_from_save_data[account_id] = "%s\t\t\t\t\t" % account_id
temp_list = [account_list_from_save_data[key] for key in sorted(account_list_from_save_data.keys())]
tool.write_file("\n".join(temp_list), SAVE_DATA_PATH, tool.WRITE_FILE_TYPE_REPLACE)
def trace(msg):
msg = tool.get_time() + " " + str(msg)
if IS_SHOW_TRACE:
tool.print_msg(msg, False)
if TRACE_LOG_PATH != "":
thread_lock.acquire()
try:
tool.write_file(msg, TRACE_LOG_PATH)
except:
raise
finally:
thread_lock.release()
def step(msg):
msg = tool.get_time() + " " + str(msg)
if IS_SHOW_STEP:
tool.print_msg(msg, False)
if STEP_LOG_PATH != "":
thread_lock.acquire()
try:
tool.write_file(msg, STEP_LOG_PATH)
except:
raise
finally:
thread_lock.release()
def error(msg):
msg = tool.get_time() + " [Error] " + str(msg)
if IS_SHOW_ERROR:
tool.print_msg(msg, False)
if ERROR_LOG_PATH != "":
thread_lock.acquire()
try:
tool.write_file(msg, ERROR_LOG_PATH)
except:
raise
finally:
thread_lock.release()
def reformat_save():
new_lines = []
for line in tool.read_file(OLD_SAVE_FILE_PATH, tool.READ_FILE_TYPE_LINE):
temp_list = line.replace("\n", "").split("\t")
new_list = list([])
# 新旧字段逻辑
new_list.append(temp_list[0])
new_list.append(temp_list[1])
new_list.append(temp_list[2])
new_lines.append("\t".join(new_list))
tool.write_file("\n".join(new_lines), NEW_SAVE_FILE_PATH,
tool.WRITE_FILE_TYPE_REPLACE)
def main():
# 存档位置
save_data_path = crawler.quickly_get_save_data_path()
account_list_from_api = get_account_from_index()
if len(account_list_from_api) > 0:
account_list_from_save_data = get_account_from_save_data(
save_data_path)
for account_id in account_list_from_api:
if account_id not in account_list_from_save_data:
account_list_from_save_data[account_id] = "%s\t\t\t%s" % (
account_id, account_list_from_api[account_id])
temp_list = [
account_list_from_save_data[key]
for key in sorted(account_list_from_save_data.keys())
]
tool.write_file("\n".join(temp_list), save_data_path,
tool.WRITE_FILE_TYPE_REPLACE)
def run(self):
global TOTAL_VIDEO_COUNT
account_id = self.account_info[0]
if len(self.account_info) >= 4 and self.account_info[3]:
account_name = self.account_info[3]
else:
account_name = self.account_info[0]
try:
log.step(account_name + " 开始")
# 如果需要重新排序则使用临时文件夹，否则直接下载到目标目录
if IS_SORT:
video_path = os.path.join(VIDEO_TEMP_PATH, account_name)
else:
video_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
video_count = 1
page_time = 0
first_video_time = "0"
need_make_video_dir = True
is_over = False
while not is_over:
# 获取一页视频信息
video_data = get_one_page_video_data(account_id, page_time)
if video_data is None:
log.error(account_name + " 视频列表获取失败")
tool.process_exit()
for video_info in video_data["info"]:
if not robot.check_sub_key(("newvideos", "id", "timestamp"), video_info):
log.error(account_name + " 视频信息 %s 解析失败" % video_info)
continue
page_time = int(video_info["timestamp"])
# 检查是否已下载到前一次的视频
if page_time <= int(self.account_info[2]):
is_over = True
break
# 将第一个视频的上传时间做为新的存档记录
if first_video_time == "0":
first_video_time = str(page_time)
# todo 处理如果有多个视频
if len(video_info["newvideos"]) != 1:
log.error(account_name + " 视频信息 %s 发现多个视频下载信息" % video_info)
continue
if not robot.check_sub_key(("vid",), video_info["newvideos"][0]):
log.error(account_name + " 视频信息 %s 解析vid失败" % video_info)
continue
# 获取视频下载地址
video_url = get_video_url(video_info["newvideos"][0]["vid"], video_info["id"])
log.step(account_name + " 开始下载第%s个视频 %s" % (video_count, video_url))
# 第一个视频，创建目录
if need_make_video_dir:
if not tool.make_dir(video_path, 0):
log.error(account_name + " 创建图片下载目录 %s 失败" % video_path)
tool.process_exit()
need_make_video_dir = False
file_type = video_url.split(".")[-1].split("?")[0]
file_path = os.path.join(video_path, "%04d.%s" % (video_count, file_type))
if tool.save_net_file(video_url, file_path):
log.step(account_name + " 第%s个视频下载成功" % video_count)
video_count += 1
else:
log.error(account_name + " 第%s个视频 %s 下载失败" % (video_count, video_url))
# 达到配置文件中的下载数量，结束
if 0 < GET_VIDEO_COUNT < video_data:
is_over = True
break
if not is_over:
if not video_data["hasNext"]:
is_over = True
log.step(account_name + " 下载完毕，总共获得%s个视频" % (video_count - 1))
# 排序
if IS_SORT:
if first_video_time != "0":
destination_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
if robot.sort_file(video_path, destination_path, int(self.account_info[3]), 4):
log.step(account_name + " 视频从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建视频保存目录 %s 失败" % destination_path)
tool.process_exit()
# 新的存档记录
if first_video_time != "0":
self.account_info[3] = str(int(self.account_info[3]) + video_count - 1)
self.account_info[4] = first_video_time
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_VIDEO_COUNT += video_count - 1
ACCOUNTS.remove(account_id)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def run(self):
global TOTAL_IMAGE_COUNT
account_id = self.account_info[0]
if len(self.account_info) >= 3 and self.account_info[2]:
account_name = self.account_info[2]
else:
account_name = self.account_info[0]
try:
log.step(account_name + " 开始")
# 获取账号对应的page_id
account_page_id = get_account_page_id(account_id)
if account_page_id is None:
log.error(account_name + " 微博主页没有获取到page_id")
tool.process_exit()
page_count = 1
this_account_total_image_count = 0
first_article_time = "0"
is_over = False
image_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
while not is_over:
# 获取一页文章预览页面
preview_article_page = get_one_page_preview_article_data(account_page_id, page_count)
if preview_article_page is None:
log.error(account_name + " 第%s页文章获取失败" % page_count)
tool.process_exit()
# 将文章预览页面内容分组
preview_article_data_list = get_preview_article_data_list(preview_article_page)
if len(preview_article_data_list) == 0:
log.error(account_name + " 第%s页文章解析失败，页面：%s" % (page_count, preview_article_page))
tool.process_exit()
for preview_article_data in preview_article_data_list:
# 获取文章的发布时间
article_time = get_article_time(preview_article_data)
if article_time is None:
log.error(account_name + " 预览 %s 中的文章发布时间解析失败" % preview_article_data)
continue
# 检查是否是上一次的最后视频
if article_time <= int(self.account_info[1]):
is_over = True
break
# 将第一个视频的地址做为新的存档记录
if first_article_time == "0":
first_article_time = str(article_time)
# 获取文章地址
article_url = get_article_url(preview_article_data)
if article_url is None:
log.error(account_name + " 预览 %s 中的文章地址解析失败" % preview_article_data)
continue
# 获取文章id
article_id = get_article_id(article_url)
if article_id is None:
log.error(account_name + " 文章地址 %s 解析文章id失败" % article_url)
continue
# 获取文章页面内容
article_page = auto_redirect_visit(article_url)
if not article_page:
log.error(account_name + " 文章 %s 获取失败" % article_url)
continue
# 文章标题
title = get_article_title(article_page, article_id[0])
# 过滤标题中不支持的字符
title = robot.filter_text(title)
if title:
article_path = os.path.join(image_path, "%s %s" % (article_id, title))
else:
article_path = os.path.join(image_path, article_id)
if not tool.make_dir(article_path, 0):
# 目录出错，把title去掉后再试一次，如果还不行退出
log.error(account_name + " 创建文章目录 %s 失败，尝试不使用title" % article_path)
article_path = os.path.join(image_path, article_id)
if not tool.make_dir(article_path, 0):
log.error(account_name + " 创建文章目录 %s 失败" % article_path)
tool.process_exit()
# 文章顶部图片
top_picture_url = get_article_top_picture_url(article_page)
if top_picture_url:
log.step(account_name + " %s 开始下载顶部图片 %s" % (title, top_picture_url))
file_type = top_picture_url.split(".")[-1]
file_path = os.path.join(article_path, "0000.%s" % file_type)
if tool.save_net_file(top_picture_url, file_path):
log.step(account_name + " %s 顶部图片下载成功" % title)
this_account_total_image_count += 1
else:
log.error(account_name + " %s 顶部图片 %s 下载失败" % (title, top_picture_url))
# 获取文章正文的图片地址列表
image_url_list = get_article_image_url_list(article_page, article_id[0])
if image_url_list is None:
log.error(account_name + " 文章 %s 正文解析失败" % article_url)
continue
image_count = 1
for image_url in list(image_url_list):
if image_url.find("/p/e_weibo_com") >= 0 or image_url.find("e.weibo.com") >= 0:
continue
log.step(account_name + " %s 开始下载第%s张图片 %s" % (title, image_count, image_url))
file_type = image_url.split(".")[-1]
file_path = os.path.join(article_path, "%s.%s" % (image_count, file_type))
if tool.save_net_file(image_url, file_path):
log.step(account_name + " %s 第%s张图片下载成功" % (title, image_count))
image_count += 1
else:
log.error(account_name + " %s 第%s张图片 %s 下载失败" % (title, image_count, image_url))
if image_count > 1:
this_account_total_image_count += image_count - 1
if not is_over:
# 获取文章总页数
if page_count >= get_max_page_count(preview_article_page):
is_over = True
else:
page_count += 1
log.step(account_name + " 下载完毕，总共获得%s张图片" % this_account_total_image_count)
# 新的存档记录
if first_article_time != "0":
self.account_info[1] = first_article_time
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_IMAGE_COUNT += this_account_total_image_count
ACCOUNTS.remove(account_id)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def run(self):
global TOTAL_VIDEO_COUNT
account_id = self.account_info[0]
if len(self.account_info) >= 3 and self.account_info[2]:
account_name = self.account_info[2]
else:
account_name = self.account_info[0]
try:
log.step(account_name + " 开始")
video_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
user_id = get_user_id(account_id)
if user_id is None:
log.error(account_name + " userid获取失败")
tool.process_exit()
page_count = 0
video_count = 1
first_audio_id = "0"
unique_list = []
is_over = False
need_make_download_dir = True
while not is_over:
# 获取指定一页的歌曲信息
audio_list = get_one_page_audio_list(user_id, page_count)
if audio_list is None:
log.step(account_name + " 第%s页歌曲列表获取失败" % page_count)
first_audio_id = "0"
break # 存档恢复
# 如果为空，表示已经取完了
if len(audio_list) == 0:
break
for audio_info in list(audio_list):
audio_id = audio_info[0]
# 检查是否歌曲id小于上次的记录
if int(audio_id) <= int(self.account_info[1]):
is_over = True
break
# 新增歌曲导致的重复判断
if audio_id in unique_list:
continue
else:
unique_list.append(audio_id)
# 将第一首歌曲id做为新的存档记录
if first_audio_id == "0":
first_audio_id = str(audio_id)
# 获取歌曲的下载地址
audio_url = get_audio_url(audio_info[2])
log.step(account_name + " 开始下载第%s首歌曲 %s" % (video_count, audio_url))
# 第一首歌曲，创建目录
if need_make_download_dir:
if not tool.make_dir(video_path, 0):
log.error(account_name + " 创建歌曲下载目录 %s 失败" % video_path)
tool.process_exit()
need_make_download_dir = False
file_path = os.path.join(video_path, "%s - %s.mp3" % (audio_id, audio_info[1]))
if tool.save_net_file(audio_url, file_path):
log.step(account_name + " 第%s首歌曲下载成功" % video_count)
video_count += 1
else:
log.error(account_name + " 第%s首歌曲 %s 下载失败" % (video_count, audio_url))
# 达到配置文件中的下载数量，结束
if 0 < GET_VIDEO_COUNT < video_count:
is_over = True
break
if not is_over:
# 获取的歌曲数量少于1页的上限，表示已经到结束了
# 如果歌曲数量正好是页数上限的倍数，则由下一页获取是否为空判断
if len(audio_list) < 20:
is_over = True
else:
page_count += 1
log.step(account_name + " 下载完毕，总共获得%s首歌曲" % (video_count - 1))
# 新的存档记录
if first_audio_id != "0":
self.account_info[1] = first_audio_id
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_VIDEO_COUNT += video_count - 1
ACCOUNTS.remove(account_id)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def run(self):
global TOTAL_IMAGE_COUNT
global TOTAL_VIDEO_COUNT
account_id = self.account_info[0]
try:
log.step(account_id + " 开始")
# 如果需要重新排序则使用临时文件夹，否则直接下载到目标目录
if IS_SORT:
image_path = os.path.join(IMAGE_TEMP_PATH, account_id)
video_path = os.path.join(VIDEO_TEMP_PATH, account_id)
else:
image_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_id)
video_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_id)
page_count = 1
image_count = 1
video_count = 1
first_post_id = ""
unique_list = []
is_over = False
need_make_image_dir = True
need_make_video_dir = True
while not is_over:
post_url_list = get_one_page_post_url_list(account_id, page_count)
if post_url_list is None:
log.error(account_id + " 无法访问第%s页相册页" % page_count)
tool.process_exit()
if len(post_url_list) == 0:
# 下载完毕了
break
log.trace(account_id + " 相册第%s页获取的所有信息页：%s" % (page_count, post_url_list))
post_url_list_group_by_post_id = filter_post_url(post_url_list)
log.trace(account_id + " 相册第%s页去重排序后的信息页：%s" % (page_count, post_url_list_group_by_post_id))
log.step(account_id + " 相册第%s页获取到%s页信息页" % (page_count, len(post_url_list_group_by_post_id)))
for post_id in sorted(post_url_list_group_by_post_id.keys(), reverse=True):
# 检查信息页id是否小于上次的记录
if post_id <= self.account_info[3]:
is_over = True
break
# 将第一个信息页的id做为新的存档记录
if first_post_id == "":
first_post_id = post_id
# 获取信息页并截取head标签内的内容
post_url = "http://%s.tumblr.com/post/%s" % (account_id, post_id)
post_page_head = get_post_page_head(post_url, post_url_list_group_by_post_id[post_id])
if post_page_head is None:
log.error(account_id + " 无法访问信息页 %s" % post_url)
continue
if not post_page_head:
log.error(account_id + " 信息页 %s 截取head标签异常" % post_url)
continue
# 获取og_type（页面类型的是视频还是图片或其他）
og_type = tool.find_sub_string(post_page_head, '<meta property="og:type" content="', '" />')
if not og_type:
log.error(account_id + " 信息页 %s，'og:type'获取异常" % post_url)
continue
# 空、音频、引用，跳过
if og_type in ["tumblr-feed:entry", "tumblr-feed:audio", "tumblr-feed:quote", "tumblr-feed:link"]:
continue
# 新增信息页导致的重复判断
if post_id in unique_list:
continue
else:
unique_list.append(post_id)
# 视频下载
if IS_DOWNLOAD_VIDEO and og_type == "tumblr-feed:video":
video_list = get_video_list(account_id, post_id)
if video_list is None:
log.error(account_id + " 第%s个视频 日志id：%s无法访问播放页" % (video_count, post_id))
else:
if len(video_list) > 0:
for video_url, video_type in list(video_list):
log.step(account_id + " 开始下载第%s个视频 %s" % (video_count, video_url))
# 第一个视频，创建目录
if need_make_video_dir:
if not tool.make_dir(video_path, 0):
log.error(account_id + " 创建视频下载目录 %s 失败" % video_path)
tool.process_exit()
need_make_video_dir = False
file_type = video_type.split("/")[-1]
video_file_path = os.path.join(video_path, "%04d.%s" % (video_count, file_type))
if tool.save_net_file(video_url, video_file_path):
log.step(account_id + " 第%s个视频下载成功" % video_count)
video_count += 1
else:
log.error(account_id + " 第%s个视频 %s 下载失败" % (video_count, video_url))
else:
log.error(account_id + " 第%s个视频 日志id：%s 中没有找到视频" % (video_count, post_id))
# 图片下载
if IS_DOWNLOAD_IMAGE:
if og_type == "tumblr-feed:video":
page_image_url_list = []
video_image_url = tool.find_sub_string(post_page_head, '<meta property="og:image" content="', '" />')
if video_image_url:
page_image_url_list.append(video_image_url)
else:
page_image_url_list = re.findall('"(http[s]?://\w*[.]?media.tumblr.com/[^"]*)"', post_page_head)
log.trace(account_id + " 信息页 %s 过滤前的所有图片：%s" % (post_url, page_image_url_list))
# 过滤头像以及页面上找到不同分辨率的同一张图
page_image_url_list = filter_different_resolution_images(page_image_url_list)
log.trace(account_id + " 信息页 %s 获取的的所有图片：%s" % (post_url, page_image_url_list))
if len(page_image_url_list) > 0:
for image_url in page_image_url_list:
log.step(account_id + " 开始下载第%s张图片 %s" % (image_count, image_url))
# 第一张图片，创建目录
if need_make_image_dir:
if not tool.make_dir(image_path, 0):
log.error(account_id + " 创建图片下载目录 %s 失败" % image_path)
tool.process_exit()
need_make_image_dir = False
file_type = image_url.split(".")[-1]
image_file_path = os.path.join(image_path, "%04d.%s" % (image_count, file_type))
if tool.save_net_file(image_url, image_file_path):
log.step(account_id + " 第%s张图片下载成功" % image_count)
image_count += 1
else:
log.error(account_id + " 第%s张图片 %s 下载失败" % (image_count, image_url))
else:
log.error(account_id + " 第%s张图片 信息页 %s 中没有找到图片" % (image_count, post_url))
if not is_over:
# 达到配置文件中的下载数量，结束
if 0 < GET_PAGE_COUNT <= page_count:
is_over = True
else:
page_count += 1
log.step(account_id + " 下载完毕，总共获得%s张图片和%s个视频" % (image_count - 1, video_count - 1))
# 排序
if IS_SORT:
if image_count > 1:
destination_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_id)
if robot.sort_file(image_path, destination_path, int(self.account_info[1]), 4):
log.step(account_id + " 图片从下载目录移动到保存目录成功")
else:
log.error(account_id + " 创建图片保存目录 %s 失败" % destination_path)
tool.process_exit()
if video_count > 1:
destination_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_id)
if robot.sort_file(video_path, destination_path, int(self.account_info[2]), 4):
log.step(account_id + " 视频从下载目录移动到保存目录成功")
else:
log.error(account_id + " 创建视频保存目录 %s 失败" % destination_path)
tool.process_exit()
# 新的存档记录
if first_post_id != "":
self.account_info[1] = str(int(self.account_info[1]) + image_count - 1)
self.account_info[2] = str(int(self.account_info[2]) + video_count - 1)
self.account_info[3] = first_post_id
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_IMAGE_COUNT += image_count - 1
TOTAL_VIDEO_COUNT += video_count - 1
ACCOUNTS.remove(account_id)
self.thread_lock.release()
log.step(account_id + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_id + " 提前退出")
else:
log.error(account_id + " 异常退出")
def save_discount_list(discount_list):
tool.write_file(tool.list_to_string(discount_list, "\n", ""), "discount.txt", 2)
def run(self):
global TOTAL_IMAGE_COUNT
global TOTAL_VIDEO_COUNT
account_name = self.account_info[0]
try:
log.step(account_name + " 开始")
# 如果需要重新排序则使用临时文件夹，否则直接下载到目标目录
if IS_SORT:
image_path = os.path.join(IMAGE_TEMP_PATH, account_name)
video_path = os.path.join(VIDEO_TEMP_PATH, account_name)
else:
image_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
video_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
image_count = 1
video_count = 1
target_id = INIT_TARGET_ID
first_post_id = "0"
is_over = False
need_make_image_dir = True
need_make_video_dir = True
while not is_over:
# 获取一页日志信息
message_page_data = get_message_page_data(account_name, target_id)
if message_page_data is None:
log.error(account_name + " 媒体列表解析异常")
tool.process_exit()
# 没有了
if len(message_page_data) == 0:
break
for message_info in message_page_data:
if not robot.check_sub_key(("post",), message_info):
log.error(account_name + " 媒体信息解析异常 %s" % message_info)
continue
if not robot.check_sub_key(("body", "postId"), message_info["post"]):
log.error(account_name + " 媒体信息解析异常 %s" % message_info)
continue
target_id = message_info["post"]["postId"]
# 检查是否已下载到前一次的记录
if int(target_id) <= int(self.account_info[3]):
is_over = True
break
# 将第一个媒体的postId做为新的存档记录
if first_post_id == "0":
first_post_id = str(target_id)
for media_info in message_info["post"]["body"]:
if not robot.check_sub_key(("bodyType",), media_info):
log.error(account_name + " 媒体列表bodyType解析异常")
continue
# bodyType = 1: text, bodyType = 3: image, bodyType = 8: video
body_type = int(media_info["bodyType"])
if body_type == 1: # 文本
pass
elif body_type == 2: # 表情
pass
elif body_type == 3: # 图片
if IS_DOWNLOAD_IMAGE:
if not robot.check_sub_key(("image",), media_info):
log.error(account_name + " 第%s张图片解析异常%s" % (image_count, media_info))
continue
image_url = str(media_info["image"])
log.step(account_name + " 开始下载第%s张图片 %s" % (image_count, image_url))
# 第一张图片，创建目录
if need_make_image_dir:
if not tool.make_dir(image_path, 0):
log.error(account_name + " 创建图片下载目录 %s 失败" % image_path)
tool.process_exit()
need_make_image_dir = False
file_type = image_url.split(".")[-1]
image_file_path = os.path.join(image_path, "%04d.%s" % (image_count, file_type))
if tool.save_net_file(image_url, image_file_path):
log.step(account_name + " 第%s张图片下载成功" % image_count)
image_count += 1
else:
log.error(account_name + " 第%s张图片 %s 下载失败" % (image_count, image_url))
elif body_type == 8: # video
if IS_DOWNLOAD_VIDEO:
if not robot.check_sub_key(("movieUrlHq",), media_info):
log.error(account_name + " 第%s个视频解析异常%s" % (video_count, media_info))
continue
video_url = str(media_info["movieUrlHq"])
log.step(account_name + " 开始下载第%s个视频 %s" % (video_count, video_url))
# 第一个视频，创建目录
if need_make_video_dir:
if not tool.make_dir(video_path, 0):
log.error(account_name + " 创建视频下载目录 %s 失败" % video_path)
tool.process_exit()
need_make_video_dir = False
file_type = video_url.split(".")[-1]
video_file_path = os.path.join(video_path, "%04d.%s" % (video_count, file_type))
if tool.save_net_file(video_url, video_file_path):
log.step(account_name + " 第%s个视频下载成功" % video_count)
video_count += 1
else:
log.error(account_name + " 第%s个视频 %s 下载失败" % (video_count, video_url))
elif body_type == 7: # 转发
pass
else:
log.error(account_name + " 第%s张图片、第%s个视频，未知bodytype %s, %s" % (image_count, video_count, body_type, media_info))
# 排序
if IS_SORT:
if image_count > 1:
destination_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
if robot.sort_file(image_path, destination_path, int(self.account_info[1]), 4):
log.step(account_name + " 图片从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建图片保存目录 %s 失败" % destination_path)
tool.process_exit()
if video_count > 1:
destination_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
if robot.sort_file(video_path, destination_path, int(self.account_info[2]), 4):
log.step(account_name + " 视频从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建视频保存目录 %s 失败" % destination_path)
tool.process_exit()
# 新的存档记录
if first_post_id != "0":
self.account_info[1] = str(int(self.account_info[1]) + image_count - 1)
self.account_info[2] = str(int(self.account_info[2]) + video_count - 1)
self.account_info[3] = first_post_id
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_IMAGE_COUNT += image_count - 1
TOTAL_VIDEO_COUNT += video_count - 1
ACCOUNTS.remove(account_name)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def run(self):
global TOTAL_IMAGE_COUNT
global TOTAL_VIDEO_COUNT
account_name = self.account_info[0]
try:
log.step(account_name + " 开始")
# 如果需要重新排序则使用临时文件夹，否则直接下载到目标目录
if IS_SORT:
image_path = os.path.join(IMAGE_TEMP_PATH, account_name)
video_path = os.path.join(VIDEO_TEMP_PATH, account_name)
else:
image_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
video_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
account_id = get_account_id(account_name)
if account_id is None:
log.error(account_name + " account id 查找失败")
tool.process_exit()
image_count = 1
video_count = 1
cursor = INIT_CURSOR
first_created_time = "0"
is_over = False
need_make_image_dir = True
need_make_video_dir = True
while not is_over:
# 获取指定时间后的一页媒体信息
media_data = get_one_page_media_data(account_id, cursor)
if media_data is None:
log.error(account_name + " 媒体列表解析异常")
tool.process_exit()
nodes_data = media_data["nodes"]
for photo_info in nodes_data:
if not robot.check_sub_key(("is_video", "display_src", "date"), photo_info):
log.error(account_name + " 媒体信息解析异常")
break
if photo_info["is_video"] and not robot.check_sub_key(("code",), photo_info):
log.error(account_name + " 视频code解析异常")
break
# 检查是否已下载到前一次的图片
if int(photo_info["date"]) <= int(self.account_info[3]):
is_over = True
break
# 将第一张图片的上传时间做为新的存档记录
if first_created_time == "0":
first_created_time = str(int(photo_info["date"]))
# 图片
if IS_DOWNLOAD_IMAGE:
image_url = str(photo_info["display_src"].split("?")[0])
log.step(account_name + " 开始下载第%s张图片 %s" % (image_count, image_url))
# 第一张图片，创建目录
if need_make_image_dir:
if not tool.make_dir(image_path, 0):
log.error(account_name + " 创建图片下载目录 %s 失败" % image_path)
tool.process_exit()
need_make_image_dir = False
file_type = image_url.split(".")[-1]
image_file_path = os.path.join(image_path, "%04d.%s" % (image_count, file_type))
if tool.save_net_file(image_url, image_file_path):
log.step(account_name + " 第%s张图片下载成功" % image_count)
image_count += 1
else:
log.error(account_name + " 第%s张图片 %s 下载失败" % (image_count, image_url))
# 视频
if IS_DOWNLOAD_VIDEO and photo_info["is_video"]:
# 根据日志ID获取视频下载地址
video_url = get_video_url(photo_info["code"])
if video_url is None:
log.error(account_name + " 第%s个视频code：%s 无法访问" % (video_count, photo_info["code"]))
continue
if not video_url:
log.error(account_name + " 第%s个视频code：%s 没有获取到下载地址" % (video_count, photo_info["code"]))
continue
log.step(account_name + " 开始下载第%s个视频 %s" % (video_count, video_url))
# 第一个视频，创建目录
if need_make_video_dir:
if not tool.make_dir(video_path, 0):
log.error(account_name + " 创建视频下载目录 %s 失败" % video_path)
tool.process_exit()
need_make_video_dir = False
file_type = video_url.split(".")[-1]
video_file_path = os.path.join(video_path, "%04d.%s" % (video_count, file_type))
if tool.save_net_file(video_url, video_file_path):
log.step(account_name + " 第%s个视频下载成功" % video_count)
video_count += 1
else:
log.error(account_name + " 第%s个视频 %s 下载失败" % (video_count, video_url))
# 达到配置文件中的下载数量，结束
if 0 < GET_IMAGE_COUNT < image_count:
is_over = True
break
if not is_over:
if media_data["page_info"]["has_next_page"]:
cursor = str(media_data["page_info"]["end_cursor"])
else:
is_over = True
log.step(account_name + " 下载完毕，总共获得%s张图片和%s个视频" % (image_count - 1, video_count - 1))
# 排序
if IS_SORT:
if image_count > 1:
destination_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
if robot.sort_file(image_path, destination_path, int(self.account_info[1]), 4):
log.step(account_name + " 图片从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建图片保存目录 %s 失败" % destination_path)
tool.process_exit()
if video_count > 1:
destination_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
if robot.sort_file(video_path, destination_path, int(self.account_info[2]), 4):
log.step(account_name + " 视频从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建视频保存目录 %s 失败" % destination_path)
tool.process_exit()
# 新的存档记录
if first_created_time != "0":
self.account_info[1] = str(int(self.account_info[1]) + image_count - 1)
self.account_info[2] = str(int(self.account_info[2]) + video_count - 1)
self.account_info[3] = first_created_time
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_IMAGE_COUNT += image_count - 1
TOTAL_VIDEO_COUNT += video_count - 1
ACCOUNTS.remove(account_name)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def run(self):
global TOTAL_IMAGE_COUNT
global TOTAL_VIDEO_COUNT
account_id = self.account_info[0]
if len(self.account_info) >= 6 and self.account_info[5]:
account_name = self.account_info[5]
else:
account_name = self.account_info[0]
try:
log.step(account_name + " 开始")
# 如果需要重新排序则使用临时文件夹，否则直接下载到目标目录
if IS_SORT:
image_path = os.path.join(IMAGE_TEMP_PATH, account_name)
video_path = os.path.join(VIDEO_TEMP_PATH, account_name)
else:
image_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
video_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
# 视频
video_count = 1
account_page_id = None
first_video_url = ""
is_over = False
need_make_video_dir = True
since_id = INIT_SINCE_ID
while IS_DOWNLOAD_VIDEO and (not is_over):
# 获取page_id
if account_page_id is None:
account_page_id = get_account_page_id(account_id)
if account_page_id is None:
log.error(account_name + " 微博主页没有获取到page_id")
break
# 获取指定时间点后的一页视频信息
video_page_data = get_one_page_video_data(account_page_id, since_id)
if video_page_data is None:
log.error(account_name + " 视频列表解析异常")
first_video_url = "" # 存档恢复
break
# 匹配获取全部的视频页面
video_play_url_list = get_video_play_url_list(video_page_data)
log.trace(account_name + "since_id：%s中的全部视频：%s" % (since_id, video_play_url_list))
for video_play_url in video_play_url_list:
# 检查是否是上一次的最后视频
if self.account_info[4] == video_play_url:
is_over = True
break
# 将第一个视频的地址做为新的存档记录
if first_video_url == "":
first_video_url = video_play_url
# 获取这个视频的下载地址
return_code, video_url_list = get_video_url(video_play_url)
if return_code != 1:
if return_code == -1:
log.error(account_name + " 第%s个视频 %s 没有获取到源地址" % (video_count, video_play_url))
elif return_code == -2:
log.error(account_name + " 第%s个视频 %s 无法访问" % (video_count, video_play_url))
elif return_code == -3:
log.error(account_name + " 第%s个视频 %s 暂不支持的视频源" % (video_count, video_play_url))
continue
log.step(account_name + " 开始下载第%s个视频 %s" % (video_count, video_play_url))
# 第一个视频，创建目录
if need_make_video_dir:
if not tool.make_dir(video_path, 0):
log.error(account_name + " 创建图片下载目录 %s 失败" % video_path)
tool.process_exit()
need_make_video_dir = False
video_file_path = os.path.join(video_path, "%04d.mp4" % video_count)
for video_url in video_url_list:
if tool.save_net_file(video_url, video_file_path):
log.step(account_name + " 第%s个视频下载成功" % video_count)
video_count += 1
else:
log.error(account_name + " 第%s个视频 %s 下载失败" % (video_count, video_url))
# 达到配置文件中的下载数量，结束
if 0 < GET_VIDEO_COUNT < video_count:
is_over = True
break
if not is_over:
# 获取下一页的since_id
since_id = tool.find_sub_string(video_page_data, "type=video&owner_uid=&since_id=", '">')
if not since_id:
break
# 有历史记录，并且此次没有获得正常结束的标记，说明历史最后的视频已经被删除了
if self.account_info[4] != "" and video_count > 1 and not is_over:
log.error(account_name + " 没有找到上次下载的最后一个视频地址")
# 图片
image_count = 1
page_count = 1
first_image_time = "0"
unique_list = []
is_over = False
need_make_image_dir = True
while IS_DOWNLOAD_IMAGE and (not is_over):
# 获取指定一页图片的信息
photo_page_data = get_one_page_photo_data(account_id, page_count)
if photo_page_data is None:
log.error(account_name + " 图片列表获取失败")
first_image_time = "0" # 存档恢复
break
log.trace(account_name + "第%s页的全部图片信息：%s" % (page_count, photo_page_data))
for image_info in photo_page_data["photo_list"]:
if not robot.check_sub_key(("pic_host", "pic_name", "timestamp"), image_info):
log.error(account_name + " 第%s张图片信息解析失败 %s" % (image_count, image_info))
continue
# 检查是否图片时间小于上次的记录
if int(image_info["timestamp"]) <= int(self.account_info[2]):
is_over = True
break
# 新增图片导致的重复判断
if image_info["pic_name"] in unique_list:
continue
else:
unique_list.append(image_info["pic_name"])
# 将第一张图片的上传时间做为新的存档记录
if first_image_time == "0":
first_image_time = str(image_info["timestamp"])
image_url = str(image_info["pic_host"]) + "/large/" + str(image_info["pic_name"])
log.step(account_name + " 开始下载第%s张图片 %s" % (image_count, image_url))
# 获取图片的二进制数据，并且判断这个图片是否是可用的
image_status, image_byte = get_image_byte(image_url)
if image_status != 1:
if image_status == -1:
log.error(account_name + " 第%s张图片 %s 下载失败" % (image_count, image_url))
elif image_status == -2:
log.error(account_name + " 第%s张图片 %s 资源已被删除，跳过" % (image_count, image_url))
continue
# 第一张图片，创建目录
if need_make_image_dir:
if not tool.make_dir(image_path, 0):
log.error(account_name + " 创建图片下载目录 %s 失败" % image_path)
tool.process_exit()
need_make_image_dir = False
file_type = image_url.split(".")[-1]
if file_type.find("/") != -1:
file_type = "jpg"
image_file_path = os.path.join(image_path, "%04d.%s" % (image_count, file_type))
save_image(image_byte, image_file_path)
log.step(account_name + " 第%s张图片下载成功" % image_count)
image_count += 1
# 达到配置文件中的下载数量，结束
if 0 < GET_IMAGE_COUNT < image_count:
is_over = True
break
if not is_over:
# 根据总的图片数量和每页显示的图片数量，计算是否还有下一页
if (photo_page_data["total"] / IMAGE_COUNT_PER_PAGE) > (page_count - 1):
page_count += 1
else:
# 全部图片下载完毕
is_over = True
log.step(account_name + " 下载完毕，总共获得%s张图片和%s个视频" % (image_count - 1, video_count - 1))
# 排序
if IS_SORT:
if first_image_time != "0":
destination_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
if robot.sort_file(image_path, destination_path, int(self.account_info[1]), 4):
log.step(account_name + " 图片从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建图片保存目录 %s 失败" % destination_path)
tool.process_exit()
if first_video_url != "":
destination_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
if robot.sort_file(video_path, destination_path, int(self.account_info[3]), 4):
log.step(account_name + " 视频从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建视频保存目录 %s 失败" % destination_path)
tool.process_exit()
# 新的存档记录
if first_image_time != "0":
self.account_info[1] = str(int(self.account_info[1]) + image_count - 1)
self.account_info[2] = first_image_time
if first_video_url != "":
self.account_info[3] = str(int(self.account_info[3]) + video_count - 1)
self.account_info[4] = first_video_url
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_IMAGE_COUNT += image_count - 1
TOTAL_VIDEO_COUNT += video_count - 1
ACCOUNTS.remove(account_id)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def rewrite_save_file(temp_save_data_path, save_data_path):
account_list = read_save_data(temp_save_data_path, 0, [])
temp_list = [account_list[key] for key in sorted(account_list.keys())]
tool.write_file(tool.list_to_string(temp_list), save_data_path, 2)
os.remove(temp_save_data_path)
def save_discount_list(discount_game_list):
tool.write_file(json.dumps(discount_game_list), DISCOUNT_DATA_PATH, tool.WRITE_FILE_TYPE_REPLACE)
def run(self):
global TOTAL_IMAGE_COUNT
global TOTAL_VIDEO_COUNT
account_name = self.account_info[0]
try:
log.step(account_name + " 开始")
# 如果需要重新排序则使用临时文件夹，否则直接下载到目标目录
if IS_SORT:
image_path = os.path.join(IMAGE_TEMP_PATH, account_name)
video_path = os.path.join(VIDEO_TEMP_PATH, account_name)
else:
image_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
video_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
image_count = 1
video_count = 1
data_tweet_id = INIT_MAX_ID
first_tweet_id = "0"
is_over = False
is_download_image = IS_DOWNLOAD_IMAGE
is_download_video = IS_DOWNLOAD_VIDEO
need_make_image_dir = True
need_make_video_dir = True
while not is_over:
# 获取指定时间点后的一页图片信息
media_page = get_media_page_data(account_name, data_tweet_id)
if media_page is None:
log.error(account_name + " 媒体列表解析异常")
tool.process_exit()
# 上一页正好获取了全部的媒体信息，所以这一页没有任何内容，完成了，直接退出
if media_page["new_latent_count"] == 0 and not media_page["has_more_items"]:
break
tweet_list = get_tweet_list(media_page["items_html"])
if len(tweet_list) == 0:
log.error(account_name + " 媒体列表拆分异常，items_html：%s" % media_page["items_html"])
tool.process_exit()
if media_page["new_latent_count"] != len(tweet_list):
log.error(account_name + " 解析的媒体数量不等于new_latent_count的数值")
# tool.process_exit()
for tweet_data in tweet_list:
tweet_id = tool.find_sub_string(tweet_data, 'data-tweet-id="', '"')
if not tweet_id:
log.error(account_name + " tweet id解析异常，tweet数据：%s" % tweet_data)
continue
# 检查是否tweet的id小于上次的记录
if int(tweet_id) <= int(self.account_info[3]):
is_over = True
break
# 将第一个tweet的id做为新的存档记录
if first_tweet_id == "0":
first_tweet_id = tweet_id
# 视频
if is_download_image:
# 这个tweet是否包含视频
if check_has_video(tweet_data):
video_file_type, video_url_list = get_video_url_list(tweet_id)
if len(video_url_list) > 0:
log.step(account_name + " 开始下载第%s个视频 %s" % (video_count, video_url_list))
# 第一个视频，创建目录
if need_make_video_dir:
if not tool.make_dir(video_path, 0):
log.error(account_name + " 创建图片下载目录 %s 失败" % video_path)
tool.process_exit()
need_make_video_dir = False
video_file_path = os.path.join(video_path, "%04d.%s" % (video_count, video_file_type))
if save_video(video_url_list, video_file_path):
log.step(account_name + " 第%s个视频下载成功" % video_count)
video_count += 1
else:
log.error(account_name + " 第%s个视频 %s 下载失败" % (video_count, video_url_list))
else:
log.error(account_name + " 第%s个视频 没有获取到源地址，tweet id：%s" % (video_count, tweet_id))
# 达到配置文件中的下载数量，结束图片下载
if 0 < GET_IMAGE_COUNT < image_count:
is_download_image = False
# 图片
if is_download_video:
# 匹配获取全部的图片地址
image_url_list = get_image_url_list(tweet_data)
for image_url in image_url_list:
image_url = str(image_url)
log.step(account_name + " 开始下载第%s张图片 %s" % (image_count, image_url))
image_return_code, image_byte = tool.http_request(image_url)[:2]
# 404，不算做错误，图片已经被删掉了
if image_return_code == -404:
log.error(account_name + " 第%s张图片 %s 已被删除，跳过" % (image_count, image_url))
elif image_return_code == 1:
# 第一张图片，创建目录
if need_make_image_dir:
if not tool.make_dir(image_path, 0):
log.error(account_name + " 创建图片下载目录 %s 失败" % image_path)
tool.process_exit()
need_make_image_dir = False
file_type = image_url.split(".")[-1].split(":")[0]
image_file_path = os.path.join(image_path, "%04d.%s" % (image_count, file_type))
save_image(image_byte, image_file_path)
log.step(account_name + " 第%s张图片下载成功" % image_count)
image_count += 1
else:
log.error(account_name + " 第%s张图片 %s 获取失败" % (image_count, image_url))
# 达到配置文件中的下载数量，结束视频下载
if 0 < GET_VIDEO_COUNT < video_count:
is_download_video = False
# 全部达到配置文件中的下载数量，结束
if not is_download_image and not is_download_video:
is_over = True
break
if not is_over:
# 查找下一页的data_tweet_id
if media_page["has_more_items"]:
data_tweet_id = str(media_page["min_position"])
else:
is_over = True
log.step(account_name + " 下载完毕，总共获得%s张图片和%s个视频" % (image_count - 1, video_count - 1))
# 排序
if IS_SORT:
if image_count > 1:
destination_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
if robot.sort_file(image_path, destination_path, int(self.account_info[1]), 4):
log.step(account_name + " 图片从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建图片子目录 %s 失败" % destination_path)
tool.process_exit()
if video_count > 1:
destination_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
if robot.sort_file(video_path, destination_path, int(self.account_info[2]), 4):
log.step(account_name + " 视频从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建视频保存目录 %s 失败" % destination_path)
tool.process_exit()
# 新的存档记录
if first_tweet_id != "0":
self.account_info[1] = str(int(self.account_info[1]) + image_count - 1)
self.account_info[2] = str(int(self.account_info[2]) + video_count - 1)
self.account_info[3] = first_tweet_id
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_IMAGE_COUNT += image_count - 1
TOTAL_VIDEO_COUNT += video_count - 1
ACCOUNTS.remove(account_name)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def run(self):
global TOTAL_VIDEO_COUNT
global GET_PAGE_COUNT
account_id = self.account_info[0]
if len(self.account_info) >= 4 and self.account_info[3]:
account_name = self.account_info[3]
else:
account_name = self.account_info[0]
# 原创、翻唱
audio_type_to_index = {"yc": 1, "fc": 2}
try:
log.step(account_name + " 开始")
video_count = 1
for audio_type in audio_type_to_index.keys():
video_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name, audio_type)
page_count = 1
first_audio_id = "0"
unique_list = []
is_over = False
need_make_download_dir = True
while not is_over:
# 获取指定一页的歌曲信息列表
audio_list = get_one_page_audio_list(account_id, audio_type, page_count)
if audio_list is None:
log.step(account_name + " 第%s页%s歌曲页面获取失败" % (page_count, audio_type))
first_audio_id = "0"
break # 存档恢复
# 如果为空，表示已经取完了
if len(audio_list) == 0:
break
for audio_info in list(audio_list):
audio_id = audio_info[0]
# 过滤标题中不支持的字符
audio_title = robot.filter_text(audio_info[1])
# 检查是否歌曲id小于上次的记录
if int(audio_id) <= int(self.account_info[audio_type_to_index[audio_type]]):
is_over = True
break
# 新增歌曲导致的重复判断
if audio_id in unique_list:
continue
else:
unique_list.append(audio_id)
# 将第一首歌曲id做为新的存档记录
if first_audio_id == "0":
first_audio_id = str(audio_id)
# 获取歌曲的下载地址
audio_url = get_audio_url(audio_id, audio_type_to_index[audio_type])
if audio_url is None:
log.step(account_name + " %s歌曲ID %s，下载地址获取失败" % (audio_type, audio_id))
continue
if not audio_url:
log.step(account_name + " %s歌曲ID %s，暂不提供下载地址" % (audio_type, audio_id))
continue
log.step(account_name + " 开始下载第%s首歌曲 %s" % (video_count, audio_url))
# 第一首歌曲，创建目录
if need_make_download_dir:
if not tool.make_dir(video_path, 0):
log.error(account_name + " 创建歌曲下载目录 %s 失败" % video_path)
tool.process_exit()
need_make_download_dir = False
file_path = os.path.join(video_path, "%s - %s.mp3" % (audio_id, audio_title))
if tool.save_net_file(audio_url, file_path):
log.step(account_name + " 第%s首歌曲下载成功" % video_count)
video_count += 1
else:
log.error(account_name + " 第%s首歌曲 %s 下载失败" % (video_count, audio_url))
# 达到配置文件中的下载数量，结束
if 0 < GET_VIDEO_COUNT < video_count:
is_over = True
break
if not is_over:
# 达到配置文件中的下载页数，结束
if 0 < GET_PAGE_COUNT <= page_count:
is_over = True
# 获取的歌曲数量少于1页的上限，表示已经到结束了
# 如果歌曲数量正好是页数上限的倍数，则由下一页获取是否为空判断
elif len(audio_list) < 20:
is_over = True
else:
page_count += 1
# 新的存档记录
if first_audio_id != "0":
self.account_info[audio_type_to_index[audio_type]] = first_audio_id
log.step(account_name + " 下载完毕，总共获得%s首歌曲" % (video_count - 1))
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_VIDEO_COUNT += video_count - 1
ACCOUNTS.remove(account_id)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def save_discount_list(review_data):
tool.write_file(json.dumps(review_data), REVIEW_DATA_PATH,
tool.WRITE_FILE_TYPE_REPLACE)
def run(self):
global TOTAL_VIDEO_COUNT
account_id = self.account_info[0]
if len(self.account_info) >= 3 and self.account_info[2]:
account_name = self.account_info[2]
else:
account_name = self.account_info[0]
try:
log.step(account_name + " 开始")
# 如果需要重新排序则使用临时文件夹，否则直接下载到目标目录
if IS_SORT:
video_path = os.path.join(VIDEO_TEMP_PATH, account_name)
else:
video_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
# 获取视频信息列表
video_info_list = get_video_info_list(account_id)
if video_info_list is None:
log.error(account_name + " 视频列表获取失败")
tool.process_exit()
video_count = 1
first_video_id = "0"
need_make_video_dir = True
for video_info in video_info_list:
if not robot.check_sub_key(("item_data",), video_info) or \
not robot.check_sub_key(("watch_id", "title"), video_info["item_data"]):
log.error(account_name + " 视频信息%s解析失败" % video_info)
tool.process_exit()
# sm30043563
video_id = str(video_info["item_data"]["watch_id"])
# 过滤标题中不支持的字符
video_title = robot.filter_text(video_info["item_data"]["title"])
# 第一个视频，创建目录
if need_make_video_dir:
if not tool.make_dir(video_path, 0):
log.error(account_name + " 创建图片下载目录 %s 失败" % video_path)
tool.process_exit()
need_make_video_dir = False
# 获取视频下载地址
video_url = get_video_url(video_id)
log.step(account_name + " 开始下载第%s个视频 %s %s" % (video_count, video_id, video_url))
print video_title
print "%s %s" % (video_id, video_title)
file_path = os.path.join(video_path, "%s %s.mp4" % (video_id, video_title))
if tool.save_net_file(video_url, file_path):
log.step(account_name + " 第%s个视频下载成功" % video_count)
video_count += 1
else:
log.error(account_name + " 第%s个视频 %s %s 下载失败" % (video_count, video_id, video_url))
log.step(account_name + " 下载完毕，总共获得%s个视频" % (video_count - 1))
# 排序
if IS_SORT:
if first_video_id != "0":
destination_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
if robot.sort_file(video_path, destination_path, int(self.account_info[3]), 4):
log.step(account_name + " 视频从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建视频保存目录 %s 失败" % destination_path)
tool.process_exit()
# 新的存档记录
if first_video_id != "0":
self.account_info[1] = first_video_id
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_VIDEO_COUNT += video_count - 1
ACCOUNTS.remove(account_id)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def run(self):
global TOTAL_IMAGE_COUNT
account_name = self.account_info[0]
try:
log.step(account_name + " 开始")
if account_name.isdigit():
site_id = account_name
else:
site_id = get_site_id(account_name)
if site_id is None:
log.error(account_name + " 主页无法访问")
tool.process_exit()
if not site_id:
log.error(account_name + " site id解析失败")
tool.process_exit()
image_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
this_account_total_image_count = 0
post_count = 0
first_post_id = "0"
post_time = "2016-11-16 14:12:00"
is_over = False
while not is_over:
# 获取一页的相册信息列表
post_info_list = get_one_page_post_info_list(site_id, post_time)
if post_info_list is None:
log.error(account_name + " 相册信息列表无法访问")
tool.process_exit()
# 如果为空，表示已经取完了
if len(post_info_list) == 0:
break
for post_info in post_info_list:
if not robot.check_sub_key(("title", "post_id", "published_at", "images"), post_info):
log.error(account_name + " 相册信息解析失败：%s" % post_info)
continue
post_id = str(post_info["post_id"])
# 检查信息页id是否小于上次的记录
if int(post_id) <= int(self.account_info[1]):
is_over = True
break
# 将第一个信息页的id做为新的存档记录
if first_post_id == "0":
first_post_id = post_id
# 过滤标题中不支持的字符
title = robot.filter_text(post_info["title"])
if title:
post_path = os.path.join(image_path, "%s %s" % (post_id, title))
else:
post_path = os.path.join(image_path, post_id)
if not tool.make_dir(post_path, 0):
# 目录出错，把title去掉后再试一次，如果还不行退出
log.error(account_name + " 创建相册目录 %s 失败，尝试不使用title" % post_path)
post_path = os.path.join(image_path, post_id)
if not tool.make_dir(post_path, 0):
log.error(account_name + " 创建相册目录 %s 失败" % post_path)
tool.process_exit()
image_count = 0
for image_info in post_info["images"]:
image_count += 1
if not robot.check_sub_key(("img_id",), image_info):
log.error(account_name + " 相册%s 第%s张图片解析失败" % (post_id, image_count))
continue
image_url = generate_large_image_url(site_id, image_info["img_id"])
log.step(account_name + " 相册%s 开始下载第%s张图片 %s" % (post_id, image_count, image_url))
file_path = os.path.join(post_path, "%s.jpg" % image_count)
if tool.save_net_file(image_url, file_path):
log.step(account_name + " 相册%s 第%s张图片下载成功" % (post_id, image_count))
else:
log.error(account_name + " 相册%s 第%s张图片 %s 下载失败" % (post_info["post_id"], image_count, image_url))
this_account_total_image_count += image_count
if not is_over:
# 达到配置文件中的下载页数，结束
if 0 < GET_PAGE_COUNT < post_count:
is_over = True
else:
# 相册发布时间
post_time = post_info["published_at"]
post_count += 1
log.step(account_name + " 下载完毕，总共获得%s张图片" % this_account_total_image_count)
# 新的存档记录
if first_post_id != "0":
self.account_info[1] = first_post_id
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_IMAGE_COUNT += this_account_total_image_count
ACCOUNTS.remove(account_name)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def run(self):
global TOTAL_IMAGE_COUNT
account_id = self.account_info[0]
if len(self.account_info) >= 4 and self.account_info[3]:
account_name = self.account_info[3]
else:
account_name = self.account_info[0]
try:
log.step(account_name + " 开始")
# 如果需要重新排序则使用临时文件夹，否则直接下载到目标目录
if IS_SORT:
image_path = os.path.join(IMAGE_TEMP_PATH, account_name)
else:
image_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
# 图片
image_count = 1
page_count = 1
first_blog_id = "0"
need_make_image_dir = True
is_over = False
is_big_image_over = False
while not is_over:
# 获取一页日志信息
blog_page = get_one_page_blog(account_id, page_count)
if blog_page is None:
log.error(account_name + " 第%s页日志获取失败" % page_count)
tool.process_exit()
if not blog_page:
log.error(account_name + " 第%s页日志解析失败" % page_count)
tool.process_exit()
blog_data_list = get_blog_data_list(blog_page)
if len(blog_data_list) == 0:
log.error(account_name + " 第%s页日志分组失败" % page_count)
tool.process_exit()
for blog_data in blog_data_list:
# 获取日志id
blog_id = get_blog_id(account_id, blog_data)
if blog_id is None:
log.error(account_name + " 日志解析日志id失败，日志内容：%s" % blog_data)
tool.process_exit()
# 检查是否已下载到前一次的日志
if blog_id <= int(self.account_info[2]):
is_over = True
break
# 将第一个日志的ID做为新的存档记录
if first_blog_id == "0":
first_blog_id = str(blog_id)
# 获取该页日志的全部图片地址列表
image_url_list = get_image_url_list(blog_data)
if len(image_url_list) == 0:
continue
# 获取日志页面中存在的所有大图显示地址，以及对应的小图地址
big_2_small_list = get_big_image_url_list(blog_data)
# 下载图片
for image_url in image_url_list:
# 检查是否存在大图可以下载
if not is_big_image_over:
image_url, is_big_image_over = check_big_image(image_url, big_2_small_list)
log.step(account_name + " 开始下载第%s张图片 %s" % (image_count, image_url))
# 第一张图片，创建目录
if need_make_image_dir:
if not tool.make_dir(image_path, 0):
log.error(account_name + " 创建图片下载目录 %s 失败" % image_path)
tool.process_exit()
need_make_image_dir = False
file_type = image_url.split(".")[-1]
if file_type.find("?") != -1:
file_type = "jpeg"
file_path = os.path.join(image_path, "%04d.%s" % (image_count, file_type))
if tool.save_net_file(image_url, file_path):
log.step(account_name + " 第%s张图片下载成功" % image_count)
image_count += 1
else:
log.error(account_name + " 第%s张图片 %s 下载失败" % (image_count, image_url))
# 达到配置文件中的下载数量，结束
if 0 < GET_IMAGE_COUNT < image_count:
is_over = True
break
if not is_over:
# 达到配置文件中的下载页数，结束
if 0 < GET_PAGE_COUNT <= page_count:
is_over = True
# 判断当前页数是否大等于总页数
elif page_count >= get_max_page_count(blog_page):
is_over = True
else:
page_count += 1
log.step(account_name + " 下载完毕，总共获得%s张图片" % (image_count - 1))
# 排序
if IS_SORT:
if first_blog_id != "0":
destination_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
if robot.sort_file(image_path, destination_path, int(self.account_info[1]), 4):
log.step(account_name + " 图片从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建图片保存目录 %s 失败" % destination_path)
tool.process_exit()
# 新的存档记录
if first_blog_id != "0":
self.account_info[1] = str(int(self.account_info[1]) + image_count - 1)
self.account_info[2] = first_blog_id
# 保存最后的信息
self.thread_lock.acquire()
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
TOTAL_IMAGE_COUNT += image_count - 1
ACCOUNTS.remove(account_id)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def run(self):
global TOTAL_IMAGE_COUNT
account_name = self.account_info[0]
try:
log.step(account_name + " 开始")
# 如果需要重新排序则使用临时文件夹，否则直接下载到目标目录
if IS_SORT:
image_path = os.path.join(IMAGE_TEMP_PATH, account_name)
else:
image_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
# 获取user id
api_info = get_api_info(account_name)
if api_info is None:
log.error(account_name + " API信息查找失败")
tool.process_exit()
if not api_info["user_id"]:
log.error(account_name + " user_id解析失败")
tool.process_exit()
if not api_info["site_key"]:
log.error(account_name + " site_key解析失败")
tool.process_exit()
# 生成一个随机的request id用作访问（使用原理暂时不明，只管模拟页面传入）
request_id = tool.generate_random_string(8)
# 图片
image_count = 1
page_count = 1
first_image_time = "0"
is_over = False
need_make_image_dir = True
while not is_over:
# 获取一页图片信息
page_data = get_one_page_image_data(api_info["user_id"], page_count, api_info["site_key"], request_id)
if page_data is None:
log.error(account_name + " 第%s页图片信息获取失败" % page_count)
tool.process_exit()
for photo_info in page_data["photos"]["photo"]:
if "dateupload" not in photo_info:
log.error(account_name + " 第%s张图片上传时间获取失败，图片信息：%s" % (image_count, photo_info))
continue
# 检查是否是上一次的最后视频
if int(self.account_info[2]) >= int(photo_info["dateupload"]):
is_over = True
break
# 将第一张图片的上传时间做为新的存档记录
if first_image_time == "0":
first_image_time = str(photo_info["dateupload"])
if "url_o_cdn" in photo_info:
image_url = str(photo_info["url_o_cdn"])
elif "url_o" in photo_info:
image_url = str(photo_info["url_o"])
else:
log.error(account_name + " 第%s张图片下载地址获取失败，图片信息：%s" % (image_count, photo_info))
continue
log.step(account_name + " 开始下载第%s张图片 %s" % (image_count, image_url))
if need_make_image_dir:
if not tool.make_dir(image_path, 0):
log.error(account_name + " 创建图片下载目录 %s 失败" % image_path)
tool.process_exit()
need_make_image_dir = False
file_type = image_url.split(".")[-1]
file_path = os.path.join(image_path, "%04d.%s" % (image_count, file_type))
if tool.save_net_file(image_url, file_path):
log.step(account_name + " 第%s张图片下载成功" % image_count)
image_count += 1
else:
log.error(account_name + " 第%s张图片 %s 下载失败" % (image_count, image_url))
# 达到配置文件中的下载数量，结束
if 0 < GET_IMAGE_COUNT < image_count:
is_over = True
break
if not is_over:
# 达到配置文件中的下载数量，结束
if 0 < GET_PAGE_COUNT <= page_count:
is_over = True
elif page_count >= int(page_data["photos"]["pages"]):
is_over = True
else:
page_count += 1
log.step(account_name + " 下载完毕，总共获得%s张图片" % (image_count - 1))
# 排序
if IS_SORT:
if first_image_time != "0":
destination_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
if robot.sort_file(image_path, destination_path, int(self.account_info[1]), 4):
log.step(account_name + " 图片从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建图片保存目录 %s 失败" % destination_path)
tool.process_exit()
# 新的存档记录
if first_image_time != "0":
self.account_info[1] = str(int(self.account_info[1]) + image_count - 1)
self.account_info[2] = first_image_time
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_IMAGE_COUNT += image_count - 1
ACCOUNTS.remove(account_name)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def run(self):
global TOTAL_IMAGE_COUNT
coser_id = self.account_info[0]
if len(self.account_info) >= 3:
cn = self.account_info[2]
else:
cn = self.account_info[0]
try:
log.step(cn + " 开始")
image_path = os.path.join(IMAGE_DOWNLOAD_PATH, cn)
# 图片下载
this_cn_total_image_count = 0
page_count = 1
total_rp_count = 1
first_rp_id = ""
unique_list = []
is_over = False
need_make_download_dir = True # 是否需要创建cn目录
while not is_over:
# 获取一页的作品信息
post_page = get_one_page_post(coser_id, page_count)
if post_page is None:
log.error(cn + " 无法访问第%s页作品" % page_count)
tool.process_exit()
# 解析作品信息，获取所有的正片信息
cp_id, rp_list = get_rp_list(post_page)
if cp_id is None:
log.error(cn + " 第%s页作品解析异常" % page_count)
tool.process_exit()
for rp_id, title in rp_list.iteritems():
# 检查是否已下载到前一次的图片
if int(rp_id) <= int(self.account_info[1]):
is_over = True
break
# 新增正片导致的重复判断
if rp_id in unique_list:
continue
else:
unique_list.append(rp_id)
# 将第一个作品的id做为新的存档记录
if first_rp_id == "":
first_rp_id = rp_id
log.trace("rp: " + rp_id)
if need_make_download_dir:
if not tool.make_dir(image_path, 0):
log.error(cn + " 创建CN目录 %s 失败" % image_path)
tool.process_exit()
need_make_download_dir = False
# 过滤标题中不支持的字符
title = robot.filter_text(title)
if title:
rp_path = os.path.join(image_path, "%s %s" % (rp_id, title))
else:
rp_path = os.path.join(image_path, rp_id)
if not tool.make_dir(rp_path, 0):
# 目录出错，把title去掉后再试一次，如果还不行退出
log.error(cn + " 创建作品目录 %s 失败，尝试不使用title" % rp_path)
rp_path = os.path.join(image_path, rp_id)
if not tool.make_dir(rp_path, 0):
log.error(cn + " 创建作品目录 %s 失败" % rp_path)
tool.process_exit()
# 获取正片页面内的所有图片地址列表
image_url_list = get_image_url_list(cp_id, rp_id)
if image_url_list is None:
log.error(cn + " 无法访问正片：%s，cp_id：%s" % (rp_id, cp_id))
continue
if len(image_url_list) == 0 and IS_AUTO_FOLLOW:
log.step(cn + " 检测到可能有私密作品且账号不是ta的粉丝，自动关注")
if follow(coser_id):
# 重新获取下正片页面内的所有图片地址列表
image_url_list = get_image_url_list(cp_id, rp_id)
if len(image_url_list) == 0:
log.error(cn + " 正片：%s没有任何图片，可能是你使用的账号没有关注ta，所以无法访问只对粉丝开放的私密作品，cp_id：%s" % (rp_id, cp_id))
continue
image_count = 1
for image_url in list(image_url_list):
# 禁用指定分辨率
image_url = "/".join(image_url.split("/")[0:-1])
log.step(cn + " %s 开始下载第%s张图片 %s" % (rp_id, image_count, image_url))
if image_url.rfind("/") < image_url.rfind("."):
file_type = image_url.split(".")[-1]
else:
file_type = "jpg"
file_path = os.path.join(rp_path, "%03d.%s" % (image_count, file_type))
if tool.save_net_file(image_url, file_path):
image_count += 1
log.step(cn + " %s 第%s张图片下载成功" % (rp_id, image_count))
else:
log.error(cn + " %s 第%s张图片 %s 下载失败" % (rp_id, image_count, image_url))
this_cn_total_image_count += image_count - 1
if 0 < GET_PAGE_COUNT < total_rp_count:
is_over = True
break
else:
total_rp_count += 1
if not is_over:
if page_count >= get_max_page_count(coser_id, post_page):
is_over = True
else:
page_count += 1
log.step(cn + " 下载完毕，总共获得%s张图片" % this_cn_total_image_count)
# 新的存档记录
if first_rp_id != "":
self.account_info[1] = first_rp_id
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_IMAGE_COUNT += this_cn_total_image_count
ACCOUNTS.remove(coser_id)
self.thread_lock.release()
log.step(cn + " 完成")
except SystemExit:
log.error(cn + " 异常退出")
except Exception, e:
log.error(cn + " 未知异常")
log.error(str(e) + "\n" + str(traceback.format_exc()))
def rewrite_save_file(temp_save_data_path, save_data_path):
account_list = read_save_data(temp_save_data_path, 0, [])
temp_list = [account_list[key] for key in sorted(account_list.keys())]
tool.write_file(tool.list_to_string(temp_list), save_data_path,
tool.WRITE_FILE_TYPE_REPLACE)
path.delete_dir_or_file(temp_save_data_path)
def run(self):
global TOTAL_IMAGE_COUNT
account_name = self.account_info[0]
try:
log.step(account_name + " 开始")
# 如果需要重新排序则使用临时文件夹，否则直接下载到目标目录
if IS_SORT:
image_path = os.path.join(IMAGE_TEMP_PATH, account_name)
else:
image_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
image_count = 1
page_count = 1
first_blog_time = "0"
is_over = False
need_make_image_dir = True
while not is_over:
# 获取一页日志
blog_data = get_blog_page_data(account_name, page_count)
if blog_data is None:
log.error(account_name + " 第%s页日志无法获取" % page_count)
tool.process_exit()
# 解析日志发布时间
blog_time = get_blog_time(blog_data)
if blog_time is None:
log.error(account_name + " 第%s页解析日志时间失败" % page_count)
tool.process_exit()
# 检查是否是上一次的最后blog
if blog_time <= int(self.account_info[2]):
break
# 将第一个日志的时间做为新的存档记录
if first_blog_time == "0":
first_blog_time = str(blog_time)
# 从日志列表中获取全部的图片
image_url_list = get_image_url_list(blog_data)
for image_url in image_url_list:
# 使用默认图片的分辨率
image_url = image_url.split("?")[0]
# 过滤表情
if image_url.find("http://emoji.ameba.jp") >= 0:
continue
log.step(account_name + " 开始下载第%s张图片 %s" % (image_count, image_url))
# 第一张图片，创建目录
if need_make_image_dir:
if not tool.make_dir(image_path, 0):
log.error(account_name + " 创建图片下载目录 %s 失败" % image_path)
tool.process_exit()
need_make_image_dir = False
file_type = image_url.split(".")[-1]
file_path = os.path.join(image_path, "%04d.%s" % (image_count, file_type))
if tool.save_net_file(image_url, file_path):
log.step(account_name + " 第%s张图片下载成功" % image_count)
image_count += 1
else:
log.error(account_name + " 第%s张图片 %s 获取失败" % (image_count, image_url))
# 达到配置文件中的下载数量，结束
if 0 < GET_IMAGE_COUNT < image_count:
is_over = True
if not is_over:
if 0 < GET_PAGE_COUNT < page_count:
is_over = True
else:
page_count += 1
log.step(account_name + " 下载完毕，总共获得%s张图片" % (image_count - 1))
# 排序
if IS_SORT and image_count > 1:
destination_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
if robot.sort_file(image_path, destination_path, int(self.account_info[1]), 4):
log.step(account_name + " 图片从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建图片子目录 %s 失败" % destination_path)
tool.process_exit()
# 新的存档记录
if first_blog_time != "0":
self.account_info[1] = str(int(self.account_info[1]) + image_count - 1)
self.account_info[2] = first_blog_time
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_IMAGE_COUNT += image_count - 1
ACCOUNTS.remove(account_name)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def main(self):
# 解析存档文件
# 寻找fkoji.save
account_list = robot.read_save_data(self.save_data_path, 0, ["", "", ""])
# 这个key的内容为总数据
if ALL_SIGN in account_list:
image_start_index = int(account_list[ALL_SIGN][1])
save_data_image_time = int(account_list[ALL_SIGN][2])
account_list.pop(ALL_SIGN)
else:
image_start_index = 0
save_data_image_time = 0
if self.is_sort:
image_path = self.image_temp_path
else:
image_path = self.image_download_path
if not tool.make_dir(image_path, 0):
# 图片保存目录创建失败
self.print_msg("图片下载目录%s创建失败！" % self.image_download_path)
tool.process_exit()
# 下载
page_index = 1
image_count = 1
first_image_time = 0
unique_list = []
is_over = False
while not is_over:
index_url = "http://jigadori.fkoji.com/?p=%s" % page_index
index_page_return_code, index_page_response = tool.http_request(index_url)[:2]
if index_page_return_code != 1:
log.error("无法访问首页地址 %s" % index_url)
tool.process_exit()
index_page = BeautifulSoup.BeautifulSoup(index_page_response)
photo_list = index_page.body.findAll("div", "photo")
# 已经下载到最后一页
if not photo_list:
break
for photo_info in photo_list:
if isinstance(photo_info, BeautifulSoup.NavigableString):
continue
# 从图片页面中解析获取推特发布时间的时间戳
tweet_created_time = get_tweet_created_time(photo_info)
if tweet_created_time is None:
log.error("第%s张图片，解析tweet-created-at失败" % image_count)
continue
# 下载完毕
if tweet_created_time <= save_data_image_time:
is_over = True
break
# 将第一张图片的上传时间做为新的存档记录
if first_image_time == 0:
first_image_time = tweet_created_time
# 从图片页面中解析获取推特发布账号
account_id = get_tweet_account_id(photo_info)
if account_id is None:
log.error("第%s张图片，解析tweet账号失败" % image_count)
continue
# 找图片
img_tags = photo_info.findAll("img")
for tag in img_tags:
tag_attr = dict(tag.attrs)
if robot.check_sub_key(("src", "alt"), tag_attr):
image_url = str(tag_attr["src"]).replace(" ", "")
# 新增图片导致的重复判断
if image_url in unique_list:
continue
else:
unique_list.append(image_url)
log.step("开始下载第%s张图片 %s" % (image_count, image_url))
file_type = image_url.split(".")[-1]
if file_type.find("/") != -1:
file_type = "jpg"
file_path = os.path.join(image_path, "%05d_%s.%s" % (image_count, account_id, file_type))
if tool.save_net_file(image_url, file_path):
log.step("第%s张图片下载成功" % image_count)
image_count += 1
else:
log.error("第%s张图片 %s，account_id：%s，下载失败" % (image_count, image_url, account_id))
if is_over:
break
if not is_over:
page_index += 1
log.step("下载完毕")
# 排序复制到保存目录
if self.is_sort:
is_check_ok = False
while not is_check_ok:
# 等待手动检测所有图片结束
input_str = raw_input(tool.get_time() + " 已经下载完毕，是否下一步操作？ (Y)es or (N)o: ")
input_str = input_str.lower()
if input_str in ["y", "yes"]:
is_check_ok = True
elif input_str in ["n", "no"]:
tool.process_exit()
all_path = os.path.join(self.image_download_path, "all")
if not tool.make_dir(all_path, 0):
log.error("创建目录 %s 失败" % all_path)
tool.process_exit()
file_list = tool.get_dir_files_name(self.image_temp_path, "desc")
for file_name in file_list:
image_path = os.path.join(self.image_temp_path, file_name)
file_name_list = file_name.split(".")
file_type = file_name_list[-1]
account_id = "_".join(".".join(file_name_list[:-1]).split("_")[1:])
# 所有
image_start_index += 1
destination_file_name = "%05d_%s.%s" % (image_start_index, account_id, file_type)
destination_path = os.path.join(all_path, destination_file_name)
tool.copy_files(image_path, destination_path)
# 单个
each_account_path = os.path.join(self.image_download_path, "single", account_id)
if not os.path.exists(each_account_path):
if not tool.make_dir(each_account_path, 0):
log.error("创建目录 %s 失败" % each_account_path)
tool.process_exit()
if account_id in account_list:
account_list[account_id][1] = int(account_list[account_id][1]) + 1
else:
account_list[account_id] = [account_id, 1]
destination_file_name = "%05d.%s" % (account_list[account_id][1], file_type)
destination_path = os.path.join(each_account_path, destination_file_name)
tool.copy_files(image_path, destination_path)
log.step("图片从下载目录移动到保存目录成功")
# 删除临时文件夹
tool.remove_dir(self.image_temp_path)
# 保存新的存档文件
temp_list = [account_list[key] for key in sorted(account_list.keys())]
# 把总数据插入列表头
temp_list.insert(0, [ALL_SIGN, str(image_start_index), str(first_image_time)])
tool.write_file(tool.list_to_string(temp_list), self.save_data_path, 2)
log.step("全部下载完毕，耗时%s秒，共计图片%s张" % (self.get_run_time(), image_count - 1))
def run(self):
global TOTAL_IMAGE_COUNT
global TOTAL_VIDEO_COUNT
account_id = self.account_info[0]
# todo 是否有需要显示不同名字
account_name = account_id
try:
log.step(account_name + " 开始")
# todo 是否需要下载图片或视频
# 如果需要重新排序则使用临时文件夹，否则直接下载到目标目录
if IS_SORT:
image_path = os.path.join(IMAGE_TEMP_PATH, account_name)
video_path = os.path.join(VIDEO_TEMP_PATH, account_name)
else:
image_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
video_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
# todo 图片下载逻辑
# 图片
image_count = 1
first_image_time = "0"
need_make_image_dir = True
if IS_DOWNLOAD_IMAGE:
pass
# todo 视频下载逻辑
# 视频
video_count = 1
first_video_time = "0"
need_make_video_dir = True
if IS_DOWNLOAD_VIDEO:
pass
log.step(account_name + " 下载完毕，总共获得%s张图片和%s个视频" % (image_count - 1, video_count - 1))
# 排序
if IS_SORT:
# todo 是否需要下载图片
if first_image_time != "0":
destination_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
if robot.sort_file(image_path, destination_path, int(self.account_info[1]), 4):
log.step(account_name + " 图片从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建图片保存目录 %s 失败" % destination_path)
tool.process_exit()
# todo 是否需要下载视频
if first_video_time != "0":
destination_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
if robot.sort_file(video_path, destination_path, int(self.account_info[3]), 4):
log.step(account_name + " 视频从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建视频保存目录 %s 失败" % destination_path)
tool.process_exit()
# todo 是否需要下载图片或视频
# 新的存档记录
if first_image_time != "0":
self.account_info[1] = str(int(self.account_info[1]) + image_count - 1)
self.account_info[2] = first_image_time
if first_video_time != "0":
self.account_info[3] = str(int(self.account_info[3]) + video_count - 1)
self.account_info[4] = first_video_time
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
# todo 是否需要下载图片或视频
TOTAL_IMAGE_COUNT += image_count - 1
TOTAL_VIDEO_COUNT += video_count - 1
ACCOUNTS.remove(account_id)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def run(self):
global TOTAL_IMAGE_COUNT
account_id = self.account_info[0]
if len(self.account_info) >= 4 and self.account_info[3]:
account_name = self.account_info[3]
else:
account_name = self.account_info[0]
if len(self.account_info) >= 5 and self.account_info[4]:
account_file_path = self.account_info[4]
else:
account_file_path = ""
try:
log.step(account_name + " 开始")
# 如果需要重新排序则使用临时文件夹，否则直接下载到目标目录
if IS_SORT:
image_path = os.path.join(IMAGE_TEMP_PATH, account_name)
else:
image_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_file_path, account_name)
# 图片下载
image_count = 1
key = ""
first_album_id = "0"
unique_list = []
is_over = False
need_make_download_dir = True
while not is_over:
# 获取一页相册
album_page = get_one_page_album(account_id, key)
if album_page is None:
log.error(account_name + " 无法访问相册页，token：%s" % key)
tool.process_exit()
# 获取相册页中的所有picasweb地址列表
picasaweb_url_list = get_picasaweb_url_list(album_page)
log.trace(account_name + " 相册获取的所有picasaweb页：%s" % picasaweb_url_list)
for picasaweb_url in picasaweb_url_list:
# 有可能拿到带authkey的，需要去掉
# https://picasaweb.google.com/116300481938868290370/2015092603?authkey\u003dGv1sRgCOGLq-jctf-7Ww#6198800191175756402
picasaweb_url = picasaweb_url.replace("\u003d", "=")
# 获取picasaweb页的album id
album_id = get_picasaweb_page_album_id(account_id, picasaweb_url)
if album_id is None:
log.error(account_name + " 第%s张图片，无法访问picasaweb页 %s" % (image_count, picasaweb_url))
continue
if not album_id:
log.error(account_name + " 第%s张图片，picasaweb页 %s 获取album id失败" % (image_count, picasaweb_url))
continue
log.trace(account_name + " picasaweb页 %s 的album id：%s" % (picasaweb_url, album_id))
# 检查是否已下载到前一次的图片
if int(album_id) <= int(self.account_info[2]):
is_over = True
break
# # 相同的album_id判断
if album_id in unique_list:
continue
else:
unique_list.append(album_id)
# 将第一个album_id做为新的存档记录
if first_album_id == "0":
first_album_id = album_id
# 获取album id对应相册存档页的全部图片地址列表
image_url_list = get_image_url_list(account_id, album_id)
if image_url_list is None:
log.error(account_name + " 第%s张图片，无法访问album id：%s 的相册存档页" % (image_count, album_id))
continue
if len(image_url_list) == 0:
log.error(account_name + " 第%s张图片，album id：%s 的相册存档页没有解析到图片" % (image_count, album_id))
continue
log.trace(account_name + " album id：%s 的相册存档页获取的所有图片：%s" % (album_id, image_url_list))
for image_url in list(image_url_list):
image_url = generate_max_resolution_image_url(image_url)
# 下载
log.step(account_name + " 开始下载第%s张图片 %s" % (image_count, image_url))
# 第一张图片，创建目录
if need_make_download_dir:
if not tool.make_dir(image_path, 0):
log.error(account_name + " 创建图片下载目录 %s 失败" % image_path)
tool.process_exit()
need_make_download_dir = False
if image_url.rfind("/") < image_url.rfind("."):
file_type = image_url.split(".")[-1]
else:
file_type = "jpg"
file_path = os.path.join(image_path, "%04d.%s" % (image_count, file_type))
if tool.save_net_file(image_url, file_path):
log.step(account_name + " 第%s张图片下载成功" % image_count)
image_count += 1
else:
log.error(account_name + " 第%s张图片 %s 下载失败" % (image_count, image_url))
# 达到配置文件中的下载数量，结束
if 0 < GET_IMAGE_COUNT < image_count:
is_over = True
break
if is_over:
break
if not is_over:
# 查找下一页的token key
key_find = re.findall('"([.]?[a-zA-Z0-9-_]*)"', album_page)
if len(key_find) > 0 and len(key_find[0]) > 80:
key = key_find[0]
else:
# 不是第一次下载
if self.account_info[2] != "0":
log.error(account_name + " 没有找到下一页的token，将该页保存：")
log.error(album_page)
is_over = True
log.step(account_name + " 下载完毕，总共获得%s张图片" % (image_count - 1))
# 排序
if IS_SORT and image_count > 1:
destination_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_file_path, account_name)
if robot.sort_file(image_path, destination_path, int(self.account_info[1]), 4):
log.step(account_name + " 图片从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建图片子目录 %s 失败" % destination_path)
tool.process_exit()
# 新的存档记录
if first_album_id != "0":
self.account_info[1] = str(int(self.account_info[1]) + image_count - 1)
self.account_info[2] = first_album_id
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_IMAGE_COUNT += image_count - 1
ACCOUNTS.remove(account_id)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def run(self):
global TOTAL_VIDEO_COUNT
account_id = self.account_info[0]
if len(self.account_info) >= 4 and self.account_info[3]:
account_name = self.account_info[3]
else:
account_name = self.account_info[0]
try:
log.step(account_name + " 开始")
# 如果需要重新排序则使用临时文件夹，否则直接下载到目标目录
if IS_SORT:
video_path = os.path.join(VIDEO_TEMP_PATH, account_name)
else:
video_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
page_count = 1
video_count = 1
first_video_id = "0"
unique_list = []
is_over = False
need_make_download_dir = True
while not is_over:
# 获取指定一页的视频信息
medias_data = get_one_page_video_data(account_id, page_count)
if medias_data is None:
log.error(account_name + " 视频列表获取失败")
tool.process_exit()
for media in medias_data:
if not robot.check_sub_key(("video", "id"), media):
log.error(account_name + " 第%s个视频信：%s解析失败" % (video_count, media))
continue
video_id = str(media["id"])
# 检查是否图片时间小于上次的记录
if int(video_id) <= int(self.account_info[2]):
is_over = True
break
# 新增视频导致的重复判断
if video_id in unique_list:
continue
else:
unique_list.append(video_id)
# 将第一张图片的上传时间做为新的存档记录
if first_video_id == "0":
first_video_id = video_id
video_url = str(media["video"])
log.step(account_name + " 开始下载第%s个视频 %s" % (video_count, video_url))
# 第一个视频，创建目录
if need_make_download_dir:
if not tool.make_dir(video_path, 0):
log.error(account_name + " 创建视频下载目录 %s 失败" % video_path)
tool.process_exit()
need_make_download_dir = False
file_path = os.path.join(video_path, "%04d.mp4" % video_count)
if tool.save_net_file(video_url, file_path):
log.step(account_name + " 第%s个视频下载成功" % video_count)
video_count += 1
else:
log.error(account_name + " 第%s个视频 %s 下载失败" % (video_count, video_url))
# 达到配置文件中的下载数量，结束
if 0 < GET_VIDEO_COUNT < video_count:
is_over = True
break
if not is_over:
if len(medias_data) >= VIDEO_COUNT_PER_PAGE:
page_count += 1
else:
# 获取的数量小于请求的数量，已经没有剩余视频了
is_over = True
log.step(account_name + " 下载完毕，总共获得%s个视频" % (video_count - 1))
# 排序
if IS_SORT and video_count > 1:
destination_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
if robot.sort_file(video_path, destination_path, int(self.account_info[1]), 4):
log.step(account_name + " 视频从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建视频保存目录 %s 失败" % destination_path)
tool.process_exit()
# 新的存档记录
if first_video_id != "":
self.account_info[1] = str(int(self.account_info[1]) + video_count - 1)
self.account_info[2] = first_video_id
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_VIDEO_COUNT += video_count - 1
ACCOUNTS.remove(account_id)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def run(self):
global TOTAL_IMAGE_COUNT
account_id = self.account_info[0]
try:
log.step(account_id + " 开始")
# 如果需要重新排序则使用临时文件夹，否则直接下载到目标目录
if IS_SORT:
image_path = os.path.join(IMAGE_TEMP_PATH, account_id)
else:
image_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_id)
# 图片下载
page_count = 1
image_count = 1
first_post_id = ""
unique_list = []
is_over = False
need_make_download_dir = True
while not is_over:
post_url_list = get_one_page_post_url_list(account_id, page_count)
# 无法获取信息首页
if post_url_list is None:
log.error(account_id + " 无法访问第%s页相册页" % page_count)
tool.process_exit()
if len(post_url_list) == 0:
# 下载完毕了
break
# 去重排序
log.trace(account_id + " 相册第%s页获取的所有信息页：%s" % (page_count, post_url_list))
post_url_list = sorted(list(set(post_url_list)), reverse=True)
log.trace(account_id + " 相册第%s页去重排序后的信息页：%s" % (page_count, post_url_list))
for post_url in post_url_list:
post_id = post_url.split("/")[-1].split("_")[-1]
# 检查是否已下载到前一次的图片
if post_id <= self.account_info[2]:
is_over = True
break
# 新增信息页导致的重复判断
if post_id in unique_list:
continue
else:
unique_list.append(post_id)
# 将第一个信息页的id做为新的存档记录
if first_post_id == "":
first_post_id = post_id
post_page_return_code, post_page = tool.http_request(post_url)[:2]
if post_page_return_code != 1:
log.error(account_id + " 第%s张图片，无法获取信息页 %s" % (image_count, post_url))
continue
image_url_list = get_image_url_list(post_page)
log.trace(account_id + " 信息页 %s 获取的所有图片：%s" % (post_url, image_url_list))
if len(image_url_list) == 0:
log.error(account_id + " 第%s张图片，信息页 %s 中没有找到图片" % (image_count, post_url))
continue
for image_url in image_url_list:
if image_url.rfind("?") > image_url.rfind("."):
image_url = image_url.split("?")[0]
log.step(account_id + " 开始下载第%s张图片 %s" % (image_count, image_url))
# 第一张图片，创建目录
if need_make_download_dir:
if not tool.make_dir(image_path, 0):
log.error(account_id + " 创建图片下载目录 %s 失败" % image_path)
tool.process_exit()
need_make_download_dir = False
file_type = image_url.split(".")[-1]
file_path = os.path.join(image_path, "%04d.%s" % (image_count, file_type))
if tool.save_net_file(image_url, file_path):
log.step(account_id + " 第%s张图片下载成功" % image_count)
image_count += 1
else:
log.error(account_id + " 第%s张图片 %s 下载失败" % (image_count, image_url))
# 达到配置文件中的下载数量，结束
if 0 < GET_IMAGE_COUNT < image_count:
is_over = True
break
if is_over:
break
if not is_over:
# 达到配置文件中的下载数量，结束
if 0 < GET_PAGE_COUNT <= page_count:
is_over = True
else:
page_count += 1
log.step(account_id + " 下载完毕，总共获得%s张图片" % (image_count - 1))
# 排序
if IS_SORT and image_count > 1:
destination_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_id)
if robot.sort_file(image_path, destination_path, int(self.account_info[1]), 4):
log.step(account_id + " 图片从下载目录移动到保存目录成功")
else:
log.error(account_id + " 创建图片子目录 %s 失败" % destination_path)
tool.process_exit()
# 新的存档记录
if first_post_id != "":
self.account_info[1] = str(int(self.account_info[1]) + image_count - 1)
self.account_info[2] = first_post_id
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_IMAGE_COUNT += image_count - 1
ACCOUNTS.remove(account_id)
self.thread_lock.release()
log.step(account_id + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_id + " 提前退出")
else:
log.error(account_id + " 异常退出")
def run(self):
global TOTAL_IMAGE_COUNT
account_id = self.account_info[0]
if len(self.account_info) >= 4 and self.account_info[3]:
account_name = self.account_info[3]
else:
account_name = self.account_info[0]
try:
log.step(account_name + " 开始")
# 如果需要重新排序则使用临时文件夹，否则直接下载到目标目录
if IS_SORT:
image_path = os.path.join(IMAGE_TEMP_PATH, account_name)
else:
image_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
image_count = 1
page_count = 1
first_diary_id = "0"
is_over = False
need_make_image_dir = True
while not is_over:
# 获取一页博客信息
diary_list = get_one_page_diary_data(account_id, page_count)
if diary_list is None:
log.error(account_name + " 第%s页日志列表解析异常" % page_count)
tool.process_exit()
# 没有获取到任何日志，所有日志已经全部获取完毕了
if len(diary_list) == 0:
break
for diary_info in list(diary_list):
# 日志id
diary_id = tool.find_sub_string(diary_info, "id=", "&")
if not diary_id:
log.error(account_name + " 日志id解析异常，日志信息：%s" % diary_info)
continue
# 检查是否是上一次的最后视频
if int(diary_id) <= int(self.account_info[2]):
is_over = True
break
# 将第一个日志的id做为新的存档记录
if first_diary_id == "0":
first_diary_id = diary_id
log.trace(account_name + " 日志id %s" % diary_id)
# 获取这个日志中的全部图片地址列表
image_url_list = get_image_url_list(diary_info)
for image_url in image_url_list:
# 如果图片地址没有域名，表示直接使用当前域名下的资源，需要拼接成完整的地址
if image_url[:7] != "http://" and image_url[:8] != "https://":
if image_url[0] == "/":
image_url = "http://www.keyakizaka46.com%s" % image_url
else:
image_url = "http://www.keyakizaka46.com/%s" % image_url
log.step(account_name + " 开始下载第%s张图片 %s" % (image_count, image_url))
# 第一张图片，创建目录
if need_make_image_dir:
if not tool.make_dir(image_path, 0):
log.error(account_name + " 创建图片下载目录 %s 失败" % image_path)
tool.process_exit()
need_make_image_dir = False
file_type = image_url.split(".")[-1]
file_path = os.path.join(image_path, "%04d.%s" % (image_count, file_type))
if tool.save_net_file(image_url, file_path):
log.step(account_name + " 第%s张图片下载成功" % image_count)
image_count += 1
else:
log.error(account_name + " 第%s张图片 %s 获取失败" % (image_count, image_url))
# 达到配置文件中的下载数量，结束
if 0 < GET_IMAGE_COUNT < image_count:
is_over = True
break
if not is_over:
# 达到配置文件中的下载页数，结束
if 0 < GET_PAGE_COUNT <= page_count:
is_over = True
else:
page_count += 1
log.step(account_name + " 下载完毕，总共获得%s张图片" % (image_count - 1))
# 排序
if IS_SORT and image_count > 1:
destination_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
if robot.sort_file(image_path, destination_path, int(self.account_info[1]), 4):
log.step(account_name + " 图片从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建图片子目录 %s 失败" % destination_path)
tool.process_exit()
# 新的存档记录
if first_diary_id != "0":
self.account_info[1] = str(int(self.account_info[1]) + image_count - 1)
self.account_info[2] = first_diary_id
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_IMAGE_COUNT += image_count - 1
ACCOUNTS.remove(account_id)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def rewrite_save_file(temp_save_data_path, save_data_path):
account_list = read_save_data(temp_save_data_path, 0, [])
temp_list = [account_list[key] for key in sorted(account_list.keys())]
tool.write_file(tool.list_to_string(temp_list), save_data_path, 2)
tool.remove_dir_or_file(temp_save_data_path)
def run(self):
global TOTAL_VIDEO_COUNT
account_id = self.account_info[0]
if len(self.account_info) >= 4 and self.account_info[3]:
account_name = self.account_info[3]
else:
account_name = self.account_info[0]
try:
log.step(account_name + " 开始")
# 如果需要重新排序则使用临时文件夹，否则直接下载到目标目录
if IS_SORT:
video_path = os.path.join(VIDEO_TEMP_PATH, account_name)
else:
video_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
suid = get_suid(account_id)
if suid is None:
log.error(account_name + " suid获取失败")
page_count = 1
video_count = 1
first_video_scid = ""
unique_list = []
is_over = False
need_make_download_dir = True
while suid != "" and (not is_over):
# 获取指定一页的视频信息
media_page = get_one_page_video_data(suid, page_count)
if media_page is None:
log.error(account_name + " 视频列表获取失败")
tool.process_exit()
# 获取视频scid列表
scid_list = get_scid_list(media_page["msg"])
if len(scid_list) == 0:
log.error(account_name + " 在视频列表：%s 中没有找到视频scid" % str(media_page["msg"]))
tool.process_exit()
for scid in scid_list:
scid = str(scid)
# 检查是否已下载到前一次的图片
if first_video_scid == self.account_info[2]:
is_over = True
break
# 新增视频导致的重复判断
if scid in unique_list:
continue
else:
unique_list.append(scid)
# 将第一个视频的id做为新的存档记录
if first_video_scid == "":
first_video_scid = scid
# 获取视频下载地址
video_url = get_video_url_by_video_id(scid)
if video_url is None:
log.error(account_name + " 第%s个视频 %s 获取下载地址失败" % (video_count, scid))
continue
log.step(account_name + " 开始下载第%s个视频 %s" % (video_count, video_url))
# 第一个视频，创建目录
if need_make_download_dir:
if not tool.make_dir(video_path, 0):
log.error(account_name + " 创建视频下载目录 %s 失败" % video_path)
tool.process_exit()
need_make_download_dir = False
file_path = os.path.join(video_path, "%04d.mp4" % video_count)
if tool.save_net_file(video_url, file_path):
log.step(account_name + " 第%s个视频下载成功" % video_count)
video_count += 1
else:
log.error(account_name + " 第%s个视频 %s 下载失败" % (video_count, video_url))
# 达到配置文件中的下载数量，结束
if 0 < GET_VIDEO_COUNT < video_count:
is_over = True
break
if not is_over:
if media_page["isall"]:
is_over = True
else:
page_count += 1
log.step(account_name + " 下载完毕，总共获得%s个视频" % (video_count - 1))
# 排序
if IS_SORT and video_count > 1:
destination_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
if robot.sort_file(video_path, destination_path, int(self.account_info[1]), 4):
log.step(account_name + " 视频从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建视频保存目录 %s 失败" % destination_path)
tool.process_exit()
# 新的存档记录
if first_video_scid != "":
self.account_info[1] = str(int(self.account_info[1]) + video_count - 1)
self.account_info[2] = first_video_scid
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_VIDEO_COUNT += video_count - 1
ACCOUNTS.remove(account_id)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")
def run(self):
global TOTAL_IMAGE_COUNT
global TOTAL_VIDEO_COUNT
account_id = self.account_info[0]
if len(self.account_info) >= 6 and self.account_info[5]:
account_name = self.account_info[5]
else:
account_name = self.account_info[0]
try:
log.step(account_name + " 开始")
# 如果需要重新排序则使用临时文件夹，否则直接下载到目标目录
if IS_SORT:
image_path = os.path.join(IMAGE_TEMP_PATH, account_name)
video_path = os.path.join(VIDEO_TEMP_PATH, account_name)
else:
image_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
video_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
image_count = 1
first_image_time = "0"
need_make_image_dir = True
while IS_DOWNLOAD_IMAGE:
# 获取全部图片地址列表
image_url_list = get_image_url_list(account_id)
if image_url_list is None:
log.error(account_name + " 图片列表获取失败")
break
for image_url in list(image_url_list):
# 不使用缩略图
image_url = image_url.split("@")[0]
image_return_code, image_byte, image_response = tool.http_request(image_url)
if image_return_code != 1:
log.step(account_name + " 第%s张图片下载失败" % image_count)
continue
# 获取图片的上传时间（字符串）
response_last_modified_time = tool.get_response_info(image_response.info(), "Last-Modified")
# 字符串转换为时间戳
image_created_time = tool.response_time_to_timestamp(response_last_modified_time)
# 检查是否已下载到前一次的图片
if int(image_created_time) <= int(self.account_info[4]):
break
# 将第一张图片的上传时间做为新的存档记录
if first_image_time == "0":
first_image_time = str(image_created_time)
log.step(account_name + " 开始下载第%s张图片 %s" % (image_count, image_url))
# 第一张图片，创建目录
if need_make_image_dir:
if not tool.make_dir(image_path, 0):
log.error(account_name + " 创建图片下载目录 %s 失败" % image_path)
tool.process_exit()
need_make_image_dir = False
file_type = image_url.split(".")[-1].split(":")[0]
image_file_path = os.path.join(image_path, "%04d.%s" % (image_count, file_type))
save_image(image_byte, image_file_path)
log.step(account_name + " 第%s张图片下载成功" % image_count)
image_count += 1
# 达到配置文件中的下载数量，结束
if 0 < GET_IMAGE_COUNT < image_count:
break
break
# 视频
video_count = 1
first_video_time = "0"
need_make_video_dir = True
while IS_DOWNLOAD_VIDEO:
# 获取全部视频ID列表
video_id_list = get_video_id_list(account_id)
if video_id_list is None:
log.error(account_name + " 视频列表获取失败")
break
for video_id in list(video_id_list):
# 获取视频的时间和下载地址
video_info = get_video_info(video_id)
if video_info is None:
log.error(account_name + " 第%s个视频 %s 信息获取失败" % (video_count, video_id))
continue
# 检查是否已下载到前一次的视频
if int(video_info["data"]["createtime"]) <= int(self.account_info[2]):
break
# 将第一个视频的上传时间做为新的存档记录
if first_video_time == "0":
first_video_time = str(video_info["data"]["createtime"])
# m3u8文件的地址
link_url = str(video_info["data"]["linkurl"])
# 视频的真实下载地址列表
ts_url_list = get_ts_url_list(link_url)
if ts_url_list is None:
log.error(account_name + " 第%s个视频下载地址列表 %s 获取失败" % (video_count, link_url))
continue
log.step(account_name + " 开始下载第%s个视频 %s" % (video_count, ts_url_list))
# 第一个视频，创建目录
if need_make_video_dir:
if not tool.make_dir(video_path, 0):
log.error(account_name + " 创建图片下载目录 %s 失败" % video_path)
tool.process_exit()
need_make_video_dir = False
video_file_path = os.path.join(video_path, "%04d.ts" % video_count)
if save_video(ts_url_list, video_file_path):
log.step(account_name + " 第%s个视频下载成功" % video_count)
video_count += 1
else:
log.error(account_name + " 第%s个视频 %s 下载失败" % (video_count, ts_url_list))
# 达到配置文件中的下载数量，结束
if 0 < GET_VIDEO_COUNT < video_count:
break
break
log.step(account_name + " 下载完毕，总共获得%s张图片和%s个视频" % (image_count - 1, video_count - 1))
# 排序
if IS_SORT:
if image_count > 1:
destination_path = os.path.join(IMAGE_DOWNLOAD_PATH, account_name)
if robot.sort_file(image_path, destination_path, int(self.account_info[3]), 4):
log.step(account_name + " 图片从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建图片保存目录 %s 失败" % destination_path)
tool.process_exit()
if video_count > 1:
destination_path = os.path.join(VIDEO_DOWNLOAD_PATH, account_name)
if robot.sort_file(video_path, destination_path, int(self.account_info[1]), 4):
log.step(account_name + " 视频从下载目录移动到保存目录成功")
else:
log.error(account_name + " 创建视频保存目录 %s 失败" % destination_path)
tool.process_exit()
if first_image_time != "0":
self.account_info[3] = str(int(self.account_info[3]) + image_count - 1)
self.account_info[4] = first_image_time
if first_video_time != "0":
self.account_info[1] = str(int(self.account_info[1]) + video_count - 1)
self.account_info[2] = first_video_time
# 保存最后的信息
tool.write_file("\t".join(self.account_info), NEW_SAVE_DATA_PATH)
self.thread_lock.acquire()
TOTAL_IMAGE_COUNT += image_count - 1
TOTAL_VIDEO_COUNT += video_count - 1
ACCOUNTS.remove(account_id)
self.thread_lock.release()
log.step(account_name + " 完成")
except SystemExit, se:
if se.code == 0:
log.step(account_name + " 提前退出")
else:
log.error(account_name + " 异常退出")