def __init__(self, root_path): self.root_path = root_path self.SELECT_SQL = "SELECT page_no, loc_url,status FROM illust WHERE illust_id={}" self.RESET_ILLUSTER_SQL = "UPDATE illust SET status = 1 WHERE illuster_id = {}" self.RESET_ILLUST_SQL = "UPDATE illust SET status = 1 WHERE illuster_id = {} AND illust_id NOT IN ({})" self.FILTER_SQL = "SELECT illust_id FROM illust WHERE status = 10 AND illust_id in ({})" # 设priority = 6代表已经检查完毕 self.MAKE_ILLUSTER_STATUS_DONE_SQL = "UPDATE illuster SET priority = 6 WHERE illuster_id = {}" self.CHECK_IF_DONE = 'SELECT priority FROM illuster WHERE illuster_id={}' self.GET_DONE_ILLUSTER = 'SELECT illuster_id FROM illuster WHERE priority=6' self.db_util = MysqlUtil() self.logger = Log(__name__, log_cate='checker').get_log() self.before_illuster_id = None
def __init__(self): self.init_pattern() self.load_config() self.logger = Log(__name__).get_log() self.GET_KEY_PAGE = "https://accounts.pixiv.net/login?lang=zh&source=pc&view_type=page&ref=wwwtop_accounts_index" self.LOGIN_PAGE = "https://accounts.pixiv.net/api/login?lang=zh" # 登陆页面 self.UGOIRA_URL = 'https://www.pixiv.net/ajax/illust/{illust_id}/ugoira_meta' # 动图url,0填写动图的id self.URL_4_GET_ALL_WORK_ID = "https://www.pixiv.net/ajax/user/{userId}/profile/all" # 画师的所有作品信息 self.URL_4_ILLUSTER_MAIN_PAGE = "https://www.pixiv.net/ajax/user/{}/profile/top" # 画师主页24个作品信息,少量画师信息 self.URL_4_GET_ILLUSTER_INFO = "https://www.pixiv.net/ajax/user/{}?full=1" # 用于获得画师信息 self.URL_ILLUST_PAGE = "https://www.pixiv.net/member_illust.php?mode=medium&illust_id={illust_id}" # 插画页面 self.URL_ILLUST_PAGE = "https://www.pixiv.net/ajax/illust/{illust_id}" # 插画页面 self.REFERER = "https://accounts.pixiv.net/login?lang=zh&source=pc&view_type=page&ref=wwwtop_accounts_index" self.USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" self.headers = {'Referer': self.REFERER, 'User-Agent': self.USER_AGENT} # 方法二:设置请求失败之后重复请求次数 # requests.adapters.DEFAULT_RETRIES = self.max_retries self.session = requests.Session() if self.cookie is not None: self.session.cookies.set("authentication", self.cookie) if self.proxies['http'] is not None or self.proxies[ 'https'] is not None: self.session.proxies = self.proxies # 设置session默认代理 # 方法1:设置请求失败之后重复请求次数 request_retry = requests.adapters.HTTPAdapter( max_retries=self.max_retries) self.session.mount('https://', request_retry) self.session.mount('http://', request_retry) # self.session.keep_alive = False self.spider_util = SpiderUtil() # 操作数据库 self.db_util = DbUtil() self.sql_4_insert = 'insert into illust(title,url,illust_id,illuster_id,page_no,status,`restrict`,x_restrict)' \ 'values ( %s, %s, %s, %s, %s, 0, %s, %s)' self.sql_4_insert_4_ugoira = 'insert into illust(title,url,illust_id,illuster_id,page_no,status)values( %s, %s, %s, %s, %s,444)' self.sql_4_insert_2_done = 'insert into illust(title,url,illust_id,illuster_id,page_no,status)values( %s, %s, %s, %s, 0, 10)'
def load_logger(self): self.logger = Log(__name__).get_log()
class PixivSpider(): def load_config(self): # 加载配置文件 self.config = ConfigureUtil('config/config.conf') try: self.thread_num = self.config.get("app", "thread_num", type_="int") # 下载图片存储的根目录 self.root_path = self.config.get("download", "path") # 下载循环等待的时间 self.waiting_time = self.config.get("download", "waiting_time", type_="int") except Exception as e: self.logger.error("请检查你配置的下载路径====》{}".format(repr(e)), exc_info=True) raise e def load_logger(self): self.logger = Log(__name__).get_log() def __init__(self): # 加载配置文件 self.sql_4_update = 'update illuster set priority = 0, modify_time = "{}" WHERE illuster_id = {}' self.load_logger() self.load_config() self.pixiv_util = PixivUtil() self.db_util = DbUtil() self.pool = threadpool.ThreadPool(self.thread_num) self.WAITING_SQL = 'select illuster_id from illuster WHERE priority > 0 ' \ ' AND illuster_id!=11 ORDER BY priority DESC LIMIT {} ' ssl._create_default_https_context = ssl._create_unverified_context requests.packages.urllib3.disable_warnings(InsecureRequestWarning) # 测试。用来解决, 'ssl3_read_bytes', 'sslv3 alert bad record mac') requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS = 'ALL' def login(self): is_login = self.pixiv_util.login() if not is_login: self.logger.info("登陆失败,请检查账号密码是否正确和网络是否连通") return False else: return True def save_concerned_illuster_info(self, type_: str): """ 根据用户id找到用户关注的画师,保存到数据库 :param type_: hide或者show hide表示未公开,show表示已公开 :return: """ if not self.login(): return page_num = self.pixiv_util.get_concerned_illusters_pagenum(type_) for current_page_num in range(1, page_num + 1): ids_, profiles, usernames = self.pixiv_util.get_concerned_illuster_info( type_, current_page_num) for illuster_id, profile, username in zip(ids_, profiles, usernames): res = self.db_util.get_one( "select id from illuster where illuster_id = " + illuster_id) str_time = get_time() if res: self.logger.debug("该画师已存在" + illuster_id) self.db_util.update( 'UPDATE illuster SET name=%s, image_url=%s, modify_time=%s, priority=%s ' 'WHERE illuster_id =%s', (username, profile, str_time, 0, illuster_id)) else: self.db_util.insert( "insert into illuster(illuster_id,name,image_url,create_time,modify_time,priority) " "value(" + illuster_id + ",'" + username + "','" + profile + "','" + str_time + "','" + str_time + "',5)") self.logger.info("信息保存完毕") def get_illuster_ids(self, num, use_data_base=False): """ 获得num个未爬取的画家id 优先使用配置文件中的画家id,然后是数据库中待爬取的 """ ids = self.config.get("download", "illuster_ids") ids = [] if ids is None else ids.split() if len(ids) > num - 1: return ids[:10] elif use_data_base: illuster_ids = self.db_util.get_all(self.WAITING_SQL.format(10)) for illuster_id in illuster_ids: ids.append(str(illuster_id[0])) temp = list(set(ids))[:10] temp.sort(key=ids.index) return temp else: return ids[:10] def main(self, batch_size=10, use_database=False): """ 自动读取配置文件或者数据库,获得需要爬取的作家的id 配置文件要更加优先 找到画家id爬取画家的作品 """ while True: illuster_ids = self.get_illuster_ids(batch_size, use_database) if len(illuster_ids) == 0: time.sleep(60 * 60 * 4) for illuster_id in illuster_ids: self.logger.info("正在下载的是" + illuster_id + "的作品") if use_database: res = self.db_util.get_one( "select id from illuster where illuster_id = %s", illuster_id) # 爬取画师有关信息 str_time = get_time() # 先判断画师是否存在于pixiv网站 # is_exist = self.pixiv_util.is_illuster_exist(illuster_id) info = self.pixiv_util.get_illuster_info(illuster_id) if info is None: self.db_util.insert( "update illuster set priority = -1 WHERE illuster_id = %s", illuster_id) continue else: name, img_url = info if res: self.logger.debug("该画师已存在数据库中" + illuster_id) self.db_util.update( "update illuster set name = %s, image_url = %s, modify_time = %s where illuster_id = %s", (name, img_url, str_time, illuster_id)) else: self.db_util.insert( "insert into illuster(illuster_id, name, image_url, create_time,modify_time,priority)" " value(%s, %s, %s, %s, %s,5)", (illuster_id, name, img_url, str_time, str_time)) dir_ = os.path.join(self.root_path, illuster_id) make_directory(dir_) illust_ids = self.pixiv_util.get_illust_ids(illuster_id) if illuster_ids is None: self.logger.error( '获得插画列表失败,画师id为{}, 开始下一个画师的信息爬取'.format(illuster_id)) continue illust_num = len(illust_ids) self.logger.info("共有插图%d个" % (illust_num)) if use_database: if res: self.logger.debug("该画师已存在数据库中" + illuster_id) self.db_util.update( "update illuster set name = %s, image_url = %s, modify_time = %s, illust_num = %s where illuster_id = %s", (name, img_url, str_time, illust_num, illuster_id)) else: self.db_util.insert( "insert into illuster(illuster_id, name, image_url, create_time,modify_time,illust_num, priority)" " value(%s, %s, %s, %s, %s, %s, 5)", (illuster_id, name, img_url, str_time, str_time, illust_num)) if illust_num > 0: if use_database: illust_ids = self.pixiv_util.filter_4_downloaded_work( illust_ids) # 准备循环的数据 var_list = [] for illust_id in illust_ids: # 循环的到的illust_id var_list.append(([dir_, illust_id], None)) if self.pool is None: self.pool = threadpool.ThreadPool(self.thread_num) tasks = threadpool.makeRequests( self.pixiv_util.download_work_by_illust_id, var_list) [self.pool.putRequest(task) for task in tasks] self.pool.wait() # 直接删除第一个id,因为第一个id最先下载 il_ids = self.config.get("download", "illuster_ids") if il_ids is not None: self.config.update("download", "illuster_ids", " ".join(il_ids.split()[1:])) if use_database: self.db_util.update( self.sql_4_update.format(get_time(), illuster_id)) self.logger.info("{}的作品下载完成".format(illuster_id)) self.logger.info('one batch is over') time.sleep(self.waiting_time) self.logger.info("所有的作品下载完成") session.close()
class PixivUtil: def __init__(self): self.init_pattern() self.load_config() self.logger = Log(__name__).get_log() self.GET_KEY_PAGE = "https://accounts.pixiv.net/login?lang=zh&source=pc&view_type=page&ref=wwwtop_accounts_index" self.LOGIN_PAGE = "https://accounts.pixiv.net/api/login?lang=zh" # 登陆页面 self.UGOIRA_URL = 'https://www.pixiv.net/ajax/illust/{illust_id}/ugoira_meta' # 动图url,0填写动图的id self.URL_4_GET_ALL_WORK_ID = "https://www.pixiv.net/ajax/user/{userId}/profile/all" # 画师的所有作品信息 self.URL_4_ILLUSTER_MAIN_PAGE = "https://www.pixiv.net/ajax/user/{}/profile/top" # 画师主页24个作品信息,少量画师信息 self.URL_4_GET_ILLUSTER_INFO = "https://www.pixiv.net/ajax/user/{}?full=1" # 用于获得画师信息 self.URL_ILLUST_PAGE = "https://www.pixiv.net/member_illust.php?mode=medium&illust_id={illust_id}" # 插画页面 self.URL_ILLUST_PAGE = "https://www.pixiv.net/ajax/illust/{illust_id}" # 插画页面 self.REFERER = "https://accounts.pixiv.net/login?lang=zh&source=pc&view_type=page&ref=wwwtop_accounts_index" self.USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" self.headers = {'Referer': self.REFERER, 'User-Agent': self.USER_AGENT} # 方法二:设置请求失败之后重复请求次数 # requests.adapters.DEFAULT_RETRIES = self.max_retries self.session = requests.Session() if self.cookie is not None: self.session.cookies.set("authentication", self.cookie) if self.proxies['http'] is not None or self.proxies[ 'https'] is not None: self.session.proxies = self.proxies # 设置session默认代理 # 方法1:设置请求失败之后重复请求次数 request_retry = requests.adapters.HTTPAdapter( max_retries=self.max_retries) self.session.mount('https://', request_retry) self.session.mount('http://', request_retry) # self.session.keep_alive = False self.spider_util = SpiderUtil() # 操作数据库 self.db_util = DbUtil() self.sql_4_insert = 'insert into illust(title,url,illust_id,illuster_id,page_no,status,`restrict`,x_restrict)' \ 'values ( %s, %s, %s, %s, %s, 0, %s, %s)' self.sql_4_insert_4_ugoira = 'insert into illust(title,url,illust_id,illuster_id,page_no,status)values( %s, %s, %s, %s, %s,444)' self.sql_4_insert_2_done = 'insert into illust(title,url,illust_id,illuster_id,page_no,status)values( %s, %s, %s, %s, 0, 10)' def get_session(self): if self.login(): return self.session else: self.logger.error("登陆失败,请检查账号密码是否正确和网络是否连通") return 'error' def load_config(self): config = ConfigureUtil('config/config.conf') http_proxy = config.get('proxy', 'http', is_error=True) https_proxy = config.get('proxy', 'https', is_error=True) self.proxies = {'http': http_proxy, 'https': https_proxy} # 重试次数必须是数值类型 self.max_retries = config.get('app', 'max_retries', 'int') # 超时时间, 单位是秒 self.timeout = config.get('app', 'time_out', type_="int", is_error=True, default=20) self.username = config.get('account', 'username') self.password = config.get('account', 'password') # self.cookie = "p_ab_id=0; p_ab_id_2=3; login_ever=yes; a_type=0; b_type=1; first_visit_datetime_pc=2018-06-06+10%3A53%3A26; _ga=GA1.2.1555034815.1497772914; p_ab_d_id=1974173592; yuid_b=NXcmaYM; module_orders_mypage=%5B%7B%22name%22%3A%22sketch_live%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22following_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22tag_follow%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22recommended_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22everyone_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22mypixiv_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22fanbox%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22featured_tags%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22contests%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22user_events%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22sensei_courses%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22spotlight%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22booth_follow_items%22%2C%22visible%22%3Atrue%7D%5D; ki_r=; __utmc=235335808; OX_plg=pm; c_type=26; __utmv=235335808.|2=login%20ever=yes=1^3=plan=normal=1^5=gender=male=1^6=user_id=27971320=1^9=p_ab_id=0=1^10=p_ab_id_2=3=1^11=lang=zh=1; gsScrollPos-422=; _td=5e2a3ce8-e5ca-4e1c-9bc5-4188deff8d9d; ki_s=197685%3A0.0.0.0.0; gsScrollPos-73=; gsScrollPos-74=0; __utmz=235335808.1558931621.48.3.utmcsr=accounts.pixiv.net|utmccn=(referral)|utmcmd=referral|utmcct=/login; limited_ads=%7B%22responsive%22%3A%22%22%7D; PHPSESSID=27971320_a3ba646dd6d6fe402a566008ca109afe; privacy_policy_agreement=1; ki_t=1539786814040%3B1564299004430%3B1564299004430%3B7%3B11; categorized_tags=0Gd_u69FDa~0VZuk18GJB~0roTzTKxJy~7cMRrOPRjW~BU9SQkS-zU~EQHKvBDRBz~IVwLyT8B6k~Ig5OcZugU6~OEXgaiEbRa~OT-C6ubi9i~RcahSSzeRf~RkTaP3d-E6~_-agXPKuAQ~b8b4-hqot7~cpt_Nk5mjc~jYnWl04aAC~l2rugVKl6u~r70NVOGJ5H~xlfjJKgpwx~y8GNntYHsi; __utmt=1; __utma=235335808.1555034815.1497772914.1564650150.1564650214.63; tag_view_ranking=0xsDLqCEW6~Ie2c51_4Sp~BU9SQkS-zU~y8GNntYHsi~RTJMXD26Ak~AI_aJCDFn0~RcahSSzeRf~i83OPEGrYw~8HRshblb4Q~4-_9de7LBH~cpt_Nk5mjc~q3eUobDMJW~skx_-I2o4Y~faHcYIP1U0~404yEt28rv~fFjokb4ZCF~-YeeMY1Yjs~FqVQndhufZ~tgP8r-gOe_~0HA6x-6rNd~Ow9mLSvmxK~KN7uxuR89w~5oPIfUbtd6~NpsIVvS-GF~gooMLQqB9a~Lt-oEicbBr~gpglyfLkWs~jYnWl04aAC~VIOKa7rioU~HBlflqJjBZ~zyKU3Q5L4C~TWrozby2UO~BB4jge2y2O~nyZIqZI1jx~MM6RXH_rlN~3gc3uGrU1V~LJo91uBPz4~laE3IylUE6~_hSAdpN9rx~n7YxiukgPF~l2rugVKl6u~A0c1GtjhvT~KhVXu5CuKx~WlKkwEuUi0~VbPCYJXdEP~2-RXlHt092~M2vKPRxAge~ehP5NJ0cy5~RokSaRBUGr~eVxus64GZU~vFXX3OXCCb~p2LP_MNOlh~pSgdr8bSLW~lhJLvPIIlV~RybylJRnhJ~T4PSuIdiwS~T53qL7THLZ~vSWEvTeZc6~iVTmZJMGJj~4i9bTBXFoE~rOnsP2Q5UN~r70NVOGJ5H~P5glpXg6VU~ie0shhAARr~JmNHQca4Km~ouiK2OKQ-A~K_WSdFXjg4~nrFOQYIh7z~7cMRrOPRjW~EWR7JDW6jH~tw8Zob-Izr~sAwDH104z0~Sbp1gmMeRy~JXmGXDx4tL~j3leh4reoN~C9_ZtBtMWU~pnCQRVigpy~dx7ljrJnxj~0roTzTKxJy~trfda46Fk8~yIg4ditfn_~XEuS3TPyCa~v3nOtgG77A~2XSW7Dtt5E~2EpPrOnc5S~QnLUXjsTk6~o2vM33GyaO~o3o9P--kXx~JL8rvDh62i~hQUvXSyZW-~LBMc5qP5TM~xlfjJKgpwx~zIv0cf5VVk~Z9XB6vYxvi~mIBxNOpKNs~B_OtVkMSZT~gmYaY_jsM2~CiSfl_AE0h~QEgdaUlAgu~JVA9YTPBgb; __utmb=235335808.3.10.1564650214" self.cookie = config.get('account', 'cookie', is_error=True) def init_pattern(self): self.illuster_id_from_user = re.compile( '<input name="id\[\]" value="(.*?)" type="checkbox"') self.illuster_profile_from_user = re.compile( 'data-profile_img="(.*?)"') self.illuster_username_from_user = re.compile( '"data-user_name="(.*?)"></a>') self.pagenum_4_show = re.compile( '<a href="\?type=user&rest=show&p=(.*?)">') self.pagenum_4_hide = re.compile( '<a href="\?type=user&rest=hide&p=(.*?)">') # 用于获得作品详情 self.get_illust_detail = re.compile( '}\)\((.*?)\);</script><link rel="apple-touch-icon"') def set_logger(self, logger): self.logger = logger def download_work_by_illust_id(self, save_dir, illust_id: str, use_databse=False): if use_databse: res = self.db_util.get_one( "select status from illust where illust_id = " + illust_id) if res is not None: if res[0] == WorkStatus.done.value: # 已被下载好 self.logger.info(" 本插画已被下载 illust_id 是 " + illust_id) return True # 获得插画信息 illust_info = self.get_img_info_by_img_id(illust_id) if illust_info is None: return (title, img_url, page_count, restrict, x_restrict, illust_type, illuster_id) = illust_info # 判断文件类型 if illust_type == IllustType.illust.value: self.download_illust(save_dir, illust_id, illuster_id, title, img_url, page_count, restrict, x_restrict) elif illust_type == IllustType.ugoira.value: # 新遇到的动图 self.logger.warning("遇到ugoira插画,id为{}".format(illust_id)) # return self.download_ugoira(save_dir, illust_id, illuster_id, title, img_url, restrict, x_restrict) elif illust_type == IllustType.manga.value: self.download_manga(illust_id, illuster_id, title, img_url, page_count, restrict, x_restrict) else: self.logger.info("暂不支持该类型的下载") def download_illust(self, save_dir, illust_id, illuster_id, title, img_url, page_count, restrict, x_restrict, use_database=False): res = True for i in range(page_count): try: temp = img_url.replace('_p0', '_p' + str(i)) # self.insert_illust(illust_id, IllustType.illust.value, title, img_url, # illuster_id, page_count, restrict, x_restrict) res = res and self.spider_util.download_img( temp, save_dir, header=self.headers) except Exception as e: res = False self.logger.error("下载失败" + repr(e) + "url是" + temp, exc_info=True) break if use_database: if res: self.insert_illust(illust_id, title, img_url, illuster_id, page_count, restrict, x_restrict, WorkStatus.done.value) else: # 数据库中保存下载失败的记录 self.insert_illust(illust_id, title, img_url, illuster_id, page_count, restrict, x_restrict, WorkStatus.failure.value) def download_illust_o(self, save_dir, illust_id, illuster_id, title, img_url, page_count, restrict, x_restrict): # temp = img_url.split("_p0") res = True for i in range(page_count): # img_url = "" try: img_url = img_url.replace('_p0', '_p' + str(i)) # img_url = temp[0] + "_p" + str(i) + temp[1] self.insert_illust(illust_id, IllustType.illust.value, title, img_url, illuster_id, page_count, restrict, x_restrict) res = res and self.spider_util.download_img( img_url, save_dir, header=self.headers) except Exception as e: res = False self.logger.error("下载失败" + repr(e) + "url是" + img_url, exc_info=True) break if res: self.db_util.update( 'update illust set status = %s where illust_id = %s', (WorkStatus.done.value, illust_id)) def filter_4_downloaded_work(self, illust_id_list): """查询数据库,看是否有对应的作品已经被下载完成了""" str_illust_id = ",".join( str(illust_id) for illust_id in illust_id_list) illust_sql = 'select illust_id from illust WHERE status = {} AND illust_id in ({})' \ .format(WorkStatus.done.value, str_illust_id) # ugoira_sql = 'select ugoira_id from ugoira WHERE status = {} AND ugoira_id in ({})' \ # .format(WorkStatus.done.value, str_illust_id) res_in_illust = self.db_util.get_all(illust_sql) # res_in_ugoira = self.db_util.get_all(ugoira_sql) downloaded_illust_ids = [i[0] for i in res_in_illust] # res_in_work.append([i[0] for i in res_in_ugoira]) for illust_id in downloaded_illust_ids: if str(illust_id) in illust_id_list: illust_id_list.remove(str(illust_id)) return illust_id_list # not_downloaded_work_ids = [] # for i in illust_id_list: # if int(i) not in downloaded_illust_ids: # not_downloaded_work_ids.append(i) # return not_downloaded_work_ids def get_postkey(self, url): """获取需要post的数据postkey""" pat = 'name="post_key" value="(.*?)"' # 不用组装headers也能拿到postKey,但是一定要是get,不能使用post content = self.session.get(url, timeout=15).text res_temp = re.findall(pat, content) if len(res_temp) != 1: self.logger.error("无法获得postKey", exc_info=True) return None postkey = re.findall(pat, content)[0] self.logger.info("your post key is " + postkey) return postkey def login(self): """模拟登陆""" try: postkey = self.get_postkey(self.GET_KEY_PAGE) if not postkey: return False post_data = \ { "pixiv_id": self.username, "password": self.password, # "captcha": "", # "g_recaptcha_response":"", "post_key": postkey, # "source":"pc", "ref": "wwwtop_accounts_index", "return_to": "https://www.pixiv.net/" } # 装个头,能解决一些问题 result = self.session.post(self.LOGIN_PAGE, data=post_data, headers=self.headers, cookies={"cookies": self.cookie}) pat = '"body":{"(.*?)"' is_login_flag = re.findall(pat, result.text)[0] if is_login_flag == 'success': self.logger.info("Log in successfully.Your username is " + self.username) return self.session else: self.logger.info("Login failed") return False except Exception as e: self.logger.error('连接无响应', exc_info=True) return False def get_illust_ids(self, illuster_id): """ 根据画师Id获得画师的插画IdList 插画包括动图和静态图 :param illuster_id: 画师id :return: """ try: # 获得所有的图片Id illust_id_json = self.get( self.URL_4_GET_ALL_WORK_ID.format(userId=illuster_id)).text illust_id_json = str2json(illust_id_json) ill_ids = illust_id_json["body"]["illusts"] # 示例:{[illust_id]:[illust_info],[illust_id]:[illust_info],[illust_id]:[illust_info]...} if ill_ids != []: return list(ill_ids.keys()) else: return [] except Exception as e: self.logger.error("搜索画师失败!" + repr(e) + "illuster_id是" + illuster_id, exc_info=True) return None def get_manga_ids(self, illuster_id): """ 根据画师Id获得画师的漫画Id :param illuster_id: :return: """ illust_id_json = self.get( self.URL_4_GET_ALL_WORK_ID.format( userId=illuster_id)).text # 获得所有的图片Id illust_id_json = str2json(illust_id_json) manga_ids = illust_id_json["body"]["manga"] return list(manga_ids.keys()) def get_img_info_by_img_id(self, illust_id): """ 根据插画或漫画的Id获得插画或者漫画的详细信息 :param img_id: :return: """ try: # info = self.session.get(self.URL_ILLUST_PAGE.format(illust_id=illust_id), timeout=self.timeout, # cookies={"cookies": self.cookie}).text info = self.get( self.URL_ILLUST_PAGE.format(illust_id=illust_id)).text illust_info_json = str2json(info) if self.isError(illust_info_json): self.logger.error( "找不到illust信息,illust_id is {},error message is {}".format( illust_id, illust_info_json['message']), exc_info=True) return None body = illust_info_json['body'] illust_type = body['illustType'] page_count = body['pageCount'] restrict = body['restrict'] x_restrict = body['xRestrict'] title = body['title'] url = body['urls']['original'] illuster_id = body['userId'] return title, url, page_count, restrict, x_restrict, illust_type, illuster_id except Exception as e: self.logger.error("获取插画信息失败" + repr(e) + "illust_id是" + illust_id, exc_info=True) return None # def is_illuster_exist(self, illuster_id): # try: # illust_id_json = self.get(self.URL_4_GET_ALL_WORK_ID.format(userId=illuster_id)).text # 获得所有的图片Id # illust_id_json = str2json(illust_id_json) # if self.isError(illust_id_json): # self.logger.error("画师不存在!illuster_id是{}".format(illuster_id), exc_info=True) # return False # else: # return True # except Exception as e: # self.logger.error("无法判断画师是否存在!illuster_id是{}".format(illuster_id), exc_info=True) # return None def get_concerned_illuster_info(self, type_, current_page_num): url = 'https://www.pixiv.net/bookmark.php?type=user&rest={}&p={}'.format( type_, current_page_num) html = self.get(url).text ids = self.illuster_id_from_user.findall(html) profiles = self.illuster_profile_from_user.findall(html) usernames = self.illuster_username_from_user.findall(html) return ids, profiles, usernames def get_show_pagenum(self, html): page = self.pagenum_4_show.findall(html) return len(page) def get_hide_pagenum(self, html): page = self.pagenum_4_hide.findall(html) return len(page) def get_concerned_illusters_pagenum(self, type_: str): """获得关注的画家的大致信息 type 为 hide或者show""" content = self.session.get( 'https://www.pixiv.net/bookmark.php?type=user&rest={}'.format( type_), cookies={ "cookies": self.cookie }).text page_num = 0 if type_ == 'hide': page_num = self.get_hide_pagenum(content) elif type_ == 'show': page_num = self.get_show_pagenum(content) return page_num def get_pagenum_hide(self, html): page = self.pagenum_4_hide.findall(html) return len(page) def get_ugoira_info(self, illust_id): # 插画id 下载illust信息 gif_info = str2json( self.session.get(self.UGOIRA_URL.format(illust_id=illust_id), cookies={ "cookies": self.cookie }).text) # print(gif_info) delays = [item["delay"] for item in gif_info["body"]["frames"]] frames = {f['file']: f['delay'] for f in gif_info["body"]['frames']} page_num = len(delays) zip_url = gif_info["body"]["originalSrc"] return frames, page_num, zip_url, delays def download_ugoira(self, save_dir, illust_id, illuster_id, title, url, restrict, x_restrict): try: frames, page_num, zip_url, delays = self.get_ugoira_info(illust_id) # self.insert_ugoira(illust_id, title, url, illuster_id, page_num=None, # restrict=restrict, x_restrict=x_restrict, status=WorkStatus.done.value) # page_num = self.spider_util.download_ugoira(illust_id, dest, self.session) gif_path = os.path.join(save_dir, illust_id + ".gif") zip_path = os.path.join(save_dir, illust_id + ".zip") res = self.spider_util.download_ugoira(zip_url, zip_path, self.headers) delays = [str(i) for i in delays] delays = ",".join(delays) if res: # 下载成功 self.insert_ugoira(illust_id, title, url, illuster_id, page_num=page_num, restrict=restrict, x_restrict=x_restrict, status=WorkStatus.done.value, delays=delays) else: self.insert_ugoira(illust_id, title, url, illuster_id, page_num=page_num, restrict=restrict, x_restrict=x_restrict, status=WorkStatus.failure.value, delays=delays) except Exception: self.logger.error('下载动图时,遇到问题', exc_info=True) def insert_illust(self, illust_id, title=None, url=None, illuster_id=None, page_num=None, restrict=None, x_restrict=None, status=None): select_sql = 'SELECT id FROM illust WHERE illust_id = %s ' res = self.db_util.get_one(select_sql, illust_id) loc_url = str(illuster_id) + "/" + url.split('/')[-1] if not res: # 数据库里没找到相关信息 insert_sql = 'INSERT INTO illust(title, url, illust_id, illuster_id, page_no, `type`, status,`restrict`,' \ 'x_restrict, loc_url) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)' self.db_util.insert(insert_sql, (title, url, illust_id, illuster_id, page_num, IllustType.illust.value, status, restrict, x_restrict, loc_url)) else: update_sql = 'UPDATE illust SET status = %s, page_no=%s, `restrict`=%s, x_restrict=%s, title=%s, ' \ 'loc_url=%s, type=%s WHERE illust_id=%s' self.db_util.update(update_sql, (status, page_num, restrict, x_restrict, title, loc_url, IllustType.illust.value, illust_id)) def insert_ugoira(self, illust_id, title, url, illuster_id, page_num, restrict=None, x_restrict=None, status=None, delays=None): select_sql = 'SELECT id FROM illust WHERE illust_id = %s AND type = {}'.format( IllustType.ugoira.value) res = self.db_util.get_one(select_sql, illust_id) loc_url = str(illuster_id) + "/" + url.split('/')[-1] if not res: # 数据库里没找到相关信息 insert_sql = "INSERT INTO ugoira( ugoira_id, delays) VALUES (%s, %s)" self.db_util.insert(insert_sql, (illust_id, delays)) insert_sql = 'INSERT INTO illust(title, url, illust_id, illuster_id, page_no, `type`, status,`restrict`,' \ 'x_restrict, loc_url) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)' self.db_util.insert(insert_sql, (title, url, illust_id, illuster_id, page_num, IllustType.ugoira.value, status, restrict, x_restrict, loc_url)) else: update_sql = 'UPDATE illust SET status = %s, page_no=%s, `restrict`=%s, x_restrict=%s, title=%s, ' \ 'loc_url=%s, type=%s WHERE illust_id=%s' self.db_util.update( update_sql, (WorkStatus.done.value, page_num, restrict, x_restrict, title, loc_url, status, illust_id)) def update_illust(self, illust_id, page_num, illust_status=WorkStatus.done.value): # select_sql = 'SELECT status FROM illust WHERE illust_id = %s ' update_sql = 'UPDATE illust SET status = %s , page_no=%s WHERE illust_id = %s' self.db_util.update(update_sql, (illust_status, page_num, illust_id)) # if self.db_util.get_one(select_sql, (illust_id)): # update_sql = 'UPDATE illust SET status = %s, page_no = %s WHERE illust_id = %s' # self.db_util.update(update_sql, (WorkStatus.done.value, page_num, illust_id)) # else: # self.logger.error("") # insert_sql = 'INSERT INTO illust(title, url, illust_id, illuster_id, page_no, `type`, status,`restrict`,' \ # 'x_restrict) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)' # self.db_util.insert(insert_sql, (title, url, illust_id, illuster_id, page_num, illust_type, # WorkStatus.waiting.value, restrict, x_restrict)) def isError(self, json): return json['error'] def download_manga(self, illust_id, illuster_id, title, img_url, page_count, restrict, x_restrict): raise NotImplementedError pass def get(self, url): return self.session.get(url, cookies={"cookies": self.cookie}, timeout=self.timeout) def get_illuster_info(self, illuster_id): # content = self.get(self.URL_4_GET_ILLUST_ID.format(userId=illuster_id)).text try: content = self.get( self.URL_4_GET_ILLUSTER_INFO.format(illuster_id)).text content_json = str2json(content) if self.isError(content_json): self.logger.error("画师不存在!illuster_id是{}".format(illuster_id), exc_info=True) return None name = content_json['body']['name'] img_url = content_json['body']['imageBig'] # print(content_json['extraData']['meta']['title']) # pat = re.compile('「(.*)」的个人资料 - pixiv') # # pat.search(content_json['extraData']['meta']['title']) # # pat = re.compile('「(.*)」的个人资料 - pixiv') # name = pat.findall(content_json['extraData']['meta']['title'])[0] return name, img_url except Exception: self.logger.error("无法判断画师是否存在!illuster_id是{}".format(illuster_id), exc_info=True) return None
def __init__(self, path): self.db_util = MysqlUtil() self.spdier_util = SpiderUtil() self.root_path = path self.logger = Log(__name__).get_log()
class Downloader: def __init__(self, path): self.db_util = MysqlUtil() self.spdier_util = SpiderUtil() self.root_path = path self.logger = Log(__name__).get_log() def get_info(self, select_sql): """ 根据sql查询要爬取的数据 """ res = self.db_util.get_all(select_sql) infos = [] for info in res: type_ = info[3] if type_ == IllustType.ugoira.value: # todo 只下载zip文件,之后再处理缩略图 url = Downloader.handle_url_4_ugoira(info[0]) else: url = str(info[0], encoding='utf-8') if '.gif' in url: # 暂不处理gif类型 continue if '.jpg' in url or '.png' in url: pass else: self.logger.warning('不支持的类型!illust_id为{}'.format(info[1])) infos.append((url, self.get_path(url, str(info[2])), info[4], info[1])) return infos def get_path(self, url: str, illuster_id: str): path = os.path.join(self.root_path, illuster_id) make_directory(path) return os.path.join(path, url.split("/")[-1]) @staticmethod def handle_url_4_ugoira(url): url = str(url, encoding="utf-8") i = url.replace('img-original', 'img-zip-ugoira') i = i.replace('0.jpg', '1920x1080.zip') i = i.replace('0.png', '1920x1080.zip') return i def download_file(self, url, save_path, headers): res = self.spdier_util.download_file(url, save_path, headers) if res == 404: self.logger.error('无法找到文件,url为{}'.format(url)) return 404 if res and valid_file(save_path, file_extension(save_path)): return True else: return False def download_file_list(self, url, save_path, headers, page_num: int, illust_id): """当一个illust_id下的所有插画都下载完成才更新状态""" res = True for i in range(page_num): url_temp = url.replace('_p0', '_p{}'.format(i)) save_path_temp = save_path.replace('_p0', '_p{}'.format(i)) res = res and self.download_file(url_temp, save_path_temp, headers) if res == 404: self.db_util.update( 'UPDATE illust SET status = {} WHERE illust_id = {}'. format(WorkStatus.unfound.value, illust_id)) return if res: self.db_util.update( 'UPDATE illust SET status = {} WHERE illust_id = {}'.format( WorkStatus.complete.value, illust_id)) def main(self, select_sql, headers, thread_num, interval): """ 查询url数据,然后自动下载 :param select_sql: 查询数据的sql语句 :param headers: 请求头设置 :param thread_num: 下载线程数 :param interval: 循环查询数据库的时间间隔 :return: """ pool = None while True: infos = self.get_info(select_sql) print('get info from database') if infos: param_list = [([i[0], i[1], headers, i[2], i[3]], None) for i in infos] if pool is None: pool = threadpool.ThreadPool(thread_num) tasks = threadpool.makeRequests(self.download_file_list, param_list) [pool.putRequest(task) for task in tasks] pool.wait() self.logger.info('one batch is over') print('one batch is over') time.sleep(interval)
class checker(): def __init__(self, root_path): self.root_path = root_path self.SELECT_SQL = "SELECT page_no, loc_url,status FROM illust WHERE illust_id={}" self.RESET_ILLUSTER_SQL = "UPDATE illust SET status = 1 WHERE illuster_id = {}" self.RESET_ILLUST_SQL = "UPDATE illust SET status = 1 WHERE illuster_id = {} AND illust_id NOT IN ({})" self.FILTER_SQL = "SELECT illust_id FROM illust WHERE status = 10 AND illust_id in ({})" # 设priority = 6代表已经检查完毕 self.MAKE_ILLUSTER_STATUS_DONE_SQL = "UPDATE illuster SET priority = 6 WHERE illuster_id = {}" self.CHECK_IF_DONE = 'SELECT priority FROM illuster WHERE illuster_id={}' self.GET_DONE_ILLUSTER = 'SELECT illuster_id FROM illuster WHERE priority=6' self.db_util = MysqlUtil() self.logger = Log(__name__, log_cate='checker').get_log() self.before_illuster_id = None def get_done_illuster(self): illuster_ids = self.db_util.get_all(self.GET_DONE_ILLUSTER) done_illuster_ids = [i[0] for i in illuster_ids] return done_illuster_ids def check_empty_dir(self, ignore=[]): try: ignore = ignore + self.get_done_illuster() for file_name in os.listdir(self.root_path): if file_name in ignore: continue print(file_name) # path 即 illuster_id if self.before_illuster_id is not None: # 将上一个设为完成 self.db_util.update( self.MAKE_ILLUSTER_STATUS_DONE_SQL.format( self.before_illuster_id)) illuster_id = file_name self.before_illuster_id = illuster_id # 如果当前的illuster已经处理过了,就跳到下一个 if self.db_util.get_one( self.CHECK_IF_DONE.format(illuster_id))[0] == 6: continue path = os.path.join(self.root_path, file_name) if os.path.isdir(path): image_files = os.listdir(path) if len(image_files) == 0: print("no images in {}".format(path)) self.db_util.update( self.RESET_ILLUSTER_SQL.format(illuster_id)) continue except Exception as e: self.logger.error('some problem happen', exc_info=1) raise e def check(self, ignore=[]): try: ignore = ignore + self.get_done_illuster() for file_name in os.listdir(self.root_path): if file_name in ignore: continue print(file_name) # path 即 illuster_id if self.before_illuster_id is not None: # 将上一个设为完成 self.db_util.update( self.MAKE_ILLUSTER_STATUS_DONE_SQL.format( self.before_illuster_id)) illuster_id = file_name self.before_illuster_id = illuster_id if self.db_util.get_one( self.CHECK_IF_DONE.format(illuster_id))[0] == 6: continue path = os.path.join(self.root_path, file_name) if os.path.isdir(path): image_files = os.listdir(path) if len(image_files) == 0: print("no images in {}".format(path)) self.db_util.update( self.RESET_ILLUSTER_SQL.format(illuster_id)) continue illust_ids = [] for file_ in image_files: if file_.endswith('.zip'): illust_id = file_.replace('.zip', '') elif file_.endswith('.gif'): # todo 暂不处理gif continue else: illust_id = file_.split("_p")[0] if illust_id not in illust_ids: illust_ids.append(illust_id) complete_illust_ids = [] # if not illust_ids: # continue # illust_id_list = self.db_util.get_all(self.FILTER_SQL.format(','.join(illust_ids))) # illust_ids = [i[0] for i in illust_id_list] # if illust_ids: # continue for illust_id in illust_ids: res = True info = self.db_util.get_one( self.SELECT_SQL.format(illust_id)) # 如果是数据库中没有信息 if not info: self.logger.info( '数据库中没有信息,illust_id为{}'.format(illust_id)) print(illust_id) continue (page_no, loc_url, status) = info if status < WorkStatus.done.value: # 10是下载成功状态,小于10 表示不成功 continue loc_url = str(loc_url, encoding='utf-8') if 'gif' in loc_url: # 暂时不处理gif,理论上。gif文件不会进入这里来 complete_illust_ids.append(illust_id) continue elif 'ugoira' in loc_url: loc_url = loc_url.replace('_ugoira0.jpg', '.zip') loc_url_temp = loc_url.replace( '_ugoira0.png', '.zip') if valid_file( os.path.join(self.root_path, loc_url_temp)): complete_illust_ids.append(illust_id) continue for i in range(page_no): loc_url_temp = loc_url.replace( "_p0", "_p{}".format(i)) res = res and valid_file( os.path.join(self.root_path, loc_url_temp)) # 理论上只有完整的jpg,png到这边来 if res: complete_illust_ids.append(illust_id) if not complete_illust_ids: # 全部设为未完成 self.db_util.update( self.RESET_ILLUSTER_SQL.format(illuster_id)) # self.db_util.update(self.RESET_ILLUST_SQL.format(",".join(incomplete_illust_ids))) else: self.db_util.update( self.RESET_ILLUST_SQL.format( illuster_id, ','.join(complete_illust_ids))) print(complete_illust_ids) # 把最后一个illuster_id进行处理 if self.before_illuster_id is not None: self.db_util.update( self.MAKE_ILLUSTER_STATUS_DONE_SQL.format( self.before_illuster_id)) except Exception as e: self.logger.error('some problem happen', exc_info=1) raise e
def __init__(self, header=None, proxies=None): self.logger = Log(__name__).get_log() self.header = header self.proxies = proxies # 防止证书警告显示 requests.packages.urllib3.disable_warnings()
class SpiderUtil: def __init__(self, header=None, proxies=None): self.logger = Log(__name__).get_log() self.header = header self.proxies = proxies # 防止证书警告显示 requests.packages.urllib3.disable_warnings() def set_header(self, header): self.header = header def download_img(self, img_url, save_path, header=None): """ 下载一张图片,如果本地有一张该图片,且图片内容完整,就不下载 """ try: img_name = img_url.split("/")[-1] full_path = os.path.join(save_path, img_name) if os.path.exists(full_path) and file_util.valid_file(full_path): self.logger.debug("已存在该图片,保存在" + full_path + "。url是" + img_url) return True else: return self.download_file(img_url, full_path, headers=header) except Exception as e: raise e def download_file(self, url, save_path, headers=None, proxies=None, time_out=16, chunk_size=1024000): """ 根据url下载文件,成功就返回True 不属于requests部分的代码只会运行try_time次 time_out的单位可能是分 :param url: 请求的url :param save_path: 文件保存的位置 :param headers: 请求时使用的头 :param proxies: 代理 :param time_out: 超时的时间 :param chunk_size: 下载的缓存的大小 :return: 没有找到文件,返回404,下载成功返回True 否则返回False """ try: (header, proxies) = self.prepare(headers, proxies) with closing( requests.get(url, verify=False, stream=True, headers=header, proxies=proxies, timeout=time_out)) as r: if r.status_code != 404: with open(save_path, 'wb') as f: for data in r.iter_content(chunk_size): if data: f.write(data) self.logger.info('download successful {}'.format(url)) else: self.logger.error( "cannot find file, url is {}".format(url), exc_info=True) return False return True except Exception as e: # raise e # time.sleep(60) # 若是最后一次报错,就打日志 self.logger.error( "some problems happened while downloading file, url is {}". format(url), exc_info=True) # if try_times > 1: # try_times = try_times - 1 # return self.download_file(url, save_path, headers=headers, proxies=proxies, chunk_size=chunk_size, # try_times=try_times) return False def easy_download_file(self, url, save_path, chunk_size=1024000): """ 没有缓存机制,很简单,但是下载大文件有可能失败,而且比较占内存 直接返回content """ # try: with closing(requests.get(url, verify=False, stream=True)) as r: if r.status_code != 404: with open(save_path, 'wb') as f: for data in r.iter_content(chunk_size): if data: f.write(data) self.logger.info('download successful {}'.format(url)) else: self.logger.error("cannot find file, url is {}".format(url), exc_info=True) return 404 # return requests.get(url, verify=False, stream=True, headers=headers).content # except Exception as e: # if try_times >= 0: # return self.easy_download_file(url, headers, --try_times) # self.logger.error("cannot download file, url is {}".format(url), exc_info=True) # return False def post(self, url, data=None, header=None, proxies=None): (header, proxies) = self.prepare(header, proxies) r = requests.post(url, headers=header, proxies=proxies, data=data) return r def json(self, url, header=None, proxies=None, except_code=None): """ 如果response的code是属于可以接受的code 就解析返回的json,最后return出去 """ (header, proxies) = self.prepare(header, proxies) response = requests.get(url, headers=header, proxies=proxies) if response.status_code == except_code: return response.json() else: return None @staticmethod def json(url, session, except_code=None): response = session.get(url) print(response.text) if response.status_code == except_code: return response.json() else: return None def save_html(self, url, path, header=None, proxies=None, encoding='utf-8'): """ 将html页面保存在文件中,用于简单的测试 """ try: (header, proxies) = self.prepare(header, proxies) r = requests.get(url, headers=header, proxies=proxies).text with open(path, 'w', encoding=encoding) as f: f.write(r) except Exception as e: raise e # def download_ugoira(self, zip_url, frames, gif_path, zip_path, headers=None, speed=1.0): # """ # 下载ugoira文件 # 下载数据之后直接保存在gif中 # :param zip_url: zip的url # :param frames: frames信息 # :param gif_path: gif保存位置 # :param zip_path: zip下载路径 # :param headers: 下载zip文件的请求头 # :param speed: 控制图片间的间隔 # :return: # """ # # 下载content的内容 # zip_content = self.easy_download_file(zip_url, headers=headers, try_times=5) # if zip_content is False: # return False # # ugoira.save('zip', zip_path, zip_content, frames, speed) # # 保存为gif # # ugoira.save(format, gif_path, zip_content, frames, speed) # return False def download_ugoira(self, url, zip_path, headers=None): """ 没有使用ugoira的下载 先把zip下载到本地 再本地解压,再组成gif 最后删去解压产生的jpg文件 """ try: # 下载zip文件,获得文件信息 # zip_path = os.path.join(dest, illust_id + ".zip") # 下载content的内容 return self.download_file(url, zip_path, headers=headers) except Exception as e: self.logger.error('动图下载失败,{} is '.format(url), exc_info=True) # raise e return False # def my_download_ugoira(self, url, frames, dest, illust_id: str, delays, headers=None): # """ # 没有使用ugoira的下载 # 先把zip下载到本地 # 再本地解压,再组成gif # 最后删去解压产生的jpg文件 # """ # try: # # 下载zip文件,获得文件信息 # zip_path = os.path.join(dest, illust_id + ".zip") # # 下载content的内容 # zipp = self.download_file(url, zip_path, headers=headers) # # gif_data = gif_data.content # file_path = os.path.join(dest, illust_id) # make_directory(file_path) # temp_file_list = [] # zipo = zipfile.ZipFile(zip_path, "r") # for file in zipo.namelist(): # temp_file_list.append(os.path.join(file_path, file)) # zipo.extract(file, file_path) # zipo.close() # # 读取所有静态图片,合成gif # image_data = [] # for file in temp_file_list: # image_data.append(imageio.imread(file)) # # ValueError: Image is not numeric, but # # Array. # # 第二种报错 ValueError: quantization error # imageio.mimsave(os.path.join(file_path, illust_id + ".gif"), image_data, "GIF", duration=delays) # # 清除所有中间文件。 # for file in temp_file_list: # os.remove(file) # return True # except Exception as e: # self.logger.error('动图下载,转化为gif失败,illust_id is ' + illust_id, exc_info=True) # raise e def download_resume(self, url, file_path, headers=None, chunk_size=1024000): """支持断点续传的文件下载方式""" try: temp = requests.get(url, stream=True, verify=False, headers=headers) # 获得文件大小 total_size = int(temp.headers['Content-Length']) self.logger.info('文件总大小为 ', total_size) # 这重要了,先看看本地文件下载了多少 if os.path.exists(file_path): temp_size = os.path.getsize(file_path) # 本地已经下载的文件大小 else: temp_size = 0 self.logger.info('已下载文件大小为 ', temp_size) # 设置下载范围 headers = {'Range': 'bytes=%d-' % temp_size} # 重新请求网址,加入新的请求头的 r = requests.get(url, stream=True, verify=False, headers=headers) # "ab"表示追加形式写入文件 with open(file_path, "ab") as f: for chunk in r.iter_content(chunk_size=chunk_size): if chunk: temp_size += len(chunk) f.write(chunk) f.flush() done = int(50 * temp_size / total_size) # 输出进度条 self.logger.info( "\r[%s%s] %d%%" % ('█' * done, ' ' * (50 - done), 100 * temp_size / total_size)) self.logger.info("done") return True except Exception as e: self.logger.info('改文件可能不支持断点续传') return False def download_aira(self, url, dest_path): pass def prepare(self, header=None, proxies=None): if header is None: header = self.header if proxies is None: proxies = self.proxies return header, proxies def is_support_continue(self, url, headers={}): """判断是否支持断点续传、多线程下载""" headers['Range'] = 'bytes=0-4' # noinspection PyBroadException try: r = requests.head(url, headers=headers) crange = r.headers['content-range'] import re int(re.match(r'^bytes 0-4/(\d+)$', crange).group(1)) return True except Exception: return False