def info_summary(movie: Movie, all_info): """汇总多个来源的在线数据生成最终数据""" final_info = MovieInfo(movie) ########## 部分字段配置了专门的选取逻辑,先处理这些字段 ########## # genre if 'javdb' in all_info: final_info.genre = all_info['javdb'].genre ########## 然后检查所有字段,如果某个字段还是默认值,则按照优先级选取数据 ########## # parser直接更新了all_info中的项目,而初始all_info是按照优先级生成的,已经符合配置的优先级顺序了 # 按照优先级取出各个爬虫获取到的信息 attrs = [i for i in dir(final_info) if not i.startswith('_')] covers, big_covers = [], [] for name, data in all_info.items(): absorbed = [] # 遍历所有属性,如果某一属性当前值为空而爬取的数据中含有该属性,则采用爬虫的属性 for attr in attrs: incoming = getattr(data, attr) if attr == 'cover': if incoming and (incoming not in covers): covers.append(incoming) absorbed.append(attr) elif attr == 'big_cover': if incoming and (incoming not in big_covers): big_covers.append(incoming) absorbed.append(attr) else: current = getattr(final_info, attr) if (not current) and (incoming): setattr(final_info, attr, incoming) absorbed.append(attr) if absorbed: logger.debug(f"从'{name}'中获取了字段: " + ' '.join(absorbed)) setattr(final_info, 'covers', covers) setattr(final_info, 'big_covers', big_covers) # 对cover和big_cover赋值,避免后续检查必须字段时出错 if covers: final_info.cover = covers[0] if big_covers: final_info.big_cover = big_covers[0] ########## 部分字段放在最后进行检查 ########## # title if cfg.Crawler.title__chinese_first and 'airav' in all_info: if all_info[ 'airav'].title and final_info.title != all_info['airav'].title: final_info.ori_title = final_info.title final_info.title = all_info['airav'].title # 检查是否所有必需的字段都已经获得了值 for attr in cfg.Crawler.required_keys: if not getattr(final_info, attr, None): logger.error(f"所有爬虫均未获取到字段: '{attr}',抓取失败") return False # 必需字段均已获得了值:将最终的数据附加到movie movie.info = final_info return True
def parse_data_raw(movie: MovieInfo, html): """解析指定番号的影片数据""" container = html.xpath("//div[@class='section product_layout_01']")[0] title = container.xpath("div/h1")[0].text_content().strip() cover = container.xpath("div/p/a[@class='sample_image']/@href")[0] # 这里使用following-sibling而不是getnext,因为getnext会获取到空格、tab等空文本 actress = container.xpath("//dt[text()='出演:']/following-sibling::dd[1]/a/text()") # 移除女优名中的空格,使女优名与其他网站保持一致 actress = [i.replace(' ', '') for i in actress] duration_str = container.xpath("//dt[text()='収録時間:']")[0].getnext().text_content() match = re.search(r'\d+', duration_str) if match: movie.duration = match.group(0) date_str = container.xpath("//dt[text()='発売日:']/following-sibling::dd[1]/a/text()")[0] publish_date = date_str.replace('/', '-') producer = container.xpath("//dt[text()='メーカー名:']/following-sibling::dd[1]/a/text()")[0] dvdid = container.xpath("//dt[text()='品番:']")[0].getnext().text_content() genre_tags = container.xpath("//dt[text()='ジャンル:']/following-sibling::dd[1]/a") genre, genre_id = [], [] for tag in genre_tags: genre.append(tag.text) genre_id.append(tag.get('href').split('=')[-1]) serial = container.xpath("//dt[text()='レーベル:']/following-sibling::dd[1]/a/text()")[0] plot = container.xpath("//h2[text()='レビュー']/following-sibling::p")[0].text.strip() preview_pics = container.xpath("//li/a[@class='sample_image']/@href") # 对于2016年开始的影片,尝试获取高清封面地址(但也并不是每部影片都有,特别是2016年早期) year = int(publish_date.split('-')[0]) if year >= 2016: # 形如'/images/corner/goods/prestige/abp/647/pb_e_abp-647.jpg'的地址,移除其中的'_e'后即为高清封面 big_cover = cover.replace('_e_', '_') movie.big_cover = big_cover movie.title = title movie.cover = cover movie.actress = actress movie.publish_date = publish_date movie.producer = producer movie.genre = genre movie.genre_id = genre_id movie.serial = serial movie.plot = plot movie.preview_pics = preview_pics movie.uncensored = False # prestige服务器在日本且面向日本国内公开发售,不会包含无码片