def extract_anime_names(sel: Selector) -> Iterable[AnimeName]: """Extract the names of the anime.""" main_title_text = get_all_text(sel.xpath("//h1/span[@itemprop='name']")) yield AnimeName(name=main_title_text, is_primary=True) alternative_titles_sel = xpath_slice_between( select_sidebar(sel), lower_xpath="./h2[. = 'Alternative Titles']", upper_xpath="./br") for alternative_title_sel in alternative_titles_sel: maybe_many_names = get_all_text( alternative_title_sel.xpath("./span/following-sibling::text()")) for name in re.split(r"\b\s*,\s+(?=[A-Z][a-z])", maybe_many_names): yield AnimeName(name=name, is_primary=False)
def extract_anime_info_fields(sel: Selector) -> Iterable[KeyValuesPair]: """Extract info fields of the anime.""" stat_sels = xpath_slice_between(select_sidebar(sel), lower_xpath="./h2[. = 'Information']", upper_xpath="./br") for stat_sel in stat_sels: key = re.sub(r"\s+", " ", get_all_text(stat_sel.xpath("./span")))\ .strip()\ .rstrip(":") # pylint: disable=line-too-long value_text = re.sub(r"\s+", " ", get_all_text(stat_sel).replace(key, "", 1))\ .lstrip(":")\ .strip() values = re.split(r"\b\s+,\s+\b", value_text) yield KeyValuesPair(key=key, values=values)
def extract_manga_roles(sel): """Extract the mangas the character appeared in.""" anime_table = \ sel.xpath("//div[text() = 'Mangaography']/following-sibling::table[1]") roles = [] for row in anime_table.xpath(".//tr"): image_sel, text_sel = row.xpath(".//td") picture = image_sel.xpath(".//img").attrib["src"] title = get_all_text(text_sel.xpath("./a")) role = get_all_text(text_sel.xpath("./div")) if re.search(r"main", role, flags=re.IGNORECASE): role = "main" elif re.search(r"support(ing)?|secondary", role, flags=re.IGNORECASE): role = "secondary" roles.append({"name": title, "picture": picture, "role": role}) return roles
def extract_info_fields(sel): """Extract the character's hair color.""" stats_sel = select_basic_stats_section(sel) return [{ "key": "Hair Color", "value": strip_field_name( get_all_text( stats_sel.xpath("div[re:test(., 'hair color', 'i')]"))) }]
def extract_anime_description(sel: Selector) -> str: """Extract the anime description.""" description_paragraph_sels = \ sel.xpath("//span[@itemprop='description']/text()") paragraphs = [] for paragraph_sel, is_last in lookahead(description_paragraph_sels): content = get_all_text(paragraph_sel) # pylint: disable=invalid-name IS_PROBABLY_SOURCE_MAX_LEN = 30 # pylint: disable=line-too-long if is_last and \ re.search(r"written|from|source|author", content, flags=re.I) and \ len(content) <= IS_PROBABLY_SOURCE_MAX_LEN: continue paragraphs.append(content) return normalize_whitespace("\n".join(paragraphs))
def extract_character_names(sel: Selector) -> Iterable[AnimeCharacter]: """Extact the names of the anime characters.""" # pylint: disable=line-too-long maybe_name_anchors = \ sel.xpath("(//h2[contains(., 'Characters')]/following-sibling::div)[1]//a[not(./img)]") for maybe_name_anchor in maybe_name_anchors: href = maybe_name_anchor.attrib["href"] match = re.search(r"/character/\d+/(?P<name>[^/]+)$", href) if not match: continue name = re.sub(r"_+|\s+", " ", match.group("name")).strip() # pylint: disable=line-too-long role = get_all_text( maybe_name_anchor.xpath("./following-sibling::div/small")) if re.search(r"main", role, flags=re.IGNORECASE): role = "main" elif re.search(r"support(ing)?|secondary", role, flags=re.IGNORECASE): role = "secondary" yield AnimeCharacter(name=name, url=href, role=role)
def extract_en_jp_name(sel): """Extract the character's full name.""" name_sel = select_name(sel) english_name = name_sel.css("::text").get().strip() japanese_name = get_all_text(name_sel.xpath("span")).strip("()") return {"en": [english_name], "jp": [japanese_name]}
def extract_heart_off_number(sel): """Extract the number of heart off emojis the character has recieved.""" return re.sub(r"[^\d]|#", "", get_all_text(sel.xpath("//h3[./span[@class = 'heartOff']]")))
def extract_top_hated_rank(sel): """Extract the character's top loved rank.""" stats_sel = select_basic_stats_section(sel) return re.sub( r"[^\d]|#", "", get_all_text(stats_sel.xpath("div/a[re:test(@href, 'top-hated')]")))
def extract_nicknames(sel): """Extract the character's nickname.""" return [strip_field_name(get_all_text(sel.xpath("//h2[@class = 'aka']")))]
def extract_names(sel): """Extract the character's full name.""" return [get_all_text(sel.xpath("//h1[@itemprop = 'name']"))]
def extract_descriptions(sel): """Extract the description of the character.""" return [get_all_text(sel.xpath("//div[@itemprop = 'description']"))]