def _parse_status(self, status): """Parses the house's state description and applies the corresponding values Parameters ---------- status: :class:`str` Plain text string containing the current renting state of the house. """ m = rented_regex.search(status) if m: self.status = HouseStatus.RENTED self.owner = m.group("owner") self.owner_sex = Sex.MALE if m.group("pronoun") == "He" else Sex.FEMALE self.paid_until = parse_tibia_datetime(m.group("paid_until")) else: self.status = HouseStatus.AUCTIONED m = transfer_regex.search(status) if m: self.transfer_date = parse_tibia_datetime(m.group("transfer_date")) self.transfer_accepted = m.group("verb") == "will" self.transferee = m.group("transferee") price = m.group("transfer_price") self.transfer_price = int(price) if price is not None else 0 m = auction_regex.search(status) if m: self.auction_end = parse_tibia_datetime(m.group("auction_end")) m = bid_regex.search(status) if m: self.highest_bid = int(m.group("highest_bid")) self.highest_bidder = m.group("bidder")
def test_character_from_content(self): """Testing parsing a character's HTML content""" character = Character.from_content( self._load_resource(FILE_CHARACTER_RESOURCE)) self._compare_character( Character("Tschas", "Gladera", Vocation.DRUID, 260, Sex.FEMALE), character) self.assertIsNotNone(character.guild_membership) self.assertEqual("Atlantis", character.guild_membership.name) self.assertEqual("Gaia", character.guild_membership.rank) self.assertIsNotNone(character.guild_url) self.assertIsNone(character.married_to_url) self.assertEqual(character.guild_name, character.guild_membership.name) self.assertEqual(character.guild_rank, character.guild_membership.rank) self.assertEqual(AccountStatus.FREE_ACCOUNT, character.account_status) self.assertEqual(182, character.achievement_points) self.assertIsNone(character.house) self.assertIsNone(character.deletion_date) self.assertIsNotNone(character.deaths) self.assertEqual(0, character.deaths.__len__()) self.assertEqual(parse_tibia_datetime("Aug 04 2019, 13:56:59 CEST"), character.last_login) self.assertEqual(character.url, Character.get_url(character.name)) self.assertEqual(5, len(character.other_characters)) self.assertFalse(character.hidden)
def from_content(cls, content): """Parse the content of the World Overview section from Tibia.com into an object of this class. Parameters ---------- content: :class:`str` The HTML content of the World Overview page in Tibia.com Returns ------- :class:`WorldOverview` An instance of this class containing all the information. Raises ------ InvalidContent If the provided content is not the HTML content of the worlds section in Tibia.com """ parsed_content = parse_tibiacom_content(content) world_overview = WorldOverview() try: record_table, *tables \ = parsed_content.find_all("table", {"class": "TableContent"}) m = record_regexp.search(record_table.text) world_overview.record_count = parse_integer(m.group("count")) world_overview.record_date = parse_tibia_datetime(m.group("date")) world_overview._parse_worlds_tables(tables) return world_overview except (AttributeError, KeyError, ValueError) as e: raise InvalidContent("content does not belong to the World Overview section in Tibia.com", e)
def _parse_account_information(self, rows): """ Parses the character's account information Parameters ---------- rows: :class:`list` of :class:`bs4.Tag`, optional A list of all rows contained in the table. """ acc_info = {} if not rows: return for row in rows: cols_raw = row.find_all('td') cols = [ele.text.strip() for ele in cols_raw] field, value = cols field = field.replace("\xa0", "_").replace(" ", "_").replace(":", "").lower() value = value.replace("\xa0", " ") acc_info[field] = value created = parse_tibia_datetime(acc_info["created"]) loyalty_title = None if acc_info[ "loyalty_title"] == "(no title)" else acc_info["loyalty_title"] position = acc_info.get("position") self.account_information = AccountInformation(created, loyalty_title, position)
def _parse_tournament_info(self, table): """Parses the tournament info table. Parameters ---------- table: :class:`bs4.BeautifulSoup` The parsed table containing the tournament's information. """ rows = table.find_all('tr') date_fields = ("start_date", "end_date") list_fields = ("worlds",) for row in rows: cols_raw = row.find_all('td') cols = [ele.text.strip() for ele in cols_raw] field, value = cols field = field.replace("\xa0", "_").replace(" ", "_").replace(":", "").lower() value = value.replace("\xa0", " ") if field in date_fields: value = parse_tibia_datetime(value) if field in list_fields: value = split_list(value, ",", ",") if field == "phase": value = try_enum(TournamentPhase, value) try: setattr(self, field, value) except AttributeError: pass
def _parse_world_info(self, world_info_table): """ Parses the World Information table from Tibia.com and adds the found values to the object. Parameters ---------- world_info_table: :class:`list`[:class:`bs4.Tag`] """ world_info = {} for row in world_info_table: cols_raw = row.find_all('td') cols = [ele.text.strip() for ele in cols_raw] field, value = cols field = field.replace("\xa0", "_").replace(" ", "_").replace(":", "").lower() value = value.replace("\xa0", " ") world_info[field] = value try: self.online_count = parse_integer(world_info.pop("players_online")) except KeyError: self.online_count = 0 self.location = try_enum(WorldLocation, world_info.pop("location")) self.pvp_type = try_enum(PvpType, world_info.pop("pvp_type")) self.transfer_type = try_enum(TransferType, world_info.pop("transfer_type", None), TransferType.REGULAR) m = record_regexp.match(world_info.pop("online_record")) if m: self.record_count = parse_integer(m.group("count")) self.record_date = parse_tibia_datetime(m.group("date")) if "world_quest_titles" in world_info: self.world_quest_titles = [ q.strip() for q in world_info.pop("world_quest_titles").split(",") ] if self.world_quest_titles and "currently has no title" in self.world_quest_titles[ 0]: self.world_quest_titles = [] self.experimental = world_info.pop("game_world_type", None) == "Experimental" self.tournament_world_type = try_enum( TournamentWorldType, world_info.pop("tournament_world_type", None)) self._parse_battleye_status(world_info.pop("battleye_status")) self.premium_only = "premium_type" in world_info month, year = world_info.pop("creation_date").split("/") month = int(month) year = int(year) if year > 90: year += 1900 else: year += 2000 self.creation_date = "%d-%02d" % (year, month) for k, v in world_info.items(): try: setattr(self, k, v) except AttributeError: pass
def test_character_from_content(self): """Testing parsing a character's HTML content""" character = Character.from_content(self.load_resource(FILE_CHARACTER_RESOURCE)) self._compare_character(Character("Tschas", "Gladera", Vocation.ELDER_DRUID, 522, Sex.FEMALE), character) self.assertIsNotNone(character.guild_membership) self.assertEqual("Bald Dwarfs", character.guild_membership.name) self.assertEqual("Emperor", character.guild_membership.rank) self.assertIsNotNone(character.guild_url) self.assertIsNone(character.married_to_url) self.assertEqual(character.guild_name, character.guild_membership.name) self.assertEqual(character.guild_rank, character.guild_membership.rank) self.assertEqual(AccountStatus.PREMIUM_ACCOUNT, character.account_status) self.assertEqual(304, character.achievement_points) self.assertIsNone(character.deletion_date) self.assertIsNotNone(character.deaths) self.assertEqual(2, character.deaths.__len__()) self.assertEqual(parse_tibia_datetime("Aug 02 2021, 17:32:07 CEST"), character.last_login) self.assertEqual(character.url, Character.get_url(character.name)) self.assertEqual(5, len(character.other_characters)) self.assertFalse(character.hidden) # Badges self.assertEqual(3, len(character.account_badges)) badge = character.account_badges[0] self.assertEqual("Ancient Hero", badge.name) self.assertEqual("The account is older than 15 years.", badge.description)
def test_character_from_content_deleted_character(self): """Testing parsing a character scheduled for deletion""" content = self.load_resource(FILE_CHARACTER_DELETION) char = Character.from_content(content) self.assertEqual("Expendable Dummy", char.name) self.assertIsNotNone(char.deletion_date) self.assertIsInstance(char.deletion_date, datetime.datetime) self.assertEqual(parse_tibia_datetime("Oct 08 2018 22:17:00 CEST"), char.deletion_date)
def test_parse_tibia_datetime(self): time = utils.parse_tibia_datetime(TIBIA_DATETIME_CEST) self.assertIsInstance(time, datetime.datetime) self.assertEqual(time.month, 7) self.assertEqual(time.day, 10) self.assertEqual(time.year, 2018) self.assertEqual(time.hour, 5) self.assertEqual(time.minute, 13) self.assertEqual(time.second, 32) time = utils.parse_tibia_datetime(TIBIA_DATETIME_CET) self.assertIsInstance(time, datetime.datetime) self.assertEqual(time.month, 1) self.assertEqual(time.day, 10) self.assertEqual(time.year, 2018) self.assertEqual(time.hour, 6) self.assertEqual(time.minute, 13) self.assertEqual(time.second, 32)
def from_content(cls, content): """Parse the content of the leaderboards page. Parameters ---------- content: :class:`str` The HTML content of the leaderboards page. Returns ------- :class:`Leaderboard` The ledaerboard if found. """ try: parsed_content = parse_tibiacom_content(content) tables = parsed_content.find_all("table", {"class": "TableContent"}) form = parsed_content.find("form") data = parse_form_data(form, include_options=True) current_world = data["world"] current_rotation = None rotations = [] for label, value in data["__options__"]["rotation"].items(): current = False if "Current" in label: label = "".join(rotation_end_pattern.findall(label)) current = True rotation_end = parse_tibia_datetime(label) rotation = LeaderboardRotation(int(value), rotation_end, current) if value == data["rotation"]: current_rotation = rotation rotations.append(rotation) leaderboard = cls(current_world, current_rotation) leaderboard.available_worlds = [ w for w in data["__options__"]["world"].values() if w ] leaderboard.available_rotations = rotations if leaderboard.rotation and leaderboard.rotation.current: last_update_table = tables[2] numbers = re.findall(r'(\d+)', last_update_table.text) if numbers: leaderboard.last_update = datetime.timedelta( minutes=int(numbers[0])) leaderboard._parse_entries(tables[-1]) pagination_block = parsed_content.find("small") pages, total, count = parse_pagination( pagination_block) if pagination_block else (0, 0, 0) leaderboard.page = pages leaderboard.total_pages = total leaderboard.results_count = count return leaderboard except (AttributeError, ValueError) as e: raise errors.InvalidContent( "content does not belong to the leaderboards", e)
def _parse_deaths(self, rows): """ Parses the character's recent deaths Parameters ---------- rows: :class:`list` of :class:`bs4.Tag` A list of all rows contained in the table. """ for row in rows: cols = row.find_all('td') if len(cols) != 2: self.deaths_truncated = True break death_time_str = cols[0].text.replace("\xa0", " ").strip() death_time = parse_tibia_datetime(death_time_str) death = str(cols[1]) death_info = death_regexp.search(death) if death_info: level = int(death_info.group("level")) killers_desc = death_info.group("killers") else: continue death = Death(self.name, level, time=death_time) assists_name_list = [] # Check if the killers list contains assists assist_match = death_assisted.search(killers_desc) if assist_match: # Filter out assists killers_desc = assist_match.group("killers") # Split assists into a list. assists_desc = assist_match.group("assists") assists_name_list = link_search.findall(assists_desc) killers_name_list = split_list(killers_desc) for killer in killers_name_list: killer = killer.replace("\xa0", " ") killer_dict = self._parse_killer(killer) death.killers.append(Killer(**killer_dict)) for assist in assists_name_list: # Extract names from character links in assists list. assist_dict = { "name": link_content.search(assist).group(1), "player": True } death.assists.append(Killer(**assist_dict)) try: self.deaths.append(death) except ValueError: # Some pvp deaths have no level, so they are raising a ValueError, they will be ignored for now. continue
def from_content(cls, content): """Parses the content of the CM Post Archive page from Tibia.com Parameters ---------- content: :class:`str` The HTML content of the CM Post Archive in Tibia.com Returns ------- :class:`CMPostArchive` The CM Post archive found in the page. Raises ------ InvalidContent If content is not the HTML content of the CM Post Archive in Tibia.com """ parsed_content = parse_tibiacom_content(content) form = parsed_content.find("form") try: start_month_selector, start_day_selector, start_year_selector, \ end_month_selector, end_day_selector, end_year_selector = form.find_all("select") start_date = cls._get_selected_date(start_month_selector, start_day_selector, start_year_selector) end_date = cls._get_selected_date(end_month_selector, end_day_selector, end_year_selector) except (AttributeError, ValueError) as e: raise errors.InvalidContent("content does not belong to the CM Post Archive in Tibia.com", e) cm_archive = cls(start_date=start_date, end_date=end_date) table = parsed_content.find("table", attrs={"class", "Table3"}) if not table: return cm_archive inner_table_container = table.find("div", attrs={"class", "InnerTableContainer"}) inner_table = inner_table_container.find("table") inner_table_rows = inner_table.find_all("tr") inner_table_rows = [e for e in inner_table_rows if e.parent == inner_table] table_content = inner_table_container.find("table", attrs={"class", "TableContent"}) header_row, *rows = table_content.find_all("tr") for row in rows: columns = row.find_all("td") date_column = columns[0] date = parse_tibia_datetime(date_column.text.replace("\xa0", " ")) board_thread_column = columns[1] convert_line_breaks(board_thread_column) board, thread = board_thread_column.text.splitlines() link_column = columns[2] post_link = link_column.find("a") post_link_url = post_link["href"] post_id = int(post_id_regex.search(post_link_url).group(1)) cm_archive.posts.append(CMPost(date=date, board=board, thread_title=thread, post_id=post_id)) if not cm_archive.posts: return cm_archive pages_column, results_column = inner_table_rows[-1].find_all("div") page_links = pages_column.find_all("a") listed_pages = [int(p.text) for p in page_links] if listed_pages: cm_archive.page = next((x for x in range(1, listed_pages[-1] + 1) if x not in listed_pages), 0) cm_archive.total_pages = max(int(page_links[-1].text), cm_archive.page) if not cm_archive.page: cm_archive.total_pages += 1 cm_archive.page = cm_archive.total_pages cm_archive.results_count = int(results_column.text.split(":")[-1]) return cm_archive
def test_parse_tibia_datetime_invalid_datetime(self): time = utils.parse_tibia_datetime(TIBIA_DATETIME_INVALID) self.assertIsNone(time)
def _parse_character_information(self, rows): """ Parses the character's basic information and applies the found values. Parameters ---------- rows: :class:`list` of :class:`bs4.Tag` A list of all rows contained in the table. """ int_rows = ["level", "achievement_points"] char = {} houses = [] for row in rows: cols_raw = row.find_all('td') cols = [ele.text.strip() for ele in cols_raw] field, value = cols field = field.replace("\xa0", "_").replace(" ", "_").replace(":", "").lower() value = value.replace("\xa0", " ") # This is a special case cause we need to see the link if field == "house": house_text = value paid_until = house_regexp.search(house_text).group(1) paid_until_date = parse_tibia_date(paid_until) house_link = cols_raw[1].find('a') url = urllib.parse.urlparse(house_link["href"]) query = urllib.parse.parse_qs(url.query) houses.append({ "id": int(query["houseid"][0]), "name": house_link.text.strip(), "town": query["town"][0], "paid_until": paid_until_date }) continue if field in int_rows: value = int(value) char[field] = value # If the character is deleted, the information is fouund with the name, so we must clean it m = deleted_regexp.match(char["name"]) if m: char["name"] = m.group(1) char["deletion_date"] = parse_tibia_datetime(m.group(2)) if "guild_membership" in char: m = guild_regexp.match(char["guild_membership"]) char["guild_membership"] = GuildMembership(m.group(2), m.group(1)) if "(traded)" in char["name"]: char["name"] = char["name"].replace("(traded)", "").strip() char["traded"] = True if "former_names" in char: former_names = [ fn.strip() for fn in char["former_names"].split(",") ] char["former_names"] = former_names if "never" in char["last_login"]: char["last_login"] = None else: char["last_login"] = parse_tibia_datetime(char["last_login"]) m = title_regexp.match(char.get("title", "")) if m: name = m.group(1).strip() unlocked = int(m.group(2)) if name == "None": name = None char["title"] = name char["unlocked_titles"] = unlocked char["vocation"] = try_enum(Vocation, char["vocation"]) char["sex"] = try_enum(Sex, char["sex"]) char["account_status"] = try_enum(AccountStatus, char["account_status"]) for k, v in char.items(): try: setattr(self, k, v) except AttributeError: # This means that there is a attribute in the character's information table that does not have a # corresponding class attribute. pass self.houses = [ CharacterHouse(h["id"], h["name"], self.world, h["town"], self.name, h["paid_until"]) for h in houses ]