def parse_monster_drops(self): """Extract monster drop information. This function parses the monsters wiki page (raw wikitext), and extracts any MediaWiki template with the name "dropsline". Each template is processed to determine the ID, name, quantity, rarity and and requirements of the specific drop. """ # Extract "dropsline" templates self.drops_templates = wikitext_parser.extract_wikitext_template(self.monster_wikitext, "dropsline") drops_dict_all = dict() drops_list_ids = list() self.monster_dict["drops"] = list() # Loop the found "dropsline" templates for template in self.drops_templates: # Parse the template template_parser = WikitextTemplateParser(self.monster_wikitext) template_parser.template = template # Initialize a null value dictionary for each item drop drop_dict = { "id": None, "name": None, "members": None, "quantity": None, "rarity": None, "drop_requirements": None } # Extract the drop information... # Extract the item drop name name = template_parser.extract_infobox_value("Name") # Skip any drop line with classified drop table if name.lower() == "drop table": continue # Determine the drop item ID item_id = None found = False for item in self.all_db_items: if item.name == name or item.wiki_name == name: found = True item_id = item.id break if not found: item_id = None # Extract the item drop quantity and if the drop is noted quantity = template_parser.extract_infobox_value("Quantity") noted = False if quantity: if "noted" in quantity.lower(): noted = True quantity = infobox_cleaner.clean_drop_quantity(quantity) # Extract, or determine, if the item is members-only members = False name_notes = template_parser.extract_infobox_value("Namenotes") if self.monster_dict["members"]: members = True elif item_id: if self.all_db_items[item_id].members: members = True elif name_notes: if "{{m}}" in name_notes: members = True # Extract the item drop rarity rarity = template_parser.extract_infobox_value("Rarity") base_value = None # If the item drop has a drop variable, fetch it if rarity: if "var:herbbase" in rarity: base_value = self.extract_dropsline_header("herbbase") elif "var:seedbase" in rarity: base_value = self.extract_dropsline_header("seedbase") elif "var:uht" in rarity: base_value = self.extract_dropsline_header("uht") if not base_value: base_value = "(22.5/250)/16" # Temp fix for Lizardman shaman elif "var:bolttipbase" in rarity: base_value = self.extract_dropsline_header("uht") if not base_value: base_value = "(2/128)/40" # Temp fix for Hydra rarity = infobox_cleaner.clean_drop_rarity(rarity, base_value) # Extract the rarity notes drop_requirements = template_parser.extract_infobox_value("Raritynotes") drop_requirements = infobox_cleaner.clean_drop_requirements(drop_requirements) # Populate the dictionary drop_dict = { "id": item_id, "name": name, "members": members, "quantity": quantity, "noted": noted, "rarity": rarity, "drop_requirements": drop_requirements } # Attach drops dict to the drops object for this monster if item_id: # self.drops[item_id] = drop_dict drops_list_ids.append(str(item_id)) drops_dict_all[str(item_id)] = drop_dict # Handle any embedded drop tables if "talismandroptable2" in self.monster_wikitext[2].lower(): items = drop_tables.talisman(self.monster_wikitext[2]) for item, item_dict in items.items(): drops_list_ids.append(str(item)) drops_dict_all[str(item)] = item_dict if "catacombsdroptable" in self.monster_wikitext[2].lower(): items = drop_tables.catacombs(self.monster_dict["name"], self.monster_dict["hitpoints"], self.monster_wikitext[2]) for item, item_dict in items.items(): drops_list_ids.append(str(item)) drops_dict_all[str(item)] = item_dict if "herbdroptable2" in self.monster_wikitext[2].lower(): items = drop_tables.herb(self.monster_dict["members"], self.monster_wikitext[2]) for item, item_dict in items.items(): drops_list_ids.append(str(item)) drops_dict_all[str(item)] = item_dict if "usefulherbdroptable2" in self.monster_wikitext[2].lower(): items = drop_tables.usefulherb(self.monster_wikitext[2]) for item, item_dict in items.items(): drops_list_ids.append(str(item)) drops_dict_all[str(item)] = item_dict if "fixedallotmentseeddroptable2" in self.monster_wikitext[2].lower(): items = drop_tables.fixedallotmentseed(self.monster_wikitext[2]) for item, item_dict in items.items(): drops_list_ids.append(str(item)) drops_dict_all[str(item)] = item_dict if "dropsallotmenttable" in self.monster_wikitext[2].lower(): items = drop_tables.fixedallotmentseed(self.monster_wikitext[2]) for item, item_dict in items.items(): drops_list_ids.append(str(item)) drops_dict_all[str(item)] = item_dict if "treeherbseeddroptable2" in self.monster_wikitext[2].lower(): items = drop_tables.treeseed(self.monster_wikitext[2]) for item, item_dict in items.items(): drops_list_ids.append(str(item)) drops_dict_all[str(item)] = item_dict if "rareseeddroptable2" in self.monster_wikitext[2].lower(): items = drop_tables.rareseed(self.monster_wikitext[2]) for item, item_dict in items.items(): drops_list_ids.append(str(item)) drops_dict_all[str(item)] = item_dict if "variableallotmentseeddroptale2" in self.monster_wikitext[2].lower(): items = drop_tables.variableallotmentseed(self.monster_wikitext[2]) for item, item_dict in items.items(): drops_list_ids.append(str(item)) drops_dict_all[str(item)] = item_dict if "manyseeddroptable2" in self.monster_wikitext[2].lower(): items = drop_tables.commonseed(self.monster_wikitext[2]) for item, item_dict in items.items(): drops_list_ids.append(str(item)) drops_dict_all[str(item)] = item_dict if "hopsdroptable2" in self.monster_wikitext[2].lower(): items = drop_tables.hopsseed(self.monster_wikitext[2]) for item, item_dict in items.items(): drops_list_ids.append(str(item)) drops_dict_all[str(item)] = item_dict if "superiordroptable" in self.monster_wikitext[2].lower(): items = drop_tables.superior(self.monster_dict["slayer_level"]) for item, item_dict in items.items(): drops_list_ids.append(str(item)) drops_dict_all[str(item)] = item_dict if ("wildernessslayerdroptable" in self.monster_wikitext[2].lower() and "krystilia" in self.monster_dict["slayer_masters"]): items = drop_tables.wildernessslayer(self.monster_dict["name"], self.monster_dict["combat_level"], self.monster_dict["hitpoints"], self.monster_dict["slayer_level"]) for item, item_dict in items.items(): drops_list_ids.append(str(item)) drops_dict_all[str(item)] = item_dict # if "mainraredroptable" in self.monster_wikitext[2].lower(): # items = drop_tables.raredroptable(self.monster_wikitext[2]) # for item, item_dict in items.items(): # drops_list_ids.append(str(item)) # drops_dict_all[str(item)] = item_dict # if "raredroptable" in self.monster_wikitext[2].lower(): # items = drop_tables.raredroptable(self.monster_wikitext[2]) # for item, item_dict in items.items(): # drops_list_ids.append(str(item)) # drops_dict_all[str(item)] = item_dict # Append all parsed drops to the drops array for item_id in drops_list_ids: self.monster_dict["drops"].append(drops_dict_all[item_id])
def fetch(): """Fetch monster drops using SMW queries. This is a request heavy method - querying about 1,000 endpoints to get monster drop data. """ # Load the monster wikitext file of processed data with open( Path(config.DATA_MONSTERS_PATH / "monsters-wiki-page-text-processed.json")) as f: all_wikitext_processed = json.load(f) # Load the raw cache data that has been processed (this is ground truth) with open(Path(config.DATA_MONSTERS_PATH / "monsters-cache-data.json")) as f: all_monster_cache_data = json.load(f) for monster_id, monster_list in all_wikitext_processed.items(): exists = all_monster_cache_data.get(monster_id, None) if not exists: continue if "dropversion" in monster_list[2].lower(): name = all_monster_cache_data[monster_id]["name"] wikitext = monster_list[2] version = monster_list[1] wikitext_template = WikitextTemplateParser(wikitext) wikitext_template.extract_infobox("infobox monster") value = wikitext_template.extract_infobox_value( f"dropversion{version}") if not value: value = wikitext_template.extract_infobox_value( f"dropversion1") multi_drop_tables[monster_id] = f"[[Dropped from::{name}#{value}]]" api_url = "https://oldschool.runescape.wiki/api.php" # Specify what the SMW query should return selection = "|?Dropped item|?Drop Quantity|?Rarity|?Rolls|limit=500" # Set parameters to run a SMW query params = {"action": "ask", "format": "json", "query": None} # Data structures for storing conditions # Conditions are used to form the SMW query conditions_set = set() conditions_dict = defaultdict(list) # Loop raw monster cache data (ground truth) for monster_id, monster in all_monster_cache_data.items(): if monster_id in multi_drop_tables: condition = multi_drop_tables[monster_id] else: condition = f"[[Dropped from::{monster['name']}]]" # Add to set of conditions to later query conditions_set.add(condition) # Add condition string for monster ID lookup conditions_dict[condition].append(monster_id) print(f">>> Fetching {len(conditions_set)} drop tables...") # Start fetching the data count = 0 for condition in conditions_set: print(f" > Processing {count}/{len(conditions_set)}: {condition}") params["query"] = f"{condition}{selection}" r = requests.get(api_url, headers=config.custom_agent, params=params) data = r.json() Path(config.DATA_MONSTERS_PATH / "monsters-drops-raw").mkdir( parents=True, exist_ok=True) for monster_id in conditions_dict[condition]: file_name = f"{monster_id}.json" file_path = Path(config.DATA_MONSTERS_PATH / "monsters-drops-raw" / file_name) with open(file_path, "w") as f: json.dump(data, f, indent=4) count += 1