def parse_monster_drops(self):
"""Extract monster drop information.
This function parses the monsters wiki page (raw wikitext), and extracts
any MediaWiki template with the name "dropsline". Each template is processed
to determine the ID, name, quantity, rarity and and requirements of the
specific drop.
"""
# Extract "dropsline" templates
self.drops_templates = wikitext_parser.extract_wikitext_template(self.monster_wikitext, "dropsline")
drops_dict_all = dict()
drops_list_ids = list()
self.monster_dict["drops"] = list()
# Loop the found "dropsline" templates
for template in self.drops_templates:
# Parse the template
template_parser = WikitextTemplateParser(self.monster_wikitext)
template_parser.template = template
# Initialize a null value dictionary for each item drop
drop_dict = {
"id": None,
"name": None,
"members": None,
"quantity": None,
"rarity": None,
"drop_requirements": None
}
# Extract the drop information...
# Extract the item drop name
name = template_parser.extract_infobox_value("Name")
# Skip any drop line with classified drop table
if name.lower() == "drop table":
continue
# Determine the drop item ID
item_id = None
found = False
for item in self.all_db_items:
if item.name == name or item.wiki_name == name:
found = True
item_id = item.id
break
if not found:
item_id = None
# Extract the item drop quantity and if the drop is noted
quantity = template_parser.extract_infobox_value("Quantity")
noted = False
if quantity:
if "noted" in quantity.lower():
noted = True
quantity = infobox_cleaner.clean_drop_quantity(quantity)
# Extract, or determine, if the item is members-only
members = False
name_notes = template_parser.extract_infobox_value("Namenotes")
if self.monster_dict["members"]:
members = True
elif item_id:
if self.all_db_items[item_id].members:
members = True
elif name_notes:
if "{{m}}" in name_notes:
members = True
# Extract the item drop rarity
rarity = template_parser.extract_infobox_value("Rarity")
base_value = None
# If the item drop has a drop variable, fetch it
if rarity:
if "var:herbbase" in rarity:
base_value = self.extract_dropsline_header("herbbase")
elif "var:seedbase" in rarity:
base_value = self.extract_dropsline_header("seedbase")
elif "var:uht" in rarity:
base_value = self.extract_dropsline_header("uht")
if not base_value:
base_value = "(22.5/250)/16" # Temp fix for Lizardman shaman
elif "var:bolttipbase" in rarity:
base_value = self.extract_dropsline_header("uht")
if not base_value:
base_value = "(2/128)/40" # Temp fix for Hydra
rarity = infobox_cleaner.clean_drop_rarity(rarity, base_value)
# Extract the rarity notes
drop_requirements = template_parser.extract_infobox_value("Raritynotes")
drop_requirements = infobox_cleaner.clean_drop_requirements(drop_requirements)
# Populate the dictionary
drop_dict = {
"id": item_id,
"name": name,
"members": members,
"quantity": quantity,
"noted": noted,
"rarity": rarity,
"drop_requirements": drop_requirements
}
# Attach drops dict to the drops object for this monster
if item_id:
# self.drops[item_id] = drop_dict
drops_list_ids.append(str(item_id))
drops_dict_all[str(item_id)] = drop_dict
# Handle any embedded drop tables
if "talismandroptable2" in self.monster_wikitext[2].lower():
items = drop_tables.talisman(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "catacombsdroptable" in self.monster_wikitext[2].lower():
items = drop_tables.catacombs(self.monster_dict["name"],
self.monster_dict["hitpoints"],
self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "herbdroptable2" in self.monster_wikitext[2].lower():
items = drop_tables.herb(self.monster_dict["members"],
self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "usefulherbdroptable2" in self.monster_wikitext[2].lower():
items = drop_tables.usefulherb(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "fixedallotmentseeddroptable2" in self.monster_wikitext[2].lower():
items = drop_tables.fixedallotmentseed(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "dropsallotmenttable" in self.monster_wikitext[2].lower():
items = drop_tables.fixedallotmentseed(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "treeherbseeddroptable2" in self.monster_wikitext[2].lower():
items = drop_tables.treeseed(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "rareseeddroptable2" in self.monster_wikitext[2].lower():
items = drop_tables.rareseed(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "variableallotmentseeddroptale2" in self.monster_wikitext[2].lower():
items = drop_tables.variableallotmentseed(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "manyseeddroptable2" in self.monster_wikitext[2].lower():
items = drop_tables.commonseed(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "hopsdroptable2" in self.monster_wikitext[2].lower():
items = drop_tables.hopsseed(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "superiordroptable" in self.monster_wikitext[2].lower():
items = drop_tables.superior(self.monster_dict["slayer_level"])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if ("wildernessslayerdroptable" in self.monster_wikitext[2].lower() and
"krystilia" in self.monster_dict["slayer_masters"]):
items = drop_tables.wildernessslayer(self.monster_dict["name"],
self.monster_dict["combat_level"], self.monster_dict["hitpoints"],
self.monster_dict["slayer_level"])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
# if "mainraredroptable" in self.monster_wikitext[2].lower():
# items = drop_tables.raredroptable(self.monster_wikitext[2])
# for item, item_dict in items.items():
# drops_list_ids.append(str(item))
# drops_dict_all[str(item)] = item_dict
# if "raredroptable" in self.monster_wikitext[2].lower():
# items = drop_tables.raredroptable(self.monster_wikitext[2])
# for item, item_dict in items.items():
# drops_list_ids.append(str(item))
# drops_dict_all[str(item)] = item_dict
# Append all parsed drops to the drops array
for item_id in drops_list_ids:
self.monster_dict["drops"].append(drops_dict_all[item_id])
def fetch():
"""Fetch monster drops using SMW queries.
This is a request heavy method - querying about 1,000 endpoints
to get monster drop data.
"""
# Load the monster wikitext file of processed data
with open(
Path(config.DATA_MONSTERS_PATH /
"monsters-wiki-page-text-processed.json")) as f:
all_wikitext_processed = json.load(f)
# Load the raw cache data that has been processed (this is ground truth)
with open(Path(config.DATA_MONSTERS_PATH /
"monsters-cache-data.json")) as f:
all_monster_cache_data = json.load(f)
for monster_id, monster_list in all_wikitext_processed.items():
exists = all_monster_cache_data.get(monster_id, None)
if not exists:
continue
if "dropversion" in monster_list[2].lower():
name = all_monster_cache_data[monster_id]["name"]
wikitext = monster_list[2]
version = monster_list[1]
wikitext_template = WikitextTemplateParser(wikitext)
wikitext_template.extract_infobox("infobox monster")
value = wikitext_template.extract_infobox_value(
f"dropversion{version}")
if not value:
value = wikitext_template.extract_infobox_value(
f"dropversion1")
multi_drop_tables[monster_id] = f"[[Dropped from::{name}#{value}]]"
api_url = "https://oldschool.runescape.wiki/api.php"
# Specify what the SMW query should return
selection = "|?Dropped item|?Drop Quantity|?Rarity|?Rolls|limit=500"
# Set parameters to run a SMW query
params = {"action": "ask", "format": "json", "query": None}
# Data structures for storing conditions
# Conditions are used to form the SMW query
conditions_set = set()
conditions_dict = defaultdict(list)
# Loop raw monster cache data (ground truth)
for monster_id, monster in all_monster_cache_data.items():
if monster_id in multi_drop_tables:
condition = multi_drop_tables[monster_id]
else:
condition = f"[[Dropped from::{monster['name']}]]"
# Add to set of conditions to later query
conditions_set.add(condition)
# Add condition string for monster ID lookup
conditions_dict[condition].append(monster_id)
print(f">>> Fetching {len(conditions_set)} drop tables...")
# Start fetching the data
count = 0
for condition in conditions_set:
print(f" > Processing {count}/{len(conditions_set)}: {condition}")
params["query"] = f"{condition}{selection}"
r = requests.get(api_url, headers=config.custom_agent, params=params)
data = r.json()
Path(config.DATA_MONSTERS_PATH / "monsters-drops-raw").mkdir(
parents=True, exist_ok=True)
for monster_id in conditions_dict[condition]:
file_name = f"{monster_id}.json"
file_path = Path(config.DATA_MONSTERS_PATH / "monsters-drops-raw" /
file_name)
with open(file_path, "w") as f:
json.dump(data, f, indent=4)
count += 1