def extract_dropsline_header(self, table_head_type: str) -> float:
"""Parse the dropstablehead template for variable drop rarity values.
:param table_head_type: Specify a seed, or herb table head search.
:return: A float of the drop rarity multiplier.
"""
# Extract "dropstablehead" templates
# This is used for extracting "herbbase" and "seedbase" values
self.drops_templates = wikitext_parser.extract_wikitext_template(
self.monster_wikitext, "dropstablehead")
table_head_value = None
# Loop the found "dropstablehead" templates
for template in self.drops_templates:
# Parse the template
template_parser = WikitextTemplateParser(self.monster_wikitext)
template_parser.template = template
if "vardefine:" + table_head_type in template:
# Example:
# {{DropsTableHead{{#vardefine:herbbase|{{#expr:9/123/128}}}}}}
table_head_value = template.split("#expr:")[1]
table_head_value = table_head_value.replace("}", "")
table_head_value = eval(table_head_value)
return table_head_value
def preprocessing(self) -> Dict:
"""Preprocess an item, and set important object variables.
This function preprocesses every item dumped from the OSRS cache. Various
properties are set to help further processing. Items are determined if
they are a linked item (noted/placeholder), or an actual item. The item
is checked if it is a valid item (has a wiki page, is an actual item etc.).
Finally, the wikitext (from the OSRS wiki) is found by looking up ID, linked
ID, name, and normalized name. The `Infobox Item` or `Infobox Pet` is then
extracted so that the wiki properties can be later processed and populated.
:return: A dictionary including success and code.
"""
# Initialize dictionary to return preprocessing status
status = {
"status": False,
"code": None
}
# Set item ID variables
self.item_id_str = str(self.item_id)
# Load item dictionary of cache data based on item ID
# This raw cache data is the baseline information about the specific item
# and can be considered 100% correct and available for every item
self.item_cache_data = self.all_items_cache_data[self.item_id_str]
# Set item name variable (directly from the cache dump)
self.item_name = self.item_cache_data["name"]
if self.verbose:
print(f">>> {self.item_id_str} {self.item_name}")
# Get the linked ID item value, if available
self.linked_id_item_int = None
self.linked_id_item_str = None
if self.item_cache_data["linked_id_item"] is not None:
self.linked_id_item_int = int(self.item_cache_data["linked_id_item"])
self.linked_id_item_str = str(self.item_cache_data["linked_id_item"])
# Determine the ID number to extract
# Noted and placeholder items should use the linked_id_item property
# to fill in additional wiki data...
item_id_to_process_int = None
if self.item_cache_data["noted"] is True or self.item_cache_data["placeholder"] is True:
item_id_to_process_int = int(self.linked_id_item_int)
else:
item_id_to_process_int = int(self.item_id)
# Find the wiki page
has_infobox = False
# Try to find the wiki data using direct ID number search
if self.all_wikitext_processed.get(self.item_id_str, None):
self.item_wikitext = self.all_wikitext_processed.get(self.item_id_str, None)
self.wikitext_found_using = "id"
status["code"] = "lookup_passed_id"
status["status"] = True
# Try to find the wiki data using linked_id_item ID number search
elif self.all_wikitext_processed.get(self.linked_id_item_str, None):
self.item_wikitext = self.all_wikitext_processed.get(self.linked_id_item_str, None)
self.wikitext_found_using = "linked_id"
status["code"] = "lookup_passed_linked_id"
status["status"] = True
# Try to find the wiki data using direct name search
elif self.all_wikitext_raw.get(self.item_name, None):
self.item_wikitext = self.all_wikitext_raw.get(self.item_name, None)
self.wikitext_found_using = "name"
status["code"] = "lookup_passed_name"
status["status"] = True
else:
status["code"] = "no_item_wikitext"
return status
# Parse the infobox item
infobox_parser = WikitextTemplateParser(self.item_wikitext)
# Try extract infobox for item, then pet
has_infobox = infobox_parser.extract_infobox("infobox item")
if not has_infobox:
has_infobox = infobox_parser.extract_infobox("infobox pet")
if not has_infobox:
self.template = None
status["code"] = "no_infobox_template"
return status
is_versioned = infobox_parser.determine_infobox_versions()
versioned_ids = infobox_parser.extract_infobox_ids()
# Set the infobox version number, default to empty string (no version number)
try:
if versioned_ids:
self.infobox_version_number = versioned_ids[item_id_to_process_int]
except KeyError:
if is_versioned:
self.infobox_version_number = "1"
else:
self.infobox_version_number = ""
# Set the template
self.template = infobox_parser.template
status["status"] = True
return status
def populate_from_wiki_data_equipment(self) -> bool:
"""Parse the wiki text template and extract item bonus values from it."""
# Hardcoded item skips - mostly unobtainable or weird items
if int(self.item_id) in infobox_cleaner.unequipable:
self.item_dict["equipable"] = False
self.item_dict["equipment"] = None
self.item_dict["equipable_by_player"] = False
self.item_dict["equipable_weapon"] = False
return
# Initialize empty equipment dictionary
self.item_dict["equipment"] = dict()
# STAGE ONE: EQUIPMENT
# Extract the infobox bonuses template
infobox_bonuses_parser = WikitextTemplateParser(self.item_wikitext)
has_infobox = infobox_bonuses_parser.extract_infobox("infobox bonuses")
if not has_infobox:
has_infobox = infobox_bonuses_parser.extract_infobox("infobox_bonuses")
if not has_infobox:
# No infobox bonuses found for the item!
print("populate_from_wiki_data_equipment: No infobox bonuses")
exit(1)
# Set the infobox bonuses template
bonuses_template = infobox_bonuses_parser.template
# This item must be equipable by a player, set to True
self.item_dict["equipable_by_player"] = True
# Initialize a dictionary that maps database_name -> property_name
# The database_name is used in this project
# The property_name is used by the OSRS Wiki
combat_bonuses = {"attack_stab": "astab",
"attack_slash": "aslash",
"attack_crush": "acrush",
"attack_magic": "amagic",
"attack_ranged": "arange",
"defence_stab": "dstab",
"defence_slash": "dslash",
"defence_crush": "dcrush",
"defence_magic": "dmagic",
"defence_ranged": "drange",
"melee_strength": "str",
"ranged_strength": "rstr",
"magic_damage": "mdmg",
"prayer": "prayer"
}
# Loop each of the combat bonuses and populate
for database_name, property_name in combat_bonuses.items():
value = None
if self.infobox_version_number is not None:
key = property_name + str(self.infobox_version_number)
value = self.extract_infobox_value(bonuses_template, key)
if value is None:
value = self.extract_infobox_value(bonuses_template, property_name)
if value is not None:
self.item_dict["equipment"][database_name] = infobox_cleaner.stats(value)
else:
self.item_dict["equipment"][database_name] = 0
self.item_dict["incomplete"] = True
# Slot
slot = None
if self.infobox_version_number is not None:
key = "slot" + str(self.infobox_version_number)
slot = self.extract_infobox_value(bonuses_template, key)
if slot is None:
slot = self.extract_infobox_value(bonuses_template, "slot")
if slot is not None:
self.item_dict["equipment"]["slot"] = infobox_cleaner.caller(slot, "slot")
else:
print(">>> populate_from_wiki_data_equipment: No slot")
exit(1)
# Skill requirements
try:
requirements = self.skill_requirements[self.item_id_str]
self.item_dict["equipment"]["requirements"] = requirements
except KeyError:
self.item_dict["equipment"]["requirements"] = None
# If item is not weapon or 2h, start set defaults and return
if (self.item_dict["equipment"]["slot"] not in ["weapon", "2h"]):
self.item_dict["equipable_weapon"] = False
return
# STAGE TWO: WEAPONS
self.item_dict["weapon"] = dict()
# Attack speed
attack_speed = None
if self.infobox_version_number is not None:
key = "speed" + str(self.infobox_version_number)
attack_speed = self.extract_infobox_value(bonuses_template, key)
if attack_speed is None:
attack_speed = self.extract_infobox_value(bonuses_template, "speed")
if attack_speed is not None:
self.item_dict["weapon"]["attack_speed"] = infobox_cleaner.caller(attack_speed, "speed")
else:
print(">>> populate_from_wiki_data_equipment: No attack_speed")
exit(1)
# Weapon type
# Extract the CombatStyles template
infobox_combat_parser = WikitextTemplateParser(self.item_wikitext)
has_infobox = infobox_combat_parser.extract_infobox("combatstyles")
if not has_infobox:
# No combatstyles template found for the item!
print("populate_from_wiki_data_equipment: No combatstyles")
exit(1)
# Set the infobox bonuses template
combat_template = infobox_combat_parser.template
weapon_type = infobox_cleaner.caller(combat_template, "weapon_type")
self.item_dict["weapon"]["weapon_type"] = weapon_type
try:
self.item_dict["weapon"]["stances"] = self.weapon_stances[weapon_type]
except KeyError:
print("populate_from_wiki_data_equipment: Weapon type error")
exit(1)
# Finally, set the equipable_weapon property to true
self.item_dict["equipable_weapon"] = True
def parse_monster_drops(self):
"""Extract monster drop information.
This function parses the monsters wiki page (raw wikitext), and extracts
any MediaWiki template with the name "dropsline". Each template is processed
to determine the ID, name, quantity, rarity and and requirements of the
specific drop.
"""
# Extract "dropsline" templates
self.drops_templates = wikitext_parser.extract_wikitext_template(self.monster_wikitext, "dropsline")
drops_dict_all = dict()
drops_list_ids = list()
self.monster_dict["drops"] = list()
# Loop the found "dropsline" templates
for template in self.drops_templates:
# Parse the template
template_parser = WikitextTemplateParser(self.monster_wikitext)
template_parser.template = template
# Initialize a null value dictionary for each item drop
drop_dict = {
"id": None,
"name": None,
"members": None,
"quantity": None,
"rarity": None,
"drop_requirements": None
}
# Extract the drop information...
# Extract the item drop name
name = template_parser.extract_infobox_value("Name")
# Skip any drop line with classified drop table
if name.lower() == "drop table":
continue
# Determine the drop item ID
item_id = None
found = False
for item in self.all_db_items:
if item.name == name or item.wiki_name == name:
found = True
item_id = item.id
break
if not found:
item_id = None
# Extract the item drop quantity and if the drop is noted
quantity = template_parser.extract_infobox_value("Quantity")
noted = False
if quantity:
if "noted" in quantity.lower():
noted = True
quantity = infobox_cleaner.clean_drop_quantity(quantity)
# Extract, or determine, if the item is members-only
members = False
name_notes = template_parser.extract_infobox_value("Namenotes")
if self.monster_dict["members"]:
members = True
elif item_id:
if self.all_db_items[item_id].members:
members = True
elif name_notes:
if "{{m}}" in name_notes:
members = True
# Extract the item drop rarity
rarity = template_parser.extract_infobox_value("Rarity")
base_value = None
# If the item drop has a drop variable, fetch it
if rarity:
if "var:herbbase" in rarity:
base_value = self.extract_dropsline_header("herbbase")
elif "var:seedbase" in rarity:
base_value = self.extract_dropsline_header("seedbase")
elif "var:uht" in rarity:
base_value = self.extract_dropsline_header("uht")
if not base_value:
base_value = "(22.5/250)/16" # Temp fix for Lizardman shaman
elif "var:bolttipbase" in rarity:
base_value = self.extract_dropsline_header("uht")
if not base_value:
base_value = "(2/128)/40" # Temp fix for Hydra
rarity = infobox_cleaner.clean_drop_rarity(rarity, base_value)
# Extract the rarity notes
drop_requirements = template_parser.extract_infobox_value("Raritynotes")
drop_requirements = infobox_cleaner.clean_drop_requirements(drop_requirements)
# Populate the dictionary
drop_dict = {
"id": item_id,
"name": name,
"members": members,
"quantity": quantity,
"noted": noted,
"rarity": rarity,
"drop_requirements": drop_requirements
}
# Attach drops dict to the drops object for this monster
if item_id:
# self.drops[item_id] = drop_dict
drops_list_ids.append(str(item_id))
drops_dict_all[str(item_id)] = drop_dict
# Handle any embedded drop tables
if "talismandroptable2" in self.monster_wikitext[2].lower():
items = drop_tables.talisman(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "catacombsdroptable" in self.monster_wikitext[2].lower():
items = drop_tables.catacombs(self.monster_dict["name"],
self.monster_dict["hitpoints"],
self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "herbdroptable2" in self.monster_wikitext[2].lower():
items = drop_tables.herb(self.monster_dict["members"],
self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "usefulherbdroptable2" in self.monster_wikitext[2].lower():
items = drop_tables.usefulherb(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "fixedallotmentseeddroptable2" in self.monster_wikitext[2].lower():
items = drop_tables.fixedallotmentseed(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "dropsallotmenttable" in self.monster_wikitext[2].lower():
items = drop_tables.fixedallotmentseed(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "treeherbseeddroptable2" in self.monster_wikitext[2].lower():
items = drop_tables.treeseed(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "rareseeddroptable2" in self.monster_wikitext[2].lower():
items = drop_tables.rareseed(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "variableallotmentseeddroptale2" in self.monster_wikitext[2].lower():
items = drop_tables.variableallotmentseed(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "manyseeddroptable2" in self.monster_wikitext[2].lower():
items = drop_tables.commonseed(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "hopsdroptable2" in self.monster_wikitext[2].lower():
items = drop_tables.hopsseed(self.monster_wikitext[2])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if "superiordroptable" in self.monster_wikitext[2].lower():
items = drop_tables.superior(self.monster_dict["slayer_level"])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
if ("wildernessslayerdroptable" in self.monster_wikitext[2].lower() and
"krystilia" in self.monster_dict["slayer_masters"]):
items = drop_tables.wildernessslayer(self.monster_dict["name"],
self.monster_dict["combat_level"], self.monster_dict["hitpoints"],
self.monster_dict["slayer_level"])
for item, item_dict in items.items():
drops_list_ids.append(str(item))
drops_dict_all[str(item)] = item_dict
# if "mainraredroptable" in self.monster_wikitext[2].lower():
# items = drop_tables.raredroptable(self.monster_wikitext[2])
# for item, item_dict in items.items():
# drops_list_ids.append(str(item))
# drops_dict_all[str(item)] = item_dict
# if "raredroptable" in self.monster_wikitext[2].lower():
# items = drop_tables.raredroptable(self.monster_wikitext[2])
# for item, item_dict in items.items():
# drops_list_ids.append(str(item))
# drops_dict_all[str(item)] = item_dict
# Append all parsed drops to the drops array
for item_id in drops_list_ids:
self.monster_dict["drops"].append(drops_dict_all[item_id])
def preprocessing(self):
"""Preprocess an monster, and set important object variables.
This function preprocesses every monster dumped from the OSRS cache. Various
properties are set to help further processing. MORE."""
# Set monster ID variables
self.monster_id_int = int(self.monster_id) # Monster ID number as an integer
self.monster_id_str = str(self.monster_id) # Monster ID number as a string
# Load monster dictionary of cache data based on monster ID
# This raw cache data is the baseline information about the specific monster
# and can be considered 100% correct and available for every monster
self.monster_cache_data = self.all_monster_cache_data[self.monster_id_str]
# Set monster name variable (directly from the cache dump)
self.monster_name = self.monster_cache_data["name"]
# Log and print monster
logging.debug(f"======================= {self.monster_id_str} {self.monster_name}")
# print(f"======================= {self.monster_id_str} {self.monster_name}")
logging.debug(f"preprocessing: using the following cache data:")
logging.debug(self.monster_cache_data)
# Set all variables to None (for invalid monsters)
self.monster_wikitext = None
self.wikitext_found_using = None
self.has_infobox = False
# Try to find the wiki data using direct ID number search
if self.all_wikitext_processed.get(self.monster_id_str, None):
self.monster_wikitext = self.all_wikitext_processed.get(self.monster_id_str, None)
self.wikitext_found_using = "id"
# Try to find the wiki data using direct name search
elif self.all_wikitext_raw.get(self.monster_name, None):
self.monster_wikitext = self.all_wikitext_raw.get(self.monster_name, None)
self.wikitext_found_using = "name"
logging.debug(f"preprocessing: self.monster_wikitext found using: {self.wikitext_found_using}")
# If there is no wikitext, and the monster is valid, raise a critical error
if not self.monster_wikitext:
logging.critical("CRITICAL: Could not find monster_wikitext by id or name...")
return False
# Parse the infobox monster
infobox_parser = WikitextTemplateParser(self.monster_wikitext)
# Try extract infobox for monster
self.has_infobox = infobox_parser.extract_infobox("infobox monster")
if not self.has_infobox:
logging.critical("CRITICAL: Could not find template...")
return False
self.is_versioned = infobox_parser.determine_infobox_versions()
logging.debug(f"preprocessing: Is the infobox versioned: {self.is_versioned}")
self.versioned_ids = infobox_parser.extract_infobox_ids()
logging.debug(f"preprocessing: Versioned IDs: {self.versioned_ids}")
# Set the infobox version number, default to empty string (no version number)
try:
if self.versioned_ids:
self.infobox_version_number = self.versioned_ids[self.monster_id_int]
except KeyError:
if self.is_versioned:
self.infobox_version_number = "1"
else:
self.infobox_version_number = ""
logging.debug(f"preprocessing: infobox_version_number: {self.infobox_version_number}")
# Set the template
self.template = infobox_parser.template
return True
def populate_equipable_properties_from_wiki_data(self) -> bool:
"""Parse the wiki text template and extract item bonus values from it."""
# Initialize empty equipment dictionary
self.item_dict["equipment"] = dict()
# Extract the infobox bonuses template
infobox_parser = WikitextTemplateParser(self.item_wikitext)
has_infobox = infobox_parser.extract_infobox("infobox bonuses")
if not has_infobox:
has_infobox = infobox_parser.extract_infobox("infobox_bonuses")
if not has_infobox:
# No infobox bonuses found for the item!
print(
"populate_equipable_properties: Item has no equipment infobox."
)
logging.critical(
"populate_equipable_properties: Item has no equipment infobox."
)
quit()
# Set the template
template = infobox_parser.template
# STAGE ONE: EQUIPABLE ITEM
# This item must be equipable by a player, set to True
self.item_dict["equipable_by_player"] = True
# Extract equipable item properties
self.item_dict["equipment"]["attack_stab"] = self.clean_bonuses_value(
template, "astab")
self.item_dict["equipment"]["attack_slash"] = self.clean_bonuses_value(
template, "aslash")
self.item_dict["equipment"]["attack_crush"] = self.clean_bonuses_value(
template, "acrush")
self.item_dict["equipment"]["attack_magic"] = self.clean_bonuses_value(
template, "amagic")
self.item_dict["equipment"][
"attack_ranged"] = self.clean_bonuses_value(template, "arange")
self.item_dict["equipment"]["defence_stab"] = self.clean_bonuses_value(
template, "dstab")
self.item_dict["equipment"][
"defence_slash"] = self.clean_bonuses_value(template, "dslash")
self.item_dict["equipment"][
"defence_crush"] = self.clean_bonuses_value(template, "dcrush")
self.item_dict["equipment"][
"defence_magic"] = self.clean_bonuses_value(template, "dmagic")
self.item_dict["equipment"][
"defence_ranged"] = self.clean_bonuses_value(template, "drange")
self.item_dict["equipment"][
"melee_strength"] = self.clean_bonuses_value(template, "str")
self.item_dict["equipment"][
"ranged_strength"] = self.clean_bonuses_value(template, "rstr")
self.item_dict["equipment"]["magic_damage"] = self.clean_bonuses_value(
template, "mdmg")
self.item_dict["equipment"]["prayer"] = self.clean_bonuses_value(
template, "prayer")
# Determine the slot for the equipable item
self.item_dict["equipment"]["slot"] = None
try:
self.item_dict["equipment"]["slot"] = self.strip_infobox(
template.get("slot").value)
self.item_dict["equipment"]["slot"] = self.item_dict["equipment"][
"slot"].lower()
except ValueError:
self.item_dict["equipment"]["slot"] = None
print(
"populate_equipable_properties: Could not determine item slot..."
)
logging.critical(
"populate_equipable_properties: Could not determine item slot..."
)
quit()
# Determine the skill requirements for the equipable item
self.item_dict["equipment"]["requirements"] = None
try:
requirements = self.skill_requirements[str(self.item_id)]
self.item_dict["equipment"]["requirements"] = requirements
except KeyError:
print(
"populate_equipable_properties: Could not determine skill requirements..."
)
logging.critical(
"populate_equipable_properties: Could not determine skill requirements..."
)
quit()
# STAGE TWO: WEAPONS
# If item is weapon, two-handed, or 2h, start processing the weapon data
if (self.item_dict["equipment"]["slot"] == "weapon"
or self.item_dict["equipment"]["slot"] == "two-handed"
or self.item_dict["equipment"]["slot"] == "2h"):
self.item_dict["weapon"] = dict()
# Try set the attack speed of the weapon
try:
self.item_dict["weapon"]["attack_speed"] = int(
self.strip_infobox(template.get("speed").value))
except ValueError:
self.item_dict["weapon"]["attack_speed"] = None
logging.critical(
"WEAPON: Could not determine weapon attack speed")
# Item IDs with no known attack speed, set to zero
if int(self.item_id) in [8871]:
self.item_dict["weapon"]["attack_speed"] = 0
# Salamander fix, set to base attack speed of 5
elif int(self.item_id) in [
10145, 10146, 10147, 10147, 10148, 10149
]:
self.item_dict["weapon"]["attack_speed"] = 5
else:
pass
# quit()
# Try to set the weapon type of the weapon
try:
weapon_type = self.weapon_types_data[str(
self.item_dict["id"])]["weapon_type"]
self.item_dict["weapon"]["weapon_type"] = weapon_type
except KeyError:
self.item_dict["weapon"]["weapon_type"] = None
print(
"populate_equipable_properties: Could not determine weapon type..."
)
logging.critical(
"populate_equipable_properties: Could not determine weapon type..."
)
quit()
# Try to set stances available for the weapon
try:
self.item_dict["weapon"]["stances"] = self.weapon_stances_data[
self.item_dict["weapon"]["weapon_type"]]
except KeyError:
self.item_dict["weapon"]["stances"] = None
print(
"populate_equipable_properties: Could not determine weapon stance..."
)
logging.critical(
"populate_equipable_properties: Could not determine weapon stance..."
)
quit()
# Finally, set the equipable_weapon property to true
self.item_dict["equipable_weapon"] = True
else:
# If the item is not a weapon, two-handed or 2h it is not a weapon
self.item_dict["equipable_weapon"] = False
return True
def populate_item(self):
"""Populate an item after preprocessing it.
This is called for every item in the OSRS cache dump. Start by populating the
raw metadata from the cache. Then process invalid items, and """
# Start by populating the item from the cache data
self.populate_from_cache_data()
# Process an invalid item
if self.is_invalid_item:
logging.debug(
"populate_item: Found and processing an invalid item...")
if self.status == "unequipable":
# Cache thinks the item is equipable, but it is not
self.populate_item_properties_from_wiki_data()
self.item_dict["equipable_by_player"] = False
self.item_dict["equipable_weapon"] = False
self.item_dict["incomplete"] = True
return True
if self.status == "normalized":
# Some items have a wiki page, but lookup by ID, linked ID and item name
# fail. So use the normalized name from the invalid-items.json file
self.item_wikitext = self.all_wikitext_raw[
self.normalized_name]
self.wikitext_found_using = "normalized_name"
infobox_parser = WikitextTemplateParser(self.item_wikitext)
infobox_parser.extract_infobox("infobox item")
self.template = infobox_parser.template
self.populate_item_properties_from_wiki_data()
self.item_dict["equipable_by_player"] = False
self.item_dict["equipable_weapon"] = False
return True
if self.status == "unobtainable":
# Some items are unobtainable, set defaults
self.populate_non_wiki_item()
return True
if self.status == "skill_guide_icon":
# Some items are actually an icon in a skill guide, set defaults
self.populate_non_wiki_item()
return True
if self.status == "construction_icon":
# Some items are actually an icon in the construction interface, set defaults
self.populate_non_wiki_item()
return True
if self.status == "unhandled":
# Some items have not been classified, set defaults
self.populate_non_wiki_item()
return True
if not self.item_dict["equipable"]:
# Process a normal, non-equipable item
logging.debug(
"populate_item: Populating a normal item using wiki data...")
self.populate_item_properties_from_wiki_data()
self.item_dict["equipable_by_player"] = False
self.item_dict["equipable_weapon"] = False
return True
if self.item_dict["equipable"]:
# Process an equipable item
logging.debug(
"populate_item: Populating an equipable item using wiki data..."
)
self.populate_item_properties_from_wiki_data()
self.populate_equipable_properties_from_wiki_data()
return True
# Return false by default, this means the item was not found or processed
logging.error("populate_item: Item was not processed...")
return False
def preprocessing(self) -> Dict:
"""Preprocess an item, and set important object variables.
This function preprocesses every item dumped from the OSRS cache. Various
properties are set to help further processing. Items are determined if
they are a linked item (noted/placeholder), or an actual item. The item
is checked if it is a valid item (has a wiki page, is an actual item etc.).
Finally, the wikitext (from the OSRS wiki) is found by looking up ID, linked
ID, name, and normalized name. The `Infobox Item` or `Infobox Pet` is then
extracted so that the wiki properties can be later processed and populated.
:return: A dictionary including success and code.
"""
# Initialize dictionary to return preprocessing status
return_status = {"status": False, "code": None}
# Set item ID variables
self.item_id_int = int(self.item_id) # Item ID number as an integer
self.item_id_str = str(self.item_id) # Item ID number as a string
# Load item dictionary of cache data based on item ID
# This raw cache data is the baseline information about the specific item
# and can be considered 100% correct and available for every item
self.item_cache_data = self.all_item_cache_data[self.item_id_str]
# Set item name variable (directly from the cache dump)
self.item_name = self.item_cache_data["name"]
# Log and print item
logging.debug(
f"======================= {self.item_id_str} {self.item_name}")
if self.verbose:
print(
f"======================= {self.item_id_str} {self.item_name}")
logging.debug("preprocessing: using the following cache data:")
logging.debug(self.item_cache_data)
# Get the linked ID item value, if available
self.linked_id_item_int = None
self.linked_id_item_str = None
if self.item_cache_data["linked_id_item"] is not None:
self.linked_id_item_int = int(
self.item_cache_data["linked_id_item"])
self.linked_id_item_str = str(
self.item_cache_data["linked_id_item"])
logging.debug(
f"preprocessing: Linked item ID: {self.linked_id_item_str}")
# Determine the ID number to extract
# Noted and placeholder items should use the linked_id_item property
# to fill in additional wiki data...
self.item_id_to_process_int = None
self.item_id_to_process_str = None
if self.item_cache_data["noted"] is True or self.item_cache_data[
"placeholder"] is True:
self.item_id_to_process_int = int(self.linked_id_item_int)
self.item_id_to_process_str = str(self.linked_id_item_str)
else:
self.item_id_to_process_int = int(self.item_id)
self.item_id_to_process_str = str(self.item_id)
logging.debug(
f"preprocessing: ID to process: {self.item_id_to_process_str}")
# Find the wiki page
# Set all variables to None (for invalid items)
self.item_wikitext = None
self.wikitext_found_using = None
self.has_infobox = False
# Try to find the wiki data using direct ID number search
if self.all_wikitext_processed.get(self.item_id_str, None):
self.item_wikitext = self.all_wikitext_processed.get(
self.item_id_str, None)
self.wikitext_found_using = "id"
return_status["code"] = "lookup_passed_id"
# Try to find the wiki data using linked_id_item ID number search
elif self.all_wikitext_processed.get(self.linked_id_item_str, None):
self.item_wikitext = self.all_wikitext_processed.get(
self.linked_id_item_str, None)
self.wikitext_found_using = "linked_id"
return_status["code"] = "lookup_passed_linked_id"
# Try to find the wiki data using direct name search
elif self.all_wikitext_raw.get(self.item_name, None):
self.item_wikitext = self.all_wikitext_raw.get(
self.item_name, None)
self.wikitext_found_using = "name"
return_status["code"] = "lookup_passed_name"
if self.item_id_to_process_str in self.invalid_items_data:
# Anything here means item cannot be found by id, linked_id, or name
# This can include not being an actual item, has no wiki page etc.
# The item must be invalid, handle it accordingly
self.is_invalid_item = True
try:
self.status = self.invalid_items_data[
self.item_id_to_process_str]["status"]
self.normalized_name = self.invalid_items_data[
self.item_id_to_process_str]["normalized_name"]
except KeyError:
self.status = None
self.normalized_name = None
logging.debug(
f"preprocessing: Invalid item details: {self.is_invalid_item} {self.status} {self.normalized_name}"
)
# Try to find the wiki data using normalized_name search
if self.all_wikitext_raw.get(self.normalized_name, None):
self.item_wikitext = self.all_wikitext_raw.get(
self.normalized_name, None)
self.wikitext_found_using = "normalized_name"
return_status["code"] = "lookup_passed_normalized_name"
else:
return_status["code"] = "lookup_failed"
logging.debug(
f"preprocessing: self.item_wikitext found using: {self.wikitext_found_using}"
)
# If there is no wikitext, and the item is valid, raise a critical error
if not self.item_wikitext and not self.is_invalid_item:
logging.critical(
"CRITICAL: Could not find item_wikitext by id, linked_id_item or name..."
)
return_status["code"] = "no_item_wikitext"
return return_status
# Parse the infobox item
infobox_parser = WikitextTemplateParser(self.item_wikitext)
# Try extract infobox for item, then pet
self.has_infobox = infobox_parser.extract_infobox("infobox item")
if not self.has_infobox:
self.has_infobox = infobox_parser.extract_infobox("infobox pet")
if not self.has_infobox:
self.template = None
logging.critical("CRITICAL: Could not find template...")
return_status["code"] = "no_infobox_template"
return return_status
self.is_versioned = infobox_parser.determine_infobox_versions()
logging.debug(
f"preprocessing: Is the infobox versioned: {self.is_versioned}")
self.versioned_ids = infobox_parser.extract_infobox_ids()
logging.debug(f"preprocessing: Versioned IDs: {self.versioned_ids}")
# Set the infobox version number, default to empty string (no version number)
try:
if self.versioned_ids:
self.infobox_version_number = self.versioned_ids[
self.item_id_to_process_int]
except KeyError:
if self.is_versioned:
self.infobox_version_number = "1"
else:
self.infobox_version_number = ""
logging.debug(
f"preprocessing: infobox_version_number: {self.infobox_version_number}"
)
# Set the template
self.template = infobox_parser.template
return_status["status"] = True
return return_status
def fetch():
"""Fetch monster drops using SMW queries.
This is a request heavy method - querying about 1,000 endpoints
to get monster drop data.
"""
# Load the monster wikitext file of processed data
with open(
Path(config.DATA_MONSTERS_PATH /
"monsters-wiki-page-text-processed.json")) as f:
all_wikitext_processed = json.load(f)
# Load the raw cache data that has been processed (this is ground truth)
with open(Path(config.DATA_MONSTERS_PATH /
"monsters-cache-data.json")) as f:
all_monster_cache_data = json.load(f)
for monster_id, monster_list in all_wikitext_processed.items():
exists = all_monster_cache_data.get(monster_id, None)
if not exists:
continue
if "dropversion" in monster_list[2].lower():
name = all_monster_cache_data[monster_id]["name"]
wikitext = monster_list[2]
version = monster_list[1]
wikitext_template = WikitextTemplateParser(wikitext)
wikitext_template.extract_infobox("infobox monster")
value = wikitext_template.extract_infobox_value(
f"dropversion{version}")
if not value:
value = wikitext_template.extract_infobox_value(
f"dropversion1")
multi_drop_tables[monster_id] = f"[[Dropped from::{name}#{value}]]"
api_url = "https://oldschool.runescape.wiki/api.php"
# Specify what the SMW query should return
selection = "|?Dropped item|?Drop Quantity|?Rarity|?Rolls|limit=500"
# Set parameters to run a SMW query
params = {"action": "ask", "format": "json", "query": None}
# Data structures for storing conditions
# Conditions are used to form the SMW query
conditions_set = set()
conditions_dict = defaultdict(list)
# Loop raw monster cache data (ground truth)
for monster_id, monster in all_monster_cache_data.items():
if monster_id in multi_drop_tables:
condition = multi_drop_tables[monster_id]
else:
condition = f"[[Dropped from::{monster['name']}]]"
# Add to set of conditions to later query
conditions_set.add(condition)
# Add condition string for monster ID lookup
conditions_dict[condition].append(monster_id)
print(f">>> Fetching {len(conditions_set)} drop tables...")
# Start fetching the data
count = 0
for condition in conditions_set:
print(f" > Processing {count}/{len(conditions_set)}: {condition}")
params["query"] = f"{condition}{selection}"
r = requests.get(api_url, headers=config.custom_agent, params=params)
data = r.json()
Path(config.DATA_MONSTERS_PATH / "monsters-drops-raw").mkdir(
parents=True, exist_ok=True)
for monster_id in conditions_dict[condition]:
file_name = f"{monster_id}.json"
file_path = Path(config.DATA_MONSTERS_PATH / "monsters-drops-raw" /
file_name)
with open(file_path, "w") as f:
json.dump(data, f, indent=4)
count += 1