def extract_dropsline_header(self, table_head_type: str) -> float:
"""Parse the dropstablehead template for variable drop rarity values.
:param table_head_type: Specify a seed, or herb table head search.
:return: A float of the drop rarity multiplier.
"""
# Extract "dropstablehead" templates
# This is used for extracting "herbbase" and "seedbase" values
self.drops_templates = wikitext_parser.extract_wikitext_template(
self.monster_wikitext, "dropstablehead")
table_head_value = None
# Loop the found "dropstablehead" templates
for template in self.drops_templates:
# Parse the template
template_parser = WikitextTemplateParser(self.monster_wikitext)
template_parser.template = template
if "vardefine:" + table_head_type in template:
# Example:
# {{DropsTableHead{{#vardefine:herbbase|{{#expr:9/123/128}}}}}}
table_head_value = template.split("#expr:")[1]
table_head_value = table_head_value.replace("}", "")
table_head_value = eval(table_head_value)
return table_head_value
def populate_equipable_properties_from_wiki_data(self) -> bool:
"""Parse the wiki text template and extract item bonus values from it."""
# Initialize empty equipment dictionary
self.item_dict["equipment"] = dict()
# Extract the infobox bonuses template
infobox_parser = WikitextTemplateParser(self.item_wikitext)
has_infobox = infobox_parser.extract_infobox("infobox bonuses")
if not has_infobox:
has_infobox = infobox_parser.extract_infobox("infobox_bonuses")
if not has_infobox:
# No infobox bonuses found for the item!
print(
"populate_equipable_properties: Item has no equipment infobox."
)
logging.critical(
"populate_equipable_properties: Item has no equipment infobox."
)
quit()
# Set the template
template = infobox_parser.template
# STAGE ONE: EQUIPABLE ITEM
# This item must be equipable by a player, set to True
self.item_dict["equipable_by_player"] = True
# Extract equipable item properties
self.item_dict["equipment"]["attack_stab"] = self.clean_bonuses_value(
template, "astab")
self.item_dict["equipment"]["attack_slash"] = self.clean_bonuses_value(
template, "aslash")
self.item_dict["equipment"]["attack_crush"] = self.clean_bonuses_value(
template, "acrush")
self.item_dict["equipment"]["attack_magic"] = self.clean_bonuses_value(
template, "amagic")
self.item_dict["equipment"][
"attack_ranged"] = self.clean_bonuses_value(template, "arange")
self.item_dict["equipment"]["defence_stab"] = self.clean_bonuses_value(
template, "dstab")
self.item_dict["equipment"][
"defence_slash"] = self.clean_bonuses_value(template, "dslash")
self.item_dict["equipment"][
"defence_crush"] = self.clean_bonuses_value(template, "dcrush")
self.item_dict["equipment"][
"defence_magic"] = self.clean_bonuses_value(template, "dmagic")
self.item_dict["equipment"][
"defence_ranged"] = self.clean_bonuses_value(template, "drange")
self.item_dict["equipment"][
"melee_strength"] = self.clean_bonuses_value(template, "str")
self.item_dict["equipment"][
"ranged_strength"] = self.clean_bonuses_value(template, "rstr")
self.item_dict["equipment"]["magic_damage"] = self.clean_bonuses_value(
template, "mdmg")
self.item_dict["equipment"]["prayer"] = self.clean_bonuses_value(
template, "prayer")
# Determine the slot for the equipable item
self.item_dict["equipment"]["slot"] = None
try:
self.item_dict["equipment"]["slot"] = self.strip_infobox(
template.get("slot").value)
self.item_dict["equipment"]["slot"] = self.item_dict["equipment"][
"slot"].lower()
except ValueError:
self.item_dict["equipment"]["slot"] = None
print(
"populate_equipable_properties: Could not determine item slot..."
)
logging.critical(
"populate_equipable_properties: Could not determine item slot..."
)
quit()
# Determine the skill requirements for the equipable item
self.item_dict["equipment"]["requirements"] = None
try:
requirements = self.skill_requirements[str(self.item_id)]
self.item_dict["equipment"]["requirements"] = requirements
except KeyError:
self.item_dict["equipment"]["requirements"] = None
# STAGE TWO: WEAPONS
# If item is weapon, two-handed, or 2h, start processing the weapon data
if (self.item_dict["equipment"]["slot"] == "weapon"
or self.item_dict["equipment"]["slot"] == "two-handed"
or self.item_dict["equipment"]["slot"] == "2h"):
self.item_dict["weapon"] = dict()
# Try set the attack speed of the weapon
try:
self.item_dict["weapon"]["attack_speed"] = int(
self.strip_infobox(template.get("aspeed").value))
except ValueError:
self.item_dict["weapon"]["attack_speed"] = None
logging.critical(
"WEAPON: Could not determine weapon attack speed")
# Item IDs with no known attack speed, set to zero
if int(self.item_id) in [8871]:
self.item_dict["weapon"]["attack_speed"] = 0
# Salamander fix, set to base attack speed of 5
elif int(self.item_id) in [
10145, 10146, 10147, 10147, 10148, 10149
]:
self.item_dict["weapon"]["attack_speed"] = 5
else:
pass
# quit()
# Try to set the weapon type of the weapon
try:
weapon_type = self.weapon_types_data[str(
self.item_dict["id"])]["weapon_type"]
self.item_dict["weapon"]["weapon_type"] = weapon_type
except KeyError:
self.item_dict["weapon"]["weapon_type"] = None
print(
"populate_equipable_properties: Could not determine weapon type..."
)
logging.critical(
"populate_equipable_properties: Could not determine weapon type..."
)
quit()
# Try to set stances available for the weapon
try:
self.item_dict["weapon"]["stances"] = self.weapon_stances_data[
self.item_dict["weapon"]["weapon_type"]]
except KeyError:
self.item_dict["weapon"]["stances"] = None
print(
"populate_equipable_properties: Could not determine weapon stance..."
)
logging.critical(
"populate_equipable_properties: Could not determine weapon stance..."
)
quit()
# Finally, set the equipable_weapon property to true
self.item_dict["equipable_weapon"] = True
else:
# If the item is not a weapon, two-handed or 2h it is not a weapon
self.item_dict["equipable_weapon"] = False
return True
def populate_item(self):
"""Populate an item after preprocessing it.
This is called for every item in the OSRS cache dump. Start by populating the
raw metadata from the cache. Then process invalid items, and """
# Start by populating the item from the cache data
self.populate_from_cache_data()
# Process an invalid item
if self.is_invalid_item:
logging.debug(
"populate_item: Found and processing an invalid item...")
if self.status == "unequipable":
# Cache thinks the item is equipable, but it is not
self.populate_item_properties_from_wiki_data()
self.item_dict["equipable_by_player"] = False
self.item_dict["equipable_weapon"] = False
self.item_dict["incomplete"] = True
return True
if self.status == "no_bonuses_available":
# Equipable item with wiki page, but does not have an Infobox Bonuses template
# This is only ever called on ring slot items, as they sometimes have a
# wiki page without an Infobox Bonuses template
self.populate_non_wiki_item()
self.item_dict["equipment"] = dict()
for equipment_property in self.equipment_properties:
self.item_dict["equipment"][equipment_property] = 0
self.item_dict["equipment"]["slot"] = "ring"
self.item_dict["equipment"]["requirements"] = None
self.item_dict["equipable_by_player"] = True
self.item_dict["equipable_weapon"] = False
self.item_dict["incomplete"] = True
return True
if self.status == "normalized":
# Some items have a wiki page, but lookup by ID, linked ID and item name
# fail. So use the normalized name from the invalid-items.json file
self.item_wikitext = self.all_wikitext_raw[
self.normalized_name]
self.wikitext_found_using = "normalized_name"
infobox_parser = WikitextTemplateParser(self.item_wikitext)
infobox_parser.extract_infobox("infobox item")
self.template = infobox_parser.template
self.populate_item_properties_from_wiki_data()
self.item_dict["equipable_by_player"] = False
self.item_dict["equipable_weapon"] = False
return True
if self.status == "unobtainable":
# Some items are unobtainable, set defaults
self.populate_non_wiki_item()
return True
if self.status == "skill_guide_icon":
# Some items are actually an icon in a skill guide, set defaults
self.populate_non_wiki_item()
return True
if self.status == "construction_icon":
# Some items are actually an icon in the construction interface, set defaults
self.populate_non_wiki_item()
return True
if self.status == "unhandled":
# Some items have not been classified, set defaults
self.populate_non_wiki_item()
return True
if not self.item_dict["equipable"]:
# Process a normal, non-equipable item
logging.debug(
"populate_item: Populating a normal item using wiki data...")
self.populate_item_properties_from_wiki_data()
self.item_dict["equipable_by_player"] = False
self.item_dict["equipable_weapon"] = False
return True
if self.item_dict["equipable"]:
# Process an equipable item
logging.debug(
"populate_item: Populating an equipable item using wiki data..."
)
self.populate_item_properties_from_wiki_data()
self.populate_equipable_properties_from_wiki_data()
return True
# Return false by default, this means the item was not found or processed
logging.error("populate_item: Item was not processed...")
return False
def preprocessing(self) -> Dict:
"""Preprocess an item, and set important object variables.
This function preprocesses every item dumped from the OSRS cache. Various
properties are set to help further processing. Items are determined if
they are a linked item (noted/placeholder), or an actual item. The item
is checked if it is a valid item (has a wiki page, is an actual item etc.).
Finally, the wikitext (from the OSRS wiki) is found by looking up ID, linked
ID, name, and normalized name. The `Infobox Item` or `Infobox Pet` is then
extracted so that the wiki properties can be later processed and populated.
:return: A dictionary including success and code.
"""
# Initialize dictionary to return preprocessing status
return_status = {"status": False, "code": None}
# Set item ID variables
self.item_id_int = int(self.item_id) # Item ID number as an integer
self.item_id_str = str(self.item_id) # Item ID number as a string
# Load item dictionary of cache data based on item ID
# This raw cache data is the baseline information about the specific item
# and can be considered 100% correct and available for every item
self.item_cache_data = self.all_item_cache_data[self.item_id_str]
# Set item name variable (directly from the cache dump)
self.item_name = self.item_cache_data["name"]
# Log and print item
logging.debug(
f"======================= {self.item_id_str} {self.item_name}")
# print(f"======================= {self.item_id_str} {self.item_name}")
logging.debug(f"preprocessing: using the following cache data:")
logging.debug(self.item_cache_data)
# Get the linked ID item value, if available
self.linked_id_item_int = None
self.linked_id_item_str = None
if self.item_cache_data["linked_id_item"] is not None:
self.linked_id_item_int = int(
self.item_cache_data["linked_id_item"])
self.linked_id_item_str = str(
self.item_cache_data["linked_id_item"])
logging.debug(
f"preprocessing: Linked item ID: {self.linked_id_item_str}")
# Determine the ID number to extract
# Noted and placeholder items should use the linked_id_item property
# to fill in additional wiki data...
self.item_id_to_process_int = None
self.item_id_to_process_str = None
if self.item_cache_data["noted"] is True or self.item_cache_data[
"placeholder"] is True:
self.item_id_to_process_int = int(self.linked_id_item_int)
self.item_id_to_process_str = str(self.linked_id_item_str)
else:
self.item_id_to_process_int = int(self.item_id)
self.item_id_to_process_str = str(self.item_id)
logging.debug(
f"preprocessing: ID to process: {self.item_id_to_process_str}")
# Find the wiki page
# Set all variables to None (for invalid items)
self.item_wikitext = None
self.wikitext_found_using = None
self.has_infobox = False
# Try to find the wiki data using direct ID number search
if self.all_wikitext_processed.get(self.item_id_str, None):
self.item_wikitext = self.all_wikitext_processed.get(
self.item_id_str, None)
self.wikitext_found_using = "id"
return_status["code"] = "lookup_passed_id"
# Try to find the wiki data using linked_id_item ID number search
elif self.all_wikitext_processed.get(self.linked_id_item_str, None):
self.item_wikitext = self.all_wikitext_processed.get(
self.linked_id_item_str, None)
self.wikitext_found_using = "linked_id"
return_status["code"] = "lookup_passed_linked_id"
# Try to find the wiki data using direct name search
elif self.all_wikitext_raw.get(self.item_name, None):
self.item_wikitext = self.all_wikitext_raw.get(
self.item_name, None)
self.wikitext_found_using = "name"
return_status["code"] = "lookup_passed_name"
if self.item_id_to_process_str in self.invalid_items_data:
# Anything here means item cannot be found by id, linked_id, or name
# This can include not being an actual item, has no wiki page etc.
# The item must be invalid, handle it accordingly
self.is_invalid_item = True
try:
self.status = self.invalid_items_data[
self.item_id_to_process_str]["status"]
self.normalized_name = self.invalid_items_data[
self.item_id_to_process_str]["normalized_name"]
except KeyError:
self.status = None
self.normalized_name = None
logging.debug(
f"preprocessing: Invalid item details: {self.is_invalid_item} {self.status} {self.normalized_name}"
)
# Try to find the wiki data using normalized_name search
if self.all_wikitext_raw.get(self.normalized_name, None):
self.item_wikitext = self.all_wikitext_raw.get(
self.normalized_name, None)
self.wikitext_found_using = "normalized_name"
return_status["code"] = "valid"
else:
return_status["code"] = "lookup_failed"
logging.debug(
f"preprocessing: self.item_wikitext found using: {self.wikitext_found_using}"
)
# If there is no wikitext, and the item is valid, raise a critical error
if not self.item_wikitext and not self.is_invalid_item:
logging.critical(
"CRITICAL: Could not find item_wikitext by id, linked_id_item or name..."
)
return_status["code"] = "no_item_wikitext"
return return_status
# Parse the infobox item
infobox_parser = WikitextTemplateParser(self.item_wikitext)
# Try extract infobox for item, then pet
self.has_infobox = infobox_parser.extract_infobox("infobox item")
if not self.has_infobox:
self.has_infobox = infobox_parser.extract_infobox("infobox pet")
if not self.has_infobox:
self.template = None
logging.critical("CRITICAL: Could not find template...")
return_status["code"] = "no_infobox_template"
return return_status
self.is_versioned = infobox_parser.determine_infobox_versions()
logging.debug(
f"preprocessing: Is the infobox versioned: {self.is_versioned}")
self.versioned_ids = infobox_parser.extract_infobox_ids()
logging.debug(f"preprocessing: Versioned IDs: {self.versioned_ids}")
# Set the infobox version number, default to empty string (no version number)
try:
if self.versioned_ids:
self.infobox_version_number = self.versioned_ids[
self.item_id_to_process_int]
except KeyError:
if self.is_versioned:
self.infobox_version_number = "1"
else:
self.infobox_version_number = ""
logging.debug(
f"preprocessing: infobox_version_number: {self.infobox_version_number}"
)
# Set the template
self.template = infobox_parser.template
return_status["status"] = True
return return_status
def parse_monster_drops(self):
"""Extract monster drop information.
This function parses the monsters wiki page (raw wikitext), and extracts
any MediaWiki template with the name "dropsline". Each template is processed
to determine the ID, name, quantity, rarity and and requirements of the
specific drop. Finally, the raw wikitext for the monster is parsed to
determine if the monster has access to the rare drop table.
"""
# Extract "dropsline" templates
self.drops_templates = wikitext_parser.extract_wikitext_template(self.monster_wikitext, "dropsline")
drops_dict_all = dict()
drops_list_ids = list()
self.monster_dict["drops"] = dict()
# Loop the found "dropsline" templates
for template in self.drops_templates:
# Parse the template
template_parser = WikitextTemplateParser(self.monster_wikitext)
template_parser.template = template
# Initialize a null value dictionary for each item drop
drop_dict = {
"id": None,
"name": None,
"members": None,
"quantity": None,
"rarity": None,
"drop_requirements": None
}
# Extract the drop information...
# Extract the item drop name
name = template_parser.extract_infobox_value("Name")
# Skip any drop line with classified drop table
if name.lower() == "drop table":
continue
# Determine the drop item ID
item_id = None
found = False
for item in self.all_db_items:
if item.name == name or item.wiki_name == name:
found = True
item_id = item.id
break
if not found:
item_id = None
# Extract the item drop quantity and if the drop is noted
quantity = template_parser.extract_infobox_value("Quantity")
noted = False
if quantity:
if "noted" in quantity.lower():
noted = True
quantity = infobox_cleaner.clean_drop_quantity(quantity)
# Extract, or determine, if the item is members-only
members = False
name_notes = template_parser.extract_infobox_value("Namenotes")
if self.monster_dict["members"]:
members = True
elif item_id:
if self.all_db_items[item_id].members:
members = True
elif name_notes:
if "{{m}}" in name_notes:
members = True
# Extract the item drop rarity
rarity = template_parser.extract_infobox_value("Rarity")
base_value = None
# If the item drop has a drop variable, fetch it
if rarity:
if "var:herbbase" in rarity:
base_value = self.extract_dropsline_header("herbbase")
elif "var:seedbase" in rarity:
base_value = self.extract_dropsline_header("seedbase")
elif "var:uht" in rarity:
base_value = self.extract_dropsline_header("uht")
if not base_value:
base_value = "(22.5/250)/16" # Temp fix for Lizardman shaman
elif "var:bolttipbase" in rarity:
base_value = self.extract_dropsline_header("uht")
if not base_value:
base_value = "(2/128)/40" # Temp fix for Hydra
rarity = infobox_cleaner.clean_drop_rarity(rarity, base_value)
# Extract the rarity notes
drop_requirements = template_parser.extract_infobox_value("Raritynotes")
drop_requirements = infobox_cleaner.clean_drop_requirements(drop_requirements)
# Populate the dictionary
drop_dict = {
"id": item_id,
"name": name,
"members": members,
"quantity": quantity,
"noted": noted,
"rarity": rarity,
"drop_requirements": drop_requirements
}
# Attach drops dict to the drops object for this monster
if item_id:
# self.drops[item_id] = drop_dict
drops_list_ids.append(str(item_id))
drops_dict_all[str(item_id)] = drop_dict
# Order list of drops using drops_list_ids
drops_list_ids.sort(key=int)
for item_id in drops_list_ids:
# Add the drops to the monster dictionary
self.monster_dict["drops"][item_id] = drops_dict_all[item_id]
# Determine if monster has access to the rare drop table
if "{{raredroptable" in self.monster_wikitext[2].lower():
self.monster_dict["rare_drop_table"] = True
elif "{{gemdroptable" in self.monster_wikitext[2].lower():
self.monster_dict["rare_drop_table"] = True
else:
self.monster_dict["rare_drop_table"] = False
def preprocessing(self):
"""Preprocess an monster, and set important object variables.
This function preprocesses every monster dumped from the OSRS cache. Various
properties are set to help further processing. MORE."""
# Set monster ID variables
self.monster_id_int = int(self.monster_id) # Monster ID number as an integer
self.monster_id_str = str(self.monster_id) # Monster ID number as a string
# Load monster dictionary of cache data based on monster ID
# This raw cache data is the baseline information about the specific monster
# and can be considered 100% correct and available for every monster
self.monster_cache_data = self.all_monster_cache_data[self.monster_id_str]
# Set monster name variable (directly from the cache dump)
self.monster_name = self.monster_cache_data["name"]
# Log and print monster
logging.debug(f"======================= {self.monster_id_str} {self.monster_name}")
# print(f"======================= {self.monster_id_str} {self.monster_name}")
logging.debug(f"preprocessing: using the following cache data:")
logging.debug(self.monster_cache_data)
# Set all variables to None (for invalid monsters)
self.monster_wikitext = None
self.wikitext_found_using = None
self.has_infobox = False
# Try to find the wiki data using direct ID number search
if self.all_wikitext_processed.get(self.monster_id_str, None):
self.monster_wikitext = self.all_wikitext_processed.get(self.monster_id_str, None)
self.wikitext_found_using = "id"
# Try to find the wiki data using direct name search
elif self.all_wikitext_raw.get(self.monster_name, None):
self.monster_wikitext = self.all_wikitext_raw.get(self.monster_name, None)
self.wikitext_found_using = "name"
logging.debug(f"preprocessing: self.monster_wikitext found using: {self.wikitext_found_using}")
# If there is no wikitext, and the monster is valid, raise a critical error
if not self.monster_wikitext:
logging.critical("CRITICAL: Could not find monster_wikitext by id or name...")
return False
# Parse the infobox monster
infobox_parser = WikitextTemplateParser(self.monster_wikitext)
# Try extract infobox for monster
self.has_infobox = infobox_parser.extract_infobox("infobox monster")
if not self.has_infobox:
logging.critical("CRITICAL: Could not find template...")
return False
self.is_versioned = infobox_parser.determine_infobox_versions()
logging.debug(f"preprocessing: Is the infobox versioned: {self.is_versioned}")
self.versioned_ids = infobox_parser.extract_infobox_ids()
logging.debug(f"preprocessing: Versioned IDs: {self.versioned_ids}")
# Set the infobox version number, default to empty string (no version number)
try:
if self.versioned_ids:
self.infobox_version_number = self.versioned_ids[self.monster_id_int]
except KeyError:
if self.is_versioned:
self.infobox_version_number = "1"
else:
self.infobox_version_number = ""
logging.debug(f"preprocessing: infobox_version_number: {self.infobox_version_number}")
# Set the template
self.template = infobox_parser.template
return True
def parse_monster_drops(self):
"""Extract monster drop information.
This function parses the monsters wiki page (raw wikitext), and extracts
any MediaWiki template with the name "dropsline". Each template is processed
to determine the ID, name, quantity, rarity and and requirements of the
specific drop. Finally, the raw wikitext for the monster is parsed to
determine if the monster has access to the rare drop table.
"""
# Extract "dropsline" templates
self.drops_templates = wikitext_parser.extract_wikitext_template(
self.monster_wikitext, "dropsline")
# Loop the found "dropsline" templates
for template in self.drops_templates:
# Parse the template
template_parser = WikitextTemplateParser(self.monster_wikitext)
template_parser.template = template
# Initialize a null value dictionary for each item drop
drop_dict = {
"id": None,
"name": None,
"quantity": None,
"rarity": None,
"drop_requirements": None
}
# Extract the drop information...
# Extract the item drop name
name = template_parser.extract_infobox_value("Name")
# Skip any drop line with rare drop table
if name.lower() == "rare drop table":
continue
# Determine the drop ID
id = None
found = False
for item in self.all_db_items:
if item.name == name or item.wiki_name == name:
found = True
id = item.id
break
if not found:
id = None
# Extract the item drop quantity and if the drop is noted
quantity = template_parser.extract_infobox_value("Quantity")
noted = False
if quantity:
if "noted" in quantity.lower():
noted = True
quantity = infobox_cleaner.clean_drop_quantity(quantity)
# Extract the item drop rarity
rarity = template_parser.extract_infobox_value("Rarity")
base_value = None
# If the item drop has a drop variable, fetch it
if rarity:
if "var:herbbase" in rarity:
base_value = self.extract_dropsline_header("herbbase")
elif "var:seedbase" in rarity:
base_value = self.extract_dropsline_header("seedbase")
rarity = infobox_cleaner.clean_drop_rarity(rarity, base_value)
# Extract the rarity notes
drop_requirements = template_parser.extract_infobox_value(
"Raritynotes")
drop_requirements = infobox_cleaner.clean_drop_requirements(
drop_requirements)
# Populate the dictionary
drop_dict = {
"id": id,
"name": name,
"quantity": quantity,
"noted": noted,
"rarity": rarity,
"drop_requirements": drop_requirements
}
# Attach drops dict to the drops list for this monster
self.drops.append(drop_dict)
# Append the drops to the monster dictionary
self.monster_dict["drops"] = self.drops
# Determine if monster has access to the rare drop table
if "{{RareDropTable}}" in self.monster_wikitext[2]:
self.monster_dict["rare_drop_table"] = True
else:
self.monster_dict["rare_drop_table"] = False