def export(self):
# Create ItemDefintion object
if "wiki_name" in self.item_dict:
del self.item_dict["wiki_name"]
if "store_price" in self.item_dict:
del self.item_dict["store_price"]
if "seller" in self.item_dict:
del self.item_dict["seller"]
if "equipable_weapon" not in self.item_dict:
self.item_dict["equipable_weapon"] = False
for prop in self.properties:
try:
self.item_dict[prop]
except KeyError:
self.item_dict[prop] = None
if self.item_dict["url"] == "https://oldschool.runescape.wiki/w/None":
self.item_dict["url"] = None
self.item_definition = ItemDefinition(**self.item_dict)
self.compare_json_files(self.item_definition)
json_out = self.item_definition.construct_json()
# Actually output a JSON file, comment out for testing
output_dir = os.path.join("..", "docs", "items-json")
self.item_definition.export_json(True, output_dir)
self.logger.debug(json_out)
return
def compare_json_files(self, item_definition: ItemDefinition) -> bool:
"""Print the difference between this item object, and the item that exists in the database.
:return changed: A boolean if the item is different, or not.
"""
changed = False
# Create JSON out object to compare
current_json = item_definition.construct_json()
# Try get existing entry (KeyError means it doesn't exist - aka a new item)
try:
existing_json = self.current_db[self.item_id]
except KeyError:
print(f">>> compare_json_files: NEW ITEM: {item_definition.id}")
print(current_json)
return changed
if current_json == existing_json:
return changed
ddiff = DeepDiff(existing_json, current_json, ignore_order=True)
print(f">>> compare_json_files: CHANGED ITEM: {item_definition.id}: {item_definition.name}")
print(f" {ddiff}")
changed = True
return changed
def export(self):
# Create ItemDefintion object
if "wiki_name" in self.item_dict:
del self.item_dict["wiki_name"]
if "store_price" in self.item_dict:
del self.item_dict["store_price"]
if "seller" in self.item_dict:
del self.item_dict["seller"]
self.itemDefinition = ItemDefinition(**self.item_dict)
self.compare_json_files(self.itemDefinition)
json_out = self.itemDefinition.construct_json()
# Actually output a JSON file, comment out for testing
output_dir = os.path.join("..", "docs", "items-json")
self.itemDefinition.export_json(True, output_dir)
self.logger.debug(json_out)
return
def _load_item(self, item_json: Dict) -> None:
"""Convert the `item_json` into a :class:`ItemDefinition` and store it."""
# Load the item using the ItemDefinition class
item_def = ItemDefinition(**item_json)
# Add item to list
self.all_items.append(item_def)
self.all_items_dict[item_def.id] = item_def
def _load_item(self, item_json: Dict) -> None:
"""Convert the `item_json` into a :class:`ItemDefinition` and store it.
:param item_json: A dict from an open and loaded JSON file.
:raises ValueError: Cannot populate item.
"""
# Load the item using the ItemDefinition class
try:
item_def = ItemDefinition.from_json(item_json)
except TypeError as e:
raise ValueError("Error: Invalid JSON structure found, check supplied input. Exiting") from e
# Add item to list
self.all_items.append(item_def)
self.all_items_dict[item_def.id] = item_def
def compare_json_files(self, itemDefinition: ItemDefinition) -> bool:
"""Print the difference between this item object, and the item that exists in the database.
:return changed: A boolean if the item is different, or not.
"""
changed = False
changes = dict()
# Create JSON out object to compare
current_json = itemDefinition.construct_json()
# Try get existing entry (KeyError means it doesn't exist - aka a new item)
try:
existing_json = self.current_db[self.item_id]
except KeyError:
return changed
for prop in self.properties:
if current_json[prop] != existing_json[prop]:
changed = True
changes[prop] = [current_json[prop], existing_json[prop]]
# Also check equipable
if itemDefinition.equipable_by_player:
for equipment_prop in self.equipment_properties:
try:
if current_json["equipment"][equipment_prop] != existing_json["equipment"][equipment_prop]:
changed = True
changes[equipment_prop] = [current_json["equipment"][equipment_prop],
existing_json["equipment"][equipment_prop]]
except KeyError:
pass # This should be fixed, when old/new item has no equipment key
# Print any item changes
if changed:
print(f">>>>>>>>>>> id: {itemDefinition.id}\tname: {itemDefinition.name}")
for prop in changes:
print("+++ MISMATCH!:", prop)
print("TYPES:", type(changes[prop][1]), type(changes[prop][0]))
print("OLD: %r" % changes[prop][1])
print("NEW: %r" % changes[prop][0])
print()
return changed
def check_duplicate_item(self) -> ItemDefinition:
"""Determine if this is a duplicate item.
:return: An ItemDeinition object.
"""
# Start by setting the duplicate property to False
self.item_dict["duplicate"] = False
# Create an ItemDefinition object
item_definition = ItemDefinition(**self.item_dict)
# Set the item properties that we want to compare
correlation_properties = {
"wiki_name": False,
"noted": False,
"placeholder": False,
"equipable": False,
"equipable_by_player": False,
"equipable_weapon": False
}
# Loop the list of currently (already processed) items
for known_item in self.known_item_names:
# Do a quick name check before deeper inspection
if item_definition.name != known_item.name:
continue
# If the cache names are equal, do further inspection
for cprop in correlation_properties:
if getattr(item_definition,
cprop) == getattr(known_item, cprop):
correlation_properties[cprop] = True
# Check is all values in correlation properties are True
correlation_result = all(
value is True for value in correlation_properties.values())
if correlation_result:
self.item_dict["duplicate"] = True
return item_definition
def compare_json_files(self, item_definition: ItemDefinition) -> bool:
"""Print the difference between this item and the database."""
# Create JSON out object to compare
current_json = item_definition.construct_json()
# Try get existing entry (KeyError means it doesn't exist - aka a new item)
try:
existing_json = self.all_db_items[self.item_id]
except KeyError:
print(f">>> compare_json_files: NEW ITEM: {item_definition.id}")
print(current_json)
return
if current_json == existing_json:
return
ddiff = DeepDiff(existing_json, current_json, ignore_order=True)
logging.debug(
f">>> compare_json_files: CHANGED ITEM: {item_definition.id}: {item_definition.name}, {item_definition.wiki_name}"
)
print(
f">>> compare_json_files: CHANGED ITEM: {item_definition.id}: {item_definition.name}"
)
# try:
# added_properties = ddiff["dictionary_item_added"]
# print(" ", added_properties)
# except KeyError:
# pass
# try:
# changed_properties = ddiff["values_changed"]
# for k, v in changed_properties.items():
# print(" ", k, v["new_value"])
# except KeyError:
# pass
print(ddiff)
return
class BuildItem:
def __init__(self, item_id, item_json, wiki_text, normalized_names, buy_limits, skill_requirements, current_db,
weapon_types, weapon_stances):
# Input item ID number
self.item_id = item_id
# Input JSON file (from RuneLite ItemScraper plugin)
self.item_json = item_json
# Input data
self.wiki_text = wiki_text # Dict of raw wiki text from OSRS Wiki
self.normalized_names = normalized_names # Maps cache names to OSRS Wiki names
self.buy_limits = buy_limits # Dictionary of item buy limits
self.skill_requirements = skill_requirements # Dictionary of item requirements
self.current_db = current_db # Dictionary dump of current database contents
self.weapon_types = weapon_types # Weapon type dictionary
self.weapon_stances = weapon_stances # Weapon stances dictionary
# For this item, create dictionary for property storage
self.item_dict = dict()
# Setup logging
logging.basicConfig(filename="builder.log",
filemode='a',
level=logging.DEBUG)
self.logger = logging.getLogger(__name__)
# If a page does not have a wiki page, it may be given a status number
self.status_code = None
self.properties = [
"id",
"name",
"members",
"tradeable",
"tradeable_on_ge",
"stackable",
"noted",
"noteable",
"linked_id",
"placeholder",
"equipable",
"equipable_by_player",
"equipable_weapon",
"cost",
"lowalch",
"highalch",
"weight",
"buy_limit",
"quest_item",
"release_date",
"examine",
"url"]
self.equipment_properties = [
"attack_stab",
"attack_slash",
"attack_crush",
"attack_magic",
"attack_ranged",
"defence_stab",
"defence_slash",
"defence_crush",
"defence_magic",
"defence_ranged",
"melee_strength",
"ranged_strength",
"magic_damage",
"prayer",
"slot",
"requirements"]
self.weapon_properties = [
"attack_speed",
"weapon_type",
"stances"]
def populate(self):
"""The primary entry and item object population function."""
# Start section in logger
self.logger.debug("============================================ START")
self.logger.debug(f"item_id: {self.item_id}")
# STAGE ONE: LOAD ITEM SCRAPER DATA
self.logger.debug("STAGE ONE: Loading item cache data data to object...")
self.populate_from_scraper()
self.logger.debug(f'id: {self.item_dict["id"]}|name: {self.item_dict["name"]}')
# print(f'>>> id: {self.item_dict["id"]}\tname: {self.item_dict["name"]}')
# STAGE TWO: DETERMINE WIKI PAGE
self.logger.debug("STAGE TWO: Determining OSRS Wiki page...")
has_wiki_page = self.determine_wiki_page()
# # This commented code can be used to determine wiki page normalization
# # You must comment out the normalization lookup in determine_wiki_page
# # WARNING: Does not check equipable item infobox extraction errors!
# if not has_wiki_page:
# if str(self.item_dict["id"]) in self.normalized_names:
# normalized_name = self.normalized_names[str(self.item_dict["id"])][1]
# if normalized_name == "" or not normalized_name:
# normalized_name = self.item_dict["name"]
# status = self.normalized_names[str(self.item_dict["id"])][2]
# print(f"{self.item_dict["id"]}|{self.item_dict["name"]}|{normalized_name}|{status}")
# else:
# print(f"TODO:{self.item_dict["id"]}|{self.item_dict["name"]}|{self.item_dict["name"]}|X")
if not has_wiki_page:
# These will be items that cannot be processed and the program should exit
print(f">>> not has_wiki_page: {self.item_dict['id']}, {self.item_dict['name']}")
if self.item_dict["name"] == "" or self.item_dict["name"].lower() == "null":
self.item_dict["equipable_by_player"] = False
self.export()
return
# STAGE THREE: EXTRACT and PARSE INFOBOX
self.logger.debug("STAGE THREE: Extracting the infobox...")
# Extract the infobox for the item
has_infobox = self.extract_infobox()
# Handle the infobox extraction, depending on the item status code
if has_infobox:
self.logger.debug("INFOBOX: Success")
self.parse_primary_infobox()
elif self.status_code in [1, 2, 3, 4, 5]:
self.logger.debug("INFOBOX: Invalid item saved")
self.item_dict["url"] = None
self.item_dict["equipable_by_player"] = False
self.export()
return
elif self.status_code == 6:
self.parse_primary_infobox()
self.item_dict["equipable_by_player"] = False
self.export()
return
else:
self.logger.critical("INFOBOX: Extraction error.")
if self.item_dict["name"] == "":
self.export()
quit()
# STAGE FOUR: PARSE INFOBOX FOR EQUIPABLE ITEMS
if self.item_dict["equipable"] and has_wiki_page:
self.logger.debug("STAGE FIVE: Parsing the bonuses...")
self.item_dict["equipment"] = dict()
# Continue processing... but only if the item is equipable
self.item_dict["equipable_by_player"] = True
has_infobox_bonuses = self.extract_bonuses()
if has_infobox_bonuses:
self.logger.debug("Item InfoBox Bonuses extracted successfully")
else:
self.logger.critical("Item InfoBox Bonuses extraction error.")
self.logger.critical("Status Code: %s" % self.status_code)
self.item_dict["equipable_by_player"] = False
self.export()
# print(">>> ERROR: Could not determine equipable item bonuses...")
return
else:
self.item_dict["equipable_by_player"] = False
# STAGE FIVE: COMPARE TO CURRENT DATABASE CONTENTS
self.logger.debug("STAGE FIVE: Compare object to existing database entry...")
self.export()
def export(self):
# Create ItemDefintion object
if "wiki_name" in self.item_dict:
del self.item_dict["wiki_name"]
if "store_price" in self.item_dict:
del self.item_dict["store_price"]
if "seller" in self.item_dict:
del self.item_dict["seller"]
if "equipable_weapon" not in self.item_dict:
self.item_dict["equipable_weapon"] = False
for prop in self.properties:
try:
self.item_dict[prop]
except KeyError:
self.item_dict[prop] = None
if self.item_dict["url"] == "https://oldschool.runescape.wiki/w/None":
self.item_dict["url"] = None
self.item_definition = ItemDefinition(**self.item_dict)
self.compare_json_files(self.item_definition)
json_out = self.item_definition.construct_json()
# Actually output a JSON file, comment out for testing
output_dir = os.path.join("..", "docs", "items-json")
self.item_definition.export_json(True, output_dir)
self.logger.debug(json_out)
return
def populate_from_scraper(self):
"""Populate the itemDefinition object from the item-scraper file content."""
self.item_dict["id"] = self.item_json["id"]
self.item_dict["name"] = self.item_json["name"]
self.item_dict["members"] = self.item_json["members"]
self.item_dict["tradeable_on_ge"] = self.item_json["tradeable_on_ge"]
self.item_dict["stackable"] = self.item_json["stackable"]
self.item_dict["noted"] = self.item_json["noted"]
self.item_dict["noteable"] = self.item_json["noteable"]
self.item_dict["linked_id"] = self.item_json["linked_id"]
self.item_dict["placeholder"] = self.item_json["placeholder"]
self.item_dict["equipable"] = self.item_json["equipable"]
self.item_dict["cost"] = self.item_json["cost"]
self.item_dict["lowalch"] = self.item_json["lowalch"]
self.item_dict["highalch"] = self.item_json["highalch"]
def determine_wiki_page(self):
"""Determine the OSRS Wiki page/url/name using the item name."""
# Set the initial wiki_name property to the actual item name
# This may change depending on the item lookup success/failure
wiki_name = self.item_dict["name"]
# PHASE ONE: Check if the item name is in the OSRS Wiki item dump using normalized name
if str(self.item_dict["id"]) in self.normalized_names:
self.logger.debug(">>> ITEM FOUND IN NORMALIZED")
# Determine normalized wiki name using lookup
normalized_name = self.normalized_names[str(self.item_dict["id"])][1]
# Set wiki URL and name
wiki_url = normalized_name.replace(" ", "_")
wiki_url = wiki_url.replace("'", "%27")
wiki_url = wiki_url.replace("&", "%26")
wiki_url = wiki_url.replace("+", "%2B")
self.item_dict["url"] = f"https://oldschool.runescape.wiki/w/{wiki_url}"
self.item_dict["wiki_name"] = normalized_name
# Set item status code
self.status_code = int(self.normalized_names[str(self.item_dict["id"])][2])
# Item name found in dump using normalization, return True
return True
# PHASE TWO: Check if the item name is in the OSRS Wiki item dump (no normalization)
if wiki_name in self.wiki_text:
self.logger.debug(">>> ITEM FOUND")
# Set wiki URL and name
wiki_url = self.item_dict["name"].replace(" ", "_")
wiki_url = wiki_url.replace("'", "%27")
wiki_url = wiki_url.replace("&", "%26")
wiki_url = wiki_url.replace("+", "%2B")
self.item_dict["url"] = f"https://oldschool.runescape.wiki/w/{wiki_url}"
self.item_dict["wiki_name"] = self.item_dict["name"]
# Set item status code, try/except as it may not be in list
try:
self.status_code = int(self.normalized_names[str(self.item_dict["name"])][2])
except KeyError:
self.status_code = 0 # Zero is no issue with item, direct lookup
# Item name found in dump without normalization, return True
return True
# If we got this far, the wiki page was not found, return false
self.logger.debug(">>> ITEM NOT FOUND")
return False
def extract_infobox(self):
"""Extract the primary properties and bonuses for the item."""
# Set templates
self.template_primary = None
self.template_bonuses = None
try:
wiki_text_entry = self.wiki_text[self.item_dict["wiki_name"]]
wikicode = mwparserfromhell.parse(wiki_text_entry)
except KeyError:
# The wiki_name was not found in the available dumped wikitext pages
# Return false to indicate no wikitext was extracted
self.logger.debug("extract_infobox: KeyError for self.wikitext")
return False
# Loop through templates in wikicode from wiki page
# Then call Inforbox Item processing method
templates = wikicode.filter_templates()
for template in templates:
template_name = template.name.strip()
template_name = template_name.lower()
if "infobox item" in template_name:
self.template_primary = template
if "infobox bonuses" in template_name:
self.template_bonuses = template
if "infobox construction" in template_name:
self.template_primary = template
if "infobox pet" in template_name:
self.template_primary = template
# If no template_primary was found, return false
if not self.template_primary:
self.logger.debug("extract_infobox: not self.template_primary")
return False
# If any equipable item, and no bonuses was found, return false
if self.item_dict["equipable"] and not self.template_bonuses:
self.logger.debug("extract_infobox: not self.template_bonuses")
return False
# If we got this far, return true
return True
def parse_primary_infobox(self):
"""Parse an actual Infobox template."""
template = self.template_primary
# Set defaults for versioned infoboxes
is_versioned = False # has multiple versions available
self.current_version = None # The version that matches the item
version_count = 0 # the number of versions available
# STAGE ONE: Determine if we have a versioned infobox, and the version count
version_identifiers = ["version",
"name",
"itemname"]
for version_identifier in version_identifiers:
# Check if the infobox is versioned, and get a version count
if version_count == 0:
try:
template.get(version_identifier + "1").value
is_versioned = True
# Now, try to determine how many versions are present
i = 1
while i <= 20: # Guessing max version number is 20
try:
template.get(version_identifier + "1").value
version_count += 1
except ValueError:
break
i += 1
except ValueError:
pass
# STAGE TWO: Match a versioned infobox to the item name
if is_versioned:
# Try determine
for version_identifier in version_identifiers:
try:
template.get(version_identifier + "1").value
i = 1
while i <= version_count:
versioned_name = version_identifier + str(i)
if self.item_dict["name"] == template.get(versioned_name).value.strip():
self.current_version = i
break
i += 1
except ValueError:
pass
self.logger.debug("NOTE: versioned infobox: %s" % self.current_version)
if is_versioned and self.current_version is None:
self.current_version = 1
# WEIGHT: Determine the weight of an item ()
weight = None
if self.current_version is not None:
key = "weight" + str(self.current_version)
weight = self.extract_infobox_value(template, key)
if weight is None:
weight = self.extract_infobox_value(template, "weight")
if weight is not None:
self.item_dict["weight"] = infobox_cleaner.clean_weight(weight, self.item_id)
# QUEST: Determine if item is associated with a quest ()
quest = None
if self.current_version is not None:
key = "quest" + str(self.current_version)
quest = self.extract_infobox_value(template, key)
if quest is None:
quest = self.extract_infobox_value(template, "quest")
if quest is not None:
self.item_dict["quest_item"] = infobox_cleaner.clean_quest(quest)
# Determine the release date of an item ()
release_date = None
if self.current_version is not None:
key = "release" + str(self.current_version)
release_date = self.extract_infobox_value(template, key)
if release_date is None:
release_date = self.extract_infobox_value(template, "release")
if release_date is not None:
self.item_dict["release_date"] = infobox_cleaner.clean_release_date(release_date)
# Determine if item has a store price ()
store_price = None
if self.current_version is not None:
key = "store" + str(self.current_version)
store_price = self.extract_infobox_value(template, key)
if store_price is None:
store_price = self.extract_infobox_value(template, "store")
if store_price is not None:
self.item_dict["store_price"] = infobox_cleaner.clean_store_price(store_price)
# Determine if item has a store price ()
seller = None
if self.current_version is not None:
key = "seller" + str(self.current_version)
seller = self.extract_infobox_value(template, key)
if seller is None:
seller = self.extract_infobox_value(template, "seller")
if seller is not None:
self.item_dict["seller"] = infobox_cleaner.clean_seller(seller)
# Determine the examine text of an item ()
tradeable = None
if self.current_version is not None:
key = "tradeable" + str(self.current_version)
tradeable = self.extract_infobox_value(template, key)
if tradeable is None:
tradeable = self.extract_infobox_value(template, "tradeable")
if tradeable is not None:
self.item_dict["tradeable"] = infobox_cleaner.clean_tradeable(tradeable)
else:
self.item_dict["tradeable"] = False
# Determine the examine text of an item ()
examine = None
if self.current_version is not None:
key = "examine" + str(self.current_version)
examine = self.extract_infobox_value(template, key)
if examine is None:
examine = self.extract_infobox_value(template, "examine")
if examine is not None:
self.item_dict["examine"] = infobox_cleaner.clean_examine(examine, self.item_dict["name"])
else:
# Being here means the extraction for "examine" failed
key = "itemexamine" + str(self.current_version)
examine = self.extract_infobox_value(template, key)
if examine is None:
examine = self.extract_infobox_value(template, "itemexamine")
if examine is not None:
self.item_dict["examine"] = infobox_cleaner.clean_examine(examine, self.item_dict["name"])
# Determine if item has a buy limit ()
if not self.item_dict["tradeable"]:
self.item_dict["buy_limit"] = None
else:
try:
self.item_dict["buy_limit"] = int(self.buy_limits[self.item_dict["name"]])
if self.item_dict["noted"]:
self.item_dict["buy_limit"] = None
except KeyError:
self.item_dict["buy_limit"] = None
return True
def extract_infobox_value(self, template: mwparserfromhell.nodes.template.Template, key: str) -> str:
"""Helper method to extract a value from a template using a specified key.
This helper method is a simple solution to repeatedly try to fetch a specific
entry from a wiki text template (a mwparserfromhell template object).
:param template: A mediawiki wiki text template.
:param key: The key to query in the template.
:return value: The extracted template value based on supplied key.
"""
value = None
try:
value = template.get(key).value
value = value.strip()
return value
except ValueError:
return value
def extract_bonuses(self) -> bool:
"""Extract the infobox bonuses template from raw wikitext.
:return: If the infobox bonuses template was extracted successfully or not.
"""
# Extract Infobox Bonuses from wikitext
try:
wikicode = mwparserfromhell.parse(self.wiki_text[self.item_dict["wiki_name"]])
except KeyError:
return False
templates = wikicode.filter_templates()
for template in templates:
if "infobox bonuses" in template.lower():
extracted_infobox = self.parse_bonuses(template)
if extracted_infobox:
return True
return False
def parse_bonuses(self, template: mwparserfromhell.nodes.template.Template) -> bool:
"""Parse the wiki text template and extract item bonus values from it.
:param template: A mediawiki wiki text template.
"""
self.item_dict["equipment"]["attack_stab"] = self.clean_bonuses_value(template, "astab")
self.item_dict["equipment"]["attack_slash"] = self.clean_bonuses_value(template, "aslash")
self.item_dict["equipment"]["attack_crush"] = self.clean_bonuses_value(template, "acrush")
self.item_dict["equipment"]["attack_magic"] = self.clean_bonuses_value(template, "amagic")
self.item_dict["equipment"]["attack_ranged"] = self.clean_bonuses_value(template, "arange")
self.item_dict["equipment"]["defence_stab"] = self.clean_bonuses_value(template, "dstab")
self.item_dict["equipment"]["defence_slash"] = self.clean_bonuses_value(template, "dslash")
self.item_dict["equipment"]["defence_crush"] = self.clean_bonuses_value(template, "dcrush")
self.item_dict["equipment"]["defence_magic"] = self.clean_bonuses_value(template, "dmagic")
self.item_dict["equipment"]["defence_ranged"] = self.clean_bonuses_value(template, "drange")
self.item_dict["equipment"]["melee_strength"] = self.clean_bonuses_value(template, "str")
self.item_dict["equipment"]["ranged_strength"] = self.clean_bonuses_value(template, "rstr")
self.item_dict["equipment"]["magic_damage"] = self.clean_bonuses_value(template, "mdmg")
self.item_dict["equipment"]["prayer"] = self.clean_bonuses_value(template, "prayer")
# Determine the slot for the equipable item
self.item_dict["equipment"]["slot"] = None
try:
self.item_dict["equipment"]["slot"] = self.strip_infobox(template.get("slot").value)
self.item_dict["equipment"]["slot"] = self.item_dict["equipment"]["slot"].lower()
except ValueError:
self.item_dict["equipment"]["slot"] = None
self.logger.critical("Could not determine equipable item slot")
quit()
# Determine the skill requirements for the equipable item
self.item_dict["equipment"]["requirements"] = None
try:
requirements = self.skill_requirements[str(self.item_id)]
self.item_dict["equipment"]["requirements"] = requirements
except KeyError:
self.item_dict["equipment"]["requirements"] = None
# Start processing only weapons
# If item is weapon, two-handed, or 2h, start processing the weapon data
if (self.item_dict["equipment"]["slot"] == "weapon" or
self.item_dict["equipment"]["slot"] == "two-handed" or
self.item_dict["equipment"]["slot"] == "2h"):
self.item_dict["weapon"] = dict()
# Try set the attack speed of the weapon
try:
self.item_dict["weapon"]["attack_speed"] = int(self.strip_infobox(template.get("aspeed").value))
except ValueError:
self.item_dict["weapon"]["attack_speed"] = None
self.logger.critical("WEAPON: Could not determine weapon attack speed")
# Item IDs with no known attack speed, set to zero
if int(self.item_id) in [8871]:
self.item_dict["weapon"]["attack_speed"] = 0
# Salamander fix, set to base attack speed of 5
elif int(self.item_id) in [10145, 10146, 10147, 10147, 10148, 10149]:
self.item_dict["weapon"]["attack_speed"] = 5
else:
quit()
# Try to set the weapon type of the weapon
try:
weapon_type = self.weapon_types[str(self.item_dict["id"])]["weapon_type"]
self.item_dict["weapon"]["weapon_type"] = weapon_type
except KeyError:
self.item_dict["weapon"]["weapon_type"] = None
self.logger.critical("WEAPON: Could not determine weapon type")
quit()
# Try to set stances available for the weapon
try:
self.item_dict["weapon"]["stances"] = self.weapon_stances[self.item_dict["weapon"]["weapon_type"]]
except KeyError:
self.item_dict["weapon"]["stances"] = None
self.logger.critical("WEAPON: Could not determine weapon stances")
quit()
# Finally, set the equipable_weapon property to true
self.item_dict["equipable_weapon"] = True
return True
def clean_bonuses_value(self, template: mwparserfromhell.nodes.template.Template, prop: str):
"""Clean a item bonuses value extracted from a wiki template.
:param template: A mediawiki wiki text template.
:param prop: The key to query in the template.
:return value: The extracted template value that has been int cast.
"""
value = None
# Try and get the versioned infobox value
if self.current_version is not None:
key = prop + str(self.current_version)
value = self.extract_infobox_value(template, key)
# If unsuccessful, try and get the normal infoxbox value
if value is None:
value = self.extract_infobox_value(template, prop)
if value is not None:
value = self.strip_infobox(value)
if isinstance(value, str):
if value[0] == "-":
if value[1:].isdigit():
value = int(value)
elif value[0] == "+":
if value[1:].isdigit():
value = int(value)
else:
if value.isdigit():
value = int(value)
else:
value = 0
return value
def strip_infobox(self, value: str) -> str:
"""Generic infobox wiki text cleaner.
:return clean_value: A cleaned wiki text value.
"""
# Clean an passed InfoBox string
clean_value = str(value)
clean_value = clean_value.strip()
clean_value = clean_value.replace("[", "")
clean_value = clean_value.replace("]", "")
return clean_value
def compare_json_files(self, item_definition: ItemDefinition) -> bool:
"""Print the difference between this item object, and the item that exists in the database.
:return changed: A boolean if the item is different, or not.
"""
changed = False
# Create JSON out object to compare
current_json = item_definition.construct_json()
# Try get existing entry (KeyError means it doesn't exist - aka a new item)
try:
existing_json = self.current_db[self.item_id]
except KeyError:
print(f">>> compare_json_files: NEW ITEM: {item_definition.id}")
print(current_json)
return changed
if current_json == existing_json:
return changed
ddiff = DeepDiff(existing_json, current_json, ignore_order=True)
print(f">>> compare_json_files: CHANGED ITEM: {item_definition.id}: {item_definition.name}")
print(f" {ddiff}")
changed = True
return changed
class BuildItem:
def __init__(self, item_id, all_item_cache_data, all_wikitext_processed,
all_wikitext_raw, all_db_items, buy_limits_data,
skill_requirements_data, weapon_types_data,
weapon_stances_data, invalid_items_data, known_item_names,
export_item):
self.item_id = item_id
self.all_item_cache_data = all_item_cache_data # Raw cache data for all items
self.all_wikitext_processed = all_wikitext_processed # Processed wikitext for all items
self.all_wikitext_raw = all_wikitext_raw # Raw data dump from OSRS Wiki
self.all_db_items = all_db_items # All current item database contents
self.buy_limits_data = buy_limits_data # Dictionary of item buy limits
self.skill_requirements = skill_requirements_data # Dictionary of item requirements
self.weapon_types_data = weapon_types_data # Weapon type dictionary
self.weapon_stances_data = weapon_stances_data # Weapon stances dictionary
self.invalid_items_data = invalid_items_data # Dictionary of invalid items
self.known_item_names = known_item_names # A list of already known (processed) items
self.export_item = export_item # If the JSON should be exported/created
# For this item instance, create dictionary for property storage
self.item_dict = dict()
# Set some important properties used for item building
self.wiki_page_name = None # The page name the wikitext is from
self.infobox_version_number = None # The version used on the wikitext page
self.status = None # Used if the item is special (invalid, normalized etc.)
self.is_invalid_item = False # If the item is not found using ID, linked ID or name lookup
# All properties that are available for all items
self.properties = [
"id", "name", "incomplete", "members", "tradeable",
"tradeable_on_ge", "stackable", "noted", "noteable",
"linked_id_item", "linked_id_noted", "linked_id_placeholder",
"placeholder", "equipable", "equipable_by_player",
"equipable_weapon", "cost", "lowalch", "highalch", "weight",
"buy_limit", "quest_item", "release_date", "examine", "wiki_name",
"wiki_url"
]
# Additional properties for all equipable items (weapons/armour)
self.equipment_properties = [
"attack_stab", "attack_slash", "attack_crush", "attack_magic",
"attack_ranged", "defence_stab", "defence_slash", "defence_crush",
"defence_magic", "defence_ranged", "melee_strength",
"ranged_strength", "magic_damage", "prayer", "slot", "requirements"
]
# Additional properties for all equipable weapons
self.weapon_properties = ["attack_speed", "weapon_type", "stances"]
def check_duplicate_item(self) -> ItemDefinition:
"""Determine if this is a duplicate item.
:return: An ItemDeinition object.
"""
# Start by setting the duplicate property to False
self.item_dict["duplicate"] = False
# Create an ItemDefinition object
item_definition = ItemDefinition(**self.item_dict)
# Set the item properties that we want to compare
correlation_properties = {
"wiki_name": False,
"noted": False,
"placeholder": False,
"equipable": False,
"equipable_by_player": False,
"equipable_weapon": False
}
# Loop the list of currently (already processed) items
for known_item in self.known_item_names:
# Do a quick name check before deeper inspection
if item_definition.name != known_item.name:
continue
# If the cache names are equal, do further inspection
for cprop in correlation_properties:
if getattr(item_definition,
cprop) == getattr(known_item, cprop):
correlation_properties[cprop] = True
# Check is all values in correlation properties are True
correlation_result = all(
value is True for value in correlation_properties.values())
if correlation_result:
self.item_dict["duplicate"] = True
return item_definition
def generate_item_object(self):
"""Generate the `ItemDefinition` object from the item_dict dictionary."""
self.item_definition = ItemDefinition(**self.item_dict)
def compare_new_vs_old_item(self):
"""Compare the newly generated item to the existing item in the database."""
self.compare_json_files(self.item_definition)
def export_item_to_json(self):
"""Export item to JSON, if requested."""
if self.export_item:
output_dir = os.path.join("..", "docs", "items-json")
self.item_definition.export_json(True, output_dir)
logging.debug(self.item_dict)
def preprocessing(self) -> Dict:
"""Preprocess an item, and set important object variables.
This function preprocesses every item dumped from the OSRS cache. Various
properties are set to help further processing. Items are determined if
they are a linked item (noted/placeholder), or an actual item. The item
is checked if it is a valid item (has a wiki page, is an actual item etc.).
Finally, the wikitext (from the OSRS wiki) is found by looking up ID, linked
ID, name, and normalized name. The `Infobox Item` or `Infobox Pet` is then
extracted so that the wiki properties can be later processed and populated.
:return: A dictionary including success and code.
"""
# Initialize dictionary to return preprocessing status
return_status = {"status": False, "code": None}
# Set item ID variables
self.item_id_int = int(self.item_id) # Item ID number as an integer
self.item_id_str = str(self.item_id) # Item ID number as a string
# Load item dictionary of cache data based on item ID
# This raw cache data is the baseline information about the specific item
# and can be considered 100% correct and available for every item
self.item_cache_data = self.all_item_cache_data[self.item_id_str]
# Set item name variable (directly from the cache dump)
self.item_name = self.item_cache_data["name"]
# Log and print item
logging.debug(
f"======================= {self.item_id_str} {self.item_name}")
# print(f"======================= {self.item_id_str} {self.item_name}")
logging.debug(f"preprocessing: using the following cache data:")
logging.debug(self.item_cache_data)
# Get the linked ID item value, if available
self.linked_id_item_int = None
self.linked_id_item_str = None
if self.item_cache_data["linked_id_item"] is not None:
self.linked_id_item_int = int(
self.item_cache_data["linked_id_item"])
self.linked_id_item_str = str(
self.item_cache_data["linked_id_item"])
logging.debug(
f"preprocessing: Linked item ID: {self.linked_id_item_str}")
# Determine the ID number to extract
# Noted and placeholder items should use the linked_id_item property
# to fill in additional wiki data...
self.item_id_to_process_int = None
self.item_id_to_process_str = None
if self.item_cache_data["noted"] is True or self.item_cache_data[
"placeholder"] is True:
self.item_id_to_process_int = int(self.linked_id_item_int)
self.item_id_to_process_str = str(self.linked_id_item_str)
else:
self.item_id_to_process_int = int(self.item_id)
self.item_id_to_process_str = str(self.item_id)
logging.debug(
f"preprocessing: ID to process: {self.item_id_to_process_str}")
# Find the wiki page
# Set all variables to None (for invalid items)
self.item_wikitext = None
self.wikitext_found_using = None
self.has_infobox = False
# Try to find the wiki data using direct ID number search
if self.all_wikitext_processed.get(self.item_id_str, None):
self.item_wikitext = self.all_wikitext_processed.get(
self.item_id_str, None)
self.wikitext_found_using = "id"
return_status["code"] = "lookup_passed_id"
# Try to find the wiki data using linked_id_item ID number search
elif self.all_wikitext_processed.get(self.linked_id_item_str, None):
self.item_wikitext = self.all_wikitext_processed.get(
self.linked_id_item_str, None)
self.wikitext_found_using = "linked_id"
return_status["code"] = "lookup_passed_linked_id"
# Try to find the wiki data using direct name search
elif self.all_wikitext_raw.get(self.item_name, None):
self.item_wikitext = self.all_wikitext_raw.get(
self.item_name, None)
self.wikitext_found_using = "name"
return_status["code"] = "lookup_passed_name"
if self.item_id_to_process_str in self.invalid_items_data:
# Anything here means item cannot be found by id, linked_id, or name
# This can include not being an actual item, has no wiki page etc.
# The item must be invalid, handle it accordingly
self.is_invalid_item = True
try:
self.status = self.invalid_items_data[
self.item_id_to_process_str]["status"]
self.normalized_name = self.invalid_items_data[
self.item_id_to_process_str]["normalized_name"]
except KeyError:
self.status = None
self.normalized_name = None
logging.debug(
f"preprocessing: Invalid item details: {self.is_invalid_item} {self.status} {self.normalized_name}"
)
# Try to find the wiki data using normalized_name search
if self.all_wikitext_raw.get(self.normalized_name, None):
self.item_wikitext = self.all_wikitext_raw.get(
self.normalized_name, None)
self.wikitext_found_using = "normalized_name"
return_status["code"] = "valid"
else:
return_status["code"] = "lookup_failed"
logging.debug(
f"preprocessing: self.item_wikitext found using: {self.wikitext_found_using}"
)
# If there is no wikitext, and the item is valid, raise a critical error
if not self.item_wikitext and not self.is_invalid_item:
logging.critical(
"CRITICAL: Could not find item_wikitext by id, linked_id_item or name..."
)
return_status["code"] = "no_item_wikitext"
return return_status
# Parse the infobox item
infobox_parser = WikitextTemplateParser(self.item_wikitext)
# Try extract infobox for item, then pet
self.has_infobox = infobox_parser.extract_infobox("infobox item")
if not self.has_infobox:
self.has_infobox = infobox_parser.extract_infobox("infobox pet")
if not self.has_infobox:
self.template = None
logging.critical("CRITICAL: Could not find template...")
return_status["code"] = "no_infobox_template"
return return_status
self.is_versioned = infobox_parser.determine_infobox_versions()
logging.debug(
f"preprocessing: Is the infobox versioned: {self.is_versioned}")
self.versioned_ids = infobox_parser.extract_infobox_ids()
logging.debug(f"preprocessing: Versioned IDs: {self.versioned_ids}")
# Set the infobox version number, default to empty string (no version number)
try:
if self.versioned_ids:
self.infobox_version_number = self.versioned_ids[
self.item_id_to_process_int]
except KeyError:
if self.is_versioned:
self.infobox_version_number = "1"
else:
self.infobox_version_number = ""
logging.debug(
f"preprocessing: infobox_version_number: {self.infobox_version_number}"
)
# Set the template
self.template = infobox_parser.template
return_status["status"] = True
return return_status
def populate_item(self):
"""Populate an item after preprocessing it.
This is called for every item in the OSRS cache dump. Start by populating the
raw metadata from the cache. Then process invalid items, and """
# Start by populating the item from the cache data
self.populate_from_cache_data()
# Process an invalid item
if self.is_invalid_item:
logging.debug(
"populate_item: Found and processing an invalid item...")
if self.status == "unequipable":
# Cache thinks the item is equipable, but it is not
self.populate_item_properties_from_wiki_data()
self.item_dict["equipable_by_player"] = False
self.item_dict["equipable_weapon"] = False
self.item_dict["incomplete"] = True
return True
if self.status == "no_bonuses_available":
# Equipable item with wiki page, but does not have an Infobox Bonuses template
# This is only ever called on ring slot items, as they sometimes have a
# wiki page without an Infobox Bonuses template
self.populate_non_wiki_item()
self.item_dict["equipment"] = dict()
for equipment_property in self.equipment_properties:
self.item_dict["equipment"][equipment_property] = 0
self.item_dict["equipment"]["slot"] = "ring"
self.item_dict["equipment"]["requirements"] = None
self.item_dict["equipable_by_player"] = True
self.item_dict["equipable_weapon"] = False
self.item_dict["incomplete"] = True
return True
if self.status == "normalized":
# Some items have a wiki page, but lookup by ID, linked ID and item name
# fail. So use the normalized name from the invalid-items.json file
self.item_wikitext = self.all_wikitext_raw[
self.normalized_name]
self.wikitext_found_using = "normalized_name"
infobox_parser = WikitextTemplateParser(self.item_wikitext)
infobox_parser.extract_infobox("infobox item")
self.template = infobox_parser.template
self.populate_item_properties_from_wiki_data()
self.item_dict["equipable_by_player"] = False
self.item_dict["equipable_weapon"] = False
return True
if self.status == "unobtainable":
# Some items are unobtainable, set defaults
self.populate_non_wiki_item()
return True
if self.status == "skill_guide_icon":
# Some items are actually an icon in a skill guide, set defaults
self.populate_non_wiki_item()
return True
if self.status == "construction_icon":
# Some items are actually an icon in the construction interface, set defaults
self.populate_non_wiki_item()
return True
if self.status == "unhandled":
# Some items have not been classified, set defaults
self.populate_non_wiki_item()
return True
if not self.item_dict["equipable"]:
# Process a normal, non-equipable item
logging.debug(
"populate_item: Populating a normal item using wiki data...")
self.populate_item_properties_from_wiki_data()
self.item_dict["equipable_by_player"] = False
self.item_dict["equipable_weapon"] = False
return True
if self.item_dict["equipable"]:
# Process an equipable item
logging.debug(
"populate_item: Populating an equipable item using wiki data..."
)
self.populate_item_properties_from_wiki_data()
self.populate_equipable_properties_from_wiki_data()
return True
# Return false by default, this means the item was not found or processed
logging.error("populate_item: Item was not processed...")
return False
def populate_non_wiki_item(self):
"""Populate an iem that has no wiki page."""
# Set all item properties to None if they have not been populated
for prop in self.properties:
try:
self.item_dict[prop]
except KeyError:
self.item_dict[prop] = None
self.item_dict["tradeable"] = False
self.item_dict["quest_item"] = False
# Set equipable item/weapon properties to false
self.item_dict["equipable_by_player"] = False
self.item_dict["equipable_weapon"] = False
self.item_dict["incomplete"] = True
def populate_from_cache_data(self):
"""Populate an item using raw cache data.
This function takes the raw OSRS cache data for the specific item and loads
all available properties (that are extracted from the cache).
"""
# Log, then populate cache properties
logging.debug(
"populate_from_cache: Loading item cache data data to object...")
self.item_dict["id"] = self.item_cache_data["id"]
self.item_dict["name"] = self.item_cache_data["name"]
self.item_dict["members"] = self.item_cache_data["members"]
self.item_dict["tradeable_on_ge"] = self.item_cache_data[
"tradeable_on_ge"]
self.item_dict["stackable"] = self.item_cache_data["stackable"]
self.item_dict["noted"] = self.item_cache_data["noted"]
self.item_dict["noteable"] = self.item_cache_data["noteable"]
self.item_dict["linked_id_item"] = self.item_cache_data[
"linked_id_item"]
self.item_dict["linked_id_noted"] = self.item_cache_data[
"linked_id_noted"]
self.item_dict["linked_id_placeholder"] = self.item_cache_data[
"linked_id_placeholder"]
self.item_dict["placeholder"] = self.item_cache_data["placeholder"]
self.item_dict["equipable"] = self.item_cache_data["equipable"]
self.item_dict["cost"] = self.item_cache_data["cost"]
self.item_dict["lowalch"] = self.item_cache_data["lowalch"]
self.item_dict["highalch"] = self.item_cache_data["highalch"]
def populate_item_properties_from_wiki_data(self):
"""Populate item data from a OSRS Wiki Infobox Item template."""
if not self.has_infobox:
# Cannot populate if there is no infobox!
self.populate_non_wiki_item()
logging.error(
"populate_item_properties_from_wiki_data: No infobox for wiki item."
)
return False
# STAGE ONE: Determine then set the wiki_name and wiki_url
# Manually set OSRS Wiki name
if self.wikitext_found_using not in ["id", "linked_id"]:
# Item found in wiki by ID, cache name is the best option
wiki_page_name = self.item_name
elif self.wikitext_found_using == "normalized":
# Item found in wiki by normalized name, normalize name is used
wiki_page_name = self.normalized_name
else:
# Item found using direct cache name lookup on wiki page names,
# So use wiki page name in the item_wikitext array
wiki_page_name = self.item_wikitext[0]
wiki_versioned_name = None
wiki_name = None
# Get the versioned, or non-versioned, name from the infobox
if self.infobox_version_number is not None:
key = "version" + str(self.infobox_version_number)
wiki_versioned_name = self.extract_infobox_value(
self.template, key)
else:
wiki_versioned_name = self.extract_infobox_value(
self.template, "version")
# Set the wiki_name property
if wiki_versioned_name is not None:
if wiki_versioned_name.startswith("("):
wiki_name = wiki_page_name + " " + wiki_versioned_name
else:
wiki_name = wiki_page_name + " (" + wiki_versioned_name + ")"
else:
wiki_name = wiki_page_name
self.item_dict["incomplete"] = True
self.item_dict["wiki_name"] = wiki_name
# Set the wiki_url property
if wiki_versioned_name is not None:
wiki_url = wiki_page_name + "#" + wiki_versioned_name
else:
wiki_url = wiki_page_name
self.item_dict["incomplete"] = True
wiki_url = wiki_url.replace(" ", "_")
self.item_dict[
"wiki_url"] = "https://oldschool.runescape.wiki/w/" + wiki_url
# STAGE TWO: Extract, process and set item properties from the infobox template
# WEIGHT: Determine the weight of an item
weight = None
if self.infobox_version_number is not None:
key = "weight" + str(self.infobox_version_number)
weight = self.extract_infobox_value(self.template, key)
if weight is None:
weight = self.extract_infobox_value(self.template, "weight")
if weight is not None:
self.item_dict["weight"] = infobox_cleaner.clean_weight(
weight, self.item_id)
else:
self.item_dict["weight"] = None
self.item_dict["incomplete"] = True
# QUEST: Determine if item is associated with a quest
quest = None
if self.infobox_version_number is not None:
key = "quest" + str(self.infobox_version_number)
quest = self.extract_infobox_value(self.template, key)
if quest is None:
quest = self.extract_infobox_value(self.template, "quest")
if quest is not None:
self.item_dict["quest_item"] = infobox_cleaner.clean_quest(quest)
else:
# Being here means the extraction for "quest" failed
key = "questrequired" + str(self.infobox_version_number)
quest = self.extract_infobox_value(self.template, key)
if quest is None:
quest = self.extract_infobox_value(self.template,
"questrequired")
if quest is not None:
self.item_dict["quest_item"] = infobox_cleaner.clean_quest(
quest)
else:
self.item_dict["quest_item"] = False
# Determine the release date of an item
release_date = None
if self.infobox_version_number is not None:
key = "release" + str(self.infobox_version_number)
release_date = self.extract_infobox_value(self.template, key)
if release_date is None:
release_date = self.extract_infobox_value(self.template, "release")
if release_date is not None:
self.item_dict[
"release_date"] = infobox_cleaner.clean_release_date(
release_date)
else:
self.item_dict["release_date"] = None
self.item_dict["incomplete"] = True
# Determine if an item is tradeable
tradeable = None
if self.infobox_version_number is not None:
key = "tradeable" + str(self.infobox_version_number)
tradeable = self.extract_infobox_value(self.template, key)
if tradeable is None:
tradeable = self.extract_infobox_value(self.template, "tradeable")
if tradeable is not None:
self.item_dict["tradeable"] = infobox_cleaner.clean_boolean(
tradeable)
else:
self.item_dict["tradeable"] = False
self.item_dict["incomplete"] = True
# Determine the examine text of an item
examine = None
if self.infobox_version_number is not None:
key = "examine" + str(self.infobox_version_number)
examine = self.extract_infobox_value(self.template, key)
if examine is None:
examine = self.extract_infobox_value(self.template, "examine")
if examine is not None:
self.item_dict["examine"] = infobox_cleaner.clean_examine(
examine, self.item_dict["name"])
else:
# Being here means the extraction for "examine" failed
key = "itemexamine" + str(self.infobox_version_number)
examine = self.extract_infobox_value(self.template, key)
if examine is None:
examine = self.extract_infobox_value(self.template,
"itemexamine")
if examine is not None:
self.item_dict["examine"] = infobox_cleaner.clean_examine(
examine, self.item_dict["name"])
else:
self.item_dict["examine"] = None
self.item_dict["incomplete"] = True
# Determine if item has a buy limit
if not self.item_dict["tradeable"]:
self.item_dict["buy_limit"] = None
else:
try:
self.item_dict["buy_limit"] = int(
self.buy_limits_data[self.item_dict["name"]])
if self.item_dict["noted"]:
self.item_dict["buy_limit"] = None
except KeyError:
self.item_dict["buy_limit"] = None
# We finished processing, set incomplete to false if not true
if not self.item_dict.get("incomplete"):
self.item_dict["incomplete"] = False
return True
def populate_equipable_properties_from_wiki_data(self) -> bool:
"""Parse the wiki text template and extract item bonus values from it."""
# Initialize empty equipment dictionary
self.item_dict["equipment"] = dict()
# Extract the infobox bonuses template
infobox_parser = WikitextTemplateParser(self.item_wikitext)
has_infobox = infobox_parser.extract_infobox("infobox bonuses")
if not has_infobox:
has_infobox = infobox_parser.extract_infobox("infobox_bonuses")
if not has_infobox:
# No infobox bonuses found for the item!
print(
"populate_equipable_properties: Item has no equipment infobox."
)
logging.critical(
"populate_equipable_properties: Item has no equipment infobox."
)
quit()
# Set the template
template = infobox_parser.template
# STAGE ONE: EQUIPABLE ITEM
# This item must be equipable by a player, set to True
self.item_dict["equipable_by_player"] = True
# Extract equipable item properties
self.item_dict["equipment"]["attack_stab"] = self.clean_bonuses_value(
template, "astab")
self.item_dict["equipment"]["attack_slash"] = self.clean_bonuses_value(
template, "aslash")
self.item_dict["equipment"]["attack_crush"] = self.clean_bonuses_value(
template, "acrush")
self.item_dict["equipment"]["attack_magic"] = self.clean_bonuses_value(
template, "amagic")
self.item_dict["equipment"][
"attack_ranged"] = self.clean_bonuses_value(template, "arange")
self.item_dict["equipment"]["defence_stab"] = self.clean_bonuses_value(
template, "dstab")
self.item_dict["equipment"][
"defence_slash"] = self.clean_bonuses_value(template, "dslash")
self.item_dict["equipment"][
"defence_crush"] = self.clean_bonuses_value(template, "dcrush")
self.item_dict["equipment"][
"defence_magic"] = self.clean_bonuses_value(template, "dmagic")
self.item_dict["equipment"][
"defence_ranged"] = self.clean_bonuses_value(template, "drange")
self.item_dict["equipment"][
"melee_strength"] = self.clean_bonuses_value(template, "str")
self.item_dict["equipment"][
"ranged_strength"] = self.clean_bonuses_value(template, "rstr")
self.item_dict["equipment"]["magic_damage"] = self.clean_bonuses_value(
template, "mdmg")
self.item_dict["equipment"]["prayer"] = self.clean_bonuses_value(
template, "prayer")
# Determine the slot for the equipable item
self.item_dict["equipment"]["slot"] = None
try:
self.item_dict["equipment"]["slot"] = self.strip_infobox(
template.get("slot").value)
self.item_dict["equipment"]["slot"] = self.item_dict["equipment"][
"slot"].lower()
except ValueError:
self.item_dict["equipment"]["slot"] = None
print(
"populate_equipable_properties: Could not determine item slot..."
)
logging.critical(
"populate_equipable_properties: Could not determine item slot..."
)
quit()
# Determine the skill requirements for the equipable item
self.item_dict["equipment"]["requirements"] = None
try:
requirements = self.skill_requirements[str(self.item_id)]
self.item_dict["equipment"]["requirements"] = requirements
except KeyError:
self.item_dict["equipment"]["requirements"] = None
# STAGE TWO: WEAPONS
# If item is weapon, two-handed, or 2h, start processing the weapon data
if (self.item_dict["equipment"]["slot"] == "weapon"
or self.item_dict["equipment"]["slot"] == "two-handed"
or self.item_dict["equipment"]["slot"] == "2h"):
self.item_dict["weapon"] = dict()
# Try set the attack speed of the weapon
try:
self.item_dict["weapon"]["attack_speed"] = int(
self.strip_infobox(template.get("aspeed").value))
except ValueError:
self.item_dict["weapon"]["attack_speed"] = None
logging.critical(
"WEAPON: Could not determine weapon attack speed")
# Item IDs with no known attack speed, set to zero
if int(self.item_id) in [8871]:
self.item_dict["weapon"]["attack_speed"] = 0
# Salamander fix, set to base attack speed of 5
elif int(self.item_id) in [
10145, 10146, 10147, 10147, 10148, 10149
]:
self.item_dict["weapon"]["attack_speed"] = 5
else:
pass
# quit()
# Try to set the weapon type of the weapon
try:
weapon_type = self.weapon_types_data[str(
self.item_dict["id"])]["weapon_type"]
self.item_dict["weapon"]["weapon_type"] = weapon_type
except KeyError:
self.item_dict["weapon"]["weapon_type"] = None
print(
"populate_equipable_properties: Could not determine weapon type..."
)
logging.critical(
"populate_equipable_properties: Could not determine weapon type..."
)
quit()
# Try to set stances available for the weapon
try:
self.item_dict["weapon"]["stances"] = self.weapon_stances_data[
self.item_dict["weapon"]["weapon_type"]]
except KeyError:
self.item_dict["weapon"]["stances"] = None
print(
"populate_equipable_properties: Could not determine weapon stance..."
)
logging.critical(
"populate_equipable_properties: Could not determine weapon stance..."
)
quit()
# Finally, set the equipable_weapon property to true
self.item_dict["equipable_weapon"] = True
else:
# If the item is not a weapon, two-handed or 2h it is not a weapon
self.item_dict["equipable_weapon"] = False
return True
def extract_infobox_value(
self, template: mwparserfromhell.nodes.template.Template,
key: str) -> str:
"""Helper method to extract a value from a template using a specified key.
This helper method is a simple solution to repeatedly try to fetch a specific
entry from a wiki text template (a mwparserfromhell template object).
:param template: A mediawiki wiki text template.
:param key: The key to query in the template.
:return value: The extracted template value based on supplied key.
"""
value = None
try:
value = template.get(key).value
value = value.strip()
return value
except ValueError:
return value
def clean_bonuses_value(self,
template: mwparserfromhell.nodes.template.Template,
prop: str) -> int:
"""Clean a item bonuses value extracted from a wiki template.
:param template: A mediawiki wiki text template.
:param prop: The key to query in the template.
:return value: The extracted template value that has been int cast.
"""
value = None
# Try and get the versioned infobox value
if self.infobox_version_number is not None:
key = prop + str(self.infobox_version_number)
value = self.extract_infobox_value(template, key)
# If unsuccessful, try and get the normal infoxbox value
if value is None:
value = self.extract_infobox_value(template, prop)
if value is not None:
value = self.strip_infobox(value)
if isinstance(value, str):
if value[0] == "-":
if value[1:].isdigit():
value = int(value)
elif value[0] == "+":
if value[1:].isdigit():
value = int(value)
else:
if value.isdigit():
value = int(value)
else:
value = 0
return value
def strip_infobox(self, value: str) -> str:
"""Generic infobox wiki text cleaner.
:return clean_value: A cleaned wiki text value.
"""
# Clean an passed InfoBox string
clean_value = str(value)
clean_value = clean_value.strip()
clean_value = clean_value.replace("[", "")
clean_value = clean_value.replace("]", "")
return clean_value
def compare_json_files(self, item_definition: ItemDefinition) -> bool:
"""Print the difference between this item and the database."""
# Create JSON out object to compare
current_json = item_definition.construct_json()
# Try get existing entry (KeyError means it doesn't exist - aka a new item)
try:
existing_json = self.all_db_items[self.item_id]
except KeyError:
print(f">>> compare_json_files: NEW ITEM: {item_definition.id}")
print(current_json)
return
if current_json == existing_json:
return
ddiff = DeepDiff(existing_json, current_json, ignore_order=True)
logging.debug(
f">>> compare_json_files: CHANGED ITEM: {item_definition.id}: {item_definition.name}, {item_definition.wiki_name}"
)
print(
f">>> compare_json_files: CHANGED ITEM: {item_definition.id}: {item_definition.name}"
)
# try:
# added_properties = ddiff["dictionary_item_added"]
# print(" ", added_properties)
# except KeyError:
# pass
# try:
# changed_properties = ddiff["values_changed"]
# for k, v in changed_properties.items():
# print(" ", k, v["new_value"])
# except KeyError:
# pass
print(ddiff)
return
def validate_item(self):
"""Use the items-schema.json file to validate the populated item."""
# Create JSON out object to validate
current_json = self.item_definition.construct_json()
# Open the JSON Schema for items
path_to_schema = Path(config.TEST_PATH / "item_schema.json")
with open(path_to_schema, 'r') as f:
schema = json.loads(f.read())
# Check the populate item object against the schema
jsonschema.validate(instance=current_json, schema=schema)
def generate_item_object(self):
"""Generate the `ItemDefinition` object from the item_dict dictionary."""
self.item_definition = ItemDefinition(**self.item_dict)