def compare_new_vs_old_item(self) -> bool: """Print the difference between this item and the database.""" # Create JSON out object to compare item_properties = ItemProperties(**self.item_dict) current_json = item_properties.construct_json() # Try get existing entry (KeyError means it doesn't exist - aka a new item) try: existing_json = self.all_db_items[self.item_id] except KeyError: print(f">>> compare_json_files: NEW ITEM: {item_properties.id}") print(current_json) self.item_dict["last_updated"] = datetime.now( timezone.utc).strftime("%Y-%m-%d") return if current_json == existing_json: self.item_dict["last_updated"] = self.all_db_items[ self.item_id]["last_updated"] return ddiff = DeepDiff(existing_json, current_json, ignore_order=True, exclude_paths="root['icon']") if ddiff: print( f">>> compare_json_files: CHANGED ITEM: {item_properties.id}: {item_properties.name}" ) print(ddiff) self.item_dict["last_updated"] = datetime.now( timezone.utc).strftime("%Y-%m-%d")
def check_duplicate_item(self) -> ItemProperties: """Determine if this is a duplicate item. :return: An ItemProperties object. """ # Start by setting the duplicate property to False self.item_dict["duplicate"] = False # Create an ItemProperties object item_properties = ItemProperties(**self.item_dict) # Check list of known bad duplicates if str(item_properties.id) in self.duplicate_items: duplicate_status = self.duplicate_items[str( item_properties.id)]["duplicate"] self.item_dict["duplicate"] = duplicate_status return None # Check noted, placeholder and noted properties # If any of these properties, it must be a duplicate if item_properties.stacked or item_properties.noted or item_properties.placeholder: self.item_dict["duplicate"] = True return None # Set the item properties that we want to compare correlation_properties = {"name": False, "wiki_name": False} # Loop the list of currently (already processed) items for known_item in self.known_items: # Skip when cache names are not the same if item_properties.name != known_item.name: continue # Check equality of each correlation property for cprop in correlation_properties: if getattr(item_properties, cprop) == getattr(known_item, cprop): correlation_properties[cprop] = True # Check all values in correlation properties are True correlation_result = all( value is True for value in correlation_properties.values()) # If name and wiki_name match, set duplicate property to True if correlation_result: item_properties.duplicate = True self.item_dict["duplicate"] = True return item_properties # If wiki_name is None, but cache names match... # The item must also be a duplicate if not item_properties.wiki_name: item_properties.duplicate = True self.item_dict["duplicate"] = True return item_properties # If we made it this far, no duplicates were found item_properties.duplicate = False self.item_dict["duplicate"] = False return item_properties
def validate_item(self): """Use the schema-items.json file to validate the populated item.""" # Create JSON out object to validate item_properties = ItemProperties(**self.item_dict) current_json = item_properties.construct_json() # Validate object with schema attached v = validator.MyValidator(self.schema_data) v.validate(current_json) # Print any validation errors if v.errors: print(v.errors) exit(1) assert v.validate(current_json)
def compare_json_files(self, item_properties: ItemProperties) -> bool: """Print the difference between this item and the database.""" # Create JSON out object to compare current_json = item_properties.construct_json() # Try get existing entry (KeyError means it doesn't exist - aka a new item) try: existing_json = self.all_db_items[self.item_id] except KeyError: print(f">>> compare_json_files: NEW ITEM: {item_properties.id}") print(current_json) return if current_json == existing_json: return ddiff = DeepDiff(existing_json, current_json, ignore_order=True) logging.debug( f">>> compare_json_files: CHANGED ITEM: {item_properties.id}: {item_properties.name}, {item_properties.wiki_name}" ) print( f">>> compare_json_files: CHANGED ITEM: {item_properties.id}: {item_properties.name}" ) print(ddiff) return
def _load_item(self, item_json: Dict) -> None: """Convert the `item_json` into a :class:`ItemProperties` and store it. :param item_json: A dict from an open and loaded JSON file. :raises ValueError: Cannot populate item. """ # Load the item using the ItemProperties class try: item_def = ItemProperties.from_json(item_json) except TypeError as e: raise ValueError( "Error: Invalid JSON structure found, check supplied input. Exiting" ) from e # Add item to list self.all_items.append(item_def) self.all_items_dict[item_def.id] = item_def
def check_duplicate_item(self) -> ItemProperties: """Determine if this is a duplicate item. :return: An ItemDeinition object. """ # Start by setting the duplicate property to False self.item_dict["duplicate"] = False # Create an ItemProperties object item_properties = ItemProperties(**self.item_dict) # Set the item properties that we want to compare correlation_properties = { "wiki_name": False, "noted": False, "placeholder": False, "equipable": False, "equipable_by_player": False, "equipable_weapon": False } # Loop the list of currently (already processed) items for known_item in self.known_items: # Do a quick name check before deeper inspection if item_properties.name != known_item.name: continue # If the cache names are equal, do further inspection for cprop in correlation_properties: if getattr(item_properties, cprop) == getattr(known_item, cprop): correlation_properties[cprop] = True # Check is all values in correlation properties are True correlation_result = all( value is True for value in correlation_properties.values()) if correlation_result: self.item_dict["duplicate"] = True return item_properties
def export_item_to_json(self): """Export item to JSON, if requested.""" item_properties = ItemProperties(**self.item_dict) output_dir = Path(config.DOCS_PATH, "items-json") item_properties.export_json(True, output_dir)
class BuildItem: def __init__(self, **kwargs): # ID number to process self.item_id = kwargs["item_id"] # Raw cache data for all items self.all_item_cache_data = kwargs["all_item_cache_data"] # Processed wikitext for all items self.all_wikitext_processed = kwargs["all_wikitext_processed"] # Raw data dump from OSRS Wiki self.all_wikitext_raw = kwargs["all_wikitext_raw"] # Dictionary of unalchable items (using item name) self.unalchable_items = kwargs["unalchable_items"] # All current item database contents self.all_db_items = kwargs["all_db_items"] # Dictionary of item buy limits self.buy_limits_data = kwargs["buy_limits_data"] # Dictionary of item requirements self.skill_requirements = kwargs["skill_requirements_data"] # Weapon type dictionary self.weapon_types_data = kwargs["weapon_types_data"] # Weapon stances dictionary self.weapon_stances_data = kwargs["weapon_stances_data"] # Dictionary of invalid items self.invalid_items_data = kwargs["invalid_items_data"] # A list of already known (processed) items self.known_items = kwargs["known_items"] # A dictionary of know duplicate items self.duplicate_items = kwargs["duplicate_items"] # Dictionary of item icons self.icons_data = kwargs["icons_data"] # The item schema self.schema_data = kwargs["schema_data"] # If the JSON should be exported/created self.export = kwargs["export"] # Specify verbosity self.verbose = kwargs["verbose"] # For this item instance, create dictionary for property storage self.item_dict = dict() # The page name the wikitext is from self.wiki_page_name = None # The version used on the wikitext page self.infobox_version_number = None # Used if the item is special (invalid, normalized etc.) self.status = None # If the item is not found using ID, linked ID or name lookup self.is_invalid_item = False # All properties that are available for all items self.properties = [ "id", "name", "incomplete", "members", "tradeable", "tradeable_on_ge", "stackable", "stacked", "noted", "noteable", "linked_id_item", "linked_id_noted", "linked_id_placeholder", "placeholder", "equipable", "equipable_by_player", "equipable_weapon", "cost", "lowalch", "highalch", "weight", "buy_limit", "quest_item", "release_date", "examine", "icon", "wiki_name", "wiki_url", "wiki_exchange" ] # Additional properties for all equipable items (weapons/armour) self.equipment_properties = [ "attack_stab", "attack_slash", "attack_crush", "attack_magic", "attack_ranged", "defence_stab", "defence_slash", "defence_crush", "defence_magic", "defence_ranged", "melee_strength", "ranged_strength", "magic_damage", "prayer", "slot", "requirements" ] # Additional properties for all equipable weapons self.weapon_properties = ["attack_speed", "weapon_type", "stances"] def check_duplicate_item(self) -> ItemProperties: """Determine if this is a duplicate item. :return: An ItemProperties object. """ # Start by setting the duplicate property to False self.item_dict["duplicate"] = False # Create an ItemProperties object item_properties = ItemProperties(**self.item_dict) # Check list of known bad duplicates if str(item_properties.id) in self.duplicate_items: duplicate_status = self.duplicate_items[str( item_properties.id)]["duplicate"] self.item_dict["duplicate"] = duplicate_status return None # Check noted, placeholder and noted properties # If any of these properties, it must be a duplicate if item_properties.stacked or item_properties.noted or item_properties.placeholder: self.item_dict["duplicate"] = True return None # Set the item properties that we want to compare correlation_properties = {"name": False, "wiki_name": False} # Loop the list of currently (already processed) items for known_item in self.known_items: # Skip when cache names are not the same if item_properties.name != known_item.name: continue # Check equality of each correlation property for cprop in correlation_properties: if getattr(item_properties, cprop) == getattr(known_item, cprop): correlation_properties[cprop] = True # Check all values in correlation properties are True correlation_result = all( value is True for value in correlation_properties.values()) # If name and wiki_name match, set duplicate property to True if correlation_result: item_properties.duplicate = True self.item_dict["duplicate"] = True return item_properties # If wiki_name is None, but cache names match... # The item must also be a duplicate if not item_properties.wiki_name: item_properties.duplicate = True self.item_dict["duplicate"] = True return item_properties # If we made it this far, no duplicates were found item_properties.duplicate = False self.item_dict["duplicate"] = False return item_properties def generate_item_object(self): """Generate the `ItemProperties` object from the item_dict dictionary.""" self.item_properties = ItemProperties(**self.item_dict) def compare_new_vs_old_item(self): """Compare the newly generated item to the existing item in the database.""" self.compare_json_files(self.item_properties) def export_item_to_json(self): """Export item to JSON, if requested.""" if self.export: output_dir = Path(config.DOCS_PATH, "items-json") self.item_properties.export_json(True, output_dir) logging.debug(self.item_dict) def preprocessing(self) -> Dict: """Preprocess an item, and set important object variables. This function preprocesses every item dumped from the OSRS cache. Various properties are set to help further processing. Items are determined if they are a linked item (noted/placeholder), or an actual item. The item is checked if it is a valid item (has a wiki page, is an actual item etc.). Finally, the wikitext (from the OSRS wiki) is found by looking up ID, linked ID, name, and normalized name. The `Infobox Item` or `Infobox Pet` is then extracted so that the wiki properties can be later processed and populated. :return: A dictionary including success and code. """ # Initialize dictionary to return preprocessing status return_status = {"status": False, "code": None} # Set item ID variables self.item_id_int = int(self.item_id) # Item ID number as an integer self.item_id_str = str(self.item_id) # Item ID number as a string # Load item dictionary of cache data based on item ID # This raw cache data is the baseline information about the specific item # and can be considered 100% correct and available for every item self.item_cache_data = self.all_item_cache_data[self.item_id_str] # Set item name variable (directly from the cache dump) self.item_name = self.item_cache_data["name"] # Log and print item logging.debug( f"======================= {self.item_id_str} {self.item_name}") if self.verbose: print( f"======================= {self.item_id_str} {self.item_name}") logging.debug("preprocessing: using the following cache data:") logging.debug(self.item_cache_data) # Get the linked ID item value, if available self.linked_id_item_int = None self.linked_id_item_str = None if self.item_cache_data["linked_id_item"] is not None: self.linked_id_item_int = int( self.item_cache_data["linked_id_item"]) self.linked_id_item_str = str( self.item_cache_data["linked_id_item"]) logging.debug( f"preprocessing: Linked item ID: {self.linked_id_item_str}") # Determine the ID number to extract # Noted and placeholder items should use the linked_id_item property # to fill in additional wiki data... self.item_id_to_process_int = None self.item_id_to_process_str = None if self.item_cache_data["noted"] is True or self.item_cache_data[ "placeholder"] is True: self.item_id_to_process_int = int(self.linked_id_item_int) self.item_id_to_process_str = str(self.linked_id_item_str) else: self.item_id_to_process_int = int(self.item_id) self.item_id_to_process_str = str(self.item_id) logging.debug( f"preprocessing: ID to process: {self.item_id_to_process_str}") # Find the wiki page # Set all variables to None (for invalid items) self.item_wikitext = None self.wikitext_found_using = None self.has_infobox = False # Try to find the wiki data using direct ID number search if self.all_wikitext_processed.get(self.item_id_str, None): self.item_wikitext = self.all_wikitext_processed.get( self.item_id_str, None) self.wikitext_found_using = "id" return_status["code"] = "lookup_passed_id" # Try to find the wiki data using linked_id_item ID number search elif self.all_wikitext_processed.get(self.linked_id_item_str, None): self.item_wikitext = self.all_wikitext_processed.get( self.linked_id_item_str, None) self.wikitext_found_using = "linked_id" return_status["code"] = "lookup_passed_linked_id" # Try to find the wiki data using direct name search elif self.all_wikitext_raw.get(self.item_name, None): self.item_wikitext = self.all_wikitext_raw.get( self.item_name, None) self.wikitext_found_using = "name" return_status["code"] = "lookup_passed_name" if self.item_id_to_process_str in self.invalid_items_data: # Anything here means item cannot be found by id, linked_id, or name # This can include not being an actual item, has no wiki page etc. # The item must be invalid, handle it accordingly self.is_invalid_item = True try: self.status = self.invalid_items_data[ self.item_id_to_process_str]["status"] self.normalized_name = self.invalid_items_data[ self.item_id_to_process_str]["normalized_name"] except KeyError: self.status = None self.normalized_name = None logging.debug( f"preprocessing: Invalid item details: {self.is_invalid_item} {self.status} {self.normalized_name}" ) # Try to find the wiki data using normalized_name search if self.all_wikitext_raw.get(self.normalized_name, None): self.item_wikitext = self.all_wikitext_raw.get( self.normalized_name, None) self.wikitext_found_using = "normalized_name" return_status["code"] = "lookup_passed_normalized_name" else: return_status["code"] = "lookup_failed" logging.debug( f"preprocessing: self.item_wikitext found using: {self.wikitext_found_using}" ) # If there is no wikitext, and the item is valid, raise a critical error if not self.item_wikitext and not self.is_invalid_item: logging.critical( "CRITICAL: Could not find item_wikitext by id, linked_id_item or name..." ) return_status["code"] = "no_item_wikitext" return return_status # Parse the infobox item infobox_parser = WikitextTemplateParser(self.item_wikitext) # Try extract infobox for item, then pet self.has_infobox = infobox_parser.extract_infobox("infobox item") if not self.has_infobox: self.has_infobox = infobox_parser.extract_infobox("infobox pet") if not self.has_infobox: self.template = None logging.critical("CRITICAL: Could not find template...") return_status["code"] = "no_infobox_template" return return_status self.is_versioned = infobox_parser.determine_infobox_versions() logging.debug( f"preprocessing: Is the infobox versioned: {self.is_versioned}") self.versioned_ids = infobox_parser.extract_infobox_ids() logging.debug(f"preprocessing: Versioned IDs: {self.versioned_ids}") # Set the infobox version number, default to empty string (no version number) try: if self.versioned_ids: self.infobox_version_number = self.versioned_ids[ self.item_id_to_process_int] except KeyError: if self.is_versioned: self.infobox_version_number = "1" else: self.infobox_version_number = "" logging.debug( f"preprocessing: infobox_version_number: {self.infobox_version_number}" ) # Set the template self.template = infobox_parser.template return_status["status"] = True return return_status def populate_item(self): """Populate an item after preprocessing it. This is called for every item in the OSRS cache dump. Start by populating the raw metadata from the cache. Then process invalid items, and """ # Start by populating the item from the cache data self.populate_from_cache_data() # Process an invalid item if self.is_invalid_item: logging.debug( "populate_item: Found and processing an invalid item...") if self.status == "unequipable": # Cache thinks the item is equipable, but it is not self.populate_item_properties_from_wiki_data() self.item_dict["equipable_by_player"] = False self.item_dict["equipable_weapon"] = False self.item_dict["incomplete"] = True return True if self.status == "normalized": # Some items have a wiki page, but lookup by ID, linked ID and item name # fail. So use the normalized name from the invalid-items.json file self.item_wikitext = self.all_wikitext_raw[ self.normalized_name] self.wikitext_found_using = "normalized_name" infobox_parser = WikitextTemplateParser(self.item_wikitext) infobox_parser.extract_infobox("infobox item") self.template = infobox_parser.template self.populate_item_properties_from_wiki_data() self.item_dict["equipable_by_player"] = False self.item_dict["equipable_weapon"] = False return True if self.status == "unobtainable": # Some items are unobtainable, set defaults self.populate_non_wiki_item() return True if self.status == "skill_guide_icon": # Some items are actually an icon in a skill guide, set defaults self.populate_non_wiki_item() return True if self.status == "construction_icon": # Some items are actually an icon in the construction interface, set defaults self.populate_non_wiki_item() return True if self.status == "unhandled": # Some items have not been classified, set defaults self.populate_non_wiki_item() return True if not self.item_dict["equipable"]: # Process a normal, non-equipable item logging.debug( "populate_item: Populating a normal item using wiki data...") self.populate_item_properties_from_wiki_data() self.item_dict["equipable_by_player"] = False self.item_dict["equipable_weapon"] = False return True if self.item_dict["equipable"]: # Process an equipable item logging.debug( "populate_item: Populating an equipable item using wiki data..." ) self.populate_item_properties_from_wiki_data() self.populate_equipable_properties_from_wiki_data() return True # Return false by default, this means the item was not found or processed logging.error("populate_item: Item was not processed...") return False def populate_non_wiki_item(self): """Populate an iem that has no wiki page.""" # Set all item properties to None if they have not been populated for prop in self.properties: try: self.item_dict[prop] except KeyError: self.item_dict[prop] = None self.item_dict["tradeable"] = False self.item_dict["quest_item"] = False # Set equipable item/weapon properties to false self.item_dict["equipable_by_player"] = False self.item_dict["equipable_weapon"] = False self.item_dict["incomplete"] = True def populate_from_cache_data(self): """Populate an item using raw cache data. This function takes the raw OSRS cache data for the specific item and loads all available properties (that are extracted from the cache). """ # Log, then populate cache properties logging.debug( "populate_from_cache: Loading item cache data data to object...") self.item_dict["id"] = self.item_cache_data["id"] self.item_dict["name"] = self.item_cache_data["name"] self.item_dict["members"] = self.item_cache_data["members"] self.item_dict["tradeable_on_ge"] = self.item_cache_data[ "tradeable_on_ge"] # Fix for invalid items that are not actually tradeable on the GE if self.item_dict["id"] in [2203, 4595, 7228, 7466, 8624, 8626, 8628]: self.item_dict["tradeable_on_ge"] = False self.item_dict["stackable"] = self.item_cache_data["stackable"] self.item_dict["stacked"] = self.item_cache_data["stacked"] self.item_dict["noted"] = self.item_cache_data["noted"] self.item_dict["noteable"] = self.item_cache_data["noteable"] self.item_dict["linked_id_item"] = self.item_cache_data[ "linked_id_item"] self.item_dict["linked_id_noted"] = self.item_cache_data[ "linked_id_noted"] self.item_dict["linked_id_placeholder"] = self.item_cache_data[ "linked_id_placeholder"] self.item_dict["placeholder"] = self.item_cache_data["placeholder"] self.item_dict["equipable"] = self.item_cache_data["equipable"] self.item_dict["cost"] = self.item_cache_data["cost"] if self.item_cache_data["placeholder"]: self.item_dict["lowalch"] = None self.item_dict["highalch"] = None else: self.item_dict["lowalch"] = self.item_cache_data["lowalch"] self.item_dict["highalch"] = self.item_cache_data["highalch"] self.item_dict["icon"] = self.icons_data[self.item_id_str] def populate_item_properties_from_wiki_data(self): """Populate item data from a OSRS Wiki Infobox Item template.""" if not self.has_infobox: # Cannot populate if there is no infobox! self.populate_non_wiki_item() logging.error( "populate_item_properties_from_wiki_data: No infobox for wiki item." ) return False # STAGE ONE: Determine then set the wiki_name, wiki_url # and wiki_exchnage properties # Manually set OSRS Wiki name if self.wikitext_found_using not in ["id", "linked_id"]: # Item found in wiki by ID, cache name is the best option wiki_page_name = self.item_name elif self.wikitext_found_using == "normalized": # Item found in wiki by normalized name, normalize name is used wiki_page_name = self.normalized_name else: # Item found using direct cache name lookup on wiki page names, # So use wiki page name in the item_wikitext array wiki_page_name = self.item_wikitext[0] wiki_versioned_name = None wiki_name = None # Get the versioned, or non-versioned, name from the infobox if self.infobox_version_number is not None: key = "version" + str(self.infobox_version_number) wiki_versioned_name = self.extract_infobox_value( self.template, key) else: wiki_versioned_name = self.extract_infobox_value( self.template, "version") # Set the wiki_name property if wiki_versioned_name is not None: if wiki_versioned_name.startswith("("): wiki_name = wiki_page_name + " " + wiki_versioned_name else: wiki_name = wiki_page_name + " (" + wiki_versioned_name + ")" else: wiki_name = wiki_page_name self.item_dict["wiki_name"] = wiki_name # Set the wiki_url property if wiki_versioned_name is not None: wiki_url = wiki_page_name + "#" + wiki_versioned_name else: wiki_url = wiki_page_name wiki_url = wiki_url.replace(" ", "_") self.item_dict[ "wiki_url"] = "https://oldschool.runescape.wiki/w/" + wiki_url # Try find the OSRS Wiki Exchange name/URL exchange_base_url = "https://oldschool.runescape.wiki/w/Exchange:" if self.item_dict["tradeable_on_ge"]: gemwname = None if self.infobox_version_number is not None: key = "gemwname" + str(self.infobox_version_number) gemwname = self.extract_infobox_value(self.template, key) if gemwname is None: gemwname = self.extract_infobox_value(self.template, "gemwname") if gemwname is not None: wiki_exchange = gemwname.replace(" ", "_") else: wiki_exchange = self.item_dict["name"].replace(" ", "_") self.item_dict["wiki_exchange"] = exchange_base_url + wiki_exchange else: self.item_dict["wiki_exchange"] = None # Check if item is not actually able to be alched if wiki_name: if wiki_name in self.unalchable_items: self.item_dict["lowalch"] = None self.item_dict["highalch"] = None elif wiki_versioned_name in self.unalchable_items: self.item_dict["lowalch"] = None self.item_dict["highalch"] = None # STAGE TWO: Extract, process and set item properties from the infobox template # WEIGHT: Determine the weight of an item weight = None if self.infobox_version_number is not None: key = "weight" + str(self.infobox_version_number) weight = self.extract_infobox_value(self.template, key) if weight is None: weight = self.extract_infobox_value(self.template, "weight") if weight is not None: self.item_dict["weight"] = infobox_cleaner.clean_weight( weight, self.item_id) else: self.item_dict["weight"] = None self.item_dict["incomplete"] = True # QUEST: Determine if item is associated with a quest quest = None if self.infobox_version_number is not None: key = "quest" + str(self.infobox_version_number) quest = self.extract_infobox_value(self.template, key) if quest is None: quest = self.extract_infobox_value(self.template, "quest") if quest is not None: self.item_dict["quest_item"] = infobox_cleaner.clean_quest(quest) else: # Being here means the extraction for "quest" failed key = "questrequired" + str(self.infobox_version_number) quest = self.extract_infobox_value(self.template, key) if quest is None: quest = self.extract_infobox_value(self.template, "questrequired") if quest is not None: self.item_dict["quest_item"] = infobox_cleaner.clean_quest( quest) else: self.item_dict["quest_item"] = False # Determine the release date of an item release_date = None if self.infobox_version_number is not None: key = "release" + str(self.infobox_version_number) release_date = self.extract_infobox_value(self.template, key) if release_date is None: release_date = self.extract_infobox_value(self.template, "release") if release_date is not None: self.item_dict[ "release_date"] = infobox_cleaner.clean_release_date( release_date) else: self.item_dict["release_date"] = None self.item_dict["incomplete"] = True # Determine if an item is tradeable tradeable = None if self.infobox_version_number is not None: key = "tradeable" + str(self.infobox_version_number) tradeable = self.extract_infobox_value(self.template, key) if tradeable is None: tradeable = self.extract_infobox_value(self.template, "tradeable") if tradeable is not None: self.item_dict["tradeable"] = infobox_cleaner.clean_boolean( tradeable) else: self.item_dict["tradeable"] = False self.item_dict["incomplete"] = True # Determine the examine text of an item examine = None if self.infobox_version_number is not None: key = "examine" + str(self.infobox_version_number) examine = self.extract_infobox_value(self.template, key) if examine is None: examine = self.extract_infobox_value(self.template, "examine") if examine is not None: self.item_dict["examine"] = infobox_cleaner.clean_examine( examine, self.item_dict["name"]) else: # Being here means the extraction for "examine" failed key = "itemexamine" + str(self.infobox_version_number) examine = self.extract_infobox_value(self.template, key) if examine is None: examine = self.extract_infobox_value(self.template, "itemexamine") if examine is not None: self.item_dict["examine"] = infobox_cleaner.clean_examine( examine, self.item_dict["name"]) else: self.item_dict["examine"] = None self.item_dict["incomplete"] = True # Set item buy limit, if it is tradeable on the GE if not self.item_dict["tradeable_on_ge"]: self.item_dict["buy_limit"] = None else: buy_limit = None try: buy_limit = self.buy_limits_data[self.item_id_str] except KeyError: print( "populate_item_properties_from_wiki_data: Error setting buy limit..." ) print(f"{self.item_id} has no buy limit available. Exiting.") exit() self.item_dict["buy_limit"] = buy_limit # We finished processing, set incomplete to false if not true if not self.item_dict.get("incomplete"): self.item_dict["incomplete"] = False return True def populate_equipable_properties_from_wiki_data(self) -> bool: """Parse the wiki text template and extract item bonus values from it.""" # Initialize empty equipment dictionary self.item_dict["equipment"] = dict() # Extract the infobox bonuses template infobox_parser = WikitextTemplateParser(self.item_wikitext) has_infobox = infobox_parser.extract_infobox("infobox bonuses") if not has_infobox: has_infobox = infobox_parser.extract_infobox("infobox_bonuses") if not has_infobox: # No infobox bonuses found for the item! print( "populate_equipable_properties: Item has no equipment infobox." ) logging.critical( "populate_equipable_properties: Item has no equipment infobox." ) quit() # Set the template template = infobox_parser.template # STAGE ONE: EQUIPABLE ITEM # This item must be equipable by a player, set to True self.item_dict["equipable_by_player"] = True # Extract equipable item properties self.item_dict["equipment"]["attack_stab"] = self.clean_bonuses_value( template, "astab") self.item_dict["equipment"]["attack_slash"] = self.clean_bonuses_value( template, "aslash") self.item_dict["equipment"]["attack_crush"] = self.clean_bonuses_value( template, "acrush") self.item_dict["equipment"]["attack_magic"] = self.clean_bonuses_value( template, "amagic") self.item_dict["equipment"][ "attack_ranged"] = self.clean_bonuses_value(template, "arange") self.item_dict["equipment"]["defence_stab"] = self.clean_bonuses_value( template, "dstab") self.item_dict["equipment"][ "defence_slash"] = self.clean_bonuses_value(template, "dslash") self.item_dict["equipment"][ "defence_crush"] = self.clean_bonuses_value(template, "dcrush") self.item_dict["equipment"][ "defence_magic"] = self.clean_bonuses_value(template, "dmagic") self.item_dict["equipment"][ "defence_ranged"] = self.clean_bonuses_value(template, "drange") self.item_dict["equipment"][ "melee_strength"] = self.clean_bonuses_value(template, "str") self.item_dict["equipment"][ "ranged_strength"] = self.clean_bonuses_value(template, "rstr") self.item_dict["equipment"]["magic_damage"] = self.clean_bonuses_value( template, "mdmg") self.item_dict["equipment"]["prayer"] = self.clean_bonuses_value( template, "prayer") # Determine the slot for the equipable item self.item_dict["equipment"]["slot"] = None try: self.item_dict["equipment"]["slot"] = self.strip_infobox( template.get("slot").value) self.item_dict["equipment"]["slot"] = self.item_dict["equipment"][ "slot"].lower() except ValueError: self.item_dict["equipment"]["slot"] = None print( "populate_equipable_properties: Could not determine item slot..." ) logging.critical( "populate_equipable_properties: Could not determine item slot..." ) quit() # Determine the skill requirements for the equipable item self.item_dict["equipment"]["requirements"] = None try: requirements = self.skill_requirements[str(self.item_id)] self.item_dict["equipment"]["requirements"] = requirements except KeyError: print( "populate_equipable_properties: Could not determine skill requirements..." ) logging.critical( "populate_equipable_properties: Could not determine skill requirements..." ) quit() # STAGE TWO: WEAPONS # If item is weapon, two-handed, or 2h, start processing the weapon data if (self.item_dict["equipment"]["slot"] == "weapon" or self.item_dict["equipment"]["slot"] == "two-handed" or self.item_dict["equipment"]["slot"] == "2h"): self.item_dict["weapon"] = dict() # Try set the attack speed of the weapon try: self.item_dict["weapon"]["attack_speed"] = int( self.strip_infobox(template.get("speed").value)) except ValueError: self.item_dict["weapon"]["attack_speed"] = None logging.critical( "WEAPON: Could not determine weapon attack speed") # Item IDs with no known attack speed, set to zero if int(self.item_id) in [8871]: self.item_dict["weapon"]["attack_speed"] = 0 # Salamander fix, set to base attack speed of 5 elif int(self.item_id) in [ 10145, 10146, 10147, 10147, 10148, 10149 ]: self.item_dict["weapon"]["attack_speed"] = 5 else: pass # quit() # Try to set the weapon type of the weapon try: weapon_type = self.weapon_types_data[str( self.item_dict["id"])]["weapon_type"] self.item_dict["weapon"]["weapon_type"] = weapon_type except KeyError: self.item_dict["weapon"]["weapon_type"] = None print( "populate_equipable_properties: Could not determine weapon type..." ) logging.critical( "populate_equipable_properties: Could not determine weapon type..." ) quit() # Try to set stances available for the weapon try: self.item_dict["weapon"]["stances"] = self.weapon_stances_data[ self.item_dict["weapon"]["weapon_type"]] except KeyError: self.item_dict["weapon"]["stances"] = None print( "populate_equipable_properties: Could not determine weapon stance..." ) logging.critical( "populate_equipable_properties: Could not determine weapon stance..." ) quit() # Finally, set the equipable_weapon property to true self.item_dict["equipable_weapon"] = True else: # If the item is not a weapon, two-handed or 2h it is not a weapon self.item_dict["equipable_weapon"] = False return True def extract_infobox_value( self, template: mwparserfromhell.nodes.template.Template, key: str) -> str: """Helper method to extract a value from a template using a specified key. This helper method is a simple solution to repeatedly try to fetch a specific entry from a wiki text template (a mwparserfromhell template object). :param template: A mediawiki wiki text template. :param key: The key to query in the template. :return value: The extracted template value based on supplied key. """ value = None try: value = template.get(key).value value = value.strip() return value except ValueError: return value def clean_bonuses_value(self, template: mwparserfromhell.nodes.template.Template, prop: str) -> int: """Clean a item bonuses value extracted from a wiki template. :param template: A mediawiki wiki text template. :param prop: The key to query in the template. :return value: The extracted template value that has been int cast. """ value = None # Try and get the versioned infobox value if self.infobox_version_number is not None: key = prop + str(self.infobox_version_number) value = self.extract_infobox_value(template, key) # If unsuccessful, try and get the normal infoxbox value if value is None: value = self.extract_infobox_value(template, prop) if value is not None: value = self.strip_infobox(value) if isinstance(value, str): if value[0] == "-": if value[1:].isdigit(): value = int(value) elif value[0] == "+": if value[1:].isdigit(): value = int(value) else: if value.isdigit(): value = int(value) else: value = 0 return value def strip_infobox(self, value: str) -> str: """Generic infobox wiki text cleaner. :return clean_value: A cleaned wiki text value. """ # Clean an passed InfoBox string clean_value = str(value) clean_value = clean_value.strip() clean_value = clean_value.replace("[", "") clean_value = clean_value.replace("]", "") return clean_value def compare_json_files(self, item_properties: ItemProperties) -> bool: """Print the difference between this item and the database.""" # Create JSON out object to compare current_json = item_properties.construct_json() # Try get existing entry (KeyError means it doesn't exist - aka a new item) try: existing_json = self.all_db_items[self.item_id] except KeyError: print(f">>> compare_json_files: NEW ITEM: {item_properties.id}") print(current_json) return if current_json == existing_json: return ddiff = DeepDiff(existing_json, current_json, ignore_order=True) logging.debug( f">>> compare_json_files: CHANGED ITEM: {item_properties.id}: {item_properties.name}, {item_properties.wiki_name}" ) print( f">>> compare_json_files: CHANGED ITEM: {item_properties.id}: {item_properties.name}" ) print(ddiff) return def validate_item(self): """Use the schema-items.json file to validate the populated item.""" # Create JSON out object to validate current_json = self.item_properties.construct_json() # Validate object with schema attached v = config.MyValidator(self.schema_data) v.validate(current_json) assert v.validate(current_json)
def generate_item_object(self): """Generate the `ItemProperties` object from the item_dict dictionary.""" self.item_properties = ItemProperties(**self.item_dict)