def run(self): # Start processing every item! for item_id in self.all_items_cache_data: # if int(item_id) < 25800: # continue # Skip any beta items if "(beta" in self.all_items_cache_data[item_id]["name"]: continue # Initialize the BuildItem class, used for all items builder = build_item.BuildItem( item_id=item_id, all_items_cache_data=self.all_items_cache_data, all_db_items=self.all_db_items, all_wikitext_raw=self.all_wikitext_raw, all_wikitext_processed=self.all_wikitext_processed, unalchable=self.unalchable, buy_limits=self.buy_limits, skill_requirements=self.skill_requirements, weapon_stances=self.weapon_stances, icons=self.icons, duplicates=self.duplicates, schema_data=self.schema_data, known_items=self.known_items, verbose=self.verbose) status = builder.preprocessing() if status["status"]: builder.populate_wiki_item() else: builder.populate_non_wiki_item() known_item = builder.check_duplicate_item() if known_item: self.known_items.append(known_item) if self.compare: builder.compare_new_vs_old_item() if self.export: builder.export_item_to_json() if self.validate: builder.validate_item() # Done processing, rejoice! print("Built.") exit(0)
def test(self): # Start processing every item! for item_id in self.all_items_cache_data: # if int(item_id) < 25000: # continue # Initialize the BuildItem class, used for all items builder = build_item.BuildItem( item_id=item_id, all_items_cache_data=self.all_items_cache_data, all_db_items=self.all_db_items, all_wikitext_raw=self.all_wikitext_raw, all_wikitext_processed=self.all_wikitext_processed, unalchable=self.unalchable, buy_limits=self.buy_limits, skill_requirements=self.skill_requirements, weapon_stances=self.weapon_stances, icons=self.icons, duplicates=self.duplicates, schema_data=self.schema_data, known_items=self.known_items, verbose=self.verbose) status = builder.preprocessing() if status["status"]: builder.populate_wiki_item() else: builder.populate_non_wiki_item() known_item = builder.check_duplicate_item() if known_item: self.known_items.append(known_item) builder.validate_item() # Done testing, rejoice! print("Tested.") exit(0)
def main(export: bool = False, verbose: bool = False): # Load the current database contents items_compltete_file_path = Path(config.DOCS_PATH / "items-complete.json") with open(items_compltete_file_path) as f: all_db_items = json.load(f) # Load the item wikitext file wiki_text_file_path = Path(config.DATA_WIKI_PATH / "page-text-items.json") with open(wiki_text_file_path) as f: all_wikitext_raw = json.load(f) # Temp loading of item ID -> wikitext processed_wikitextfile_path = Path(config.DATA_WIKI_PATH / "processed-wikitext-items.json") with open(processed_wikitextfile_path) as f: all_wikitext_processed = json.load(f) # Load the invalid items file invalid_items_file_path = Path(config.DATA_ITEMS_PATH / "invalid-items.json") with open(invalid_items_file_path) as f: invalid_items_data = json.load(f) # Load buy limit data buy_limits_file_path = Path(config.DATA_ITEMS_PATH / "ge-limits-names.json") with open(buy_limits_file_path) as f: buy_limits_data = json.load(f) # Load skill requirement data skill_requirements_file_path = Path(config.DATA_ITEMS_PATH / "skill-requirements.json") with open(skill_requirements_file_path) as f: skill_requirements_data = json.load(f) # Load weapon_type data weapon_type_file_path = Path(config.DATA_ITEMS_PATH / "weapon-types.json") with open(weapon_type_file_path) as f: weapon_types_data = json.load(f) # Load stances data weapon_stance_file_path = Path(config.DATA_ITEMS_PATH / "weapon-stances.json") with open(weapon_stance_file_path) as f: weapon_stances_data = json.load(f) # Load the raw OSRS cache item data # This is the final data load, and used as baseline data for database population all_item_cache_data_path = Path(config.DATA_ITEMS_PATH / "items-cache-data.json") with open(all_item_cache_data_path) as f: all_item_cache_data = json.load(f) # Load schema data schema_file_path = Path(config.DATA_SCHEMAS_PATH / "schema-items.json") with open(schema_file_path) as f: schema_data = json.load(f) # Initialize a list of known items known_items = list() # Start processing every item! for item_id in all_item_cache_data: # Toggle to start, stop at a specific item ID # if int(item_id) < 24300: # continue # Initialize the BuildItem class, used for all items builder = build_item.BuildItem( item_id=item_id, all_item_cache_data=all_item_cache_data, all_wikitext_processed=all_wikitext_processed, all_wikitext_raw=all_wikitext_raw, all_db_items=all_db_items, buy_limits_data=buy_limits_data, skill_requirements_data=skill_requirements_data, weapon_types_data=weapon_types_data, weapon_stances_data=weapon_stances_data, invalid_items_data=invalid_items_data, known_items=known_items, schema_data=schema_data, export=export, verbose=verbose) preprocessing_status = builder.preprocessing() if preprocessing_status["status"]: builder.populate_item() known_item = builder.check_duplicate_item() known_items.append(known_item) builder.generate_item_object() builder.compare_new_vs_old_item() builder.export_item_to_json() builder.validate_item() else: builder.populate_from_cache_data() builder.populate_non_wiki_item() known_item = builder.check_duplicate_item() known_items.append(known_item) builder.generate_item_object() builder.compare_new_vs_old_item() builder.export_item_to_json() builder.validate_item() # Done processing, rejoice! print("Done.")
def main(): # Load the current database contents items_compltete_file_path = Path(config.DOCS_PATH / "items-complete.json") with open(items_compltete_file_path) as f: all_db_items = json.load(f) # Load the item wikitext file wiki_text_file_path = Path(config.DATA_WIKI_PATH / "page-text-items.json") with open(wiki_text_file_path) as f: all_wikitext_raw = json.load(f) # Temp loading of item ID -> wikitext processed_wikitextfile_path = Path(config.DATA_WIKI_PATH / "processed-wikitext-items.json") with open(processed_wikitextfile_path) as f: all_wikitext_processed = json.load(f) # Load the invalid items file invalid_items_file_path = Path(config.DATA_ITEMS_PATH / "invalid-items.json") with open(invalid_items_file_path) as f: invalid_items_data = json.load(f) # Load buy limit data buy_limits_file_path = Path(config.DATA_ITEMS_PATH / "ge-limits-names.json") with open(buy_limits_file_path) as f: buy_limits_data = json.load(f) # Load skill requirement data skill_requirements_file_path = Path(config.DATA_ITEMS_PATH / "skill-requirements.json") with open(skill_requirements_file_path) as f: skill_requirements_data = json.load(f) # Load weapon_type data weapon_type_file_path = Path(config.DATA_ITEMS_PATH / "weapon-types.json") with open(weapon_type_file_path) as f: weapon_types_data = json.load(f) # Load stances data weapon_stance_file_path = Path(config.DATA_ITEMS_PATH / "weapon-stances.json") with open(weapon_stance_file_path) as f: weapon_stances_data = json.load(f) # Load the raw OSRS cache item data # This is the final data load, and used as baseline data for database population all_item_cache_data_path = Path(config.DATA_ITEMS_PATH / "items-cache-data.json") with open(all_item_cache_data_path) as f: all_item_cache_data = json.load(f) # Set export to false export = False # Initialize a list of known items known_items = list() # Start processing every item! for item_id in all_item_cache_data: # Toggle to start, stop at a specific item ID # if int(item_id) < 24000: # continue # Initialize the BuildItem class, used for all items builder = build_item.BuildItem( item_id=item_id, all_item_cache_data=all_item_cache_data, all_wikitext_processed=all_wikitext_processed, all_wikitext_raw=all_wikitext_raw, all_db_items=all_db_items, buy_limits_data=buy_limits_data, skill_requirements_data=skill_requirements_data, weapon_types_data=weapon_types_data, weapon_stances_data=weapon_stances_data, invalid_items_data=invalid_items_data, known_items=known_items, export=export) preprocessing_status = builder.preprocessing() item_id_original = builder.item_id_str item_id_to_process = builder.item_id_to_process_str # preprocessing_status is a dictionary # {'status': True, 'code': 'valid'} # A valid item with an OSRS Wiki page (found using id, linked_id, name): # preprocessing_status["code"] == "valid" # An invalid item without an OSRS Wiki page will have a variety of errors # See the builders/items/build_item.py module for error codes if preprocessing_status["code"] == "valid": # Check for valid items in the invalid-items.json file if item_id_to_process in invalid_items_data: print(item_id_original, item_id_to_process, builder.item_name, invalid_items_data[item_id_to_process]["status"]) else: print(item_id_original, item_id_to_process, builder.item_name, preprocessing_status["code"]) # Done processing, rejoice! print("Done.")
def main(): # Load the current database contents items_compltete_file_path = Path(config.DOCS_PATH / "items-complete.json") with open(items_compltete_file_path) as f: all_db_items = json.load(f) # Load the item wikitext file wiki_text_file_path = Path(config.DATA_WIKI_PATH / "page-text-items.json") with open(wiki_text_file_path) as f: all_wikitext_raw = json.load(f) # Temp loading of item ID -> wikitext processed_wikitextfile_path = Path(config.DATA_WIKI_PATH / "processed-wikitext-items.json") with open(processed_wikitextfile_path) as f: all_wikitext_processed = json.load(f) # Load the invalid items file invalid_items_file_path = Path(config.DATA_ITEMS_PATH / "invalid-items.json") with open(invalid_items_file_path) as f: invalid_items_data = json.load(f) # Load buy limit data buy_limits_file_path = Path(config.DATA_ITEMS_PATH / "ge-limits-names.json") with open(buy_limits_file_path) as f: buy_limits_data = json.load(f) # Load skill requirement data skill_requirements_file_path = Path(config.DATA_ITEMS_PATH / "skill-requirements.json") with open(skill_requirements_file_path) as f: skill_requirements_data = json.load(f) # Load weapon_type data weapon_type_file_path = Path(config.DATA_ITEMS_PATH / "weapon-types.json") with open(weapon_type_file_path) as f: weapon_types_data = json.load(f) # Load stances data weapon_stance_file_path = Path(config.DATA_ITEMS_PATH / "weapon-stances.json") with open(weapon_stance_file_path) as f: weapon_stances_data = json.load(f) # Load the raw OSRS cache item data # This is the final data load, and used as baseline data for database population all_item_cache_data_path = Path(config.DATA_ITEMS_PATH / "items-cache-data.json") with open(all_item_cache_data_path) as f: all_item_cache_data = json.load(f) # Load schema data schema_file_path = Path(config.DATA_SCHEMAS_PATH / "schema-items.json") with open(schema_file_path) as f: schema_data = json.load(f) # Set export to false export = False # Set verbose to false verbose = False # Initialize a list of known items known_items = list() # Start processing every item! for item_id in all_item_cache_data: # Toggle to start, stop at a specific item ID if int(item_id) < 24000: continue # Initialize the BuildItem class, used for all items builder = build_item.BuildItem( item_id=item_id, all_item_cache_data=all_item_cache_data, all_wikitext_processed=all_wikitext_processed, all_wikitext_raw=all_wikitext_raw, all_db_items=all_db_items, buy_limits_data=buy_limits_data, skill_requirements_data=skill_requirements_data, weapon_types_data=weapon_types_data, weapon_stances_data=weapon_stances_data, invalid_items_data=invalid_items_data, known_items=known_items, schema_data=schema_data, export=export, verbose=verbose) preprocessing_status = builder.preprocessing() item_id_original = builder.item_id_str item_id_to_process = builder.item_id_to_process_str # Preprocessing codes: # {'status': True, 'code': 'various_options_here_listed_below'} # lookup_passed_id: Found item in wiki using ID # lookup_passed_linked_id: Found item in wiki using linked ID # lookup_passed_name: Found item in wiki using name # Anything else is invalid... valid_item_codes = [ "lookup_passed_id", "lookup_passed_linked_id", "lookup_passed_name" ] # Check for valid items in invalid-items.json file... if preprocessing_status["code"] in valid_item_codes: if item_id_to_process in invalid_items_data: print("ERROR: Item should not be invalid:") print(item_id_original, item_id_to_process, builder.item_name, invalid_items_data[item_id_to_process]["status"]) # Preprocessing codes: # lookup_failed: No wiki entry # no_item_wikitext: No wikitext data # no_infobox_template: No wiki infobox data invalid_item_codes = [ "lookup_failed", "no_item_wikitext", "no_infobox_template" ] # Check for items that failed lookup and are not in invalid-items.json file... if preprocessing_status["code"] in invalid_item_codes: try: invalid_items_data[item_id_to_process] except KeyError: print("ERROR: Item not marked as invalid:") print(item_id_original, item_id_to_process, builder.item_name) # Done processing, rejoice! print("Done.")