def run(self):
# Start processing every item!
for item_id in self.all_items_cache_data:
# if int(item_id) < 25800:
# continue
# Skip any beta items
if "(beta" in self.all_items_cache_data[item_id]["name"]:
continue
# Initialize the BuildItem class, used for all items
builder = build_item.BuildItem(
item_id=item_id,
all_items_cache_data=self.all_items_cache_data,
all_db_items=self.all_db_items,
all_wikitext_raw=self.all_wikitext_raw,
all_wikitext_processed=self.all_wikitext_processed,
unalchable=self.unalchable,
buy_limits=self.buy_limits,
skill_requirements=self.skill_requirements,
weapon_stances=self.weapon_stances,
icons=self.icons,
duplicates=self.duplicates,
schema_data=self.schema_data,
known_items=self.known_items,
verbose=self.verbose)
status = builder.preprocessing()
if status["status"]:
builder.populate_wiki_item()
else:
builder.populate_non_wiki_item()
known_item = builder.check_duplicate_item()
if known_item:
self.known_items.append(known_item)
if self.compare:
builder.compare_new_vs_old_item()
if self.export:
builder.export_item_to_json()
if self.validate:
builder.validate_item()
# Done processing, rejoice!
print("Built.")
exit(0)
def test(self):
# Start processing every item!
for item_id in self.all_items_cache_data:
# if int(item_id) < 25000:
# continue
# Initialize the BuildItem class, used for all items
builder = build_item.BuildItem(
item_id=item_id,
all_items_cache_data=self.all_items_cache_data,
all_db_items=self.all_db_items,
all_wikitext_raw=self.all_wikitext_raw,
all_wikitext_processed=self.all_wikitext_processed,
unalchable=self.unalchable,
buy_limits=self.buy_limits,
skill_requirements=self.skill_requirements,
weapon_stances=self.weapon_stances,
icons=self.icons,
duplicates=self.duplicates,
schema_data=self.schema_data,
known_items=self.known_items,
verbose=self.verbose)
status = builder.preprocessing()
if status["status"]:
builder.populate_wiki_item()
else:
builder.populate_non_wiki_item()
known_item = builder.check_duplicate_item()
if known_item:
self.known_items.append(known_item)
builder.validate_item()
# Done testing, rejoice!
print("Tested.")
exit(0)
def main(export: bool = False, verbose: bool = False):
# Load the current database contents
items_compltete_file_path = Path(config.DOCS_PATH / "items-complete.json")
with open(items_compltete_file_path) as f:
all_db_items = json.load(f)
# Load the item wikitext file
wiki_text_file_path = Path(config.DATA_WIKI_PATH / "page-text-items.json")
with open(wiki_text_file_path) as f:
all_wikitext_raw = json.load(f)
# Temp loading of item ID -> wikitext
processed_wikitextfile_path = Path(config.DATA_WIKI_PATH /
"processed-wikitext-items.json")
with open(processed_wikitextfile_path) as f:
all_wikitext_processed = json.load(f)
# Load the invalid items file
invalid_items_file_path = Path(config.DATA_ITEMS_PATH /
"invalid-items.json")
with open(invalid_items_file_path) as f:
invalid_items_data = json.load(f)
# Load buy limit data
buy_limits_file_path = Path(config.DATA_ITEMS_PATH /
"ge-limits-names.json")
with open(buy_limits_file_path) as f:
buy_limits_data = json.load(f)
# Load skill requirement data
skill_requirements_file_path = Path(config.DATA_ITEMS_PATH /
"skill-requirements.json")
with open(skill_requirements_file_path) as f:
skill_requirements_data = json.load(f)
# Load weapon_type data
weapon_type_file_path = Path(config.DATA_ITEMS_PATH / "weapon-types.json")
with open(weapon_type_file_path) as f:
weapon_types_data = json.load(f)
# Load stances data
weapon_stance_file_path = Path(config.DATA_ITEMS_PATH /
"weapon-stances.json")
with open(weapon_stance_file_path) as f:
weapon_stances_data = json.load(f)
# Load the raw OSRS cache item data
# This is the final data load, and used as baseline data for database population
all_item_cache_data_path = Path(config.DATA_ITEMS_PATH /
"items-cache-data.json")
with open(all_item_cache_data_path) as f:
all_item_cache_data = json.load(f)
# Load schema data
schema_file_path = Path(config.DATA_SCHEMAS_PATH / "schema-items.json")
with open(schema_file_path) as f:
schema_data = json.load(f)
# Initialize a list of known items
known_items = list()
# Start processing every item!
for item_id in all_item_cache_data:
# Toggle to start, stop at a specific item ID
# if int(item_id) < 24300:
# continue
# Initialize the BuildItem class, used for all items
builder = build_item.BuildItem(
item_id=item_id,
all_item_cache_data=all_item_cache_data,
all_wikitext_processed=all_wikitext_processed,
all_wikitext_raw=all_wikitext_raw,
all_db_items=all_db_items,
buy_limits_data=buy_limits_data,
skill_requirements_data=skill_requirements_data,
weapon_types_data=weapon_types_data,
weapon_stances_data=weapon_stances_data,
invalid_items_data=invalid_items_data,
known_items=known_items,
schema_data=schema_data,
export=export,
verbose=verbose)
preprocessing_status = builder.preprocessing()
if preprocessing_status["status"]:
builder.populate_item()
known_item = builder.check_duplicate_item()
known_items.append(known_item)
builder.generate_item_object()
builder.compare_new_vs_old_item()
builder.export_item_to_json()
builder.validate_item()
else:
builder.populate_from_cache_data()
builder.populate_non_wiki_item()
known_item = builder.check_duplicate_item()
known_items.append(known_item)
builder.generate_item_object()
builder.compare_new_vs_old_item()
builder.export_item_to_json()
builder.validate_item()
# Done processing, rejoice!
print("Done.")
def main():
# Load the current database contents
items_compltete_file_path = Path(config.DOCS_PATH / "items-complete.json")
with open(items_compltete_file_path) as f:
all_db_items = json.load(f)
# Load the item wikitext file
wiki_text_file_path = Path(config.DATA_WIKI_PATH / "page-text-items.json")
with open(wiki_text_file_path) as f:
all_wikitext_raw = json.load(f)
# Temp loading of item ID -> wikitext
processed_wikitextfile_path = Path(config.DATA_WIKI_PATH /
"processed-wikitext-items.json")
with open(processed_wikitextfile_path) as f:
all_wikitext_processed = json.load(f)
# Load the invalid items file
invalid_items_file_path = Path(config.DATA_ITEMS_PATH /
"invalid-items.json")
with open(invalid_items_file_path) as f:
invalid_items_data = json.load(f)
# Load buy limit data
buy_limits_file_path = Path(config.DATA_ITEMS_PATH /
"ge-limits-names.json")
with open(buy_limits_file_path) as f:
buy_limits_data = json.load(f)
# Load skill requirement data
skill_requirements_file_path = Path(config.DATA_ITEMS_PATH /
"skill-requirements.json")
with open(skill_requirements_file_path) as f:
skill_requirements_data = json.load(f)
# Load weapon_type data
weapon_type_file_path = Path(config.DATA_ITEMS_PATH / "weapon-types.json")
with open(weapon_type_file_path) as f:
weapon_types_data = json.load(f)
# Load stances data
weapon_stance_file_path = Path(config.DATA_ITEMS_PATH /
"weapon-stances.json")
with open(weapon_stance_file_path) as f:
weapon_stances_data = json.load(f)
# Load the raw OSRS cache item data
# This is the final data load, and used as baseline data for database population
all_item_cache_data_path = Path(config.DATA_ITEMS_PATH /
"items-cache-data.json")
with open(all_item_cache_data_path) as f:
all_item_cache_data = json.load(f)
# Set export to false
export = False
# Initialize a list of known items
known_items = list()
# Start processing every item!
for item_id in all_item_cache_data:
# Toggle to start, stop at a specific item ID
# if int(item_id) < 24000:
# continue
# Initialize the BuildItem class, used for all items
builder = build_item.BuildItem(
item_id=item_id,
all_item_cache_data=all_item_cache_data,
all_wikitext_processed=all_wikitext_processed,
all_wikitext_raw=all_wikitext_raw,
all_db_items=all_db_items,
buy_limits_data=buy_limits_data,
skill_requirements_data=skill_requirements_data,
weapon_types_data=weapon_types_data,
weapon_stances_data=weapon_stances_data,
invalid_items_data=invalid_items_data,
known_items=known_items,
export=export)
preprocessing_status = builder.preprocessing()
item_id_original = builder.item_id_str
item_id_to_process = builder.item_id_to_process_str
# preprocessing_status is a dictionary
# {'status': True, 'code': 'valid'}
# A valid item with an OSRS Wiki page (found using id, linked_id, name):
# preprocessing_status["code"] == "valid"
# An invalid item without an OSRS Wiki page will have a variety of errors
# See the builders/items/build_item.py module for error codes
if preprocessing_status["code"] == "valid":
# Check for valid items in the invalid-items.json file
if item_id_to_process in invalid_items_data:
print(item_id_original, item_id_to_process, builder.item_name,
invalid_items_data[item_id_to_process]["status"])
else:
print(item_id_original, item_id_to_process, builder.item_name,
preprocessing_status["code"])
# Done processing, rejoice!
print("Done.")
def main():
# Load the current database contents
items_compltete_file_path = Path(config.DOCS_PATH / "items-complete.json")
with open(items_compltete_file_path) as f:
all_db_items = json.load(f)
# Load the item wikitext file
wiki_text_file_path = Path(config.DATA_WIKI_PATH / "page-text-items.json")
with open(wiki_text_file_path) as f:
all_wikitext_raw = json.load(f)
# Temp loading of item ID -> wikitext
processed_wikitextfile_path = Path(config.DATA_WIKI_PATH /
"processed-wikitext-items.json")
with open(processed_wikitextfile_path) as f:
all_wikitext_processed = json.load(f)
# Load the invalid items file
invalid_items_file_path = Path(config.DATA_ITEMS_PATH /
"invalid-items.json")
with open(invalid_items_file_path) as f:
invalid_items_data = json.load(f)
# Load buy limit data
buy_limits_file_path = Path(config.DATA_ITEMS_PATH /
"ge-limits-names.json")
with open(buy_limits_file_path) as f:
buy_limits_data = json.load(f)
# Load skill requirement data
skill_requirements_file_path = Path(config.DATA_ITEMS_PATH /
"skill-requirements.json")
with open(skill_requirements_file_path) as f:
skill_requirements_data = json.load(f)
# Load weapon_type data
weapon_type_file_path = Path(config.DATA_ITEMS_PATH / "weapon-types.json")
with open(weapon_type_file_path) as f:
weapon_types_data = json.load(f)
# Load stances data
weapon_stance_file_path = Path(config.DATA_ITEMS_PATH /
"weapon-stances.json")
with open(weapon_stance_file_path) as f:
weapon_stances_data = json.load(f)
# Load the raw OSRS cache item data
# This is the final data load, and used as baseline data for database population
all_item_cache_data_path = Path(config.DATA_ITEMS_PATH /
"items-cache-data.json")
with open(all_item_cache_data_path) as f:
all_item_cache_data = json.load(f)
# Load schema data
schema_file_path = Path(config.DATA_SCHEMAS_PATH / "schema-items.json")
with open(schema_file_path) as f:
schema_data = json.load(f)
# Set export to false
export = False
# Set verbose to false
verbose = False
# Initialize a list of known items
known_items = list()
# Start processing every item!
for item_id in all_item_cache_data:
# Toggle to start, stop at a specific item ID
if int(item_id) < 24000:
continue
# Initialize the BuildItem class, used for all items
builder = build_item.BuildItem(
item_id=item_id,
all_item_cache_data=all_item_cache_data,
all_wikitext_processed=all_wikitext_processed,
all_wikitext_raw=all_wikitext_raw,
all_db_items=all_db_items,
buy_limits_data=buy_limits_data,
skill_requirements_data=skill_requirements_data,
weapon_types_data=weapon_types_data,
weapon_stances_data=weapon_stances_data,
invalid_items_data=invalid_items_data,
known_items=known_items,
schema_data=schema_data,
export=export,
verbose=verbose)
preprocessing_status = builder.preprocessing()
item_id_original = builder.item_id_str
item_id_to_process = builder.item_id_to_process_str
# Preprocessing codes:
# {'status': True, 'code': 'various_options_here_listed_below'}
# lookup_passed_id: Found item in wiki using ID
# lookup_passed_linked_id: Found item in wiki using linked ID
# lookup_passed_name: Found item in wiki using name
# Anything else is invalid...
valid_item_codes = [
"lookup_passed_id", "lookup_passed_linked_id", "lookup_passed_name"
]
# Check for valid items in invalid-items.json file...
if preprocessing_status["code"] in valid_item_codes:
if item_id_to_process in invalid_items_data:
print("ERROR: Item should not be invalid:")
print(item_id_original, item_id_to_process, builder.item_name,
invalid_items_data[item_id_to_process]["status"])
# Preprocessing codes:
# lookup_failed: No wiki entry
# no_item_wikitext: No wikitext data
# no_infobox_template: No wiki infobox data
invalid_item_codes = [
"lookup_failed", "no_item_wikitext", "no_infobox_template"
]
# Check for items that failed lookup and are not in invalid-items.json file...
if preprocessing_status["code"] in invalid_item_codes:
try:
invalid_items_data[item_id_to_process]
except KeyError:
print("ERROR: Item not marked as invalid:")
print(item_id_original, item_id_to_process, builder.item_name)
# Done processing, rejoice!
print("Done.")
