def get_base_jsons():
"""Loads all the JSONs needed for parsing base skills,
and returns both of them.
If a JSON fails to load, this function will return an empty
dictionary in place of the JSON file.
"""
# with open("building_data_zh.json", "r", encoding="utf8") as f:
# base_skills_json = json.load(f) # debug
# Fetch the jsons
base_skills_req = scrape_json(
read_line_from_file("./info/scraper/baseSkillsJsonUrl.txt"))
riic_req = scrape_json(
read_line_from_file("./info/scraper/riicJsonUrl.txt"))
base_skills_json, riic_json = {}, {}
# Make sure we retrieve the JSONs correctly
if base_skills_req is not None:
base_skills_json = base_skills_req.json()
if riic_req is not None:
riic_json = riic_req.json()
# with open("riic.json", "r", encoding="utf8") as f:
# riic_json = json.load(f)
return base_skills_json, riic_json
def initialize_operator_list() -> Optional[List[Type[TaggedOperator]]]:
"""Initializes a list of TaggedOperators with names and tags and
returns said list.
This function uses a recruitment json created by Aceship to find
each operator's tags, so if this function is unable to find the
json, it will return None which should be caught.
Otherwise, it will generate a list of TaggedOperator objects, each
with a name, rarity, and recruitment tags.
Note that hidden operators (globalHidden or hidden) will not be
included in this list.
"""
operatortags_rawjson = scrape_json(
read_line_from_file("./info/recruitops/recruitTagJsonUrl.txt"))
# with open("tags_zh.json", "r", encoding="utf8") as f:
# operatortags_rawjson = json.load(f) # debug
if operatortags_rawjson is None:
return None
# Getting the json content returns a list, so we call it a
# list appropriately
operatortags_list = operatortags_rawjson.json()
# operatortags_list = operatortags_rawjson # debug
operator_list = []
name_replacements = read_lines_into_dict(
"./info/recruitops/operatorNameReplacements.txt")
# initialize an easy to access list of operators and their tags
#
# Some operators aren't available in HH and thus they are labelled
# as 'hidden' or 'globalHidden' in the json.
for operator in operatortags_list:
if (operator["hidden"] or
("globalHidden" in operator.keys() and operator["globalHidden"])):
continue
operator_list.append(
TaggedOperator(
operator["name_en"], operator["level"], operator["tags"] +
[operator["type"].rstrip()]) if operator["name_en"] not in
name_replacements else TaggedOperator(
name_replacements[operator["name_en"]], operator["level"],
operator["tags"] + [operator["type"].rstrip()]))
return operator_list
def get_operator_dict(operator):
"""Searches the Aceship character JSON for a specified operator,
and returns the associated character dict and the character key
in the JSON if found.
If the operator was not found, return an empty dict and None
for the key, indicating that the scraper should try using
Gamepress instead of the JSON, as the character is not
in the JSON yet.
"""
# Since the JSON I first use to find info uses
# properly formatted names, I have to convert any name
# to a properly formatted one
# TODO: Does this break DRY? This appears in gamepress too
replacement_names = read_lines_into_dict(
"./info/scraper/jsonOperatorReplacements.txt")
formatted_name = operator.replace("-", " ").title()
proper_name = (formatted_name
if formatted_name not in replacement_names.keys() else
replacement_names[formatted_name])
# with open("character_table.json", "r", encoding="utf8") as f:
# operator_raw_json = json.load(f) # debug
operator_raw_json = scrape_json(
read_line_from_file("./info/scraper/operatorJsonUrl.txt"))
if operator_raw_json is None:
return {}, None
operator_dict = {}
operator_key = None
operator_json = operator_raw_json.json()
# operator_json = operator_raw_json #debug
for operator in operator_json.keys():
# So that names like "SilverAsh" don't screw up the parser,
# we take the key and convert it to the title form (Silverash)
if operator_json[operator]["name"].title() == proper_name:
operator_key = operator
operator_dict = operator_json[operator]
break
return operator_dict, operator_key
def scrape_for_operator(operator):
"""Sends a GET request for a certain operator and returns the
Response object if status code is 200.
Returns None (as per scrape_website() implementation) if server
responds with a different code.
"""
url_replacement_names = read_lines_into_dict(
"./info/scraper/urlOperatorReplacements.txt")
operator_url = read_line_from_file("./info/scraper/url.txt")
operator_url = (
operator_url + "operator/" + operator
if operator not in url_replacement_names.keys()
else operator_url + "operator/"
+ url_replacement_names[operator]
)
return scrape_website(operator_url)
def get_skill_jsons():
"""Loads the skill JSON needed to properly parse operator skills,
and returns it.
If the JSON fails to load, this function will return an empty
dictionary in place of the JSON file.
"""
# with open("skill_table.json", "r", encoding="utf8") as f:
# skills_json = json.load(f) # debug
skills_req = scrape_json(
read_line_from_file("./info/scraper/skillsJsonUrl.txt"))
skills_json = {}
# Make sure the request didn't fail, cause if it did, we can simply
# provide an empty dict and have them catch it.
if skills_req is not None:
skills_json = skills_req.json()
return skills_json
def create_stats_json(soup, operator):
"""Creates the JSON file (dictionary) containing all the operator's stats, and returns it.
This dictionary MUST have the basic operator stats
(ie. atk, def, hp) and will return `None` if these
stats are not found (if the JSON file I use for these stats is
down, or the operator cannot be found.)
This function will first look and load the basic stats
(ATK, DEF, HP) for each stage.
Then it will attempt to load (Block, Cost, Res) from the
operator website under the variable myStats.
Finally, it will attempt to load (Redeploy Time, Attack Interval)
from the operator website from a different section.
Thus, this function will look 3 times for the specified attributes.
If any of these searches fails (except for the first one,
which is essential), this function will simply set
that attribute's value as -1, indicating failure to retrieve.
"""
# Load in the json file for all operators
# TODO: should I make a response obj to hold any possible errors?
# Do this seperately because if I did it every time
# that would be a waste of time
# for requests that don't need this info.
stats_url = read_line_from_file(
"./info/scraper/url.txt") + "/stat-rankings?_format=json"
stats_info = scrape_website(stats_url)
if stats_info is None:
# print("Could not get the JSON file!")
return {} # Request failed
stats_json = None
for json_file in stats_info.json():
# for json_file in stats_info: # debugging
# Find the title so that names like GreyThroat don't screw
# up the parser.
if json_file["title"].title() == operator:
stats_json = json_file
break # whoa a bad break
if stats_json is None:
# print("No operator found!")
return {} # No operator found in the big JSON file
# Find myStats from the operator's site
# This is so we can find res, cost, and block
good_scripts = soup.find_all("script", "")
for script in good_scripts:
if re.search(r"myStats =", str(script)):
mystats_script = script
break # oh no another bad break
levels = ["ne", "e1", "e2"]
wanted_attributes = ["arts", "cost", "block"]
# We're gonna find every stage of each wanted attribute,
# and then add each stage to our stats json.
for attr in wanted_attributes:
all_stats = (re.findall(fr'"{attr}": "(\d+)"', str(mystats_script)))
# Since 'block' has both a base and max (which doesn't matter),
# we're gonna only take every other index.
if attr == "block":
all_stats = all_stats[::2]
# Make sure that all_stats has SOMETHING in it.
# We can't just check for 3 cause some operators don't have E2s
# or E1s
if len(all_stats) != 0:
for stat, lvl in zip(all_stats, levels):
# We're gonna manually add the stat to our
# stats_json dictionary so we can format it later
stats_json[lvl + "_" + attr] = \
int(stat)
else:
# Just to make sure we don't break the program later
# if there's a bad parse somewhere
#
# We'll set every level to -1, because when the program
# later parses these stats, it checks to see if
# level_atk in stats_dict is empty. If it has something in
# level_atk but nothing here, it'll break, but if it has
# nothing in level_atk but something here, nothing happens.
for lvl in levels:
# -1 to symbolize failure
stats_json[lvl + "_" + attr] = -1
# other_stats will provide us with the atk speed
# and redeploy time stats.
other_stats = soup.find_all("div", "other-stat-value-cell")
if len(other_stats) > 0:
# Convert every object into their stripped strings counterpart
for stat in other_stats:
cur_stats_list = []
for string in stat.stripped_strings:
cur_stats_list.append(string)
if "Attack Interval" in cur_stats_list:
stats_json["atk_int"] = float(cur_stats_list[-1])
elif "Redeploy Time" in cur_stats_list:
stats_json["deploy_time"] = int(cur_stats_list[-1])
# Failsafes, in case
stats_json["atk_int"] = (-1 if "atk_int" not in stats_json.keys() else
stats_json["atk_int"])
stats_json["deploy_time"] = (-1 if "deploy_time" not in stats_json.keys()
else stats_json["deploy_time"])
return stats_json