def load():
session.query(ChatWheelMessage).delete()
print("chat_wheel")
print("- loading chat_wheel stuff from scripts")
# load all of the item scripts data information
data = valve_readfile(config.vpk_path,
paths['chat_wheel_scripts_file'],
"kv",
encoding="utf-8")["chat_wheel"]
for key in data["messages"]:
msg_data = data["messages"][key]
message = ChatWheelMessage()
message.id = int(msg_data["message_id"])
message.name = key
message.label = msg_data.get("label")
message.message = msg_data.get("message")
message.sound = msg_data.get("sound")
message.image = msg_data.get("image")
message.all_chat = msg_data.get("all_chat") == "1"
if message.sound:
if message.sound == "soundboard.crash":
message.sound = "soundboard.crash_burn"
message.sound = f"/sounds/misc/soundboard/{message.sound.replace('soundboard.', '')}.wav"
if message.image:
message.image = f"/panorama/images/{message.image}"
session.add(message)
for category in data["categories"]:
for msg in data["categories"][category]["messages"]:
for message in session.query(ChatWheelMessage).filter_by(name=msg):
if message.category is not None:
raise ValueError(
f"More than one category for chatwheel: {message.name}"
)
message.category = category
print("- loading chat wheel data from dota_english")
# Load additional information from the dota_english.txt file
data = valve_readfile(config.vpk_path,
paths['dota_english_file'],
"kv",
encoding="UTF-16")["lang"]["Tokens"]
for message in session.query(ChatWheelMessage):
if message.label is None or message.message is None:
continue
if message.label.startswith("#") and message.label[1:] in data:
message.label = data[message.label[1:]]
if message.message.startswith("#") and message.message[1:] in data:
message.message = data[message.message[1:]]
if message.id in [71, 72]:
message.message = message.message.replace("%s1", "A hero")
session.commit()
def load():
session.query(ChatWheelMessage).delete()
print("chat_wheel")
print("- loading chat_wheel stuff from scripts")
# load sounds info from vsndevts file
sounds_data = valve_readfile(config.vpk_path, paths['chat_wheel_vsndevts_file'], "vsndevts")
# load all of the item scripts data information
data = valve_readfile(config.vpk_path, paths['chat_wheel_scripts_file'], "kv", encoding="utf-8")["chat_wheel"]
for key in data["messages"]:
msg_data = data["messages"][key]
message = ChatWheelMessage()
message.id = int(msg_data["message_id"])
message.name = key
message.label = msg_data.get("label")
message.message = msg_data.get("message")
message.sound = msg_data.get("sound")
message.image = msg_data.get("image")
message.all_chat = msg_data.get("all_chat") == "1"
if message.sound:
if message.sound not in sounds_data:
printerr(f"Couldn't find vsndevts entry for {message.sound}, skipping")
continue
if "vsnd_files" not in sounds_data[message.sound]:
printerr(f"no associated vsnd files found for {message.sound}, skipping")
continue
message.sound = "/" + sounds_data[message.sound]["vsnd_files"][0].replace(".vsnd", ".wav")
if not os.path.exists(config.vpk_path + message.sound):
printerr(f"Missing file: {message.sound}")
if message.image:
message.image = f"/panorama/images/{message.image}"
session.add(message)
for category in data["categories"]:
for msg in data["categories"][category]["messages"]:
for message in session.query(ChatWheelMessage).filter_by(name=msg):
if message.category is not None:
raise ValueError(f"More than one category for chatwheel: {message.name}")
message.category = category
print("- loading chat wheel data from dota_english")
# Load additional information from the dota_english.txt file
data = valve_readfile(config.vpk_path, paths['dota_english_file'], "kv", encoding="UTF-8")["lang"]["Tokens"]
for message in session.query(ChatWheelMessage):
if message.label is None or message.message is None:
continue
if message.label.startswith("#") and message.label[1:] in data:
message.label = data[message.label[1:]]
if message.message.startswith("#") and message.message[1:] in data:
message.message = data[message.message[1:]]
if message.id in [ 71, 72 ]:
message.message = message.message.replace("%s1", "A hero")
session.commit()
def load():
session.query(Item).delete()
print("items")
print("- loading items from item scripts")
# load all of the item scripts data information
data = valve_readfile(config.vpk_path, paths['item_scripts_file'], "kv")["DOTAAbilities"]
for itemname in data:
if itemname == "Version":
continue
item_data = data[itemname]
item = Item()
item.name = itemname
item.id = item_data['ID']
item.cost = item_data.get('ItemCost')
item.aliases = "|".join(item_data.get("ItemAliases", "").split(";"))
item.quality = item_data.get("ItemQuality")
item.mana_cost = clean_values(item_data.get('AbilityManaCost'))
item.cooldown = clean_values(item_data.get('AbilityCooldown'))
item.base_level = item_data.get("ItemBaseLevel")
item.ability_special = json.dumps(get_ability_special(item_data.get("AbilitySpecial"), itemname), indent=4)
item.json_data = json.dumps(item_data, indent=4)
session.add(item)
print("- loading item data from dota_english")
# Load additional information from the dota_english.txt file
data = valve_readfile(config.vpk_path, paths['localization_abilities'], "kv", encoding="UTF-16")["lang"]["Tokens"]
for item in session.query(Item):
item_tooltip = "DOTA_Tooltip_Ability_" + item.name
item_tooltip2 = "DOTA_Tooltip_ability_" + item.name
item.localized_name = data.get(item_tooltip, item.name)
item.description = data.get(item_tooltip + "_Description", data.get(item_tooltip2 + "_Description", ""))
item.lore = data.get(item_tooltip + "_Lore", data.get(item_tooltip2 + "_Lore", ""))
ability_special = json.loads(item.ability_special, object_pairs_hook=OrderedDict)
ability_special = ability_special_add_header(ability_special, data, item.name)
item.ability_special = json.dumps(ability_special, indent=4)
item.description = clean_description(item.description, ability_special, base_level=item.base_level)
print("- adding item icon files")
# Add img files to item
for item in session.query(Item):
if os.path.isfile(config.vpk_path + paths['item_img_path'] + item.name.replace("item_", "") + ".png"):
item.icon = paths['item_img_path'] + item.name.replace("item_", "") + ".png"
else:
if "recipe" in item.name:
item.icon = paths['item_img_path'] + "recipe.png"
else:
print(f"icon file not found for {item.name}")
session.commit()
def load():
session.query(Emoticon).delete()
print("emoticons")
print("- loading emoticons from scripts")
# load all of the item scripts data information
data = valve_readfile(config.vpk_path,
paths['emoticon_scripts_file'],
"kv",
encoding="UTF-16")["emoticons"]
for emoticonid in data:
if int(emoticonid) >= 1000:
continue # These are team emoticons
emoticon = Emoticon()
emoticon.id = int(emoticonid)
emoticon.name = data[emoticonid]['aliases']['0']
emoticon.ms_per_frame = data[emoticonid]['ms_per_frame']
emoticon.url = paths['emoticon_image_path'] + data[emoticonid][
'image_name']
try:
img = Image.open(config.vpk_path + emoticon.url)
emoticon.frames = int(img.size[0] / img.size[1])
except:
# Error loading this image, so dont add it to the database
continue
session.add(emoticon)
session.commit()
def load():
session.query(LoadingScreen).delete()
print("loadingscreens")
items_data = valve_readfile(config.vpk_path,
paths['cosmetics_scripts_file'],
"kv_nocomment",
encoding="UTF-8")["items_game"]["items"]
custom_paths = {
"Default Loading Screen":
"/panorama/images/loadingscreens/default/startup_background_logo_png.png"
}
# this will be used later for assigning category
couriers = []
print("- loading loadingscreens from items_game")
# load all of the item scripts data information
for key in items_data:
data = items_data[key]
if data.get("prefab") == "courier":
couriers.append(data.get("name"))
if data.get("prefab") != "loading_screen":
continue # These are team loadingscreens
loadingscreen = LoadingScreen()
loadingscreen.id = int(key)
loadingscreen.name = data.get("name")
date_array = list(map(int, data.get("creation_date").split("-")))
loadingscreen.creation_date = datetime.date(date_array[0],
date_array[1],
date_array[2])
loadingscreen.category = "other"
if loadingscreen.name in custom_paths:
loadingscreen.image = custom_paths[loadingscreen.name]
else:
for asset in data["visuals"]:
asset_data = data["visuals"][asset]
if not isinstance(asset_data, str) and asset_data.get(
"type") == "loading_screen":
image_path = asset_data["asset"]
if ".vtex" in image_path:
image_path = image_path.replace(".vtex", "")
else:
image_path += "_tga"
loadingscreen.image = f"/panorama/images/{image_path}.png"
loadingscreen.thumbnail = os.path.dirname(
loadingscreen.image) + "/thumbnail.png"
if not os.path.exists(config.vpk_path + loadingscreen.image):
continue # skip this loadingscreen because it doesn't exist
session.add(loadingscreen)
progress = ProgressBar(session.query(LoadingScreen).count(),
title="- making thumbnails and retrieving colors")
for loadingscreen in session.query(LoadingScreen):
progress.tick()
if not os.path.exists(config.vpk_path + loadingscreen.thumbnail):
image = Image.open(config.vpk_path + loadingscreen.image)
image.thumbnail((128, 64), Image.ANTIALIAS)
image.save(config.vpk_path + loadingscreen.thumbnail, format="PNG")
colors = colorgram.extract(config.vpk_path + loadingscreen.thumbnail,
5)
loadingscreen.color = "#{0:02x}{1:02x}{2:02x}".format(*colors[0].rgb)
hsv = rgb_to_hsv(colors[0].rgb)
loadingscreen.hue = hsv[0]
loadingscreen.saturation = hsv[1]
loadingscreen.value = hsv[2]
## Categories:
# hero_set
# hud_skin
# tournament
# courier
# other
item_type_to_category = {
"#DOTA_WearableType_Hud_Skin_Bundle": "hud_skin",
"#DOTA_WearableType_Tournament_Bundle": "tournament"
}
print("- associating item packs")
for key in items_data:
data = items_data[key]
if data.get("prefab") != "bundle":
continue
for name in data.get("bundle", []):
for loadingscreen in session.query(LoadingScreen).filter_by(
name=name):
heroes = data.get("used_by_heroes", {})
for hero_name in heroes:
hero = session.query(Hero).filter_by(
full_name=hero_name).first()
if hero:
loadingscreen.hero_ids = str(hero.id)
loadingscreen.category = "hero_set"
if loadingscreen.category == "hero_set":
continue
category = item_type_to_category.get(
data.get("item_type_name"))
if category:
loadingscreen.category = category
continue
if any(x in couriers for x in data.get("bundle", [])):
loadingscreen.category = "courier"
continue
print("- linking heroes")
data = read_json("builderdata/loadingscreen_heroes.json")
for screen in session.query(LoadingScreen):
if screen.name in data:
heroes = []
if screen.hero_ids:
heroes.append(screen.hero_ids)
for heroname in data[screen.name]:
hero = session.query(Hero).filter_by(name=heroname).first()
heroes.append(str(hero.id))
screen.hero_ids = "|".join(heroes)
session.commit()
def load():
session.query(Item).delete()
print("items")
added_ids = []
item_name_fixes = {"item_trident1": "item_trident"}
print("- loading items from item scripts")
# load all of the item scripts data information
data = valve_readfile(config.vpk_path, paths['item_scripts_file'],
"kv")["DOTAAbilities"]
for itemname in data:
if itemname == "Version":
continue
item_data = data[itemname]
if item_data.get('IsObsolete') == "1":
continue # ignore obsolete items
item = Item()
item.name = item_name_fixes.get(itemname, itemname)
item.id = int(item_data['ID'])
item.cost = item_data.get('ItemCost')
item.aliases = "|".join(item_data.get("ItemAliases", "").split(";"))
item.quality = item_data.get("ItemQuality")
item.mana_cost = clean_values(item_data.get('AbilityManaCost'))
item.cooldown = clean_values(item_data.get('AbilityCooldown'))
item.cast_range = clean_values(item_data.get('AbilityCastRange'))
item.base_level = item_data.get("ItemBaseLevel")
item.secret_shop = item_data.get("SecretShop") == "1"
item.ability_special = json.dumps(get_ability_special(
item_data.get("AbilitySpecial"), item.name),
indent=4)
item.json_data = json.dumps(item_data, indent=4)
if item.id in added_ids:
print(f"duplicate id on: {itemname}")
continue
added_ids.append(item.id)
session.add(item)
print("- loading item data from dota_english")
# Load additional information from the dota_english.txt file
data = valve_readfile(config.vpk_path,
paths['localization_abilities'],
"kv",
encoding="UTF-8")["lang"]["Tokens"]
for item in session.query(Item):
item_tooltip = "DOTA_Tooltip_Ability_" + item.name
item_tooltip2 = "DOTA_Tooltip_ability_" + item.name
item.localized_name = data.get(item_tooltip, item.name)
item.description = data.get(
item_tooltip + "_Description",
data.get(item_tooltip2 + "_Description", ""))
item.lore = data.get(item_tooltip + "_Lore",
data.get(item_tooltip2 + "_Lore", ""))
ability_special = json.loads(item.ability_special,
object_pairs_hook=OrderedDict)
ability_special = ability_special_add_header(ability_special, data,
item.name)
item.ability_special = json.dumps(ability_special, indent=4)
item.description = clean_description(item.description,
build_replacements_dict(item),
base_level=item.base_level)
print("- adding neutral item data")
data = valve_readfile(config.vpk_path, paths['neutral_item_scripts_file'],
"kv")["neutral_items"]
item_tier_map = {}
for tier in data:
for name in data[tier]["items"]:
item_tier_map[name] = tier
for item in session.query(Item):
if item.name in item_tier_map:
item.neutral_tier = item_tier_map[item.name]
print("- linking recipes")
for recipe in session.query(Item):
json_data = json.loads(recipe.json_data)
if json_data.get("ItemRecipe", "0") != "0":
components = list(
json_data.get("ItemRequirements", {
"01": None
}).values())[0]
if components is None:
continue
components = components.replace(";", " ").strip().split(" ")
if recipe.cost != 0:
components.append(recipe.name)
crafted_item_name = json_data.get("ItemResult")
crafted_item = session.query(Item).filter_by(
name=crafted_item_name).first()
if not crafted_item:
raise ValueError(
f"Can't find crafted item {crafted_item_name}")
crafted_item.recipe = "|".join(components)
if recipe.neutral_tier is not None: # stuff like trident
crafted_item.neutral_tier = recipe.neutral_tier
if recipe.cost == 0 and not json_data.get("ItemIsNeutralDrop"):
session.delete(recipe)
print("- adding item icon files")
# Add img files to item
for item in session.query(Item):
iconpath = paths['item_img_path'] + item.name.replace("item_",
"") + "_png.png"
if os.path.isfile(config.vpk_path + iconpath):
item.icon = iconpath
else:
if "recipe" in item.name:
item.icon = paths['item_img_path'] + "recipe.png"
else:
printerr(f"icon file not found for {item.name}", flush=True)
session.commit()
def load():
session.query(Response).delete()
session.query(Criterion).delete()
print("Responses")
progress = ProgressBar(session.query(Voice).count(), title="- loading from vsnd files:")
for voice in session.query(Voice):
progress.tick()
if not voice.media_name:
continue
vsndevts_path = f"/soundevents/voscripts/game_sounds_vo_{voice.media_name}.vsndevts"
vsndevts_data = valve_readfile(config.vpk_path, vsndevts_path, "vsndevts")
captionsFilename = f"{config.vpk_path}/resource/subtitles/subtitles_{voice.media_name}_english.dat"
if os.path.exists(captionsFilename):
captionsFile = ClosedCaptionFile(captionsFilename)
else:
printerr(f"missing {captionsFilename}")
captionsFile = None
for key in vsndevts_data:
data = vsndevts_data[key]
filename = "/" + data["vsnd_files"][0].replace("vsnd", "mp3")
response = Response()
response.fullname = key
response.name = os.path.basename(filename).replace(".mp3", "")
for ext in file_types:
newname = filename.replace(".mp3", f".{ext}")
if os.path.exists(config.vpk_path + newname):
filename = newname
break
if not os.path.exists(config.vpk_path + filename):
printerr(f"Missing file: {filename}")
response.mp3 = filename
response.voice_id = voice.id
response.hero_id = voice.hero_id
response.criteria = ""
if captionsFile:
text = captionsFile.lookup(response.fullname)
if text:
response.text = text
response.text_simple = text.replace("...", " ")
response.text_simple = " " + re.sub(r'[^a-z^0-9^A-Z^\s]', r'', response.text_simple).lower() + " "
response.text_simple = re.sub(r'\s+', r' ', response.text_simple)
else:
response.text = ""
session.add(response)
print("- loading criteria")
rules = {}
groups = {}
criteria = {}
# Load response_rules
for root, dirs, files in os.walk(config.vpk_path + paths['response_rules_path']):
for file in files:
data = valve_readfile(config.vpk_path, paths['response_rules_path'] + file, "rules")
for key in data:
if key.startswith("rule_"):
rules[key[5:]] = data[key]
elif key.startswith("response_"):
groups[key[9:]] = data[key]
elif key.startswith("criterion_"):
criteria[key[10:]] = data[key]
for key in criteria:
criterion = Criterion()
criterion.name = key
vals = criteria[key].split(" ")
criterion.matchkey = vals[0]
criterion.matchvalue = vals[1]
if "weight" in vals:
criterion.weight = float(vals[vals.index("weight") + 1])
else:
criterion.weight = 1.0
criterion.required = "required" in vals
session.add(criterion)
voice_linker = {}
custom_voice_criteria = { # because valve did customresponse:arcana for 2 things
"Tempest Helm of the Thundergod": "IsZeusEconArcana"
}
# fix up voice.criteria
for voice in session.query(Voice):
if voice.criteria:
if voice.name in custom_voice_criteria:
voice.criteria = custom_voice_criteria[voice.name]
continue
crits = []
for crit in voice.criteria.split("|"):
key, value = crit.split(":")
realcrit = session.query(Criterion).filter_by(matchkey=key).filter_by(matchvalue=value).first()
if realcrit:
crits.append(realcrit.name)
voice.criteria = "|".join(crits)
pattern = f"(^|\|| ){voice.criteria}($|\|| )"
voice_linker[pattern] = voice
progress = ProgressBar(len(rules) + session.query(Response).count(), title="- linking rules:")
pre_responses = {}
for key in rules:
progress.tick()
response_criteria = rules[key]['criteria'].rstrip()
for fullname in groups[rules[key]['response']]:
if fullname not in pre_responses:
pre_responses[fullname] = response_criteria
else:
pre_responses[fullname] += "|" + response_criteria
for response in session.query(Response):
progress.tick()
if response.fullname in pre_responses:
response.criteria = pre_responses[response.fullname]
for pattern, voice in voice_linker.items():
if re.search(pattern, response.criteria):
response.voice_id = voice.id
print("- generating pretty criteria")
criteria_sentancing.load_pretty_criteria(session)
session.commit()
def load():
session.query(Ability).delete()
print("Abilities")
added_ids = []
print("- loading abilities from ability scripts")
# load all of the ability scripts data information
data = valve_readfile(config.vpk_path, paths['ability_scripts_file'],
"kv")["DOTAAbilities"]
for abilityname in data:
if (abilityname == "Version" or abilityname == "ability_deward"
or abilityname == "dota_base_ability"
or not data[abilityname]['ID'].isdigit()):
continue
ability_data = data[abilityname]
ability = Ability()
def get_val(key, default_base=False):
if key in ability_data:
val = ability_data[key]
if ' ' in val and all(x == val.split(' ')[0]
for x in val.split(' ')):
return val.split(' ')[0]
return val
elif default_base:
return data["ability_base"][key]
else:
return None
ability.name = abilityname
ability.id = int(ability_data['ID'])
ability.type = get_val('AbilityType', default_base=True)
ability.behavior = get_val('AbilityBehavior', default_base=True)
ability.cast_range = clean_values(get_val('AbilityCastRange'))
ability.cast_point = clean_values(get_val('AbilityCastPoint'))
ability.channel_time = clean_values(get_val('AbilityChannelTime'))
ability.cooldown = clean_values(get_val('AbilityCooldown'))
ability.duration = clean_values(get_val('AbilityDuration'))
ability.damage = clean_values(get_val('AbilityDamage'))
ability.mana_cost = clean_values(get_val('AbilityManaCost'))
ability.ability_special = json.dumps(get_ability_special(
ability_data.get("AbilitySpecial"), abilityname),
indent=4)
# link talents
link_fixes = {
"axe_counter_culling_blade":
"axe_culling_blade",
"troll_warlord_whirling_axes":
"troll_warlord_whirling_axes_ranged troll_warlord_whirling_axes_melee",
"invoker_sunstrike":
"invoker_sun_strike",
"morphling_adaptive_strike":
"morphling_adaptive_strike_agi morphling_adaptive_strike_str"
}
for special in ability_data.get("AbilitySpecial", {}).values():
link = special.get("ad_linked_ability")
if link:
if link in ["multi_linked_ability", "multi_linked_or_ability"]:
link = special["linked_ad_abilities"].split(" ")[0]
if link in ["special_bonus_inherent"]:
continue # doesn't link to a different ability
if link in link_fixes:
link = link_fixes[link]
link = link.replace(" ", "|")
ability.linked_abilities = link
if ability.id in added_ids:
print(f"duplicate id on: {abilityname}")
continue
added_ids.append(ability.id)
def get_enum_val(key, prefix):
value = get_val(key)
if value:
return re.sub(prefix, "", value).lower().replace(" ", "")
else:
return value
ability.behavior = get_enum_val('AbilityBehavior',
"DOTA_ABILITY_BEHAVIOR_")
ability.damage_type = get_enum_val('AbilityUnitDamageType',
"DAMAGE_TYPE_")
ability.spell_immunity = get_enum_val(
'SpellImmunityType', "SPELL_IMMUNITY_(ENEMIES|ALLIES)_")
ability.target_team = get_enum_val('AbilityUnitTargetTeam',
"DOTA_UNIT_TARGET_TEAM_")
ability.dispellable = get_enum_val('SpellDispellableType',
"SPELL_DISPELLABLE_")
ability.json_data = json.dumps(ability_data, indent=4)
session.add(ability)
print("- loading ability data from dota_english")
# Load additional information from the dota_english.txt file
data = valve_readfile(config.vpk_path,
paths['localization_abilities'],
"kv",
encoding="UTF-8")["lang"]["Tokens"]
for ability in session.query(Ability):
ability_tooltip = "DOTA_Tooltip_ability_" + ability.name
ability.localized_name = data.get(ability_tooltip, ability.name)
ability.description = data.get(ability_tooltip + "_Description", "")
ability.lore = data.get(ability_tooltip + "_Lore", "")
ability.aghanim = data.get(ability_tooltip + "_aghanim_description",
"")
notes = []
for i in range(8):
key = f"{ability_tooltip}_Note{i}"
if key in data:
notes.append(data[key])
ability.note = "" if len(notes) == 0 else "\n".join(notes)
ability_special = json.loads(ability.ability_special,
object_pairs_hook=OrderedDict)
ability_special = ability_special_add_talent(ability_special,
session.query(Ability))
ability_special = ability_special_add_header(ability_special, data,
ability.name)
ability.ability_special = json.dumps(ability_special, indent=4)
replacements_dict = build_replacements_dict(ability)
ability.localized_name = clean_description(ability.localized_name,
replacements_dict,
value_bolding=False)
ability.description = clean_description(ability.description,
replacements_dict)
ability.note = clean_description(ability.note, replacements_dict)
ability.aghanim = clean_description(ability.aghanim, replacements_dict)
if ability.localized_name.startswith(": "):
ability.localized_name = ability.localized_name[2:]
print("- adding ability icon files")
# Add img files to ability
for ability in session.query(Ability):
iconpath = paths['ability_icon_path'] + ability.name + "_png.png"
if os.path.isfile(config.vpk_path + iconpath):
ability.icon = iconpath
else:
ability.icon = paths['ability_icon_path'] + "wisp_empty1_png.png"
session.commit()
def load():
session.query(Response).delete()
session.query(Criterion).delete()
print("Responses")
progress = ProgressBar(session.query(Voice).count(),
title="- loading from vsnd files:")
for voice in session.query(Voice):
progress.tick()
if not voice.vsndevts_path:
continue
vsndevts_data = valve_readfile(config.vpk_path, voice.vsndevts_path,
"vsndevts")
for key in vsndevts_data:
data = vsndevts_data[key]
filename = "/" + data["vsnd_files"][0].replace("vsnd", "mp3")
response = Response()
response.fullname = key
response.name = os.path.basename(filename).replace(".mp3", "")
for ext in file_types:
newname = filename.replace(".mp3", f".{ext}")
if os.path.exists(config.vpk_path + newname):
filename = newname
break
if not os.path.exists(config.vpk_path + filename):
print(f"Missing file: {filename}")
response.mp3 = filename
response.voice_id = voice.id
response.hero_id = voice.hero_id
response.criteria = ""
session.add(response)
fulldata = valve_readfile(paths['scraped_responses_dir'],
paths['scraped_responses_file'],
"scrapedresponses")
progress = ProgressBar(len(fulldata), title="- loading response texts")
for voiceid in fulldata:
progress.tick()
voice = session.query(Voice).filter_by(id=int(voiceid)).first()
data = fulldata[voiceid]
for response in session.query(Response).filter_by(voice_id=voice.id):
text = ""
# these to help with some weird shit
fullname = re.sub(r'^announcer_', '', response.fullname)
fullname = re.sub(r'defensegrid', 'defense_grid', fullname)
fullname = re.sub(r'techies_tech_ann', 'tech_ann', fullname)
fullname = re.sub(r'dlc_tusk_tusk_ann', 'greevling_tusk_ann',
fullname)
if voice.id == 49: # dragon knight
fullname = f"dk_{response.name}"
if response.fullname in data:
text = data[response.fullname]
elif fullname in data:
text = data[fullname]
elif response.name in data:
text = data[response.name]
if text != "":
text = re.sub(r'<!--.*-->', r'', text)
text = re.sub(r'{{Tooltip\|([^|]+)\|(.*)}}', r'\1 (\2)', text)
text = re.sub(r'{{tooltip\|\?\|(.*)}}', r'(\1)', text)
text = re.sub(r'{{.*}}', r'', text)
response.text = text
response.text_simple = text.replace("...", " ")
response.text_simple = " " + re.sub(
r'[^a-z^0-9^A-Z^\s]', r'',
response.text_simple).lower() + " "
response.text_simple = re.sub(r'\s+', r' ',
response.text_simple)
else:
response.text = ""
print("- loading criteria")
rules = {}
groups = {}
criteria = {}
# Load response_rules
for root, dirs, files in os.walk(config.vpk_path +
paths['response_rules_path']):
for file in files:
data = valve_readfile(config.vpk_path,
paths['response_rules_path'] + file, "rules")
for key in data:
if key.startswith("rule_"):
rules[key[5:]] = data[key]
elif key.startswith("response_"):
groups[key[9:]] = data[key]
elif key.startswith("criterion_"):
criteria[key[10:]] = data[key]
for key in criteria:
criterion = Criterion()
criterion.name = key
vals = criteria[key].split(" ")
criterion.matchkey = vals[0]
criterion.matchvalue = vals[1]
criterion.weight = vals[3] if "weight" in vals else 1.0
criterion.required = "required" in vals
session.add(criterion)
progress = ProgressBar(len(rules), title="- linking rules:")
for key in rules:
response_criteria = rules[key]['criteria'].rstrip()
progress.tick()
for fullname in groups[rules[key]['response']]:
response = session.query(Response).filter_by(
fullname=fullname).first()
if response is not None:
if response.criteria == "":
response.criteria = response_criteria
else:
response.criteria += "|" + response_criteria
print("- generating pretty criteria")
criteria_sentancing.load_pretty_criteria(session)
session.commit()
def load():
session.query(Ability).delete()
print("Abilities")
print("- loading abilities from ability scripts")
# load all of the ability scripts data information
data = valve_readfile(config.vpk_path, paths['ability_scripts_file'],
"kv")["DOTAAbilities"]
for abilityname in data:
if (abilityname == "Version" or abilityname == "ability_deward"
or abilityname == "dota_base_ability"):
continue
ability_data = data[abilityname]
ability = Ability()
def get_val(key, default_base=False):
if key in ability_data:
val = ability_data[key]
if ' ' in val and all(x == val.split(' ')[0]
for x in val.split(' ')):
return val.split(' ')[0]
return val
elif default_base:
return data["ability_base"][key]
else:
return None
ability.name = abilityname
ability.id = ability_data['ID']
ability.type = get_val('AbilityType', default_base=True)
ability.behavior = get_val('AbilityBehavior', default_base=True)
ability.cast_range = clean_values(get_val('AbilityCastRange'))
ability.cast_point = clean_values(get_val('AbilityCastPoint'))
ability.channel_time = clean_values(get_val('AbilityChannelTime'))
ability.cooldown = clean_values(get_val('AbilityCooldown'))
ability.duration = clean_values(get_val('AbilityDuration'))
ability.damage = clean_values(get_val('AbilityDamage'))
ability.mana_cost = clean_values(get_val('AbilityManaCost'))
ability.ability_special = json.dumps(get_ability_special(
ability_data.get("AbilitySpecial"), abilityname),
indent=4)
def get_enum_val(key, prefix):
value = get_val(key)
if value:
return re.sub(prefix, "", value).lower().replace(" ", "")
else:
return value
ability.behavior = get_enum_val('AbilityBehavior',
"DOTA_ABILITY_BEHAVIOR_")
ability.damage_type = get_enum_val('AbilityUnitDamageType',
"DAMAGE_TYPE_")
ability.spell_immunity = get_enum_val(
'SpellImmunityType', "SPELL_IMMUNITY_(ENEMIES|ALLIES)_")
ability.target_team = get_enum_val('AbilityUnitTargetTeam',
"DOTA_UNIT_TARGET_TEAM_")
ability.dispellable = get_enum_val('SpellDispellableType',
"SPELL_DISPELLABLE_")
ability.json_data = json.dumps(ability_data, indent=4)
session.add(ability)
print("- loading ability data from dota_english")
# Load additional information from the dota_english.txt file
data = valve_readfile(config.vpk_path,
paths['localization_abilities'],
"kv",
encoding="UTF-16")["lang"]["Tokens"]
for ability in session.query(Ability):
ability_tooltip = "DOTA_Tooltip_ability_" + ability.name
ability.localized_name = data.get(ability_tooltip, ability.name)
ability.description = data.get(ability_tooltip + "_Description", "")
ability.lore = data.get(ability_tooltip + "_Lore", "")
ability.aghanim = data.get(ability_tooltip + "_aghanim_description",
"")
notes = []
for i in range(8):
key = f"{ability_tooltip}_Note{i}"
if key in data:
notes.append(data[key])
ability.note = "" if len(notes) == 0 else "\n".join(notes)
ability_special = json.loads(ability.ability_special,
object_pairs_hook=OrderedDict)
ability_special = ability_special_add_talent(ability_special,
session.query(Ability))
ability_special = ability_special_add_header(ability_special, data,
ability.name)
ability.ability_special = json.dumps(ability_special, indent=4)
ability.description = clean_description(ability.description,
ability_special)
print("- adding ability icon files")
# Add img files to ability
for ability in session.query(Ability):
if os.path.isfile(config.vpk_path + paths['ability_icon_path'] +
ability.name + ".png"):
ability.icon = paths['ability_icon_path'] + ability.name + ".png"
else:
ability.icon = paths['ability_icon_path'] + "wisp_empty1.png"
session.commit()
def load():
session.query(Hero).delete()
print("Heroes")
# load all of the hero scripts data information
data = valve_readfile(config.vpk_path, paths['hero_scripts_file'],
"kv")["DOTAHeroes"]
progress = ProgressBar(len(data), title="- loading from hero scripts")
for heroname in data:
progress.tick()
if (heroname == "Version" or heroname == "npc_dota_hero_target_dummy"
or heroname == "npc_dota_hero_base"):
continue
hero = Hero()
hero_data = data[heroname]
def get_val(key):
if key in hero_data:
return hero_data[key]
else:
return data["npc_dota_hero_base"].get(key)
hero.full_name = heroname
hero.media_name = hero_data['VoiceFile'][37:-9]
hero.name = heroname.replace("npc_dota_hero_", "")
hero.id = get_val('HeroID')
hero.team = get_val('Team')
hero.base_health_regen = get_val('StatusHealthRegen')
hero.base_movement = get_val('MovementSpeed')
hero.turn_rate = get_val('MovementTurnRate')
hero.base_armor = get_val('ArmorPhysical')
hero.magic_resistance = get_val('MagicalResistance')
hero.attack_range = get_val('AttackRange')
hero.attack_projectile_speed = get_val('ProjectileSpeed')
hero.attack_damage_min = get_val('AttackDamageMin')
hero.attack_damage_max = get_val('AttackDamageMax')
hero.attack_rate = get_val('AttackRate')
hero.attack_point = get_val('AttackAnimationPoint')
hero.attr_primary = attribute_dict[get_val('AttributePrimary')]
hero.attr_strength_base = get_val('AttributeBaseStrength')
hero.attr_strength_gain = get_val('AttributeStrengthGain')
hero.attr_intelligence_base = get_val('AttributeBaseIntelligence')
hero.attr_intelligence_gain = get_val('AttributeIntelligenceGain')
hero.attr_agility_base = get_val('AttributeBaseAgility')
hero.attr_agility_gain = get_val('AttributeAgilityGain')
hero.vision_day = get_val('VisionDaytimeRange')
hero.vision_night = get_val('VisionNighttimeRange')
hero.is_melee = get_val(
'AttackCapabilities') == "DOTA_UNIT_CAP_MELEE_ATTACK"
hero.material = get_val('GibType')
hero.roles = hero_data.get('Role', '').replace(',', '|')
hero.role_levels = hero_data.get('Rolelevels', '').replace(',', '|')
glow_color = hero_data.get('HeroGlowColor', None)
if glow_color:
hero.color = "#{0:02x}{1:02x}{2:02x}".format(
*map(int, glow_color.split(' ')))
hero.json_data = json.dumps(hero_data, indent=4)
talents = []
# Link abilities and add talents
for slot in range(1, 30):
if "Ability" + str(slot) in hero_data:
ability = session.query(Ability).filter_by(
name=hero_data["Ability" + str(slot)]).first()
if ability.name.startswith("special_bonus"):
talents.append(ability.localized_name)
else:
ability.hero_id = hero.id
ability.ability_slot = slot
if len(talents) != 8:
raise ValueError("{} only has {} talents?".format(
hero.localized_name, len(talents)))
hero.talents = "|".join(talents)
session.add(hero)
print("- loading hero names from dota_english file")
# Load hero names from dota_english file
data = valve_readfile(config.vpk_path,
paths['dota_english_file'],
"kv",
encoding="UTF-16")["lang"]["Tokens"]
for hero in session.query(Hero):
hero.localized_name = data[hero.full_name]
hero.hype = data[hero.full_name + "_hype"]
print("- loading bio from hero lore file")
# Load bio from hero lore file
data = valve_readfile(config.vpk_path,
paths['localization_hero_lore'],
"kv",
encoding="UTF-16")["hero_lore"]
for hero in session.query(Hero):
hero.bio = data[hero.full_name + "_bio"].replace("<br>", "\n")
print("- adding hero image files")
# Add img files to hero
for hero in session.query(Hero):
hero.icon = paths['hero_icon_path'] + hero.name + ".png"
hero.image = paths['hero_image_path'] + hero.name + ".png"
hero.portrait = paths['hero_selection_path'] + hero.full_name + ".png"
print("- adding hero real names")
data = read_json("builderdata/hero_names.json")
for hero in session.query(Hero):
hero.real_name = data.get(hero.name, "")
print("- adding hero aliases")
data = read_json("builderdata/hero_aliases.json")
for hero in session.query(Hero):
aliases = []
aliases.append(hero.name.replace("_", " "))
text = re.sub(r'[^a-z^\s]', r'',
hero.localized_name.replace("_", " ").lower())
if text not in aliases:
aliases.append(text)
if hero.real_name != "":
aliases.append(re.sub(r'[^a-z^\s]', r'', hero.real_name.lower()))
aliases.extend(data.get(hero.name, []))
hero.aliases = "|".join(aliases)
print("- adding hero colors")
data = read_json("builderdata/hero_colors.json")
for hero_name in data:
hero = session.query(Hero).filter_by(name=hero_name).first()
hero.color = data[hero_name]
session.commit()
def load():
session.query(Voice).delete()
print("Voices")
print("- loading from heroes")
for hero in session.query(Hero):
voice = Voice()
voice.id = hero.id
voice.name = hero.localized_name
voice.icon = hero.icon
voice.image = hero.portrait
voice.url = name_to_url(hero.localized_name) + "/Responses"
voice.vsndevts_path = "/" + json.loads(hero.json_data).get("VoiceFile")
voice.hero_id = hero.id
session.add(voice)
print("- loading cosmetics file (takes a bit)")
data = valve_readfile(config.vpk_path,
paths['cosmetics_scripts_file'],
"kv_nocomment",
encoding="UTF-8")["items_game"]["items"]
custom_urls = {
"Announcer: Tuskar": "Announcer:_Tusk",
"Default Announcer": "Announcer_responses",
"Default Mega-Kill Announcer": "Announcer_responses",
"Announcer: Bristleback": "Bristleback_Announcer_Pack",
"Mega-Kills: Bristleback": "Bristleback_Announcer_Pack"
}
custom_vsndevts = {
"Default Announcer":
"/soundevents/voscripts/game_sounds_vo_announcer.vsndevts",
"Default Mega-Kill Announcer":
"/soundevents/voscripts/game_sounds_vo_announcer_killing_spree.vsndevts",
"Announcer: Kunkka & Tidehunter":
"/soundevents/voscripts/game_sounds_vo_announcer_dlc_kunkka_tide.vsndevts",
"Mega-Kills: Kunkka & Tidehunter":
"/soundevents/voscripts/game_sounds_vo_announcer_dlc_kunkka_tide_killing_spree.vsndevts"
}
print("- loading from announcers")
for key in data:
announcer = data[key]
if announcer.get("prefab") != "announcer":
continue
voice = Voice()
# the first announcer has id = 586, so this will not interfere with hero ids
voice.id = int(key)
voice.name = announcer["name"]
voice.icon = "/panorama/images/icon_announcer_psd.png"
voice.image = f"/panorama/images/{announcer['image_inventory']}_png.png"
if voice.name in custom_urls:
voice.url = custom_urls[voice.name]
else:
voice.url = name_to_url(announcer["name"])
if voice.name in custom_vsndevts:
voice.vsndevts_path = custom_vsndevts[voice.name]
else:
for asset in announcer["visuals"]:
if announcer["visuals"][asset]["type"] == "announcer":
voice.vsndevts_path = "/" + announcer["visuals"][asset][
"modifier"]
session.add(voice)
print("- associating announcer packs")
for key in data:
pack = data[key]
if pack.get("prefab") != "bundle" or pack.get(
"name") == "Assembly of Announcers Pack":
continue
for name in pack.get("bundle", []):
for voice in session.query(Voice).filter_by(name=name):
voice.url = name_to_url(pack["name"])
data = read_json("builderdata/voice_actors.json")
print("- adding voice actors")
for voice in session.query(Voice):
if str(voice.id) in data:
voice.voice_actor = data[str(voice.id)]
session.commit()