def load():
session.query(Emoticon).delete()
print("emoticons")
print("- loading emoticons from scripts")
# load all of the item scripts data information
data = valve_readfile(config.vpk_path,
paths['emoticon_scripts_file'],
"kv",
encoding="UTF-16")["emoticons"]
for emoticonid in data:
if int(emoticonid) >= 1000:
continue # These are team emoticons
emoticon = Emoticon()
emoticon.id = int(emoticonid)
emoticon.name = data[emoticonid]['aliases']['0']
emoticon.ms_per_frame = data[emoticonid]['ms_per_frame']
emoticon.url = paths['emoticon_image_path'] + data[emoticonid][
'image_name']
try:
img = Image.open(config.vpk_path + emoticon.url)
emoticon.frames = int(img.size[0] / img.size[1])
except:
# Error loading this image, so dont add it to the database
continue
session.add(emoticon)
session.commit()
def dump_responses(directory):
os.makedirs(directory)
for voice in session.query(Voice):
data = dump_table(Response, voice.responses)
filename = voice.name.lower().replace(" ", "_")
filename = re.sub(r"[^a-z_]", "", filename)
filename = os.path.join(directory, f"{filename}.json")
write_json(filename, data)
def dump_table(table, query=None):
full_data = []
if query is None:
query = session.query(table)
for item in query:
data = OrderedDict()
for col in table.__table__.columns:
value = getattr(item, col.name)
if col.name in ["json_data", "ability_special"]:
data[col.name] = json.loads(value,
object_pairs_hook=OrderedDict)
elif value is None or value == "":
continue
elif isinstance(value, int) or isinstance(
value, bool) or isinstance(value, float):
data[col.name] = value
else:
data[col.name] = str(value)
full_data.append(data)
return full_data
def load():
session.query(Response).delete()
session.query(Criterion).delete()
print("Responses")
progress = ProgressBar(session.query(Voice).count(),
title="- loading from vsnd files:")
for voice in session.query(Voice):
progress.tick()
if not voice.vsndevts_path:
continue
vsndevts_data = valve_readfile(config.vpk_path, voice.vsndevts_path,
"vsndevts")
for key in vsndevts_data:
data = vsndevts_data[key]
filename = "/" + data["vsnd_files"][0].replace("vsnd", "mp3")
response = Response()
response.fullname = key
response.name = os.path.basename(filename).replace(".mp3", "")
for ext in file_types:
newname = filename.replace(".mp3", f".{ext}")
if os.path.exists(config.vpk_path + newname):
filename = newname
break
if not os.path.exists(config.vpk_path + filename):
print(f"Missing file: {filename}")
response.mp3 = filename
response.voice_id = voice.id
response.hero_id = voice.hero_id
response.criteria = ""
session.add(response)
fulldata = valve_readfile(paths['scraped_responses_dir'],
paths['scraped_responses_file'],
"scrapedresponses")
progress = ProgressBar(len(fulldata), title="- loading response texts")
for voiceid in fulldata:
progress.tick()
voice = session.query(Voice).filter_by(id=int(voiceid)).first()
data = fulldata[voiceid]
for response in session.query(Response).filter_by(voice_id=voice.id):
text = ""
# these to help with some weird shit
fullname = re.sub(r'^announcer_', '', response.fullname)
fullname = re.sub(r'defensegrid', 'defense_grid', fullname)
fullname = re.sub(r'techies_tech_ann', 'tech_ann', fullname)
fullname = re.sub(r'dlc_tusk_tusk_ann', 'greevling_tusk_ann',
fullname)
if voice.id == 49: # dragon knight
fullname = f"dk_{response.name}"
if response.fullname in data:
text = data[response.fullname]
elif fullname in data:
text = data[fullname]
elif response.name in data:
text = data[response.name]
if text != "":
text = re.sub(r'<!--.*-->', r'', text)
text = re.sub(r'{{Tooltip\|([^|]+)\|(.*)}}', r'\1 (\2)', text)
text = re.sub(r'{{tooltip\|\?\|(.*)}}', r'(\1)', text)
text = re.sub(r'{{.*}}', r'', text)
response.text = text
response.text_simple = text.replace("...", " ")
response.text_simple = " " + re.sub(
r'[^a-z^0-9^A-Z^\s]', r'',
response.text_simple).lower() + " "
response.text_simple = re.sub(r'\s+', r' ',
response.text_simple)
else:
response.text = ""
print("- loading criteria")
rules = {}
groups = {}
criteria = {}
# Load response_rules
for root, dirs, files in os.walk(config.vpk_path +
paths['response_rules_path']):
for file in files:
data = valve_readfile(config.vpk_path,
paths['response_rules_path'] + file, "rules")
for key in data:
if key.startswith("rule_"):
rules[key[5:]] = data[key]
elif key.startswith("response_"):
groups[key[9:]] = data[key]
elif key.startswith("criterion_"):
criteria[key[10:]] = data[key]
for key in criteria:
criterion = Criterion()
criterion.name = key
vals = criteria[key].split(" ")
criterion.matchkey = vals[0]
criterion.matchvalue = vals[1]
criterion.weight = vals[3] if "weight" in vals else 1.0
criterion.required = "required" in vals
session.add(criterion)
progress = ProgressBar(len(rules), title="- linking rules:")
for key in rules:
response_criteria = rules[key]['criteria'].rstrip()
progress.tick()
for fullname in groups[rules[key]['response']]:
response = session.query(Response).filter_by(
fullname=fullname).first()
if response is not None:
if response.criteria == "":
response.criteria = response_criteria
else:
response.criteria += "|" + response_criteria
print("- generating pretty criteria")
criteria_sentancing.load_pretty_criteria(session)
session.commit()
def load():
session.query(Ability).delete()
print("Abilities")
print("- loading abilities from ability scripts")
# load all of the ability scripts data information
data = valve_readfile(config.vpk_path, paths['ability_scripts_file'],
"kv")["DOTAAbilities"]
for abilityname in data:
if (abilityname == "Version" or abilityname == "ability_deward"
or abilityname == "dota_base_ability"):
continue
ability_data = data[abilityname]
ability = Ability()
def get_val(key, default_base=False):
if key in ability_data:
val = ability_data[key]
if ' ' in val and all(x == val.split(' ')[0]
for x in val.split(' ')):
return val.split(' ')[0]
return val
elif default_base:
return data["ability_base"][key]
else:
return None
ability.name = abilityname
ability.id = ability_data['ID']
ability.type = get_val('AbilityType', default_base=True)
ability.behavior = get_val('AbilityBehavior', default_base=True)
ability.cast_range = clean_values(get_val('AbilityCastRange'))
ability.cast_point = clean_values(get_val('AbilityCastPoint'))
ability.channel_time = clean_values(get_val('AbilityChannelTime'))
ability.cooldown = clean_values(get_val('AbilityCooldown'))
ability.duration = clean_values(get_val('AbilityDuration'))
ability.damage = clean_values(get_val('AbilityDamage'))
ability.mana_cost = clean_values(get_val('AbilityManaCost'))
ability.ability_special = json.dumps(get_ability_special(
ability_data.get("AbilitySpecial"), abilityname),
indent=4)
def get_enum_val(key, prefix):
value = get_val(key)
if value:
return re.sub(prefix, "", value).lower().replace(" ", "")
else:
return value
ability.behavior = get_enum_val('AbilityBehavior',
"DOTA_ABILITY_BEHAVIOR_")
ability.damage_type = get_enum_val('AbilityUnitDamageType',
"DAMAGE_TYPE_")
ability.spell_immunity = get_enum_val(
'SpellImmunityType', "SPELL_IMMUNITY_(ENEMIES|ALLIES)_")
ability.target_team = get_enum_val('AbilityUnitTargetTeam',
"DOTA_UNIT_TARGET_TEAM_")
ability.dispellable = get_enum_val('SpellDispellableType',
"SPELL_DISPELLABLE_")
ability.json_data = json.dumps(ability_data, indent=4)
session.add(ability)
print("- loading ability data from dota_english")
# Load additional information from the dota_english.txt file
data = valve_readfile(config.vpk_path,
paths['localization_abilities'],
"kv",
encoding="UTF-16")["lang"]["Tokens"]
for ability in session.query(Ability):
ability_tooltip = "DOTA_Tooltip_ability_" + ability.name
ability.localized_name = data.get(ability_tooltip, ability.name)
ability.description = data.get(ability_tooltip + "_Description", "")
ability.lore = data.get(ability_tooltip + "_Lore", "")
ability.aghanim = data.get(ability_tooltip + "_aghanim_description",
"")
notes = []
for i in range(8):
key = f"{ability_tooltip}_Note{i}"
if key in data:
notes.append(data[key])
ability.note = "" if len(notes) == 0 else "\n".join(notes)
ability_special = json.loads(ability.ability_special,
object_pairs_hook=OrderedDict)
ability_special = ability_special_add_talent(ability_special,
session.query(Ability))
ability_special = ability_special_add_header(ability_special, data,
ability.name)
ability.ability_special = json.dumps(ability_special, indent=4)
ability.description = clean_description(ability.description,
ability_special)
print("- adding ability icon files")
# Add img files to ability
for ability in session.query(Ability):
if os.path.isfile(config.vpk_path + paths['ability_icon_path'] +
ability.name + ".png"):
ability.icon = paths['ability_icon_path'] + ability.name + ".png"
else:
ability.icon = paths['ability_icon_path'] + "wisp_empty1.png"
session.commit()
def load():
session.query(Hero).delete()
print("Heroes")
# load all of the hero scripts data information
data = valve_readfile(config.vpk_path, paths['hero_scripts_file'],
"kv")["DOTAHeroes"]
progress = ProgressBar(len(data), title="- loading from hero scripts")
for heroname in data:
progress.tick()
if (heroname == "Version" or heroname == "npc_dota_hero_target_dummy"
or heroname == "npc_dota_hero_base"):
continue
hero = Hero()
hero_data = data[heroname]
def get_val(key):
if key in hero_data:
return hero_data[key]
else:
return data["npc_dota_hero_base"].get(key)
hero.full_name = heroname
hero.media_name = hero_data['VoiceFile'][37:-9]
hero.name = heroname.replace("npc_dota_hero_", "")
hero.id = get_val('HeroID')
hero.team = get_val('Team')
hero.base_health_regen = get_val('StatusHealthRegen')
hero.base_movement = get_val('MovementSpeed')
hero.turn_rate = get_val('MovementTurnRate')
hero.base_armor = get_val('ArmorPhysical')
hero.magic_resistance = get_val('MagicalResistance')
hero.attack_range = get_val('AttackRange')
hero.attack_projectile_speed = get_val('ProjectileSpeed')
hero.attack_damage_min = get_val('AttackDamageMin')
hero.attack_damage_max = get_val('AttackDamageMax')
hero.attack_rate = get_val('AttackRate')
hero.attack_point = get_val('AttackAnimationPoint')
hero.attr_primary = attribute_dict[get_val('AttributePrimary')]
hero.attr_strength_base = get_val('AttributeBaseStrength')
hero.attr_strength_gain = get_val('AttributeStrengthGain')
hero.attr_intelligence_base = get_val('AttributeBaseIntelligence')
hero.attr_intelligence_gain = get_val('AttributeIntelligenceGain')
hero.attr_agility_base = get_val('AttributeBaseAgility')
hero.attr_agility_gain = get_val('AttributeAgilityGain')
hero.vision_day = get_val('VisionDaytimeRange')
hero.vision_night = get_val('VisionNighttimeRange')
hero.is_melee = get_val(
'AttackCapabilities') == "DOTA_UNIT_CAP_MELEE_ATTACK"
hero.material = get_val('GibType')
hero.roles = hero_data.get('Role', '').replace(',', '|')
hero.role_levels = hero_data.get('Rolelevels', '').replace(',', '|')
glow_color = hero_data.get('HeroGlowColor', None)
if glow_color:
hero.color = "#{0:02x}{1:02x}{2:02x}".format(
*map(int, glow_color.split(' ')))
hero.json_data = json.dumps(hero_data, indent=4)
talents = []
# Link abilities and add talents
for slot in range(1, 30):
if "Ability" + str(slot) in hero_data:
ability = session.query(Ability).filter_by(
name=hero_data["Ability" + str(slot)]).first()
if ability.name.startswith("special_bonus"):
talents.append(ability.localized_name)
else:
ability.hero_id = hero.id
ability.ability_slot = slot
if len(talents) != 8:
raise ValueError("{} only has {} talents?".format(
hero.localized_name, len(talents)))
hero.talents = "|".join(talents)
session.add(hero)
print("- loading hero names from dota_english file")
# Load hero names from dota_english file
data = valve_readfile(config.vpk_path,
paths['dota_english_file'],
"kv",
encoding="UTF-16")["lang"]["Tokens"]
for hero in session.query(Hero):
hero.localized_name = data[hero.full_name]
hero.hype = data[hero.full_name + "_hype"]
print("- loading bio from hero lore file")
# Load bio from hero lore file
data = valve_readfile(config.vpk_path,
paths['localization_hero_lore'],
"kv",
encoding="UTF-16")["hero_lore"]
for hero in session.query(Hero):
hero.bio = data[hero.full_name + "_bio"].replace("<br>", "\n")
print("- adding hero image files")
# Add img files to hero
for hero in session.query(Hero):
hero.icon = paths['hero_icon_path'] + hero.name + ".png"
hero.image = paths['hero_image_path'] + hero.name + ".png"
hero.portrait = paths['hero_selection_path'] + hero.full_name + ".png"
print("- adding hero real names")
data = read_json("builderdata/hero_names.json")
for hero in session.query(Hero):
hero.real_name = data.get(hero.name, "")
print("- adding hero aliases")
data = read_json("builderdata/hero_aliases.json")
for hero in session.query(Hero):
aliases = []
aliases.append(hero.name.replace("_", " "))
text = re.sub(r'[^a-z^\s]', r'',
hero.localized_name.replace("_", " ").lower())
if text not in aliases:
aliases.append(text)
if hero.real_name != "":
aliases.append(re.sub(r'[^a-z^\s]', r'', hero.real_name.lower()))
aliases.extend(data.get(hero.name, []))
hero.aliases = "|".join(aliases)
print("- adding hero colors")
data = read_json("builderdata/hero_colors.json")
for hero_name in data:
hero = session.query(Hero).filter_by(name=hero_name).first()
hero.color = data[hero_name]
session.commit()
def load():
session.query(Voice).delete()
print("Voices")
print("- loading from heroes")
for hero in session.query(Hero):
voice = Voice()
voice.id = hero.id
voice.name = hero.localized_name
voice.icon = hero.icon
voice.image = hero.portrait
voice.url = name_to_url(hero.localized_name) + "/Responses"
voice.vsndevts_path = "/" + json.loads(hero.json_data).get("VoiceFile")
voice.hero_id = hero.id
session.add(voice)
print("- loading cosmetics file (takes a bit)")
data = valve_readfile(config.vpk_path,
paths['cosmetics_scripts_file'],
"kv_nocomment",
encoding="UTF-8")["items_game"]["items"]
custom_urls = {
"Announcer: Tuskar": "Announcer:_Tusk",
"Default Announcer": "Announcer_responses",
"Default Mega-Kill Announcer": "Announcer_responses",
"Announcer: Bristleback": "Bristleback_Announcer_Pack",
"Mega-Kills: Bristleback": "Bristleback_Announcer_Pack"
}
custom_vsndevts = {
"Default Announcer":
"/soundevents/voscripts/game_sounds_vo_announcer.vsndevts",
"Default Mega-Kill Announcer":
"/soundevents/voscripts/game_sounds_vo_announcer_killing_spree.vsndevts",
"Announcer: Kunkka & Tidehunter":
"/soundevents/voscripts/game_sounds_vo_announcer_dlc_kunkka_tide.vsndevts",
"Mega-Kills: Kunkka & Tidehunter":
"/soundevents/voscripts/game_sounds_vo_announcer_dlc_kunkka_tide_killing_spree.vsndevts"
}
print("- loading from announcers")
for key in data:
announcer = data[key]
if announcer.get("prefab") != "announcer":
continue
voice = Voice()
# the first announcer has id = 586, so this will not interfere with hero ids
voice.id = int(key)
voice.name = announcer["name"]
voice.icon = "/panorama/images/icon_announcer_psd.png"
voice.image = f"/panorama/images/{announcer['image_inventory']}_png.png"
if voice.name in custom_urls:
voice.url = custom_urls[voice.name]
else:
voice.url = name_to_url(announcer["name"])
if voice.name in custom_vsndevts:
voice.vsndevts_path = custom_vsndevts[voice.name]
else:
for asset in announcer["visuals"]:
if announcer["visuals"][asset]["type"] == "announcer":
voice.vsndevts_path = "/" + announcer["visuals"][asset][
"modifier"]
session.add(voice)
print("- associating announcer packs")
for key in data:
pack = data[key]
if pack.get("prefab") != "bundle" or pack.get(
"name") == "Assembly of Announcers Pack":
continue
for name in pack.get("bundle", []):
for voice in session.query(Voice).filter_by(name=name):
voice.url = name_to_url(pack["name"])
data = read_json("builderdata/voice_actors.json")
print("- adding voice actors")
for voice in session.query(Voice):
if str(voice.id) in data:
voice.voice_actor = data[str(voice.id)]
session.commit()
