def load_data_csv(self, parent_map : DataMap, data_file, *, key=None, groups=[], leaftype):
"""Loads a data file, using a base map to anchor it to id
The parent_map is updated to map id -> data row.
Returns the parent_map to support chaining.
Language is automatically determined by the name of the first column.
"""
data_file = self.get_data_path(data_file)
if leaftype == 'list' and not key:
raise ValueError("key is required if leaftype is list")
rows = read_csv(data_file)
if not rows:
return parent_map
# Auto detect language
first_column = next(iter(rows[0].keys()))
match = re.match('name_([a-zA-Z]+)', first_column)
if not match:
raise Exception("First column needs to be a name_{lang} column")
lang = match.group(1)
data = unflatten(rows, nest=[first_column], groups=groups, leaftype=leaftype)
return parent_map.merge(data, lang=lang, key=key)
def load_base_csv(self, data_file, languages, groups=[], translation_filename=None, translation_extra=[], keys_ex=[], validate=True):
"""Loads a base data map object from a csv
groups is a list of additional fields (name is automatically include)
that nest via groupname_subfield.
"""
data_file = self.get_data_path(data_file)
groups = ['name'] + groups
rows = [group_fields(row, groups=groups) for row in read_csv(data_file)]
basemap = DataMap(languages=languages, keys_ex=keys_ex)
basemap.extend(rows)
if translation_filename:
try:
translations = fix_id(self.load_list_csv(translation_filename))
groups = set(['name'] + translation_extra)
merge_list(basemap, translations, groups=groups, many=False)
except FileNotFoundError:
print(f"Warning: Could not find translation file {translation_filename}")
if languages:
self._validate_base_map(data_file, basemap, languages, error=validate)
return basemap
def update_monsters(mhdata):
monster_keys = read_csv(dirname(abspath(__file__)) + '/monster_map.csv')
monster_keys = dict((r['name_en'], r) for r in monster_keys)
monster_name_text = load_text('common/text/em_names')
monster_info_text = load_text('common/text/em_info')
for monster_entry in mhdata.monster_map.values():
name_en = monster_entry.name('en')
if name_en not in monster_keys:
print(f'Warning: {name_en} not mapped, skipping')
monster_key_entry = monster_keys[monster_entry.name('en')]
key = monster_key_entry['key']
info_key = monster_key_entry['key_info_override'] or key
monster_entry['name'] = monster_name_text[key]
if info_key != 'NONE':
monster_entry['description'] = monster_info_text[f'NOTE_{info_key}_DESC']
# Write new data
writer = create_writer()
writer.save_base_map_csv(
"monsters/monster_base.csv",
mhdata.monster_map,
schema=schema.MonsterBaseSchema(),
translation_filename="monsters/monster_base_translations.csv",
translation_extra=['description']
)
print("Monsters updated\n")
def __init__(self):
id_alt_keys = ['id_alt', 'id_alt2']
this_dir = dirname(abspath(__file__))
# Load data from the quest data dump project
# Note that since the key is a FILEPATH it can't be joined with the rest of the data
self.monster_data_ext = json.load(
open(this_dir + '/metadata_files/MonsterData.json'))
monster_keys_csv = read_csv(this_dir +
'/metadata_files/monster_map.csv')
monster_entries = [
MonsterMetaEntry(
name=r['name_en'].strip(),
id=int(r['id'], 16),
id_alt=[int(r[key], 16) for key in id_alt_keys if r[key]],
key_name=r['key_name'],
key_description=r['key_description']) for r in monster_keys_csv
]
self._map = dict((r.name, r) for r in monster_entries)
self._map_by_id = dict((r.id, r) for r in monster_entries)
# Add alt keys. Note that they only go one way and cannot be reverse associated
for r in monster_entries:
for alt_id in r.id_alt:
self._map_by_id[alt_id] = r
def load_area_map():
this_dir = dirname(abspath(__file__))
area_map = {
int(r['id'], 16): r['name']
for r in read_csv(this_dir + '/metadata_files/area_map.csv')
}
return area_map
def load_list_csv(self, data_file, *, schema=None):
"""Loads a simple csv without processing.
Accepts marshmallow schema to transform and validate it"""
data_file = self.get_data_path(data_file)
data = read_csv(data_file)
if schema:
# When version 3 is released, this api will change
# load will just return converted and errors will auto-raise
(converted, errors) = schema.load(data, many=True)
if errors:
raise Exception(str(errors))
data = converted
return data
def update_all():
"Updates all supported entity types using merged chunk data from ingame binaries."
from .items import ItemUpdater
from mhdata.load import load_data
from mhdata.merge.binary.load import MonsterMetadata
from .armor import update_armor
from .weapons import update_weapons, update_weapon_songs, update_kinsects
from .monsters import update_monsters
from .quests import update_quests
from mhdata.io.csv import read_csv
from os.path import dirname, abspath
mhdata = load_data()
print("Existing Data loaded. Using it as a base to merge new data")
this_dir = dirname(abspath(__file__))
area_map = {
int(r['id']): r['name']
for r in read_csv(this_dir + '/area_map.csv')
}
print("Area Map Loaded")
# validate area map
error = False
for name in area_map.values():
if name not in mhdata.location_map.names('en'):
print(f"Error: Area map has invalid location name {name}.")
error = True
if error:
return
print("Area Map validated")
item_updater = ItemUpdater()
monster_meta = MonsterMetadata()
print() # newline
update_armor(mhdata, item_updater)
update_weapons(mhdata, item_updater)
update_weapon_songs(mhdata)
update_kinsects(mhdata, item_updater)
update_monsters(mhdata, item_updater, monster_meta)
update_quests(mhdata, item_updater, monster_meta, area_map)
# Now finalize the item updates from parsing the rest of the data
item_updater.update_items()
def load_base_csv(self, data_file, groups=[], validate=True):
"""Loads a base data map object from a csv
groups is a list of additional fields (name is automatically include)
that nest via groupname_subfield.
"""
data_file = self.get_data_path(data_file)
groups = ['name'] + groups
rows = read_csv(data_file)
rows = [group_fields(row, groups=groups) for row in rows]
basemap = DataMap()
basemap.extend(rows)
self._validate_base_map(data_file, basemap, error=validate)
return basemap
def load_data_csv(self,
parent_map: DataMap,
data_file,
*,
key=None,
groups=[],
leaftype):
"""Loads a data file, using a base map to anchor it to id
The parent_map is updated to map id -> data row.
Returns the parent_map to support chaining.
:param key: The dictionary key name in the base map to add the new data under.
If none, it'll extend the current list
:param groups: Additional fields in the data map to group together into one field based on suffix.
:param leaftype: Either list or dict, deciding on whether the result of the new data should be a list or just a dict.
Use list if the data is one to many, or dict if its one to one
Language is automatically determined by the name of the first column.
"""
data_file = self.get_data_path(data_file)
if leaftype == 'list' and not key:
raise ValueError("key is required if leaftype is list")
rows = read_csv(data_file)
if not rows:
return parent_map
# Auto detect language
first_column = next(iter(rows[0].keys()))
match = re.match('(?:base_)?([a-zA-Z]+)(?:_([a-zA-Z]+))?',
first_column)
if not match:
raise Exception(
"First column needs to be a base_{field} or base_{field}_{lang} column"
)
fieldname = match.group(1)
lang = match.group(2)
data = unflatten(rows,
nest=[first_column],
groups=groups,
leaftype=leaftype)
return parent_map.merge(data, field=fieldname, lang=lang, key=key)
def extend_decoration_chances(decoration_map: DataMap):
"""Calculates the drop tables given the decoration map.
Each decoration is part of a drop table (decided by rarity), and feystones
will individually land on a drop table. Once on a drop table, each decoration in that drop table
has an "equal" chance within that drop table.
Odds are listed here, with one typo (gleaming is actually glowing).
https://docs.google.com/spreadsheets/d/1ysj6c2boC6GarFvMah34e6VviZeaoKB6QWovWLSGlsY/htmlview?usp=sharing&sle=true#
"""
jewel_to_table_odds = {}
droprates = read_csv(
join(data_path, "decorations/decoration_droprates.csv"))
for row in droprates:
entries = {}
for i in range(5, 14):
entries[i] = int(row[str(i)] or '0')
jewel_to_table_odds[row['feystone']] = entries
# Calculate how many entries there are per drop table type
table_counts = {table: 0 for table in range(5, 14)}
for entry in decoration_map.values():
table_counts[entry['rarity']] += 1
# Create an odds map for each drop table level
# This maps droptable -> feystone -> probability
# This is necessary because all decorations are assigned to a droptable
odds_map = {}
for table in range(5, 14):
odds_map[table] = {}
for feystone, feystone_odds in jewel_to_table_odds.items():
count = table_counts[table]
if count == 0:
continue
value = Decimal(feystone_odds[table]) / Decimal(count)
odds_map[table][feystone] = value.quantize(Decimal('1.00000'))
# Assign the odds map for the drop table level to the decoration itself
for entry in decoration_map.values():
entry['chances'] = odds_map[entry['rarity']]
def __init__(self):
id_alt_keys = ['id_alt', 'id_alt2']
this_dir = dirname(abspath(__file__))
monster_keys_csv = read_csv(this_dir + '/monster_map.csv')
monster_entries = [
MonsterMetaEntry(
name=r['name_en'].strip(),
id=int(r['id'], 16),
id_alt=[int(r[key], 16) for key in id_alt_keys if r[key]],
key_name=r['key_name'],
key_description=r['key_description']) for r in monster_keys_csv
]
self._map = dict((r.name, r) for r in monster_entries)
self._map_by_id = dict((r.id, r) for r in monster_entries)
# Add alt keys. Note that they only go one way and cannot be reverse associated
for r in monster_entries:
for alt_id in r.id_alt:
self._map_by_id[alt_id] = r
