def load_names(self):
translated = load_keyed_db_file(private_data_path("translated.csv"))
overrides = load_keyed_db_file(private_data_path("overrides.csv"))
names = load_keyed_db_file(transient_data_path("names.csv"))
if not names:
# then we can't get a schema
names.update(translated)
names.update(overrides)
return names
schema = next(iter(names.values())).__class__
names_keys = set(schema._fields)
overrides_keys = set(schema._fields)
if not overrides_keys <= names_keys:
raise Exception('names.csv schema error: all of "chara_id","kanji","kanji_spaced","kana_spaced","conventional" must be present in the header')
# maps kanji -> chara id
by_kanji = {v.kanji: k for k, v in names.items()}
for key in overrides:
if key < 0:
# a negative chara id in the override entry means we should match on kanji.
real_key = by_kanji.get(overrides[key].kanji)
intermediate = names.get(real_key)
else:
real_key = key
intermediate = names.get(key)
if intermediate is None:
continue
d = intermediate._asdict()
override_vals = overrides[key]._asdict()
# chara_id may differ if we indexed on kanji, so remove it
del override_vals["chara_id"]
d.update(override_vals)
names[real_key] = schema(**d)
# copies
for key in translated:
if key < 0:
# a negative chara id in the override entry means we should match on kanji.
real_key = by_kanji.get(translated[key].kanji)
intermediate = names.get(real_key)
else:
real_key = key
intermediate = names.get(key)
if intermediate is None:
continue
d = intermediate._asdict()
translated_vals = translated[key]._asdict()
# chara_id may differ if we indexed on kanji, so remove it
del translated_vals["chara_id"]
d.update(translated_vals)
names[real_key] = schema(**d)
return names
def main(file1, file2):
data_a = csvloader.load_keyed_db_file(file1)
data_b = csvloader.load_keyed_db_file(file2)
ksa, ksb = map(lambda x: set(x.keys()), (data_a, data_b))
added = ksb - ksa
if added:
diff_obj = {"date": str(date.today()), "cids": sorted(added)}
print(json.dumps(diff_obj))
def main(file1, file2, title_key):
loc = os.path.basename(file2)
if "." in loc:
loc = loc[:loc.rfind(".")]
data_a = csvloader.load_keyed_db_file(file1)
data_b = csvloader.load_keyed_db_file(file2)
#assert schema_b == schema_a, "Schema mismatch."
ksa, ksb = map(lambda x: set(x.keys()), (data_a, data_b))
added = ksb - ksa
removed = ksa - ksb
changed = set()
maybe_changed = ksb & ksa
for key in maybe_changed:
if data_a[key] != data_b[key]:
changed.add(key)
diff = []
for key in removed:
diff.append({
"_k": key,
"change_type": "Removed",
"where": "{0}:{1}".format(loc, key),
"title": getattr(data_a[key], title_key)
})
for key in added:
diff.append({
"_k": key,
"change_type": "Added",
"where": "{0}:{1}".format(loc, key),
"title": getattr(data_b[key], title_key)
})
for key in changed:
diff.append({
"_k": key,
"change_type": "Changed",
"where": "{0}:{1}".format(loc, key),
"title": getattr(data_b[key], title_key),
"extra": "\n".join(values_changed(data_a[key], data_b[key]))
})
diff.sort(key=lambda x: x["_k"])
for x in diff:
print(json.dumps(x, sort_keys=1))
def prime_caches(self):
self.names = self.load_names()
self.kanji_to_name = {v.kanji: v.conventional for v in self.names.values()}
self.ea_overrides = list(load_db_file(private_data_path("event_availability_overrides.csv")))
self.fix_limited = load_keyed_db_file(private_data_path("gacha_availability_overrides.csv"))
self.overridden_events = set(x.event_id for x in self.ea_overrides)
prob_def = self.keyed_prime_from_table("probability_type")
time_def = self.keyed_prime_from_table("available_time_type")
self._skills = self.keyed_prime_from_table("skill_data",
chance=lambda obj: partial(skill_chance, prob_def, obj.probability_type),
dur=lambda obj: partial(skill_dur, time_def, obj.available_time_type),
max_chance=lambda obj: prob_def[obj.probability_type].probability_max,
max_duration=lambda obj: time_def[obj.available_time_type].available_time_max)
self._lead_skills = self.keyed_prime_from_table("leader_skill_data")
self.rarity_dep = self.keyed_prime_from_table("card_rarity")
self.chain_id = {}
self.id_chain = defaultdict(lambda: [])
chain_cur = self.hnd.execute("SELECT id, series_id FROM card_data WHERE album_id > 0")
for p in self.prime_from_cursor("chain_id_t", chain_cur):
self.chain_id[p.id] = p.series_id
self.id_chain[p.series_id].append(p.id)
self.char_cache = {}
self.card_cache = {}
def load_names(self):
overrides = load_keyed_db_file(private_data_path("overrides.csv"))
names = load_keyed_db_file(transient_data_path("names.csv"))
if not names:
# then we can't get a schema
names.update(overrides)
return names
schema = next(iter(names.values())).__class__
names_keys = set(schema._fields)
overrides_keys = set(schema._fields)
if not overrides_keys <= names_keys:
raise Exception(
'names.csv schema error: all of "chara_id","kanji","kanji_spaced","kana_spaced","conventional" must be present in the header'
)
# maps kanji -> chara id
by_kanji = {v.kanji: k for k, v in names.items()}
for key in overrides:
if key < 0:
# a negative chara id in the override entry means we should match on kanji.
real_key = by_kanji.get(overrides[key].kanji)
intermediate = names.get(real_key)
else:
real_key = key
intermediate = names.get(key)
if intermediate is None:
continue
d = intermediate._asdict()
override_vals = overrides[key]._asdict()
# chara_id may differ if we indexed on kanji, so remove it
del override_vals["chara_id"]
d.update(override_vals)
names[real_key] = schema(**d)
# Filter out any special-purpose char entries.
valid_char_ids = set(id for id, in self.hnd.execute(
"SELECT chara_id FROM chara_data WHERE base_card_id != 0"))
names = {k: v for k, v in names.items() if k in valid_char_ids}
return names
def prime_caches(self):
self.names = self.load_names()
self.kanji_to_name = {
v.kanji: v.conventional
for v in self.names.values()
}
self.ea_overrides = list(
load_db_file(
private_data_path("event_availability_overrides.csv")))
self.fix_limited = load_keyed_db_file(
private_data_path("gacha_availability_overrides.csv"))
self.overridden_events = set(x.event_id for x in self.ea_overrides)
prob_def = self.keyed_prime_from_table("probability_type")
time_def = self.keyed_prime_from_table("available_time_type")
self._skills = self.keyed_prime_from_table(
"skill_data",
chance=lambda obj: partial(skill_chance, prob_def, obj.
probability_type),
dur=lambda obj: partial(skill_dur, time_def, obj.
available_time_type),
max_chance=lambda obj: prob_def[obj.probability_type
].probability_max,
max_duration=lambda obj: time_def[obj.available_time_type
].available_time_max)
self._lead_skills = self.keyed_prime_from_table("leader_skill_data")
self.rarity_dep = self.keyed_prime_from_table("card_rarity")
self.chain_id = {}
self.id_chain = defaultdict(lambda: [])
chain_cur = self.hnd.execute(
"SELECT id, series_id FROM card_data WHERE album_id > 0")
for p in self.prime_from_cursor("chain_id_t", chain_cur):
self.chain_id[p.id] = p.series_id
self.id_chain[p.series_id].append(p.id)
self.char_cache = {}
self.card_cache = {}
a = sqlite3.connect(new_db)
c = a.execute("SELECT chara_id, name, name_kana FROM chara_data")
ret = {}
for r in c:
ret[r[0]] = chara_stub_t(*r[1:])
a.close()
return ret
if __name__ == '__main__':
new_db = sys.argv[2]
name_tab = sys.argv[3]
charas = load_from_db(new_db)
try:
have_names = csvloader.load_keyed_db_file(name_tab)
except IOError:
have_names = {}
missing = set(charas.keys()) - set(have_names.keys())
for key in sorted(missing):
chara = charas[key]
print("---", chara.name, "----------")
res = EnamdictHandle(sys.argv[1]).find_name(chara.name, chara.name_kana)
if not res:
print("warning: No solution found at all")
res = [(Word(chara.name, chara.name_kana, set(), ""),)]
try:
import json
import sqlite3
from datetime import datetime
import locale
locale.setlocale(locale.LC_ALL, "en_US.UTF-8")
from pytz import timezone, utc
from collections import namedtuple
import models
from starlight import JST, private_data_path
import csvloader
overrides = csvloader.load_keyed_db_file(
private_data_path("gacha_availability_overrides.csv"))
gacha_stub_t = namedtuple(
"gacha_stub_t",
("id", "name", "start_date", "end_date", "type", "subtype"))
def gacha_ids(f):
gachas = []
a = sqlite3.connect(f)
for id, n, ss, es, t, t2 in a.execute(
"SELECT id, name, start_date, end_date, type, type_detail FROM gacha_data where type = 3 and type_detail = 1"
):
ss, es = JST(ss), JST(es)
gachas.append(gacha_stub_t(id, n, ss, es, t, t2))
def load_names(self):
translated = load_keyed_db_file(private_data_path("translated.csv"))
overrides = load_keyed_db_file(private_data_path("overrides.csv"))
names = load_keyed_db_file(transient_data_path("names.csv"))
if not names:
# then we can't get a schema
names.update(translated)
names.update(overrides)
return names
schema = next(iter(names.values())).__class__
names_keys = set(schema._fields)
overrides_keys = set(schema._fields)
if not overrides_keys <= names_keys:
raise Exception(
'names.csv schema error: all of "chara_id","kanji","kanji_spaced","kana_spaced","conventional" must be present in the header'
)
# maps kanji -> chara id
by_kanji = {v.kanji: k for k, v in names.items()}
for key in overrides:
if key < 0:
# a negative chara id in the override entry means we should match on kanji.
real_key = by_kanji.get(overrides[key].kanji)
intermediate = names.get(real_key)
else:
real_key = key
intermediate = names.get(key)
if intermediate is None:
continue
d = intermediate._asdict()
override_vals = overrides[key]._asdict()
# chara_id may differ if we indexed on kanji, so remove it
del override_vals["chara_id"]
d.update(override_vals)
names[real_key] = schema(**d)
# copies
for key in translated:
if key < 0:
# a negative chara id in the override entry means we should match on kanji.
real_key = by_kanji.get(translated[key].kanji)
intermediate = names.get(real_key)
else:
real_key = key
intermediate = names.get(key)
if intermediate is None:
continue
d = intermediate._asdict()
translated_vals = translated[key]._asdict()
# chara_id may differ if we indexed on kanji, so remove it
del translated_vals["chara_id"]
d.update(translated_vals)
names[real_key] = schema(**d)
return names
def cached_keyed_db(path, cache={}, **kwargs):
if path not in cache:
cache[path] = csvloader.load_keyed_db_file(path, **kwargs)
return cache[path]
for given in self.find_given_name_candidates(kanji_str[len(candy.kanji):], kana_str[len(candy.kana):]):
possible.append((candy, given))
# if there is a non-inferred given name in there, delete all inferred matches
# this is so we don't provide guesses when real data is available
if not all(map(lambda x: "i" in x[1].classifier, possible)):
return list(filter(lambda x: "i" not in x[1].classifier, possible))
else:
return possible
def final_fixups(string):
return FIX_PAT.sub(lambda match: match.group(1), string)
if __name__ == '__main__':
try:
have_names = csvloader.load_keyed_db_file("_data/private/names.csv")
except IOError:
have_names = {}
charas = csvloader.load_keyed_db_file("_data/ark/chara_data.csv")
missing = set(charas.keys()) - set(have_names.keys())
f = open("_data/private/names.csv", "w")
c = csv.writer(f, delimiter=",", quotechar="\"", quoting=csv.QUOTE_NONNUMERIC, lineterminator="\n")
c.writerow(("chara_id", "kanji", "kanji_spaced", "kana_spaced", "conventional"))
for key in sorted(missing):
chara = charas[key]
print("---", chara.name, "----------")
res = EnamdictHandle(sys.argv[1]).find_name(chara.name, chara.name_kana)