def copy_images(args):
base_dir = args.base_dir
output_dir = args.output_dir
jp_icon_input_dir = os.path.join(base_dir, 'jp', 'portrait', 'local')
na_icon_input_dir = os.path.join(base_dir, 'na', 'portrait', 'local')
jp_portrait_input_dir = os.path.join(base_dir, 'jp', 'full',
'corrected_data')
na_portrait_input_dir = os.path.join(base_dir, 'na', 'full',
'corrected_data')
for jp_id in range(1, 6000):
monster_no = monster_id_mapping.jp_id_to_monster_no(jp_id)
monster_no_filled = str(monster_no).zfill(4)
monster_no_filled_long = str(monster_no).zfill(5)
do_copy(jp_icon_input_dir, '{}.png'.format(jp_id), output_dir,
'icon_{}.png'.format(monster_no_filled))
do_copy(jp_portrait_input_dir, '{}.png'.format(jp_id), output_dir,
'portrait_{}.png'.format(monster_no_filled))
do_copy(jp_portrait_input_dir, '{}.png'.format(jp_id), output_dir,
'texture_{}.png'.format(monster_no_filled_long))
for na_id in monster_id_mapping.NA_VOLTRON_IDS:
monster_no = monster_id_mapping.na_id_to_monster_no(na_id)
monster_no_filled = str(monster_no).zfill(4)
monster_no_filled_long = str(monster_no).zfill(5)
do_copy(na_icon_input_dir, '{}.png'.format(na_id), output_dir,
'icon_{}.png'.format(monster_no_filled))
do_copy(na_portrait_input_dir, '{}.png'.format(na_id), output_dir,
'portrait_{}.png'.format(monster_no_filled))
do_copy(na_portrait_input_dir, '{}.png'.format(na_id), output_dir,
'texture_{}.png'.format(monster_no_filled_long))
def lookup_existing_skill_id(field_name, skill_id):
alt_monster_no = monster_id_mapping.jp_id_to_monster_no(
min(skill_id_to_card_ids[skill_id]))
if alt_monster_no == monster_no:
# No existing monster (by monster id order) has this skill
return
# An existing card already has this skill, look it up
ts_seq = db_wrapper.get_single_value(
"select {} from monster_list where monster_no = {}".format(
field_name, alt_monster_no),
op=int)
logger.warn('Looked up existing skill id %s from %s for %s',
ts_seq, alt_monster_no, merged_card)
return ts_seq
def make_cross_server_card(jp_card: MergedCard,
na_card: MergedCard) -> (CrossServerCard, str):
card_id = jp_card.card.card_id
if card_id <= 0 or card_id > 6000:
return None, 'crazy id: {}'.format(repr(card))
if '***' in jp_card.card.name or '???' in jp_card.card.name:
return None, 'Skipping debug card: {}'.format(repr(card))
if '***' in na_card.card.name or '???' in na_card.card.name:
# Card probably exists in JP but not in NA
na_card = jp_card
# Apparently some monsters can be ported to NA before their skills are
if jp_card.leader_skill and not na_card.leader_skill:
na_card.leader_skill = jp_card.leader_skill
if jp_card.active_skill and not na_card.active_skill:
na_card.active_skill = jp_card.active_skill
monster_no = monster_id_mapping.jp_id_to_monster_no(card_id)
return CrossServerCard(monster_no, jp_card, na_card), None
def database_diff_cards(db_wrapper, jp_database, na_database):
jp_card_ids = [mc.card.card_id for mc in jp_database.cards]
jp_id_to_card = {mc.card.card_id: mc for mc in jp_database.cards}
na_id_to_card = {mc.card.card_id: mc for mc in na_database.cards}
if len(jp_card_ids) != len(jp_id_to_card):
logger.error('Mismatched sizes: %s %s', len(jp_card_ids),
len(jp_id_to_card))
# This is the list of cards we could potentially update
combined_cards = [] # List[CrossServerCard]
for card_id in jp_card_ids:
jp_card = jp_id_to_card.get(card_id)
na_card = na_id_to_card.get(monster_id_mapping.jp_id_to_na_id(card_id),
jp_card)
csc, err_msg = make_cross_server_card(jp_card, na_card)
if csc:
combined_cards.append(csc)
elif err_msg:
fail_logger.debug('Skipping card, %s', err_msg)
def insert_or_update(item: monster.SqlItem):
# Check if the item exists by key
if db_wrapper.check_existing(item.exists_sql()):
# It exists, check if the updatable values have changed
update_sql = item.needs_update_sql()
if update_sql and not db_wrapper.check_existing(update_sql):
logger.warn('Updating: %s', repr(item))
db_wrapper.insert_item(item.update_sql())
else:
fail_logger.debug(
'Skipping existing item that needs no updates: %s',
repr(item))
else:
# This is a new item, so populate it
logger.warn('Inserting new item: %s', repr(item))
db_wrapper.insert_item(item.insert_sql())
# Base monster
for csc in combined_cards:
insert_or_update(
monster.MonsterItem(csc.jp_card.card, csc.na_card.card))
# Monster info
for csc in combined_cards:
insert_or_update(
monster.MonsterInfoItem(csc.jp_card.card, csc.na_card.card))
# Additional monster info
for csc in combined_cards:
insert_or_update(monster.MonsterAddInfoItem(csc.jp_card.card))
# Monster prices
for csc in combined_cards:
insert_or_update(monster.MonsterPriceItem(csc.jp_card.card))
# Awakenings
awakening_name_and_id = db_wrapper.fetch_data(monster.awoken_name_id_sql())
awoken_name_to_id = {
row['name']: row['ts_seq']
for row in awakening_name_and_id
}
next_awakening_id = db_wrapper.get_single_value(
'SELECT 1 + COALESCE(MAX(CAST(tma_seq AS SIGNED)), 20000) FROM awoken_skill_list',
op=int)
for csc in combined_cards:
awakenings = monster.card_to_awakenings(awoken_name_to_id,
csc.jp_card.card)
for item in awakenings:
tma_seq = db_wrapper.check_existing_value(
item.exists_by_values_sql())
if tma_seq:
item.tma_seq = tma_seq
else:
item.tma_seq = next_awakening_id
next_awakening_id += 1
insert_or_update(item)
# Evolutions
next_evo_id = db_wrapper.get_single_value(
'SELECT 1 + COALESCE(MAX(CAST(tv_seq AS SIGNED)), 4000) FROM evolution_list',
op=int)
for csc in combined_cards:
evolution = monster.EvolutionItem(csc.jp_card.card)
if not evolution.is_valid():
continue
if db_wrapper.check_existing(evolution.exists_sql()):
fail_logger.debug('Skipping existing evolution: %s',
repr(evolution))
else:
logger.warn('Inserting new evolution: %s', repr(evolution))
db_wrapper.insert_item(evolution.insert_sql(next_evo_id))
next_evo_id += 1
# Try to populate series if missing.
# First stage.
# 1) Pull the list of monster_no -> series_id from the DB.
# 2) For monsters with tsr_seq = 42, find the series of it's ancestor
# 3) If that monster has a series != 42, apply it and save.
monster_no_to_series_id = db_wrapper.load_to_key_value(
'monster_no', 'tsr_seq', 'monster_info_list') # type Map<int, int>
for csc in combined_cards:
if monster_no_to_series_id[csc.monster_no] != 42:
continue
ancestor_id = csc.jp_card.card.ancestor_id
if ancestor_id == 0:
continue
ancestor_monster_no = monster_id_mapping.jp_id_to_monster_no(
ancestor_id)
ancestor_series = monster_no_to_series_id[ancestor_monster_no]
if ancestor_series != 42:
logger.warn('Detected new group ID for %s, %s',
repr(csc.na_card.card), ancestor_series)
db_wrapper.insert_item(
monster.update_series_by_monster_no_sql(
csc.monster_no, ancestor_series))
# Second stage.
# 1) Pull the list of monster_no -> series_id from the DB (again, may have been updated in step 1)
# 2) Compile a list of group_id -> series_id (excluding premium, tsr_seq=42).
# 3) Discard any group_id with more than one series_id.
# 4) Iterate over every monster with series_id = 42. If the group_id has a series_id mapped,
# assign it and save.
monster_no_to_series_id = db_wrapper.load_to_key_value(
'monster_no', 'tsr_seq', 'monster_info_list') # type Map<int, int>
group_id_to_cards = defaultdict(
list) # type DefaultDict<GroupId, List[CrossServerCard]>
for csc in combined_cards:
group_id_to_cards[csc.jp_card.card.group_id].append(csc)
group_id_to_series_id = {} # type Map<int, int
for group_id, card_list in group_id_to_cards.items():
series_id = None
for card in card_list:
new_series_id = monster_no_to_series_id[card.monster_no]
if new_series_id == 42:
continue # Skip premium
if series_id != new_series_id:
if series_id is None:
series_id = new_series_id
else:
series_id = None
break
if series_id is not None:
group_id_to_series_id[group_id] = series_id
for csc in combined_cards:
series_id = monster_no_to_series_id[csc.monster_no]
if series_id != 42:
continue
group_id = csc.jp_card.card.group_id
if group_id in group_id_to_series_id:
new_series_id = group_id_to_series_id[group_id]
logger.warn('Detected new group ID for %s, %s',
repr(csc.na_card.card), new_series_id)
db_wrapper.insert_item(
monster.update_series_by_monster_no_sql(
csc.monster_no, new_series_id))
# Evo mats
next_evo_mat_id = db_wrapper.get_single_value(
'SELECT 1 + COALESCE(MAX(CAST(tem_seq AS SIGNED)), 15000) FROM evo_material_list',
op=int)
for csc in combined_cards:
card = csc.jp_card.card
if not card.ancestor_id:
continue
tv_seq = db_wrapper.get_single_value(monster.lookup_evo_id_sql(card),
op=int)
evo_mat_items = monster.card_to_evo_mats(card, tv_seq)
for item in evo_mat_items:
tem_seq = db_wrapper.check_existing_value(
item.exists_by_values_sql())
if tem_seq:
item.tem_seq = tem_seq
else:
item.tem_seq = next_evo_mat_id
next_evo_mat_id += 1
insert_or_update(item)
# Skills
next_skill_id = db_wrapper.get_single_value(
'SELECT 1 + COALESCE(MAX(CAST(ts_seq AS SIGNED)), 20000) FROM skill_list',
op=int)
# Compute English skill text
calc_skills = skill_info.reformat_json_info(jp_database.raw_skills)
# Create a list of SkillIds to CardIds
skill_id_to_card_ids = defaultdict(
list) # type DefaultDict<SkillId, List[CardId]>
for merged_card in jp_database.cards:
for skill in [merged_card.active_skill, merged_card.leader_skill]:
if skill is None:
continue
skill_id_to_card_ids[skill.skill_id].append(
merged_card.card.card_id)
for csc in combined_cards:
merged_card = csc.jp_card
merged_card_na = csc.na_card
monster_no = csc.monster_no
info = db_wrapper.get_single_or_no_row(
monster_skill.get_monster_skill_ids(merged_card))
if not info:
fail_logger.warn('Unexpected empty skill lookup: %s',
repr(merged_card))
continue
def lookup_existing_skill_id(field_name, skill_id):
alt_monster_no = monster_id_mapping.jp_id_to_monster_no(
min(skill_id_to_card_ids[skill_id]))
if alt_monster_no == monster_no:
# No existing monster (by monster id order) has this skill
return
# An existing card already has this skill, look it up
ts_seq = db_wrapper.get_single_value(
"select {} from monster_list where monster_no = {}".format(
field_name, alt_monster_no),
op=int)
logger.warn('Looked up existing skill id %s from %s for %s',
ts_seq, alt_monster_no, merged_card)
return ts_seq
def find_or_create_skill(monster_field_name, skill_value,
na_skill_value, calc_skill_description):
# Try to look up another monster with that skill
ts_seq = lookup_existing_skill_id(monster_field_name,
skill_value.skill_id)
if ts_seq is None:
# Lookup failed, insert a new skill
nonlocal next_skill_id
item = monster_skill.MonsterSkillItem(next_skill_id,
skill_value,
na_skill_value,
calc_skill_description)
logger.warn('Inserting new monster skill: %s - %s',
repr(merged_card), repr(item))
db_wrapper.insert_item(item.insert_sql())
ts_seq = next_skill_id
next_skill_id += 1
return ts_seq
def maybe_update_skill(ts_seq, skill_value, na_skill_value,
calc_skill_description):
# Skill exists, check if it needs an update
item = monster_skill.MonsterSkillItem(ts_seq, skill_value,
na_skill_value,
calc_skill_description)
if not db_wrapper.check_existing(item.exists_sql()):
fail_logger.fatal('Unexpected empty skill lookup: %s',
repr(item))
exit()
# It exists, check if the updatable values have changed
if not db_wrapper.check_existing(item.needs_update_sql()):
logger.warn('Updating: %s', repr(item))
db_wrapper.insert_item(item.update_sql())
else:
fail_logger.debug(
'Skipping existing item that needs no updates: %s',
repr(item))
update_monster = False
ts_seq_leader = info['ts_seq_leader']
if merged_card.leader_skill:
calc_ls_skill = calc_skills.get(merged_card.leader_skill.skill_id,
'')
calc_ls_skill_description = calc_ls_skill.description.strip(
) or None if calc_ls_skill else None
if ts_seq_leader:
# Monster already has a skill attached, see if it needs to be updated
maybe_update_skill(ts_seq_leader, merged_card.leader_skill,
merged_card_na.leader_skill,
calc_ls_skill_description)
else:
# Skill needs to be attached
ts_seq_leader = find_or_create_skill(
'ts_seq_leader', merged_card.leader_skill,
merged_card_na.leader_skill, calc_ls_skill_description)
update_monster = True
if calc_ls_skill:
leader_data_item = monster_skill.MonsterSkillLeaderDataItem(
ts_seq_leader, calc_ls_skill.params)
if leader_data_item.leader_data:
insert_or_update(leader_data_item)
ts_seq_skill = info['ts_seq_skill']
if merged_card.active_skill:
calc_as_skill = calc_skills.get(merged_card.active_skill.skill_id,
'')
calc_as_skill_description = calc_as_skill.description.strip(
) or None if calc_as_skill else None
if ts_seq_skill:
# Monster already has a skill attached, see if it needs to be updated
maybe_update_skill(ts_seq_skill, merged_card.active_skill,
merged_card_na.active_skill,
calc_as_skill_description)
else:
ts_seq_skill = find_or_create_skill(
'ts_seq_skill', merged_card.active_skill,
merged_card_na.active_skill, calc_as_skill_description)
update_monster = True
if update_monster:
logger.warn('Updating monster skill info: %s - %s - %s',
repr(merged_card), ts_seq_leader, ts_seq_skill)
db_wrapper.insert_item(
monster_skill.get_update_monster_skill_ids(
merged_card, ts_seq_leader, ts_seq_skill))