def copy_images(args): base_dir = args.base_dir output_dir = args.output_dir jp_icon_input_dir = os.path.join(base_dir, 'jp', 'portrait', 'local') na_icon_input_dir = os.path.join(base_dir, 'na', 'portrait', 'local') jp_portrait_input_dir = os.path.join(base_dir, 'jp', 'full', 'corrected_data') na_portrait_input_dir = os.path.join(base_dir, 'na', 'full', 'corrected_data') for jp_id in range(1, 6000): monster_no = monster_id_mapping.jp_id_to_monster_no(jp_id) monster_no_filled = str(monster_no).zfill(4) monster_no_filled_long = str(monster_no).zfill(5) do_copy(jp_icon_input_dir, '{}.png'.format(jp_id), output_dir, 'icon_{}.png'.format(monster_no_filled)) do_copy(jp_portrait_input_dir, '{}.png'.format(jp_id), output_dir, 'portrait_{}.png'.format(monster_no_filled)) do_copy(jp_portrait_input_dir, '{}.png'.format(jp_id), output_dir, 'texture_{}.png'.format(monster_no_filled_long)) for na_id in monster_id_mapping.NA_VOLTRON_IDS: monster_no = monster_id_mapping.na_id_to_monster_no(na_id) monster_no_filled = str(monster_no).zfill(4) monster_no_filled_long = str(monster_no).zfill(5) do_copy(na_icon_input_dir, '{}.png'.format(na_id), output_dir, 'icon_{}.png'.format(monster_no_filled)) do_copy(na_portrait_input_dir, '{}.png'.format(na_id), output_dir, 'portrait_{}.png'.format(monster_no_filled)) do_copy(na_portrait_input_dir, '{}.png'.format(na_id), output_dir, 'texture_{}.png'.format(monster_no_filled_long))
def lookup_existing_skill_id(field_name, skill_id): alt_monster_no = monster_id_mapping.jp_id_to_monster_no( min(skill_id_to_card_ids[skill_id])) if alt_monster_no == monster_no: # No existing monster (by monster id order) has this skill return # An existing card already has this skill, look it up ts_seq = db_wrapper.get_single_value( "select {} from monster_list where monster_no = {}".format( field_name, alt_monster_no), op=int) logger.warn('Looked up existing skill id %s from %s for %s', ts_seq, alt_monster_no, merged_card) return ts_seq
def make_cross_server_card(jp_card: MergedCard, na_card: MergedCard) -> (CrossServerCard, str): card_id = jp_card.card.card_id if card_id <= 0 or card_id > 6000: return None, 'crazy id: {}'.format(repr(card)) if '***' in jp_card.card.name or '???' in jp_card.card.name: return None, 'Skipping debug card: {}'.format(repr(card)) if '***' in na_card.card.name or '???' in na_card.card.name: # Card probably exists in JP but not in NA na_card = jp_card # Apparently some monsters can be ported to NA before their skills are if jp_card.leader_skill and not na_card.leader_skill: na_card.leader_skill = jp_card.leader_skill if jp_card.active_skill and not na_card.active_skill: na_card.active_skill = jp_card.active_skill monster_no = monster_id_mapping.jp_id_to_monster_no(card_id) return CrossServerCard(monster_no, jp_card, na_card), None
def database_diff_cards(db_wrapper, jp_database, na_database): jp_card_ids = [mc.card.card_id for mc in jp_database.cards] jp_id_to_card = {mc.card.card_id: mc for mc in jp_database.cards} na_id_to_card = {mc.card.card_id: mc for mc in na_database.cards} if len(jp_card_ids) != len(jp_id_to_card): logger.error('Mismatched sizes: %s %s', len(jp_card_ids), len(jp_id_to_card)) # This is the list of cards we could potentially update combined_cards = [] # List[CrossServerCard] for card_id in jp_card_ids: jp_card = jp_id_to_card.get(card_id) na_card = na_id_to_card.get(monster_id_mapping.jp_id_to_na_id(card_id), jp_card) csc, err_msg = make_cross_server_card(jp_card, na_card) if csc: combined_cards.append(csc) elif err_msg: fail_logger.debug('Skipping card, %s', err_msg) def insert_or_update(item: monster.SqlItem): # Check if the item exists by key if db_wrapper.check_existing(item.exists_sql()): # It exists, check if the updatable values have changed update_sql = item.needs_update_sql() if update_sql and not db_wrapper.check_existing(update_sql): logger.warn('Updating: %s', repr(item)) db_wrapper.insert_item(item.update_sql()) else: fail_logger.debug( 'Skipping existing item that needs no updates: %s', repr(item)) else: # This is a new item, so populate it logger.warn('Inserting new item: %s', repr(item)) db_wrapper.insert_item(item.insert_sql()) # Base monster for csc in combined_cards: insert_or_update( monster.MonsterItem(csc.jp_card.card, csc.na_card.card)) # Monster info for csc in combined_cards: insert_or_update( monster.MonsterInfoItem(csc.jp_card.card, csc.na_card.card)) # Additional monster info for csc in combined_cards: insert_or_update(monster.MonsterAddInfoItem(csc.jp_card.card)) # Monster prices for csc in combined_cards: insert_or_update(monster.MonsterPriceItem(csc.jp_card.card)) # Awakenings awakening_name_and_id = db_wrapper.fetch_data(monster.awoken_name_id_sql()) awoken_name_to_id = { row['name']: row['ts_seq'] for row in awakening_name_and_id } next_awakening_id = db_wrapper.get_single_value( 'SELECT 1 + COALESCE(MAX(CAST(tma_seq AS SIGNED)), 20000) FROM awoken_skill_list', op=int) for csc in combined_cards: awakenings = monster.card_to_awakenings(awoken_name_to_id, csc.jp_card.card) for item in awakenings: tma_seq = db_wrapper.check_existing_value( item.exists_by_values_sql()) if tma_seq: item.tma_seq = tma_seq else: item.tma_seq = next_awakening_id next_awakening_id += 1 insert_or_update(item) # Evolutions next_evo_id = db_wrapper.get_single_value( 'SELECT 1 + COALESCE(MAX(CAST(tv_seq AS SIGNED)), 4000) FROM evolution_list', op=int) for csc in combined_cards: evolution = monster.EvolutionItem(csc.jp_card.card) if not evolution.is_valid(): continue if db_wrapper.check_existing(evolution.exists_sql()): fail_logger.debug('Skipping existing evolution: %s', repr(evolution)) else: logger.warn('Inserting new evolution: %s', repr(evolution)) db_wrapper.insert_item(evolution.insert_sql(next_evo_id)) next_evo_id += 1 # Try to populate series if missing. # First stage. # 1) Pull the list of monster_no -> series_id from the DB. # 2) For monsters with tsr_seq = 42, find the series of it's ancestor # 3) If that monster has a series != 42, apply it and save. monster_no_to_series_id = db_wrapper.load_to_key_value( 'monster_no', 'tsr_seq', 'monster_info_list') # type Map<int, int> for csc in combined_cards: if monster_no_to_series_id[csc.monster_no] != 42: continue ancestor_id = csc.jp_card.card.ancestor_id if ancestor_id == 0: continue ancestor_monster_no = monster_id_mapping.jp_id_to_monster_no( ancestor_id) ancestor_series = monster_no_to_series_id[ancestor_monster_no] if ancestor_series != 42: logger.warn('Detected new group ID for %s, %s', repr(csc.na_card.card), ancestor_series) db_wrapper.insert_item( monster.update_series_by_monster_no_sql( csc.monster_no, ancestor_series)) # Second stage. # 1) Pull the list of monster_no -> series_id from the DB (again, may have been updated in step 1) # 2) Compile a list of group_id -> series_id (excluding premium, tsr_seq=42). # 3) Discard any group_id with more than one series_id. # 4) Iterate over every monster with series_id = 42. If the group_id has a series_id mapped, # assign it and save. monster_no_to_series_id = db_wrapper.load_to_key_value( 'monster_no', 'tsr_seq', 'monster_info_list') # type Map<int, int> group_id_to_cards = defaultdict( list) # type DefaultDict<GroupId, List[CrossServerCard]> for csc in combined_cards: group_id_to_cards[csc.jp_card.card.group_id].append(csc) group_id_to_series_id = {} # type Map<int, int for group_id, card_list in group_id_to_cards.items(): series_id = None for card in card_list: new_series_id = monster_no_to_series_id[card.monster_no] if new_series_id == 42: continue # Skip premium if series_id != new_series_id: if series_id is None: series_id = new_series_id else: series_id = None break if series_id is not None: group_id_to_series_id[group_id] = series_id for csc in combined_cards: series_id = monster_no_to_series_id[csc.monster_no] if series_id != 42: continue group_id = csc.jp_card.card.group_id if group_id in group_id_to_series_id: new_series_id = group_id_to_series_id[group_id] logger.warn('Detected new group ID for %s, %s', repr(csc.na_card.card), new_series_id) db_wrapper.insert_item( monster.update_series_by_monster_no_sql( csc.monster_no, new_series_id)) # Evo mats next_evo_mat_id = db_wrapper.get_single_value( 'SELECT 1 + COALESCE(MAX(CAST(tem_seq AS SIGNED)), 15000) FROM evo_material_list', op=int) for csc in combined_cards: card = csc.jp_card.card if not card.ancestor_id: continue tv_seq = db_wrapper.get_single_value(monster.lookup_evo_id_sql(card), op=int) evo_mat_items = monster.card_to_evo_mats(card, tv_seq) for item in evo_mat_items: tem_seq = db_wrapper.check_existing_value( item.exists_by_values_sql()) if tem_seq: item.tem_seq = tem_seq else: item.tem_seq = next_evo_mat_id next_evo_mat_id += 1 insert_or_update(item) # Skills next_skill_id = db_wrapper.get_single_value( 'SELECT 1 + COALESCE(MAX(CAST(ts_seq AS SIGNED)), 20000) FROM skill_list', op=int) # Compute English skill text calc_skills = skill_info.reformat_json_info(jp_database.raw_skills) # Create a list of SkillIds to CardIds skill_id_to_card_ids = defaultdict( list) # type DefaultDict<SkillId, List[CardId]> for merged_card in jp_database.cards: for skill in [merged_card.active_skill, merged_card.leader_skill]: if skill is None: continue skill_id_to_card_ids[skill.skill_id].append( merged_card.card.card_id) for csc in combined_cards: merged_card = csc.jp_card merged_card_na = csc.na_card monster_no = csc.monster_no info = db_wrapper.get_single_or_no_row( monster_skill.get_monster_skill_ids(merged_card)) if not info: fail_logger.warn('Unexpected empty skill lookup: %s', repr(merged_card)) continue def lookup_existing_skill_id(field_name, skill_id): alt_monster_no = monster_id_mapping.jp_id_to_monster_no( min(skill_id_to_card_ids[skill_id])) if alt_monster_no == monster_no: # No existing monster (by monster id order) has this skill return # An existing card already has this skill, look it up ts_seq = db_wrapper.get_single_value( "select {} from monster_list where monster_no = {}".format( field_name, alt_monster_no), op=int) logger.warn('Looked up existing skill id %s from %s for %s', ts_seq, alt_monster_no, merged_card) return ts_seq def find_or_create_skill(monster_field_name, skill_value, na_skill_value, calc_skill_description): # Try to look up another monster with that skill ts_seq = lookup_existing_skill_id(monster_field_name, skill_value.skill_id) if ts_seq is None: # Lookup failed, insert a new skill nonlocal next_skill_id item = monster_skill.MonsterSkillItem(next_skill_id, skill_value, na_skill_value, calc_skill_description) logger.warn('Inserting new monster skill: %s - %s', repr(merged_card), repr(item)) db_wrapper.insert_item(item.insert_sql()) ts_seq = next_skill_id next_skill_id += 1 return ts_seq def maybe_update_skill(ts_seq, skill_value, na_skill_value, calc_skill_description): # Skill exists, check if it needs an update item = monster_skill.MonsterSkillItem(ts_seq, skill_value, na_skill_value, calc_skill_description) if not db_wrapper.check_existing(item.exists_sql()): fail_logger.fatal('Unexpected empty skill lookup: %s', repr(item)) exit() # It exists, check if the updatable values have changed if not db_wrapper.check_existing(item.needs_update_sql()): logger.warn('Updating: %s', repr(item)) db_wrapper.insert_item(item.update_sql()) else: fail_logger.debug( 'Skipping existing item that needs no updates: %s', repr(item)) update_monster = False ts_seq_leader = info['ts_seq_leader'] if merged_card.leader_skill: calc_ls_skill = calc_skills.get(merged_card.leader_skill.skill_id, '') calc_ls_skill_description = calc_ls_skill.description.strip( ) or None if calc_ls_skill else None if ts_seq_leader: # Monster already has a skill attached, see if it needs to be updated maybe_update_skill(ts_seq_leader, merged_card.leader_skill, merged_card_na.leader_skill, calc_ls_skill_description) else: # Skill needs to be attached ts_seq_leader = find_or_create_skill( 'ts_seq_leader', merged_card.leader_skill, merged_card_na.leader_skill, calc_ls_skill_description) update_monster = True if calc_ls_skill: leader_data_item = monster_skill.MonsterSkillLeaderDataItem( ts_seq_leader, calc_ls_skill.params) if leader_data_item.leader_data: insert_or_update(leader_data_item) ts_seq_skill = info['ts_seq_skill'] if merged_card.active_skill: calc_as_skill = calc_skills.get(merged_card.active_skill.skill_id, '') calc_as_skill_description = calc_as_skill.description.strip( ) or None if calc_as_skill else None if ts_seq_skill: # Monster already has a skill attached, see if it needs to be updated maybe_update_skill(ts_seq_skill, merged_card.active_skill, merged_card_na.active_skill, calc_as_skill_description) else: ts_seq_skill = find_or_create_skill( 'ts_seq_skill', merged_card.active_skill, merged_card_na.active_skill, calc_as_skill_description) update_monster = True if update_monster: logger.warn('Updating monster skill info: %s - %s - %s', repr(merged_card), ts_seq_leader, ts_seq_skill) db_wrapper.insert_item( monster_skill.get_update_monster_skill_ids( merged_card, ts_seq_leader, ts_seq_skill))