def process(self, db: DbWrapper): ready_dungeons = db.fetch_data(FIND_DUNGEONS_SQL) for row in ready_dungeons: dungeon_id = row['dungeon_id'] dungeon = self.data.dungeon_by_id(dungeon_id) self._process_dungeon(db, dungeon)
def _print_bad_enemies(self, desc: str, dungeon, sub_dungeon, db: DbWrapper, sql: str): bad_stored_encounters = db.fetch_data(sql) if not bad_stored_encounters: return encounter_list_str = ','.join( [str(x['encounter_id']) for x in bad_stored_encounters]) encounter_info_list_str = ','.join([ '{}/{}'.format(x['encounter_id'], x['enemy_id']) for x in bad_stored_encounters ]) delete_drops_sql = 'DELETE FROM drops WHERE encounter_id IN ({});'.format( encounter_list_str) delete_encounters_sql = 'DELETE FROM encounters WHERE encounter_id IN ({});'.format( encounter_list_str) human_fix_logger.warning( 'Found bad %s stored encounters for %s: [%s] - %s\n%s\n%s', desc, dungeon.na_dungeon.clean_name, sub_dungeon.na_sub_dungeon.clean_name, encounter_info_list_str, delete_drops_sql, delete_encounters_sql) deleted_drops = db.update_item(delete_drops_sql) deleted_encounters = db.update_item(delete_encounters_sql) human_fix_logger.warning( 'Auto deleted {} drops and {} encounters'.format( deleted_drops, deleted_encounters))
def do_migration(csd: crossed_data.CrossServerDatabase, db: DbWrapper): get_series_sql = ('SELECT tsr_seq AS series_id, name_jp, name_us AS name_na, name_kr' ' FROM padguide.series_list' ' WHERE del_yn = 0') data = db.fetch_data(get_series_sql) for row in data: series_id = row['series_id'] if series_id == 42: # Not importing 42 (premium) which was used as the old unsorted continue item = Series(series_id=series_id, name_jp=row['name_jp'], name_na=row['name_na'], name_kr=row['name_kr']) db.insert_or_update(item) monsterno_seriesid_map = db.load_to_key_value('monster_no', 'tsr_seq', 'padguide.monster_info_list') monsterno_nameoverride_map = db.load_to_key_value('monster_no', 'tm_name_us_override', 'padguide.monster_list') monsterno_regdate_map = db.load_to_key_value('monster_no', 'reg_date', 'padguide.monster_list') for csc in csd.ownable_cards: monster_id = csc.monster_id if csc.jp_card.server == Server.na: # Handle Voltron. monster_no = na_id_to_monster_no(csc.na_card.card.monster_no) elif csc.jp_card.server != Server.jp: raise ValueError('Unexpected failure') else: # Since this is a JP card for sure, monster_id == JP monster_no monster_no = jp_id_to_monster_no(monster_id) # Series processing series_id = monsterno_seriesid_map[monster_no] if series_id == 42: series_id = 0 # Map premium to unsorted update_sql = 'UPDATE monsters SET series_id={} WHERE monster_id={}'.format(series_id, monster_id) db.insert_item(update_sql) # Reg Date processing reg_date = monsterno_regdate_map[monster_no] update_sql = "UPDATE monsters SET reg_date='{}' WHERE monster_id={}".format(reg_date.date().isoformat(), monster_id) db.insert_item(update_sql) # NA name override processing name_override = monsterno_nameoverride_map.get(monster_no, None) if name_override: name_override = name_override.replace("'", "''") update_sql = "UPDATE monsters SET name_na_override='{}' WHERE monster_id={}".format(name_override, monster_id) db.insert_item(update_sql)
def process(self, db: DbWrapper): print('Starting deletion of old records') print('schedule size:', db.get_single_value('select count(*) from schedule')) print('deleted_rows size', db.get_single_value('select count(*) from deleted_rows')) # This is a hint to mysql that we shouldn't insert into deleted_rows # while purging. The client should handle deleting old events in bulk. db.fetch_data('set @TRIGGER_DISABLED=true') delete_timestamp = date2tstamp(datetime.now() - timedelta(weeks=4)) print('deleting before', delete_timestamp) schedule_deletes = db.update_item( "DELETE FROM `schedule` WHERE end_timestamp < {}".format( delete_timestamp)) deleted_row_deletes = db.update_item( "DELETE FROM `deleted_rows` WHERE tstamp < {}".format( delete_timestamp)) logger.info("purged {} old schedules and {} old deleted_rows".format( schedule_deletes, deleted_row_deletes))
def process(self, db: DbWrapper): db.fetch_data("DELETE FROM `schedule` WHERE end_timestamp < {}" \ .format(date2tstamp(datetime.now()-timedelta(weeks=4)))) schedule = db.fetch_data("SELECT ROW_COUNT()")[0]['ROW_COUNT()'] db.fetch_data("DELETE FROM `deleted_rows` WHERE tstamp < {}" \ .format(date2tstamp(datetime.now()-timedelta(weeks=4)))) del_rows = db.fetch_data("SELECT ROW_COUNT()")[0]['ROW_COUNT()'] logger.info("purged {} old schedules and {} old deleted_rows".format( schedule, del_rows))
def load_data(args): logger.info('Connecting to database') with open(args.db_config) as f: db_config = json.load(f) dry_run = not args.doupdates db_wrapper = DbWrapper(dry_run) db_wrapper.connect(db_config) data = db_wrapper.fetch_data("SELECT * FROM egg_machines") for machine_sql in data: egg_machine_id = machine_sql['egg_machine_id'] contents = ast.literal_eval(machine_sql['contents']) for monster_id in contents.keys(): real_monster_id = int(monster_id.strip("()")) emm = EggMachinesMonster(egg_machine_monster_id=None, monster_id=real_monster_id, roll_chance=contents.get(monster_id), egg_machine_id=egg_machine_id) db_wrapper.insert_or_update(emm)
def load_data(args): logger.info('Connecting to database') with open(args.db_config) as f: db_config = json.load(f) db_wrapper = DbWrapper() db_wrapper.connect(db_config) data = db_wrapper.fetch_data(ENCOUNTER_QUERY) output = {} for encounter in data: sdgid = encounter['sdgid'] floor = encounter['floor'] spawn = {'id': encounter['enemy_id'], 'lv': encounter['level']} if sdgid not in output: output[sdgid] = {'name': encounter['stage_name'], 'floors': {}} if floor not in output[sdgid]['floors']: output[sdgid]['floors'][floor] = {'spawns': []} if spawn not in output[sdgid]['floors'][floor]['spawns']: output[sdgid]['floors'][floor]['spawns'].append(spawn) with open(os.path.join(args.output_dir, "encounter_data.json"), 'w+') as f: json.dump(output, f)
def _maybe_insert_encounters(self, db: DbWrapper, dungeon: CrossServerDungeon, sub_dungeon: CrossServerSubDungeon, result_floor: ResultFloor): for stage in result_floor.stages: seen_enemies = set() for slot in stage.slots: csc = self.data.card_by_monster_id(slot.monster_id) card = csc.jp_card.card enemy = card.enemy() seen_enemies.add(slot.monster_id) turns = card.enemy_turns if dungeon.jp_dungeon.full_dungeon_type == RawDungeonType.TECHNICAL and card.enemy_turns_alt: turns = card.enemy_turns_alt sd = sub_dungeon.jp_sub_dungeon hp = int( round(sd.hp_mult * enemy.hp.value_at(slot.monster_level))) atk = int( round(sd.atk_mult * enemy.atk.value_at(slot.monster_level))) defence = int( round(sd.def_mult * enemy.defense.value_at(slot.monster_level))) # TODO: add comments based on slot data encounter = Encounter( dungeon_id=dungeon.dungeon_id, sub_dungeon_id=sub_dungeon.sub_dungeon_id, enemy_id=slot.monster_id, monster_id=slot.visible_monster_id(), stage=stage.stage_idx, comment_jp=None, comment_na=None, comment_kr=None, amount=slot.min_spawn if slot.min_spawn == slot.max_spawn else None, order_idx=slot.order, turns=turns, level=slot.monster_level, hp=hp, atk=atk, defence=defence) sql = ''' SELECT encounter_id FROM encounters WHERE dungeon_id={} AND sub_dungeon_id={} AND stage={} AND enemy_id={} '''.format(dungeon.dungeon_id, sub_dungeon.sub_dungeon_id, stage.stage_idx, slot.monster_id) stored_encounter_id = db.get_single_value(sql, int, fail_on_empty=False) if stored_encounter_id: encounter.encounter_id = stored_encounter_id db.insert_or_update(encounter) drops = Drop.from_slot(slot, encounter) for drop in drops: db.insert_or_update(drop) if seen_enemies: sql = ''' SELECT encounter_id, enemy_id FROM encounters WHERE dungeon_id={} AND sub_dungeon_id={} AND stage={} AND enemy_id not in ({}) '''.format(dungeon.dungeon_id, sub_dungeon.sub_dungeon_id, stage.stage_idx, ','.join(map(str, seen_enemies))) bad_stored_encounters = db.fetch_data(sql) if bad_stored_encounters: encounter_list_str = ','.join([ str(x['encounter_id']) for x in bad_stored_encounters ]) delete_drops_sql = 'DELETE FROM drops WHERE encounter_id IN ({});'.format( encounter_list_str) delete_encounters_sql = 'DELETE FROM encounters WHERE encounter_id IN ({});'.format( encounter_list_str) human_fix_logger.warning( 'Found bad stored encounters for %s: [%s] - %s\n%s\n%s', dungeon.na_dungeon.clean_name, sub_dungeon.na_sub_dungeon.clean_name, encounter_list_str, delete_drops_sql, delete_encounters_sql)