def process(self, db: DbWrapper):
ready_dungeons = db.fetch_data(FIND_DUNGEONS_SQL)
for row in ready_dungeons:
dungeon_id = row['dungeon_id']
dungeon = self.data.dungeon_by_id(dungeon_id)
self._process_dungeon(db, dungeon)
def _print_bad_enemies(self, desc: str, dungeon, sub_dungeon,
db: DbWrapper, sql: str):
bad_stored_encounters = db.fetch_data(sql)
if not bad_stored_encounters:
return
encounter_list_str = ','.join(
[str(x['encounter_id']) for x in bad_stored_encounters])
encounter_info_list_str = ','.join([
'{}/{}'.format(x['encounter_id'], x['enemy_id'])
for x in bad_stored_encounters
])
delete_drops_sql = 'DELETE FROM drops WHERE encounter_id IN ({});'.format(
encounter_list_str)
delete_encounters_sql = 'DELETE FROM encounters WHERE encounter_id IN ({});'.format(
encounter_list_str)
human_fix_logger.warning(
'Found bad %s stored encounters for %s: [%s] - %s\n%s\n%s', desc,
dungeon.na_dungeon.clean_name,
sub_dungeon.na_sub_dungeon.clean_name, encounter_info_list_str,
delete_drops_sql, delete_encounters_sql)
deleted_drops = db.update_item(delete_drops_sql)
deleted_encounters = db.update_item(delete_encounters_sql)
human_fix_logger.warning(
'Auto deleted {} drops and {} encounters'.format(
deleted_drops, deleted_encounters))
def do_migration(csd: crossed_data.CrossServerDatabase, db: DbWrapper):
get_series_sql = ('SELECT tsr_seq AS series_id, name_jp, name_us AS name_na, name_kr'
' FROM padguide.series_list'
' WHERE del_yn = 0')
data = db.fetch_data(get_series_sql)
for row in data:
series_id = row['series_id']
if series_id == 42:
# Not importing 42 (premium) which was used as the old unsorted
continue
item = Series(series_id=series_id,
name_jp=row['name_jp'],
name_na=row['name_na'],
name_kr=row['name_kr'])
db.insert_or_update(item)
monsterno_seriesid_map = db.load_to_key_value('monster_no', 'tsr_seq', 'padguide.monster_info_list')
monsterno_nameoverride_map = db.load_to_key_value('monster_no', 'tm_name_us_override', 'padguide.monster_list')
monsterno_regdate_map = db.load_to_key_value('monster_no', 'reg_date', 'padguide.monster_list')
for csc in csd.ownable_cards:
monster_id = csc.monster_id
if csc.jp_card.server == Server.na:
# Handle Voltron.
monster_no = na_id_to_monster_no(csc.na_card.card.monster_no)
elif csc.jp_card.server != Server.jp:
raise ValueError('Unexpected failure')
else:
# Since this is a JP card for sure, monster_id == JP monster_no
monster_no = jp_id_to_monster_no(monster_id)
# Series processing
series_id = monsterno_seriesid_map[monster_no]
if series_id == 42:
series_id = 0 # Map premium to unsorted
update_sql = 'UPDATE monsters SET series_id={} WHERE monster_id={}'.format(series_id, monster_id)
db.insert_item(update_sql)
# Reg Date processing
reg_date = monsterno_regdate_map[monster_no]
update_sql = "UPDATE monsters SET reg_date='{}' WHERE monster_id={}".format(reg_date.date().isoformat(),
monster_id)
db.insert_item(update_sql)
# NA name override processing
name_override = monsterno_nameoverride_map.get(monster_no, None)
if name_override:
name_override = name_override.replace("'", "''")
update_sql = "UPDATE monsters SET name_na_override='{}' WHERE monster_id={}".format(name_override,
monster_id)
db.insert_item(update_sql)
def process(self, db: DbWrapper):
print('Starting deletion of old records')
print('schedule size:',
db.get_single_value('select count(*) from schedule'))
print('deleted_rows size',
db.get_single_value('select count(*) from deleted_rows'))
# This is a hint to mysql that we shouldn't insert into deleted_rows
# while purging. The client should handle deleting old events in bulk.
db.fetch_data('set @TRIGGER_DISABLED=true')
delete_timestamp = date2tstamp(datetime.now() - timedelta(weeks=4))
print('deleting before', delete_timestamp)
schedule_deletes = db.update_item(
"DELETE FROM `schedule` WHERE end_timestamp < {}".format(
delete_timestamp))
deleted_row_deletes = db.update_item(
"DELETE FROM `deleted_rows` WHERE tstamp < {}".format(
delete_timestamp))
logger.info("purged {} old schedules and {} old deleted_rows".format(
schedule_deletes, deleted_row_deletes))
def process(self, db: DbWrapper):
db.fetch_data("DELETE FROM `schedule` WHERE end_timestamp < {}" \
.format(date2tstamp(datetime.now()-timedelta(weeks=4))))
schedule = db.fetch_data("SELECT ROW_COUNT()")[0]['ROW_COUNT()']
db.fetch_data("DELETE FROM `deleted_rows` WHERE tstamp < {}" \
.format(date2tstamp(datetime.now()-timedelta(weeks=4))))
del_rows = db.fetch_data("SELECT ROW_COUNT()")[0]['ROW_COUNT()']
logger.info("purged {} old schedules and {} old deleted_rows".format(
schedule, del_rows))
def load_data(args):
logger.info('Connecting to database')
with open(args.db_config) as f:
db_config = json.load(f)
dry_run = not args.doupdates
db_wrapper = DbWrapper(dry_run)
db_wrapper.connect(db_config)
data = db_wrapper.fetch_data("SELECT * FROM egg_machines")
for machine_sql in data:
egg_machine_id = machine_sql['egg_machine_id']
contents = ast.literal_eval(machine_sql['contents'])
for monster_id in contents.keys():
real_monster_id = int(monster_id.strip("()"))
emm = EggMachinesMonster(egg_machine_monster_id=None, monster_id=real_monster_id,
roll_chance=contents.get(monster_id),
egg_machine_id=egg_machine_id)
db_wrapper.insert_or_update(emm)
def load_data(args):
logger.info('Connecting to database')
with open(args.db_config) as f:
db_config = json.load(f)
db_wrapper = DbWrapper()
db_wrapper.connect(db_config)
data = db_wrapper.fetch_data(ENCOUNTER_QUERY)
output = {}
for encounter in data:
sdgid = encounter['sdgid']
floor = encounter['floor']
spawn = {'id': encounter['enemy_id'], 'lv': encounter['level']}
if sdgid not in output:
output[sdgid] = {'name': encounter['stage_name'], 'floors': {}}
if floor not in output[sdgid]['floors']:
output[sdgid]['floors'][floor] = {'spawns': []}
if spawn not in output[sdgid]['floors'][floor]['spawns']:
output[sdgid]['floors'][floor]['spawns'].append(spawn)
with open(os.path.join(args.output_dir, "encounter_data.json"), 'w+') as f:
json.dump(output, f)
def _maybe_insert_encounters(self, db: DbWrapper,
dungeon: CrossServerDungeon,
sub_dungeon: CrossServerSubDungeon,
result_floor: ResultFloor):
for stage in result_floor.stages:
seen_enemies = set()
for slot in stage.slots:
csc = self.data.card_by_monster_id(slot.monster_id)
card = csc.jp_card.card
enemy = card.enemy()
seen_enemies.add(slot.monster_id)
turns = card.enemy_turns
if dungeon.jp_dungeon.full_dungeon_type == RawDungeonType.TECHNICAL and card.enemy_turns_alt:
turns = card.enemy_turns_alt
sd = sub_dungeon.jp_sub_dungeon
hp = int(
round(sd.hp_mult * enemy.hp.value_at(slot.monster_level)))
atk = int(
round(sd.atk_mult *
enemy.atk.value_at(slot.monster_level)))
defence = int(
round(sd.def_mult *
enemy.defense.value_at(slot.monster_level)))
# TODO: add comments based on slot data
encounter = Encounter(
dungeon_id=dungeon.dungeon_id,
sub_dungeon_id=sub_dungeon.sub_dungeon_id,
enemy_id=slot.monster_id,
monster_id=slot.visible_monster_id(),
stage=stage.stage_idx,
comment_jp=None,
comment_na=None,
comment_kr=None,
amount=slot.min_spawn
if slot.min_spawn == slot.max_spawn else None,
order_idx=slot.order,
turns=turns,
level=slot.monster_level,
hp=hp,
atk=atk,
defence=defence)
sql = '''
SELECT encounter_id
FROM encounters
WHERE dungeon_id={}
AND sub_dungeon_id={}
AND stage={}
AND enemy_id={}
'''.format(dungeon.dungeon_id, sub_dungeon.sub_dungeon_id,
stage.stage_idx, slot.monster_id)
stored_encounter_id = db.get_single_value(sql,
int,
fail_on_empty=False)
if stored_encounter_id:
encounter.encounter_id = stored_encounter_id
db.insert_or_update(encounter)
drops = Drop.from_slot(slot, encounter)
for drop in drops:
db.insert_or_update(drop)
if seen_enemies:
sql = '''
SELECT encounter_id, enemy_id
FROM encounters
WHERE dungeon_id={}
AND sub_dungeon_id={}
AND stage={}
AND enemy_id not in ({})
'''.format(dungeon.dungeon_id, sub_dungeon.sub_dungeon_id,
stage.stage_idx,
','.join(map(str, seen_enemies)))
bad_stored_encounters = db.fetch_data(sql)
if bad_stored_encounters:
encounter_list_str = ','.join([
str(x['encounter_id']) for x in bad_stored_encounters
])
delete_drops_sql = 'DELETE FROM drops WHERE encounter_id IN ({});'.format(
encounter_list_str)
delete_encounters_sql = 'DELETE FROM encounters WHERE encounter_id IN ({});'.format(
encounter_list_str)
human_fix_logger.warning(
'Found bad stored encounters for %s: [%s] - %s\n%s\n%s',
dungeon.na_dungeon.clean_name,
sub_dungeon.na_sub_dungeon.clean_name,
encounter_list_str, delete_drops_sql,
delete_encounters_sql)