def finish(self): database_cursor = self.database.cursor() # variants left in this list must now be deleted for row in self.existing_variants.values(): variant_id = row[2] database_cursor.execute( "DELETE FROM game_variant WHERE id = ?", (variant_id,)) # games left in this list must now be deleted for row in self.existing_games.values(): game_id = row[2] database_cursor.execute( "DELETE FROM game WHERE id = ?", (game_id,)) database_cursor.execute( "SELECT count(*) FROM game WHERE uuid IS NOT NULL " "AND (have IS NULL OR have != (SELECT coalesce(max(have), 0) " "FROM game_variant WHERE game_uuid = game.uuid))") update_rows = database_cursor.fetchone()[0] print(update_rows, "game entries need update for have field") if update_rows > 0: database_cursor.execute( "UPDATE game SET have = (SELECT coalesce(max(have), 0) FROM " "game_variant WHERE game_uuid = game.uuid) WHERE uuid IS NOT " "NULL AND (have IS NULL OR have != (SELECT coalesce(max(" "have), 0) FROM game_variant WHERE game_uuid = game.uuid))") FileDatabase.get_instance().get_last_event_stamps() self.database.update_last_file_event_stamps(self.file_stamps)
def finish(self): database_cursor = self.database.cursor() # variants left in this list must now be deleted for row in self.existing_variants.values(): variant_id = row[2] database_cursor.execute("DELETE FROM game_variant WHERE id = ?", (variant_id, )) # games left in this list must now be deleted for row in self.existing_games.values(): game_id = row[2] database_cursor.execute("DELETE FROM game WHERE id = ?", (game_id, )) database_cursor.execute( "SELECT count(*) FROM game WHERE uuid IS NOT NULL " "AND (have IS NULL OR have != (SELECT coalesce(max(have), 0) " "FROM game_variant WHERE game_uuid = game.uuid))") update_rows = database_cursor.fetchone()[0] print(update_rows, "game entries need update for have field") if update_rows > 0: database_cursor.execute( "UPDATE game SET have = (SELECT coalesce(max(have), 0) FROM " "game_variant WHERE game_uuid = game.uuid) WHERE uuid IS NOT " "NULL AND (have IS NULL OR have != (SELECT coalesce(max(" "have), 0) FROM game_variant WHERE game_uuid = game.uuid))") FileDatabase.get_instance().get_last_event_stamps() self.database.update_last_file_event_stamps(self.file_stamps)
def kickstart_startup_scan(cls): if cls._kickstart_scanned: return cls._kickstart_scanned = True print("kickstart_startup_scan") kickstarts_dir = FSGSDirectories.get_kickstarts_dir() if LauncherSettings.get("kickstarts_dir_mtime") == \ cls.get_dir_mtime_str(kickstarts_dir): print("... mtime not changed") else: file_database = FileDatabase.get_instance() print("... database.find_local_roms") local_roms = file_database.find_local_roms() print("... walk kickstarts_dir") for dir_path, dir_names, file_names in os.walk(kickstarts_dir): for file_name in file_names: if not file_name.lower().endswith(".rom") and not \ file_name.lower().endswith(".bin"): continue path = Paths.join(dir_path, file_name) if path in local_roms: local_roms[path] = None # already exists in database continue print("[startup] adding kickstart", path) ROMManager.add_rom_to_database(path, file_database) print(local_roms) for path, id in local_roms.items(): if id is not None: print("[startup] removing kickstart", path) file_database.delete_file(id=id) print("... commit") file_database.commit() LauncherSettings.set( "kickstarts_dir_mtime", cls.get_dir_mtime_str(kickstarts_dir)) amiga = Amiga.get_model_config("A500") for sha1 in amiga["kickstarts"]: if fsgs.file.find_by_sha1(sha1=sha1): break else: file_database = FileDatabase.get_instance() cls.amiga_forever_kickstart_scan() file_database.commit()
def run(self): self.defragment(FileDatabase.get_instance(), "Files.sqlite") self.stop_check() self.defragment(LockerDatabase.instance(), "Locker.sqlite") self.stop_check() self.defragment(fsgs.get_game_database(), "Amiga.sqlite") self.stop_check() self.defragment(Database.get_instance(), "Database.sqlite")
def build_media_list(self, floppies=False, cds=False, hds=False): media_list = [] added = set() for file_item in self.get_file_list(): name = file_item["name"] url = file_item.get("url", "") if name.startswith("DH0/"): if hds: # p = os.path.join(self.path, "HardDrive") p = "hd://game/" + self.uuid + "/DH0" if p in added: # already added continue added.add(p) # FIXME: hack for now sha1 = self.values.get("dh0_sha1", "") media_list.append((p, sha1)) else: continue sha1 = file_item["sha1"] base, ext = os.path.splitext(name) ext = ext.lower() if hds: if ext not in [".zip"]: continue elif cds: if ext not in [".cue", ".iso"]: continue if "(Track" in base: # probably part of a split multi-track cue continue elif floppies: if ext not in [".adf", ".adz", ".dms", ".ipf"]: continue path = "" found_sha1 = "" if sha1: print(sha1) file = FileDatabase.get_instance().find_file(sha1=sha1) if file: found_sha1 = sha1 path = file["path"] if url and not path: path = url found_sha1 = sha1 if path: media_list.append((path, found_sha1)) else: pass # return False # FIXME: handle it with a visible error message # raise Exception("could not find file " + repr(name)) return media_list
def kickstart_startup_scan(cls): if cls._kickstart_scanned: return cls._kickstart_scanned = True print("kickstart_startup_scan") kickstarts_dir = FSGSDirectories.get_kickstarts_dir() if LauncherSettings.get("kickstarts_dir_mtime") == \ cls.get_dir_mtime_str(kickstarts_dir): print("... mtime not changed") else: file_database = FileDatabase.get_instance() print("... database.find_local_roms") local_roms = file_database.find_local_roms() print("... walk kickstarts_dir") for dir_path, dir_names, file_names in os.walk(kickstarts_dir): for file_name in file_names: if not file_name.lower().endswith(".rom") and not \ file_name.lower().endswith(".bin"): continue path = Paths.join(dir_path, file_name) if path in local_roms: local_roms[path] = None # already exists in database continue print("[startup] adding kickstart", path) ROMManager.add_rom_to_database(path, file_database) print(local_roms) for path, file_id in local_roms.items(): if file_id is not None: print("[startup] removing kickstart", path) file_database.delete_file(id=file_id) print("... commit") file_database.commit() LauncherSettings.set("kickstarts_dir_mtime", cls.get_dir_mtime_str(kickstarts_dir)) amiga = Amiga.get_model_config("A500") for sha1 in amiga["kickstarts"]: if fsgs.file.find_by_sha1(sha1=sha1): break else: file_database = FileDatabase.get_instance() cls.amiga_forever_kickstart_scan() file_database.commit()
def config_startup_scan(cls): if cls._config_scanned: return cls._config_scanned = True configs_dir = FSGSDirectories.get_configurations_dir() print("config_startup_scan", configs_dir) print(LauncherSettings.settings) settings_mtime = LauncherSettings.get("configurations_dir_mtime") dir_mtime = cls.get_dir_mtime_str(configs_dir) if settings_mtime == dir_mtime + "+" + str(Database.VERSION): print("... mtime not changed", settings_mtime, dir_mtime) return database = Database.get_instance() file_database = FileDatabase.get_instance() print("... database.find_local_configurations") local_configs = Database.get_instance().find_local_configurations() print("... walk configs_dir") for dir_path, dir_names, file_names in os.walk(configs_dir): for file_name in file_names: if not file_name.endswith(".fs-uae"): continue path = Paths.join(dir_path, file_name) if path in local_configs: local_configs[path] = None # already exists in database continue name, ext = os.path.splitext(file_name) # search = ConfigurationScanner.create_configuration_search( # name) scanner = ConfigurationScanner() print("[startup] adding config", path) file_database.delete_file(path=path) with open(path, "rb") as f: sha1 = hashlib.sha1(f.read()).hexdigest() file_database.add_file(path=path, sha1=sha1) game_id = database.add_game( path=path, name=scanner.create_configuration_name(name)) database.update_game_search_terms( game_id, scanner.create_search_terms(name)) for path, id in local_configs.items(): if id is not None: print("[startup] removing configuration", path) database.delete_game(id=id) file_database.delete_file(path=path) print("... commit") database.commit() LauncherSettings.set( "configurations_dir_mtime", cls.get_dir_mtime_str(configs_dir) + "+" + str( Database.VERSION))
def config_startup_scan(cls): if cls._config_scanned: return cls._config_scanned = True configs_dir = FSGSDirectories.get_configurations_dir() print("config_startup_scan", configs_dir) print(LauncherSettings.settings) settings_mtime = LauncherSettings.get("configurations_dir_mtime") dir_mtime = cls.get_dir_mtime_str(configs_dir) if settings_mtime == dir_mtime + "+" + str(Database.VERSION): print("... mtime not changed", settings_mtime, dir_mtime) return database = Database.get_instance() file_database = FileDatabase.get_instance() print("... database.find_local_configurations") local_configs = Database.get_instance().find_local_configurations() print("... walk configs_dir") for dir_path, dir_names, file_names in os.walk(configs_dir): for file_name in file_names: if not file_name.endswith(".fs-uae"): continue path = Paths.join(dir_path, file_name) if path in local_configs: local_configs[path] = None # already exists in database continue name, ext = os.path.splitext(file_name) # search = ConfigurationScanner.create_configuration_search( # name) scanner = ConfigurationScanner() print("[startup] adding config", path) file_database.delete_file(path=path) with open(path, "rb") as f: sha1 = hashlib.sha1(f.read()).hexdigest() file_database.add_file(path=path, sha1=sha1) game_id = database.add_game( path=path, name=scanner.create_configuration_name(name)) database.update_game_search_terms( game_id, scanner.create_search_terms(name)) for path, config_id in local_configs.items(): if config_id is not None: print("[startup] removing configuration", path) database.delete_game(id=config_id) file_database.delete_file(path=path) print("... commit") database.commit() LauncherSettings.set( "configurations_dir_mtime", cls.get_dir_mtime_str(configs_dir) + "+" + str(Database.VERSION))
def check_sha1(self, sha1): database = FileDatabase.instance() result = database.check_sha1(sha1) if not result and is_locker_enabled(): database = LockerDatabase.instance() result = database.check_sha1(sha1) # print("check sha1", sha1, "in locker database - result", result) # if not result: # result = self.context.get_game_database().find_file_by_sha1(sha1) return result
def update_file_database_timestamps(self): # This isn't very elegant, but in order to force the game scanner to # refresh the game list based on files, we update the stamps in the # file database. Also, since we haven't keep track of additions / # deletions, we set both stamps for now... file_database = FileDatabase.instance() file_database.last_file_insert = time.time() file_database.last_file_delete = time.time() file_database.update_last_event_stamps() file_database.commit()
def scan_dir_for_kickstarts(scan_dir): file_database = FileDatabase.get_instance() for dir_path, dir_names, file_names in os.walk(scan_dir): for file_name in file_names: if not file_name.endswith(".rom"): continue path = Paths.join(dir_path, file_name) if file_database.find_file(path=path): continue print("[startup] adding kickstart", path) ROMManager.add_rom_to_database(path, file_database)
def find_by_sha1(self, sha1): database = FileDatabase.instance() result = database.find_file(sha1=sha1)["path"] if not result: path = Downloader.get_cache_path(sha1) if os.path.exists(path): result = path # result = self.context.get_game_database().find_file_by_sha1(sha1) # print("find by sha1", sha1, "in file database - result", result) if not result and is_locker_enabled(): database = LockerDatabase.instance() if database.check_sha1(sha1): result = "locker://" + sha1 # print("find by sha1", sha1, "in locker database - result", # result) return result
def scan(self): self.set_status(gettext("Scanning files"), gettext("Scanning files")) database = FileDatabase.get_instance() # database.clear() scan_dirs = self.get_scan_dirs() if self.purge_other_dirs: all_database_file_ids = database.get_file_ids() else: all_database_file_ids = None for dir_ in scan_dirs: if not os.path.exists(dir_): print("[SCANNER] Directory does not exist:", dir_) continue # this is important to make sure the database is portable across # operating systems dir_ = Paths.get_real_case(dir_) self.database_file_ids = database.get_file_hierarchy_ids(dir_) if self.purge_other_dirs: all_database_file_ids.difference_update( self.database_file_ids) self.scan_dir(database, dir_) # print("Remaining files:", self.database_file_ids) self.purge_file_ids(self.database_file_ids) self.set_status(gettext("Scanning files"), gettext("Committing data...")) # update last_file_insert and last_file_delete database.update_last_event_stamps() print("FileScanner.scan - committing data") database.commit() if self.purge_other_dirs: self.purge_file_ids(all_database_file_ids) if self.stop_check(): database.rollback() else: self.set_status(gettext("Scanning files"), gettext("Committing data...")) print("FileScanner.scan - committing data") database.commit()
def scan_fs_uae_files(self, database): cursor = database.cursor() cursor.execute("SELECT id, path FROM game WHERE path IS NOT NULL") config_path_ids = {} for row in cursor: config_path_ids[database.decode_path(row[1])] = row[0] file_database = FileDatabase.get_instance() configurations = file_database.find_files(ext=".fs-uae") for c in configurations: if self.stop_check(): break # name = os.path.basename(c["path"]) # name = c["name"] # name = name[:-7] # search = name.lower() path = c["path"] name = os.path.basename(path) name, ext = os.path.splitext(name) sort_key = name.lower() # search = self.create_configuration_search(name) name = self.create_configuration_name(name) game_id = database.add_game(path=path, name=name, sort_key=sort_key) database.update_game_search_terms(game_id, self.create_search_terms(name)) print(config_path_ids) try: del config_path_ids[path] except KeyError: pass # FIXME: try splitting name into name, variant pair, and # actually add game_variant for config files as well, where # possible for id in config_path_ids.values(): database.delete_game(id=id)
def upload_check(self, prefix): self.progressed( gettext("Finding files eligible for OAGD.net Locker") + " ({:0.0%})".format((prefix / 16.0))) file_database = FileDatabase.instance() cursor = file_database.cursor() # FIXME: prefix p = "0123456789ABCDEF"[prefix] cursor.execute( "SELECT DISTINCT sha1 FROM file " "WHERE hex(sha1) LIKE ?", (p + "%", )) string_io = StringIO() for row in cursor: string_io.write(row[0]) # print(prefix, len(string_io.getvalue())) self.stop_check() retry_seconds = 1 while True: try: result = self.client.post("/api/locker-upload-check", data=string_io.getvalue()) except OGDClient.ForbiddenError: raise Task.Failure( gettext("OAGD.net Locker is not enabled for your user. " "It may be available only to a few select beta " "users.")) except OGDClient.NonRetryableHTTPError as e: raise e except Exception: traceback.print_exc() self.progressed( gettext("Re-trying in {0} seconds...").format( retry_seconds)) for _ in range(retry_seconds): self.stop_check() time.sleep(1.0) retry_seconds = min(retry_seconds * 2, 60 * 10) else: return result
def scan_fs_uae_files(self, database): cursor = database.cursor() cursor.execute("SELECT id, path FROM game WHERE path IS NOT NULL") config_path_ids = {} for row in cursor: config_path_ids[database.decode_path(row[1])] = row[0] file_database = FileDatabase.get_instance() configurations = file_database.find_files(ext=".fs-uae") for c in configurations: if self.stop_check(): break # name = os.path.basename(c["path"]) # name = c["name"] # name = name[:-7] # search = name.lower() path = c["path"] name = os.path.basename(path) name, ext = os.path.splitext(name) sort_key = name.lower() # search = self.create_configuration_search(name) name = self.create_configuration_name(name) game_id = database.add_game(path=path, name=name, sort_key=sort_key) database.update_game_search_terms( game_id, self.create_search_terms(name)) print(config_path_ids) try: del config_path_ids[path] except KeyError: pass # FIXME: try splitting name into name, variant pair, and # actually add game_variant for config files as well, where # possible for id in config_path_ids.values(): database.delete_game(id=id)
def copy_roms(self, src, dst): count = 0 if not os.path.isdir(src): self.log("{0} is not a directory".format(src)) return count src_file = os.path.join(src, "rom.key") if os.path.exists(src_file): dst_file = os.path.join(dst, "rom.key") self.copy_file(src_file, dst_file) for file_name in os.listdir(src): name, ext = os.path.splitext(file_name) if ext not in [".rom"]: continue src_file = os.path.join(src, file_name) dst_file = os.path.join(dst, file_name) self.copy_file(src_file, dst_file) database = FileDatabase.get_instance() ROMManager.add_rom_to_database(dst_file, database, self.log) database.commit() count += 1 return count
def __init__(self, database): self.database = database database_cursor = self.database.cursor() database_cursor.execute( "SELECT uuid, update_stamp, have, id FROM game " "WHERE uuid IS NOT NULL") self.existing_games = {} for row in database_cursor: self.existing_games[row[0]] = row[1], row[2], row[3] database_cursor.execute( "SELECT uuid, update_stamp, have, id FROM game_variant") self.existing_variants = {} for row in database_cursor: self.existing_variants[row[0]] = row[1], row[2], row[3] self.file_stamps = FileDatabase.get_instance().get_last_event_stamps() cached_file_stamps = self.database.get_last_file_event_stamps() self.added_files = self.file_stamps["last_file_insert"] != \ cached_file_stamps["last_file_insert"] self.deleted_files = self.file_stamps["last_file_delete"] != \ cached_file_stamps["last_file_delete"]
def upload_check(self, prefix): self.progressed( gettext("Finding files eligible for OAGD.net Locker") + " ({:0.0%})".format((prefix / 16.0))) file_database = FileDatabase.instance() cursor = file_database.cursor() # FIXME: prefix p = "0123456789ABCDEF"[prefix] cursor.execute("SELECT DISTINCT sha1 FROM file " "WHERE hex(sha1) LIKE ?", (p + "%",)) string_io = StringIO() for row in cursor: string_io.write(row[0]) # print(prefix, len(string_io.getvalue())) self.stop_check() retry_seconds = 1 while True: try: result = self.client.post("/api/locker-upload-check", data=string_io.getvalue()) except OGDClient.ForbiddenError: raise Task.Failure( gettext("OAGD.net Locker is not enabled for your user. " "It may be available only to a few select beta " "users.")) except OGDClient.NonRetryableHTTPError as e: raise e except Exception: traceback.print_exc() self.progressed(gettext( "Re-trying in {0} seconds...").format(retry_seconds)) for _ in range(retry_seconds): self.stop_check() time.sleep(1.0) retry_seconds = min(retry_seconds * 2, 60 * 10) else: return result
def save_config(self): print("SaveButton.save_config") database = Database.get_instance() name = LauncherSettings.get("config_name").strip() if not name: print("no config_name") # FIXME: notify user return file_name = name + ".fs-uae" path = os.path.join(FSGSDirectories.get_configurations_dir(), file_name) with io.open(path, "w", encoding="UTF-8") as f: f.write("# FS-UAE configuration saved by FS-UAE Launcher\n") f.write("# Last saved: {0}\n".format( datetime.datetime.today().strftime("%Y-%m-%d %H:%M:%S"))) f.write("\n[fs-uae]\n") keys = sorted(fsgs.config.values.keys()) for key in keys: value = LauncherConfig.get(key) if key.startswith("__"): continue if key in LauncherConfig.no_save_keys_set: continue # elif key == "joystick_port_2_mode" and value == "nothing": # continue # elif key == "joystick_port_3_mode" and value == "nothing": # continue if value == LauncherConfig.default_config.get(key, ""): continue if value: f.write("{0} = {1}\n".format(key, value)) # scanner = ConfigurationScanner() # search = ConfigurationScanner.create_configuration_search(name) # name = scanner.create_configuration_name(name) # print("adding", path) # # deleting the path from the database first in case it already exists # database.delete_configuration(path=path) # database.delete_file(path=path) # database.add_file(path=path) # database.add_configuration( # path=path, uuid="", name=name, scan=0, search=search) file_database = FileDatabase.get_instance() scanner = ConfigurationScanner() print("[save config] adding config", path) file_database.delete_file(path=path) with open(path, "rb") as f: sha1 = hashlib.sha1(f.read()).hexdigest() file_database.add_file(path=path, sha1=sha1) game_id = database.add_game( path=path, name=scanner.create_configuration_name(name)) database.update_game_search_terms(game_id, scanner.create_search_terms(name)) database.commit() file_database.commit() LauncherSettings.set("config_refresh", str(time.time())) # Settings.set("config_changed", "0") LauncherConfig.set("__changed", "0")
def purge_file_ids(self, file_ids): self.set_status(gettext("Scanning files"), gettext("Purging old entries...")) database = FileDatabase.get_instance() for file_id in file_ids: database.delete_file(id=file_id)
def save_config(): print("SaveButton.save_config") database = Database.get_instance() name = LauncherSettings.get("config_name").strip() if not name: print("no config_name") # FIXME: notify user return file_name = name + ".fs-uae" path = os.path.join( FSGSDirectories.get_configurations_dir(), file_name) with io.open(path, "w", encoding="UTF-8") as f: f.write("# FS-UAE configuration saved by FS-UAE Launcher\n") f.write("# Last saved: {0}\n".format( datetime.datetime.today().strftime("%Y-%m-%d %H:%M:%S"))) f.write("\n[fs-uae]\n") keys = sorted(fsgs.config.values.keys()) for key in keys: value = LauncherConfig.get(key) if key.startswith("__"): continue if key in LauncherConfig.no_save_keys_set: continue # elif key == "joystick_port_2_mode" and value == "nothing": # continue # elif key == "joystick_port_3_mode" and value == "nothing": # continue if value == LauncherConfig.default_config.get(key, ""): continue if value: f.write("{0} = {1}\n".format(key, value)) # scanner = ConfigurationScanner() # search = ConfigurationScanner.create_configuration_search(name) # name = scanner.create_configuration_name(name) # print("adding", path) # # deleting the path from the database first in case it already exists # database.delete_configuration(path=path) # database.delete_file(path=path) # database.add_file(path=path) # database.add_configuration( # path=path, uuid="", name=name, scan=0, search=search) file_database = FileDatabase.get_instance() scanner = ConfigurationScanner() print("[save config] adding config", path) file_database.delete_file(path=path) with open(path, "rb") as f: sha1 = hashlib.sha1(f.read()).hexdigest() file_database.add_file(path=path, sha1=sha1) game_id = database.add_game( path=path, name=scanner.create_configuration_name(name)) database.update_game_search_terms( game_id, scanner.create_search_terms(name)) database.commit() file_database.commit() LauncherSettings.set("__config_refresh", str(time.time())) # Settings.set("config_changed", "0") LauncherConfig.set("__changed", "0")