def finish(self):
database_cursor = self.database.cursor()
# variants left in this list must now be deleted
for row in self.existing_variants.values():
variant_id = row[2]
database_cursor.execute("DELETE FROM game_variant WHERE id = ?",
(variant_id, ))
# games left in this list must now be deleted
for row in self.existing_games.values():
game_id = row[2]
database_cursor.execute("DELETE FROM game WHERE id = ?",
(game_id, ))
database_cursor.execute(
"SELECT count(*) FROM game WHERE uuid IS NOT NULL "
"AND (have IS NULL OR have != (SELECT coalesce(max(have), 0) "
"FROM game_variant WHERE game_uuid = game.uuid))")
update_rows = database_cursor.fetchone()[0]
print(update_rows, "game entries need update for have field")
if update_rows > 0:
database_cursor.execute(
"UPDATE game SET have = (SELECT coalesce(max(have), 0) FROM "
"game_variant WHERE game_uuid = game.uuid) WHERE uuid IS NOT "
"NULL AND (have IS NULL OR have != (SELECT coalesce(max("
"have), 0) FROM game_variant WHERE game_uuid = game.uuid))")
# FIXME: Remove this line?
FileDatabase.get_instance().get_last_event_stamps()
self.file_stamps["database_locker"] = LauncherSettings.get(
Option.DATABASE_LOCKER)
self.database.update_last_file_event_stamps(self.file_stamps)
def __init__(self, database):
self.database = database
database_cursor = self.database.cursor()
database_cursor.execute(
"SELECT uuid, update_stamp, have, id FROM game "
"WHERE uuid IS NOT NULL")
self.existing_games = {}
for row in database_cursor:
self.existing_games[row[0]] = row[1], row[2], row[3]
database_cursor.execute(
"SELECT uuid, update_stamp, have, id FROM game_variant")
self.existing_variants = {}
for row in database_cursor:
self.existing_variants[row[0]] = row[1], row[2], row[3]
self.file_stamps = FileDatabase.get_instance().get_last_event_stamps()
cached_file_stamps = self.database.get_last_file_event_stamps()
self.added_files = (self.file_stamps["last_file_insert"] !=
cached_file_stamps["last_file_insert"])
self.deleted_files = (self.file_stamps["last_file_delete"] !=
cached_file_stamps["last_file_delete"])
# print(LauncherSettings.get(Option.DATABASE_LOCKER),
# cached_file_stamps["database_locker"])
# assert 0
if (LauncherSettings.get(Option.DATABASE_LOCKER) !=
cached_file_stamps["database_locker"]):
# Assume that files could be deleted or removed...
if LauncherSettings.get(Option.DATABASE_LOCKER) == "0":
self.deleted_files = True
else:
self.added_files = True
def build_media_list(self, floppies=False, cds=False, hds=False):
media_list = []
added = set()
for file_item in self.get_file_list():
name = file_item["name"]
url = file_item.get("url", "")
if name.startswith("DH0/"):
if hds:
# p = os.path.join(self.path, "HardDrive")
# p = "hd://game/" + self.uuid + "/DH0"
p = "file_list://DH0"
if p in added:
# already added
continue
added.add(p)
# FIXME: hack for now
sha1 = self.values.get("dh0_sha1", "")
media_list.append((p, sha1))
else:
continue
sha1 = file_item["sha1"]
base, ext = os.path.splitext(name)
ext = ext.lower()
if hds:
if ext not in [".zip"]:
continue
elif cds:
if ext not in [".cue", ".iso"]:
continue
if "(Track" in base:
# probably part of a split multi-track cue
continue
elif floppies:
if ext not in [".adf", ".adz", ".dms", ".ipf"]:
continue
path = ""
found_sha1 = ""
if sha1:
print(sha1)
file = FileDatabase.get_instance().find_file(sha1=sha1)
if file:
found_sha1 = sha1
path = file["path"]
if url and not path:
path = url
found_sha1 = sha1
if path:
media_list.append((path, found_sha1))
else:
pass
# return False
# FIXME: handle it with a visible error message
# raise Exception("could not find file " + repr(name))
return media_list
def run(self):
self.defragment(FileDatabase.get_instance(), "Files.sqlite")
self.stop_check()
self.defragment(LockerDatabase.instance(), "Locker.sqlite")
self.stop_check()
self.defragment(fsgs.get_game_database(), "Amiga.sqlite")
self.stop_check()
self.defragment(Database.get_instance(), "Database.sqlite")
def kickstart_startup_scan(cls):
if cls._kickstart_scanned:
return
cls._kickstart_scanned = True
print("kickstart_startup_scan")
kickstarts_dir = FSGSDirectories.get_kickstarts_dir()
if LauncherSettings.get("kickstarts_dir_mtime"
) == cls.get_dir_mtime_str(kickstarts_dir):
print("... mtime not changed")
else:
file_database = FileDatabase.get_instance()
print("... database.find_local_roms")
local_roms = file_database.find_local_roms()
print("... walk kickstarts_dir")
for dir_path, dir_names, file_names in os.walk(kickstarts_dir):
for file_name in file_names:
if not file_name.lower().endswith(
".rom") and not file_name.lower().endswith(".bin"):
continue
path = Paths.join(dir_path, file_name)
if path in local_roms:
local_roms[path] = None
# already exists in database
continue
print("[startup] adding kickstart", path)
ROMManager.add_rom_to_database(path, file_database)
print(local_roms)
for path, file_id in local_roms.items():
if file_id is not None:
print("[startup] removing kickstart", path)
file_database.delete_file(id=file_id)
print("... commit")
file_database.commit()
LauncherSettings.set("kickstarts_dir_mtime",
cls.get_dir_mtime_str(kickstarts_dir))
amiga = Amiga.get_model_config("A500")
for sha1 in amiga["kickstarts"]:
if fsgs.file.find_by_sha1(sha1=sha1):
break
else:
file_database = FileDatabase.get_instance()
cls.amiga_forever_kickstart_scan()
file_database.commit()
def update_file_database_timestamps(self):
# This isn't very elegant, but in order to force the game scanner to
# refresh the game list based on files, we update the stamps in the
# file database. Also, since we haven't keep track of additions /
# deletions, we set both stamps for now...
file_database = FileDatabase.instance()
file_database.last_file_insert = time.time()
file_database.last_file_delete = time.time()
file_database.update_last_event_stamps()
file_database.commit()
def scan_dir_for_kickstarts(scan_dir):
file_database = FileDatabase.get_instance()
for dir_path, dir_names, file_names in os.walk(scan_dir):
for file_name in file_names:
if not file_name.endswith(".rom"):
continue
path = Paths.join(dir_path, file_name)
if file_database.find_file(path=path):
continue
print("[startup] adding kickstart", path)
ROMManager.add_rom_to_database(path, file_database)
def check_sha1(cls, sha1):
# FIXME: check_sha1 should check with PluginManager directly?
database = FileDatabase.instance()
result = database.check_sha1(sha1)
if not result and is_locker_enabled():
database = LockerDatabase.instance()
result = database.check_sha1(sha1)
# print("check sha1", sha1, "in locker database - result", result)
# if not result:
# result = self.context.get_game_database().find_file_by_sha1(sha1)
return result
def scan_fs_uae_files(self, database):
cursor = database.cursor()
cursor.execute("SELECT id, path FROM game WHERE path IS NOT NULL")
config_path_ids = {}
for row in cursor:
config_path_ids[database.decode_path(row[1])] = row[0]
file_database = FileDatabase.get_instance()
# FIXME: This is not good enough, we want to support multiple
# extensions when using OpenRetro Launcher at least.
if Product.base_name == "FS-Fuse":
extension = ".fs-fuse"
else:
extension = ".fs-uae"
configurations = file_database.find_files(ext=extension)
for c in configurations:
if self.stop_check():
break
# name = os.path.basename(c["path"])
# name = c["name"]
# name = name[:-7]
# search = name.lower()
path = c["path"]
name = os.path.basename(path)
name, ext = os.path.splitext(name)
sort_key = name.lower()
# search = self.create_configuration_search(name)
name = self.create_configuration_name(name)
game_id = database.add_configuration(path=path,
name=name,
sort_key=sort_key)
database.update_game_search_terms(game_id,
self.create_search_terms(name))
print(config_path_ids)
try:
del config_path_ids[path]
except KeyError:
pass
# FIXME: try splitting name into name, variant pair, and
# actually add game_variant for config files as well, where
# possible
for id in config_path_ids.values():
database.delete_game(id=id)
def find_by_sha1(cls, sha1):
# FIXME: check_sha1 should check with PluginManager directly?
database = FileDatabase.instance()
result = database.find_file(sha1=sha1)["path"]
if not result:
path = Downloader.get_cache_path(sha1)
if os.path.exists(path):
result = path
# result = self.context.get_game_database().find_file_by_sha1(sha1)
# print("find by sha1", sha1, "in file database - result", result)
if not result and is_locker_enabled() and not offline_mode():
database = LockerDatabase.instance()
if database.check_sha1(sha1):
result = "locker://" + sha1
# print("find by sha1", sha1, "in locker database - result",
# result)
return result
def scan(self):
self.set_status(gettext("Scanning files"), gettext("Scanning files"))
file_database = FileDatabase.get_instance()
# database.clear()
scan_dirs = self.get_scan_dirs()
if self.purge_other_dirs:
all_database_file_ids = file_database.get_file_ids()
else:
all_database_file_ids = None
for dir_ in scan_dirs:
if not os.path.exists(dir_):
print("[FILES] Scanner: Directory does not exist:", dir_)
continue
# this is important to make sure the database is portable across
# operating systems
dir_ = Paths.get_real_case(dir_)
self.database_file_ids = file_database.get_file_hierarchy_ids(dir_)
if self.purge_other_dirs:
all_database_file_ids.difference_update(self.database_file_ids)
self.scan_dir(file_database, dir_)
print("Remaining files:", self.database_file_ids)
self.purge_file_ids(self.database_file_ids)
self.set_status(gettext("Scanning files"),
gettext("Committing data..."))
# update last_file_insert and last_file_delete
file_database.update_last_event_stamps()
print("[FILES] FileScanner.scan - committing data")
file_database.commit()
if self.purge_other_dirs:
self.purge_file_ids(all_database_file_ids)
if self.stop_check():
file_database.rollback()
else:
self.set_status(gettext("Scanning files"),
gettext("Committing data..."))
print("[FILES] FileScanner.scan - committing data")
file_database.commit()
def upload_check(self, prefix):
self.progressed(
gettext("Finding files eligible for OpenRetro Locker") +
" ({:0.0%})".format((prefix / 16.0)))
file_database = FileDatabase.instance()
cursor = file_database.cursor()
# FIXME: prefix
p = "0123456789ABCDEF"[prefix]
cursor.execute(
"SELECT DISTINCT sha1 FROM file "
"WHERE hex(sha1) LIKE ?",
(p + "%", ),
)
string_io = StringIO()
for row in cursor:
string_io.write(row[0])
# print(prefix, len(string_io.getvalue()))
self.stop_check()
retry_seconds = 1
while True:
try:
result = self.client.post("/api/locker-upload-check",
data=string_io.getvalue())
except OGDClient.ForbiddenError:
raise Task.Failure(
gettext("OpenRetro Locker is not enabled for your user. "
"It may be available only to a few select beta "
"users."))
except OGDClient.NonRetryableHTTPError as e:
raise e
except Exception:
traceback.print_exc()
self.progressed(
gettext("Re-trying in {0} seconds...").format(
retry_seconds))
for _ in range(retry_seconds):
self.stop_check()
time.sleep(1.0)
retry_seconds = min(retry_seconds * 2, 60 * 10)
else:
return result
def copy_roms(self, src, dst):
count = 0
if not os.path.isdir(src):
self.log("{0} is not a directory".format(src))
return count
src_file = os.path.join(src, "rom.key")
if os.path.exists(src_file):
dst_file = os.path.join(dst, "rom.key")
self.copy_file(src_file, dst_file)
for file_name in os.listdir(src):
name, ext = os.path.splitext(file_name)
if ext not in [".rom"]:
continue
src_file = os.path.join(src, file_name)
dst_file = os.path.join(dst, file_name)
self.copy_file(src_file, dst_file)
database = FileDatabase.get_instance()
ROMManager.add_rom_to_database(dst_file, database, self.log)
database.commit()
count += 1
return count
def run_config_or_game(cls):
gscontext = FSGameSystemContext()
config_path = None
archive_path = None
floppy_image_paths = []
cdrom_image_paths = []
config_uuid = None
floppy_extensions = (".adf", ".ipf", ".dms", ".adz")
cdrom_extensions = (".cue", ".iso")
archive_extensions = (".zip", ".lha")
# FIXME: replace argument "parsing" with use of argparse module
# at some point
last_arg = sys.argv[-1]
file_ext = os.path.splitext(last_arg)[-1].lower()
if file_ext == ".fs-uae":
config_path = last_arg
elif file_ext in archive_extensions:
archive_path = last_arg
# elif file_ext in floppy_extensions:
# floppy_image_paths = [last_arg]
elif is_uuid(last_arg):
config_uuid = last_arg.lower()
for arg in sys.argv[1:]:
if not arg.startswith("--"):
_, ext = os.path.splitext(arg)
if ext in floppy_extensions:
floppy_image_paths.append(arg)
elif ext in cdrom_extensions:
cdrom_image_paths.append(arg)
if config_path:
print("[STARTUP] Config path given:", config_path)
if not os.path.exists(config_path):
print("[STARTUP] Config path does not exist", file=sys.stderr)
return True
LauncherConfig.load_file(config_path)
gscontext.config.add_from_argv()
return cls.run_config_directly(gscontext=gscontext)
if archive_path:
print("[STARTUP] Archive path given:", archive_path)
if not os.path.exists(archive_path):
print("[STARTUP] Archive path does not exist", file=sys.stderr)
return True
archive = Archive(os.path.realpath(archive_path))
archive_name = os.path.basename(archive_path)
# We want to exclude pure directory entries when checking for
# archives with only floppies.
arc_files = [
x for x in archive.list_files() if not x.endswith("/")
]
if all(
map(lambda f: f.lower().endswith(floppy_extensions),
arc_files)):
print("[STARTUP] Archive contains floppy disk images only")
floppy_image_paths = arc_files
else:
if cls.auto_detect_game:
# FIXME: Could also do this for floppy file archives.
archive_util = ArchiveUtil(archive_path)
archive_uuid = archive_util.create_variant_uuid()
print(
"[STARTUP] Try auto-detecting variant, uuid =",
archive_uuid,
)
if gscontext.load_game_variant(archive_uuid):
print("[STARTUP] Auto-detected variant", archive_uuid)
print("[STARTUP] Adding archive files to file index")
for archive_file in archive.list_files():
stream = archive.open(archive_file)
data = stream.read()
size = len(data)
sha1 = hashlib.sha1(data).hexdigest()
FileDatabase.add_static_file(archive_file,
size=size,
sha1=sha1)
gscontext.config.add_from_argv()
gscontext.config.set("__config_name", archive_name)
LauncherConfig.post_load_values(gscontext.config)
return cls.run_config_directly(gscontext=gscontext)
values = whdload.generate_config_for_archive(archive_path)
values["hard_drive_0"] = archive_path
values.update(gscontext.config.config_from_argv())
# archive_name, archive_ext = os.path.splitext(archive_name)
values["__config_name"] = archive_name
return cls.run_config_directly_with_values(values,
gscontext=gscontext)
if floppy_image_paths:
enum_paths = tuple(enumerate(floppy_image_paths))
values = {}
values.update(gscontext.config.config_from_argv())
max_drives = int(values.get("floppy_drive_count", "4"))
values.update({
"floppy_drive_{0}".format(k): v
for k, v in enum_paths[:max_drives]
})
values.update(
{"floppy_image_{0}".format(k): v
for k, v in enum_paths[:20]})
# FIXME: Generate a better config name for save dir?
values["__config_name"] = "Default"
return cls.run_config_directly_with_values(values,
gscontext=gscontext)
if cdrom_image_paths:
enum_paths = tuple(enumerate(cdrom_image_paths))
values = {"amiga_model": "CD32"}
values.update(gscontext.config.config_from_argv())
max_drives = int(values.get("cdrom_drive_count", "1"))
values.update({
"cdrom_drive_{0}".format(k): v
for k, v in enum_paths[:max_drives]
})
values.update(
{"cdrom_image_{0}".format(k): v
for k, v in enum_paths[:20]})
# FIXME: Generate a better config name for save dir?
values["__config_name"] = "Default"
return cls.run_config_directly_with_values(values,
gscontext=gscontext)
if config_uuid:
print("[STARTUP] Config uuid given:", config_uuid)
variant_uuid = config_uuid
# values = gscontext.game.set_from_variant_uuid(variant_uuid)
if gscontext.load_game_variant(variant_uuid):
print("[STARTUP] Loaded variant")
else:
print("[STARTUP] Could not load variant, try to load game")
game_uuid = config_uuid
variant_uuid = gscontext.find_preferred_game_variant(game_uuid)
print("[STARTUP] Preferred variant:", variant_uuid)
gscontext.load_game_variant(variant_uuid)
gscontext.config.add_from_argv()
LauncherConfig.post_load_values(gscontext.config)
return cls.run_config_directly(gscontext=gscontext)
def save_config():
print("SaveButton.save_config")
config = get_config(self)
database = Database.get_instance()
name = LauncherSettings.get("config_name").strip()
if not name:
print("no config_name")
# FIXME: notify user
return
file_name = name + get_extension_for_config(config)
path = os.path.join(
FSGSDirectories.get_configurations_dir(), file_name
)
with io.open(path, "w", encoding="UTF-8") as f:
f.write("# FS-UAE configuration saved by FS-UAE Launcher\n")
f.write(
"# Last saved: {0}\n".format(
datetime.datetime.today().strftime("%Y-%m-%d %H:%M:%S")
)
)
f.write("\n[fs-uae]\n")
keys = sorted(fsgs.config.values.keys())
for key in keys:
value = config.get(key)
if key.startswith("__"):
continue
if key in LauncherConfig.no_save_keys_set:
continue
# elif key == "joystick_port_2_mode" and value == "nothing":
# continue
# elif key == "joystick_port_3_mode" and value == "nothing":
# continue
if value == LauncherConfig.default_config.get(key, ""):
continue
if value:
f.write("{0} = {1}\n".format(key, value))
# scanner = ConfigurationScanner()
# search = ConfigurationScanner.create_configuration_search(name)
# name = scanner.create_configuration_name(name)
# print("adding", path)
# # deleting the path from the database first in case it already exists
# database.delete_configuration(path=path)
# database.delete_file(path=path)
# database.add_file(path=path)
# database.add_configuration(
# path=path, uuid="", name=name, scan=0, search=search)
file_database = FileDatabase.get_instance()
scanner = ConfigurationScanner()
print("[save config] adding config", path)
file_database.delete_file(path=path)
with open(path, "rb") as f:
sha1 = hashlib.sha1(f.read()).hexdigest()
file_database.add_file(path=path, sha1=sha1)
game_id = database.add_configuration(
path=path, name=scanner.create_configuration_name(name)
)
database.update_game_search_terms(
game_id, scanner.create_search_terms(name)
)
database.commit()
file_database.commit()
LauncherSettings.set("__config_refresh", str(time.time()))
# Settings.set("config_changed", "0")
LauncherConfig.set("__changed", "0")
def purge_file_ids(self, file_ids):
self.set_status(gettext("Scanning files"),
gettext("Purging old entries..."))
database = FileDatabase.get_instance()
for file_id in file_ids:
database.delete_file(id=file_id)
def config_startup_scan(cls):
if cls._config_scanned:
return
cls._config_scanned = True
configs_dir = FSGSDirectories.get_configurations_dir()
print("config_startup_scan", configs_dir)
print(LauncherSettings.settings)
settings_mtime = LauncherSettings.get("configurations_dir_mtime")
dir_mtime = cls.get_dir_mtime_str(configs_dir)
if settings_mtime == dir_mtime + "+" + str(Database.VERSION):
print("... mtime not changed", settings_mtime, dir_mtime)
return
database = Database.get_instance()
file_database = FileDatabase.get_instance()
print("... database.find_local_configurations")
local_configs = Database.get_instance().find_local_configurations()
print("... walk configs_dir")
for dir_path, dir_names, file_names in os.walk(configs_dir):
for file_name in file_names:
# if not file_name.endswith(".fs-uae"):
# continue
# FIXME: Move extension list somewhere else
if not file_name.endswith(
".fs-uae") and not file_name.endswith(".fs-fuse"):
continue
path = Paths.join(dir_path, file_name)
if path in local_configs:
local_configs[path] = None
# already exists in database
continue
name, ext = os.path.splitext(file_name)
# search = ConfigurationScanner.create_configuration_search(
# name)
scanner = ConfigurationScanner()
print("[startup] adding config", path)
file_database.delete_file(path=path)
with open(path, "rb") as f:
sha1 = hashlib.sha1(f.read()).hexdigest()
file_database.add_file(path=path, sha1=sha1)
game_id = database.add_configuration(
path=path, name=scanner.create_configuration_name(name))
database.update_game_search_terms(
game_id, scanner.create_search_terms(name))
for path, config_id in local_configs.items():
if config_id is not None:
print("[startup] removing configuration", path)
database.delete_game(id=config_id)
file_database.delete_file(path=path)
print("... commit")
database.commit()
LauncherSettings.set(
"configurations_dir_mtime",
cls.get_dir_mtime_str(configs_dir) + "+" + str(Database.VERSION),
)