def scan_dir_for_kickstarts(scan_dir):
file_database = FileDatabase.get_instance()
for dir_path, dir_names, file_names in os.walk(scan_dir):
for file_name in file_names:
if not file_name.endswith(".rom"):
continue
path = Paths.join(dir_path, file_name)
if file_database.find_file(path=path):
continue
print("[startup] adding kickstart", path)
ROMManager.add_rom_to_database(path, file_database)
def scan_dir_for_kickstarts(scan_dir):
file_database = FileDatabase.get_instance()
for dir_path, dir_names, file_names in os.walk(scan_dir):
for file_name in file_names:
if not file_name.endswith(".rom"):
continue
path = Paths.join(dir_path, file_name)
if file_database.find_file(path=path):
continue
print("[startup] adding kickstart", path)
ROMManager.add_rom_to_database(path, file_database)
def kickstart_startup_scan(cls):
if cls._kickstart_scanned:
return
cls._kickstart_scanned = True
print("kickstart_startup_scan")
kickstarts_dir = FSGSDirectories.get_kickstarts_dir()
if LauncherSettings.get(
"kickstarts_dir_mtime"
) == cls.get_dir_mtime_str(kickstarts_dir):
print("... mtime not changed")
else:
file_database = FileDatabase.get_instance()
print("... database.find_local_roms")
local_roms = file_database.find_local_roms()
print("... walk kickstarts_dir")
for dir_path, dir_names, file_names in os.walk(kickstarts_dir):
for file_name in file_names:
if not file_name.lower().endswith(
".rom"
) and not file_name.lower().endswith(".bin"):
continue
path = Paths.join(dir_path, file_name)
if path in local_roms:
local_roms[path] = None
# already exists in database
continue
print("[startup] adding kickstart", path)
ROMManager.add_rom_to_database(path, file_database)
print(local_roms)
for path, file_id in local_roms.items():
if file_id is not None:
print("[startup] removing kickstart", path)
file_database.delete_file(id=file_id)
print("... commit")
file_database.commit()
LauncherSettings.set(
"kickstarts_dir_mtime", cls.get_dir_mtime_str(kickstarts_dir)
)
amiga = Amiga.get_model_config("A500")
for sha1 in amiga["kickstarts"]:
if fsgs.file.find_by_sha1(sha1=sha1):
break
else:
file_database = FileDatabase.get_instance()
cls.amiga_forever_kickstart_scan()
file_database.commit()
def upload_prefix(self, prefix):
self.stop_check()
result = self.upload_check(prefix)
print(len(result))
for k in range(0, len(result), 20):
self.stop_check()
sha1 = result[k:k + 20]
path = fsgs.file.find_by_sha1(bytes_to_hex(sha1))
if not path:
continue
try:
# this is done to properly handle encrypted ROMs
archive = Archive(path)
# FIXME: Use Archive.open to get support for filter functions
# FIXME: Also use stream api, do not buffer entire file
data = ROMManager.decrypt_archive_rom(archive, path)["data"]
except Exception:
traceback.print_exc()
uri = "sha1://{0}".format(bytes_to_hex(sha1))
print(uri)
try:
input_stream = fsgs.file.open(uri)
data = input_stream.read()
except Exception:
continue
assert not input_stream.read()
print("uploading file of size ", len(data))
# self.progressed(gettext("Verifying {name}").format(
# name=bytes_to_hex(sha1)))
self.progressed(
gettext("Uploading {name}").format(name=bytes_to_hex(sha1)))
import hashlib
new_hash = hashlib.sha1(data).hexdigest()
print(new_hash, "vs", bytes_to_hex(sha1))
if hashlib.sha1(data).hexdigest() != bytes_to_hex(sha1):
print("hash mismatch, probably Cloanto ROM...")
continue
retry_seconds = 1
while True:
try:
self.client.post("/api/locker-upload-file", data=data)
except OGDClient.NonRetryableHTTPError as e:
raise e
except Exception:
traceback.print_exc()
self.progressed(
gettext("Re-trying in {0} seconds...").format(
retry_seconds))
for _ in range(retry_seconds):
self.stop_check()
time.sleep(1.0)
retry_seconds = min(retry_seconds * 2, 60 * 10)
else:
break
def kickstart_startup_scan(cls):
if cls._kickstart_scanned:
return
cls._kickstart_scanned = True
print("kickstart_startup_scan")
kickstarts_dir = FSGSDirectories.get_kickstarts_dir()
if LauncherSettings.get("kickstarts_dir_mtime") == \
cls.get_dir_mtime_str(kickstarts_dir):
print("... mtime not changed")
else:
file_database = FileDatabase.get_instance()
print("... database.find_local_roms")
local_roms = file_database.find_local_roms()
print("... walk kickstarts_dir")
for dir_path, dir_names, file_names in os.walk(kickstarts_dir):
for file_name in file_names:
if not file_name.lower().endswith(".rom") and not \
file_name.lower().endswith(".bin"):
continue
path = Paths.join(dir_path, file_name)
if path in local_roms:
local_roms[path] = None
# already exists in database
continue
print("[startup] adding kickstart", path)
ROMManager.add_rom_to_database(path, file_database)
print(local_roms)
for path, file_id in local_roms.items():
if file_id is not None:
print("[startup] removing kickstart", path)
file_database.delete_file(id=file_id)
print("... commit")
file_database.commit()
LauncherSettings.set(
"kickstarts_dir_mtime",
cls.get_dir_mtime_str(kickstarts_dir))
amiga = Amiga.get_model_config("A500")
for sha1 in amiga["kickstarts"]:
if fsgs.file.find_by_sha1(sha1=sha1):
break
else:
file_database = FileDatabase.get_instance()
cls.amiga_forever_kickstart_scan()
file_database.commit()
def copy_roms(self, src, dst):
count = 0
if not os.path.isdir(src):
self.log("{0} is not a directory".format(src))
return count
src_file = os.path.join(src, "rom.key")
if os.path.exists(src_file):
dst_file = os.path.join(dst, "rom.key")
self.copy_file(src_file, dst_file)
for file_name in os.listdir(src):
name, ext = os.path.splitext(file_name)
if ext not in [".rom"]:
continue
src_file = os.path.join(src, file_name)
dst_file = os.path.join(dst, file_name)
self.copy_file(src_file, dst_file)
database = FileDatabase.get_instance()
ROMManager.add_rom_to_database(dst_file, database, self.log)
database.commit()
count += 1
return count
def copy_roms(self, src, dst):
count = 0
if not os.path.isdir(src):
self.log("{0} is not a directory".format(src))
return count
src_file = os.path.join(src, "rom.key")
if os.path.exists(src_file):
dst_file = os.path.join(dst, "rom.key")
self.copy_file(src_file, dst_file)
for file_name in os.listdir(src):
name, ext = os.path.splitext(file_name)
if ext not in [".rom"]:
continue
src_file = os.path.join(src, file_name)
dst_file = os.path.join(dst, file_name)
self.copy_file(src_file, dst_file)
database = FileDatabase.get_instance()
ROMManager.add_rom_to_database(dst_file, database, self.log)
database.commit()
count += 1
return count
def scan_archive_stream(self,
database,
archive,
path,
name,
size,
mtime,
parent=None):
self.set_status(
gettext("Scanning files ({count} scanned)").format(
count=self.scan_count),
name,
)
f = None
sha1 = None
raw_sha1_obj = hashlib.sha1()
filter_sha1_obj = hashlib.sha1()
filter_name = ""
filter_size = 0
base_name, ext = os.path.splitext(name)
ext = ext.lower()
if ext == ".nes":
# FIXME: NES header hack. Would be better to add a proper notion of
# file filters to the file database.
# FIXME: This will confuse some functionality, such as the
# Locker uploader or other tools expecting on-disk data to match
# the database checksum (this also applies to the Cloanto ROM
# hack). Should be done properly.
f = archive.open(path)
data = f.read(16)
if len(data) == 16 and data.startswith(b"NES\x1a"):
print("Stripping iNES header for", path)
filter_name = "Skip(16)"
raw_sha1_obj.update(data)
elif ext == ".a78":
# FIXME: Check if 128 is a fixed or variable number of bytes
f = archive.open(path)
data = f.read(128)
if len(data) == 128 and data[1:10] == b"ATARI7800":
print("Stripping A78 header for", path)
filter_name = "Skip(128)"
raw_sha1_obj.update(data)
elif ext == ".smc":
f = archive.open(path)
data = f.read(512)
if len(data) == 512 and all(map(lambda x: x == 0, data[48:])):
print("Stripping SMC header for", path)
filter_name = "Skip(512)"
raw_sha1_obj.update(data)
elif ext in [".v64", ".n64"]:
filter_name = "ByteSwapWords"
def is_sha1(name):
if not len(name) == 40:
return False
name = name.lower()
for c in name:
if c not in "0123456789abcdef":
return False
return True
if filter_name:
# We have a filter, so we must calculate checksum of filtered contents
pass
else:
# Try to see if we can deduce the checksum of the contained file.
# Supports symlinks to compressed files.
real_path = os.path.realpath(archive.path)
real_name = os.path.basename(real_path)
real_name = real_name.lower()
real_name, real_ext = os.path.splitext(real_name)
if real_ext in [".xz", ".gz"]:
if is_sha1(real_name):
if parent is not None:
# We assume this is the correct SHA-1, avoids having
# to decompress and calculate the SHA-1
sha1 = real_name
else:
# Assume we are not interested in the parent's SHA-1
sha1 = "ffffffffffffffffffffffffffffffffffffffff"
if sha1 is None:
if f is None:
f = archive.open(path)
while True:
if self.stop_check():
return
data = f.read(65536)
if not data:
break
raw_sha1_obj.update(data)
if filter_name.startswith("Skip("):
filter_sha1_obj.update(data)
filter_size += len(data)
elif filter_name == "ByteSwapWords":
# We don't really expect odd number of bytes when
# byteswapping words, but this handles the cases where we
# have "false positives" that aren't supposed to be
# byteswapped, and this prevents errors.
data_size = len(data)
for i in range(0, len(data), 2):
if data_size - i >= 2:
filter_sha1_obj.update(data[i + 1:i + 2])
filter_sha1_obj.update(data[i:i + 1])
filter_size += 2
else:
filter_sha1_obj.update(data[i:i + 1])
filter_size += 1
sha1 = raw_sha1_obj.hexdigest()
filter_sha1 = filter_sha1_obj.hexdigest()
if ext == ".rom":
try:
filter_data = ROMManager.decrypt_archive_rom(archive, path)
filter_sha1 = filter_data["sha1"]
filter_size = len(filter_data["data"])
except Exception:
import traceback
traceback.print_exc()
filter_sha1 = None
if filter_sha1:
if filter_sha1 != sha1:
print("[Files] Found encrypted rom {0} => {1}".format(
sha1, filter_sha1))
# sha1 is now the decrypted sha1, not the actual sha1 of the
# file itself, a bit ugly, since md5 and crc32 are still
# encrypted hashes, but it works well with the kickstart
# lookup mechanism
# FIXME: Enable use of filter mechanism for Cloanto ROMs
sha1 = filter_sha1
# filter_name = "Cloanto"
else:
# If the ROM was encrypted and could not be decrypted, we
# don't add it to the database. This way, the file will be
# correctly added on later scans if rom.key is added to the
# directory.
return None
if parent:
path = "#/" + path.rsplit("#/", 1)[1]
if parent is not None:
# FIXME: size is incorrect here -- it's the size of the parent
# file currently (not a big problem), setting it to -1 instead
# for now.
size = -1
file_id = database.add_file(path=path,
sha1=sha1,
mtime=mtime,
size=size,
parent=parent)
self.files_added += 1
if parent is None:
self.bytes_added += size
if filter_name:
if parent:
# We want to add to the previous archive path
pass
else:
# Reset path
path = ""
path += "#?Filter=" + filter_name
# If not already in an archive (has a real file parent), set
# parent to the real file
database.add_file(
path=path,
sha1=filter_sha1,
mtime=mtime,
size=filter_size,
parent=(parent or file_id),
)
if ext == ".rom":
if self.on_rom_found:
self.on_rom_found(path, sha1)
return file_id
def checksum_rom(self, path):
print("[CHECKSUM] ROM:", repr(path))
archive = Archive(path)
return ROMManager.decrypt_archive_rom(archive, path)["sha1"]
def scan_archive_stream(
self, database, archive, path, name, size, mtime, parent=None
):
self.set_status(
gettext("Scanning files ({count} scanned)").format(
count=self.scan_count
),
name,
)
f = archive.open(path)
raw_sha1_obj = hashlib.sha1()
filter_sha1_obj = hashlib.sha1()
filter_name = ""
filter_size = 0
base_name, ext = os.path.splitext(name)
ext = ext.lower()
if ext == ".nes":
# FIXME: NES header hack. Would be better to add a proper notion of
# file filters to the file database.
# FIXME: This will confuse some functionality, such as the
# Locker uploader or other tools expecting on-disk data to match
# the database checksum (this also applies to the Cloanto ROM
# hack). Should be done properly.
data = f.read(16)
if len(data) == 16 and data.startswith(b"NES\x1a"):
print("Stripping iNES header for", path)
filter_name = "Skip(16)"
raw_sha1_obj.update(data)
elif ext == ".a78":
# FIXME: Check if 128 is a fixed or variable number of bytes
data = f.read(128)
if len(data) == 128 and data[1:10] == b"ATARI7800":
print("Stripping A78 header for", path)
filter_name = "Skip(128)"
raw_sha1_obj.update(data)
elif ext == ".smc":
data = f.read(512)
if len(data) == 512 and all(map(lambda x: x == 0, data[48:])):
print("Stripping SMC header for", path)
filter_name = "Skip(512)"
raw_sha1_obj.update(data)
elif ext in [".v64", ".n64"]:
filter_name = "ByteSwapWords"
while True:
if self.stop_check():
return
data = f.read(65536)
if not data:
break
raw_sha1_obj.update(data)
if filter_name.startswith("Skip("):
filter_sha1_obj.update(data)
filter_size += len(data)
elif filter_name == "ByteSwapWords":
for i in range(0, len(data), 2):
filter_sha1_obj.update(data[i + 1 : i + 2])
filter_sha1_obj.update(data[i : i + 1])
filter_size += 2
# FIXME: Handle it when we get an odd number of bytes..
assert len(data) % 2 == 0
sha1 = raw_sha1_obj.hexdigest()
filter_sha1 = filter_sha1_obj.hexdigest()
if ext == ".rom":
try:
filter_data = ROMManager.decrypt_archive_rom(archive, path)
filter_sha1 = filter_data["sha1"]
filter_size = len(filter_data["data"])
except Exception:
import traceback
traceback.print_exc()
filter_sha1 = None
if filter_sha1:
if filter_sha1 != sha1:
print(
"[Files] Found encrypted rom {0} => {1}".format(
sha1, filter_sha1
)
)
# sha1 is now the decrypted sha1, not the actual sha1 of the
# file itself, a bit ugly, since md5 and crc32 are still
# encrypted hashes, but it works well with the kickstart
# lookup mechanism
# FIXME: Enable use of filter mechanism for Cloanto ROMs
sha1 = filter_sha1
# filter_name = "Cloanto"
else:
# If the ROM was encrypted and could not be decrypted, we
# don't add it to the database. This way, the file will be
# correctly added on later scans if rom.key is added to the
# directory.
return None
if parent:
path = "#/" + path.rsplit("#/", 1)[1]
file_id = database.add_file(
path=path, sha1=sha1, mtime=mtime, size=size, parent=parent
)
self.files_added += 1
self.bytes_added += size
if filter_name:
if parent:
# We want to add to the previous archive path
pass
else:
# Reset path
path = ""
path += "#?Filter=" + filter_name
# If not already in an archive (has a real file parent), set
# parent to the real file
database.add_file(
path=path,
sha1=filter_sha1,
mtime=mtime,
size=filter_size,
parent=(parent or file_id),
)
if ext == ".rom":
if self.on_rom_found:
self.on_rom_found(path, sha1)
return file_id
def upload_prefix(self, prefix):
self.stop_check()
result = self.upload_check(prefix)
print(len(result))
for k in range(0, len(result), 20):
self.stop_check()
sha1 = result[k : k + 20]
path = fsgs.file.find_by_sha1(bytes_to_hex(sha1))
if not path:
continue
try:
# this is done to properly handle encrypted ROMs
archive = Archive(path)
# FIXME: Use Archive.open to get support for filter functions
# FIXME: Also use stream api, do not buffer entire file
data = ROMManager.decrypt_archive_rom(archive, path)["data"]
except Exception:
traceback.print_exc()
uri = "sha1://{0}".format(bytes_to_hex(sha1))
print(uri)
try:
input_stream = fsgs.file.open(uri)
data = input_stream.read()
except Exception:
continue
assert not input_stream.read()
print("uploading file of size ", len(data))
# self.progressed(gettext("Verifying {name}").format(
# name=bytes_to_hex(sha1)))
self.progressed(
gettext("Uploading {name}").format(name=bytes_to_hex(sha1))
)
import hashlib
new_hash = hashlib.sha1(data).hexdigest()
print(new_hash, "vs", bytes_to_hex(sha1))
if hashlib.sha1(data).hexdigest() != bytes_to_hex(sha1):
print("hash mismatch, probably Cloanto ROM...")
continue
retry_seconds = 1
while True:
try:
self.client.post("/api/locker-upload-file", data=data)
except OGDClient.NonRetryableHTTPError as e:
raise e
except Exception:
traceback.print_exc()
self.progressed(
gettext("Re-trying in {0} seconds...").format(
retry_seconds
)
)
for _ in range(retry_seconds):
self.stop_check()
time.sleep(1.0)
retry_seconds = min(retry_seconds * 2, 60 * 10)
else:
break
def scan_archive_stream(self,
database,
archive,
path,
name,
size,
mtime,
parent=None):
self.set_status(
gettext("Scanning files ({count} scanned)").format(
count=self.scan_count), name)
f = archive.open(path)
raw_sha1_obj = hashlib.sha1()
filter_sha1_obj = hashlib.sha1()
filter_name = ""
filter_size = 0
base_name, ext = os.path.splitext(name)
ext = ext.lower()
if ext == ".nes":
# FIXME: NES header hack. Would be better to add a proper notion of
# file filters to the file database.
# FIXME: This will confuse some functionality, such as the
# Locker uploader or other tools expecting on-disk data to match
# the database checksum (this also applies to the Cloanto ROM
# hack). Should be done properly.
data = f.read(16)
if len(data) == 16 and data.startswith(b"NES\x1a"):
print("Stripping iNES header for", path)
filter_name = "Skip(16)"
raw_sha1_obj.update(data)
elif ext == ".a78":
# FIXME: Check if 128 is a fixed or variable number of bytes
data = f.read(128)
if len(data) == 128 and data[1:10] == b"ATARI7800":
print("Stripping A78 header for", path)
filter_name = "Skip(128)"
raw_sha1_obj.update(data)
elif ext == ".smc":
data = f.read(512)
if len(data) == 512 and all(map(lambda x: x == 0, data[48:])):
print("Stripping SMC header for", path)
filter_name = "Skip(512)"
raw_sha1_obj.update(data)
elif ext in [".v64", ".n64"]:
filter_name = "ByteSwapWords"
while True:
if self.stop_check():
return
data = f.read(65536)
if not data:
break
raw_sha1_obj.update(data)
if filter_name.startswith("Skip("):
filter_sha1_obj.update(data)
filter_size += len(data)
elif filter_name == "ByteSwapWords":
for i in range(0, len(data), 2):
filter_sha1_obj.update(data[i + 1:i + 2])
filter_sha1_obj.update(data[i:i + 1])
filter_size += 2
# FIXME: Handle it when we get an odd number of bytes..
assert len(data) % 2 == 0
sha1 = raw_sha1_obj.hexdigest()
filter_sha1 = filter_sha1_obj.hexdigest()
if ext == ".rom":
try:
filter_data = ROMManager.decrypt_archive_rom(archive, path)
filter_sha1 = filter_data["sha1"]
filter_size = len(filter_data["data"])
except Exception:
import traceback
traceback.print_exc()
filter_sha1 = None
if filter_sha1:
if filter_sha1 != sha1:
print("[Files] Found encrypted rom {0} => {1}".format(
sha1, filter_sha1))
# sha1 is now the decrypted sha1, not the actual sha1 of the
# file itself, a bit ugly, since md5 and crc32 are still
# encrypted hashes, but it works well with the kickstart
# lookup mechanism
# FIXME: Enable use of filter mechanism for Cloanto ROMs
sha1 = filter_sha1
# filter_name = "Cloanto"
else:
# If the ROM was encrypted and could not be decrypted, we
# don't add it to the database. This way, the file will be
# correctly added on later scans if rom.key is added to the
# directory.
return None
if parent:
path = "#/" + path.rsplit("#/", 1)[1]
file_id = database.add_file(path=path,
sha1=sha1,
mtime=mtime,
size=size,
parent=parent)
self.files_added += 1
self.bytes_added += size
if filter_name:
if parent:
# We want to add to the previous archive path
pass
else:
# Reset path
path = ""
path += "#?Filter=" + filter_name
# If not already in an archive (has a real file parent), set
# parent to the real file
database.add_file(path=path,
sha1=filter_sha1,
mtime=mtime,
size=filter_size,
parent=(parent or file_id))
if ext == ".rom":
if self.on_rom_found:
self.on_rom_found(path, sha1)
return file_id