def scan_dir_for_kickstarts(scan_dir):
file_database = FileDatabase.get_instance()
for dir_path, dir_names, file_names in os.walk(scan_dir):
for file_name in file_names:
if not file_name.endswith(".rom"):
continue
path = Paths.join(dir_path, file_name)
if file_database.find_file(path=path):
continue
print("[startup] adding kickstart", path)
ROMManager.add_rom_to_database(path, file_database)
def upload_prefix(self, prefix):
self.stop_check()
result = self.upload_check(prefix)
print(len(result))
for k in range(0, len(result), 20):
self.stop_check()
sha1 = result[k:k + 20]
path = fsgs.file.find_by_sha1(bytes_to_hex(sha1))
if not path:
continue
try:
# this is done to properly handle encrypted ROMs
archive = Archive(path)
# FIXME: Use Archive.open to get support for filter functions
# FIXME: Also use stream api, do not buffer entire file
data = ROMManager.decrypt_archive_rom(archive, path)["data"]
except Exception:
traceback.print_exc()
uri = "sha1://{0}".format(bytes_to_hex(sha1))
print(uri)
try:
input_stream = fsgs.file.open(uri)
data = input_stream.read()
except Exception:
continue
assert not input_stream.read()
print("uploading file of size ", len(data))
# self.progressed(gettext("Verifying {name}").format(
# name=bytes_to_hex(sha1)))
self.progressed(
gettext("Uploading {name}").format(name=bytes_to_hex(sha1)))
import hashlib
new_hash = hashlib.sha1(data).hexdigest()
print(new_hash, "vs", bytes_to_hex(sha1))
if hashlib.sha1(data).hexdigest() != bytes_to_hex(sha1):
print("hash mismatch, probably Cloanto ROM...")
continue
retry_seconds = 1
while True:
try:
self.client.post("/api/locker-upload-file", data=data)
except OGDClient.NonRetryableHTTPError as e:
raise e
except Exception:
traceback.print_exc()
self.progressed(
gettext("Re-trying in {0} seconds...").format(
retry_seconds))
for _ in range(retry_seconds):
self.stop_check()
time.sleep(1.0)
retry_seconds = min(retry_seconds * 2, 60 * 10)
else:
break
def kickstart_startup_scan(cls):
if cls._kickstart_scanned:
return
cls._kickstart_scanned = True
print("kickstart_startup_scan")
kickstarts_dir = FSGSDirectories.get_kickstarts_dir()
if LauncherSettings.get("kickstarts_dir_mtime"
) == cls.get_dir_mtime_str(kickstarts_dir):
print("... mtime not changed")
else:
file_database = FileDatabase.get_instance()
print("... database.find_local_roms")
local_roms = file_database.find_local_roms()
print("... walk kickstarts_dir")
for dir_path, dir_names, file_names in os.walk(kickstarts_dir):
for file_name in file_names:
if not file_name.lower().endswith(
".rom") and not file_name.lower().endswith(".bin"):
continue
path = Paths.join(dir_path, file_name)
if path in local_roms:
local_roms[path] = None
# already exists in database
continue
print("[startup] adding kickstart", path)
ROMManager.add_rom_to_database(path, file_database)
print(local_roms)
for path, file_id in local_roms.items():
if file_id is not None:
print("[startup] removing kickstart", path)
file_database.delete_file(id=file_id)
print("... commit")
file_database.commit()
LauncherSettings.set("kickstarts_dir_mtime",
cls.get_dir_mtime_str(kickstarts_dir))
amiga = Amiga.get_model_config("A500")
for sha1 in amiga["kickstarts"]:
if fsgs.file.find_by_sha1(sha1=sha1):
break
else:
file_database = FileDatabase.get_instance()
cls.amiga_forever_kickstart_scan()
file_database.commit()
def decrypt_amiromtype1_stream(stream, data):
# If we open a stream starting with AMIROMTYPE1, we assume that the file
# has been opened via Archive and stored in the database with the
# decrypted checksum.
# FIXME: Better to implement this properly with archive filters...
path = stream.archive_path
archive = Archive(path)
key_path = archive.join(archive.dirname(path), "rom.key")
key_archive = Archive(key_path)
try:
f2 = key_archive.open(key_path)
except Exception:
raise Exception("Did not find rom.key to decrypt ROM with")
print("Using key file", key_path)
key_data = f2.read()
f2.close()
out_data = []
result = {}
f = BytesIO(data[len("AMIROMTYPE1"):] + stream.read())
sha1_obj = hashlib.sha1()
while True:
data = f.read(len(key_data))
if not data:
break
dec = []
for i in range(len(data)):
dec.append(data[i] ^ key_data[i])
dec_data = bytes(dec)
# if file is not None:
# file.write(dec_data)
out_data.append(dec_data)
# if sha1 is not None:
sha1_obj.update(dec_data)
result["data"] = b"".join(out_data)
result["sha1"] = sha1_obj.hexdigest()
# print(result)
ROMManager.patch_rom(result)
# if file is not None:
# file.write(result["data"])
# return result
return BytesIO(result["data"])
def write_stream(stream, path, sha1=None):
dst = path
dst_partial = os.path.join(os.path.dirname(dst),
"~" + os.path.basename(dst) + ".tmp")
sha1_obj = hashlib.sha1()
written_size = 0
with open(dst_partial, "wb") as f:
while True:
data = stream.read(CHUNK_SIZE)
if data.startswith(b"AMIROMTYPE1"):
# FIXME: Better to implement this properly with archive
# filters...
stream = decrypt_amiromtype1_stream(stream, data)
data = stream.read(CHUNK_SIZE)
if not data:
break
f.write(data)
sha1_obj.update(data)
written_size += len(data)
written_sha1 = sha1_obj.hexdigest()
# If SHA-1 did not match, see if the written file matches one of the ROMs
# than ROMManager can transform into the correct one.
if (sha1 is not None and written_sha1 != sha1
and (written_sha1, sha1) in ROMManager.rom_transformations):
print(f"Patching ROM {written_sha1} -> {sha1}")
rom = {"sha1": written_sha1, "data": b""}
with open(dst_partial, "rb") as f:
rom["data"] = f.read()
ROMManager.patch_rom(rom)
with open(dst_partial, "wb") as f:
f.write(rom["data"])
written_sha1 = rom["sha1"]
if sha1 is not None:
if written_sha1 != sha1:
raise Exception(
f"Written SHA-1 ({written_sha1}) does not match expected SHA-1 ({sha1})"
)
os.rename(dst_partial, dst)
return written_sha1, written_size
def copy_roms(self, src, dst):
count = 0
if not os.path.isdir(src):
self.log("{0} is not a directory".format(src))
return count
src_file = os.path.join(src, "rom.key")
if os.path.exists(src_file):
dst_file = os.path.join(dst, "rom.key")
self.copy_file(src_file, dst_file)
for file_name in os.listdir(src):
name, ext = os.path.splitext(file_name)
if ext not in [".rom"]:
continue
src_file = os.path.join(src, file_name)
dst_file = os.path.join(dst, file_name)
self.copy_file(src_file, dst_file)
database = FileDatabase.get_instance()
ROMManager.add_rom_to_database(dst_file, database, self.log)
database.commit()
count += 1
return count
def scan_archive_stream(self,
database,
archive,
path,
name,
size,
mtime,
parent=None):
self.set_status(
gettext("Scanning files ({count} scanned)").format(
count=self.scan_count),
name,
)
f = None
sha1 = None
raw_sha1_obj = hashlib.sha1()
filter_sha1_obj = hashlib.sha1()
filter_name = ""
filter_size = 0
base_name, ext = os.path.splitext(name)
ext = ext.lower()
if ext == ".nes":
# FIXME: NES header hack. Would be better to add a proper notion of
# file filters to the file database.
# FIXME: This will confuse some functionality, such as the
# Locker uploader or other tools expecting on-disk data to match
# the database checksum (this also applies to the Cloanto ROM
# hack). Should be done properly.
f = archive.open(path)
data = f.read(16)
if len(data) == 16 and data.startswith(b"NES\x1a"):
print("Stripping iNES header for", path)
filter_name = "Skip(16)"
raw_sha1_obj.update(data)
elif ext == ".a78":
# FIXME: Check if 128 is a fixed or variable number of bytes
f = archive.open(path)
data = f.read(128)
if len(data) == 128 and data[1:10] == b"ATARI7800":
print("Stripping A78 header for", path)
filter_name = "Skip(128)"
raw_sha1_obj.update(data)
elif ext == ".smc":
f = archive.open(path)
data = f.read(512)
if len(data) == 512 and all(map(lambda x: x == 0, data[48:])):
print("Stripping SMC header for", path)
filter_name = "Skip(512)"
raw_sha1_obj.update(data)
elif ext in [".v64", ".n64"]:
filter_name = "ByteSwapWords"
def is_sha1(name):
if not len(name) == 40:
return False
name = name.lower()
for c in name:
if c not in "0123456789abcdef":
return False
return True
if filter_name:
# We have a filter, so we must calculate checksum of filtered contents
pass
else:
# Try to see if we can deduce the checksum of the contained file.
# Supports symlinks to compressed files.
real_path = os.path.realpath(archive.path)
real_name = os.path.basename(real_path)
real_name = real_name.lower()
real_name, real_ext = os.path.splitext(real_name)
if real_ext in [".xz", ".gz"]:
if is_sha1(real_name):
if parent is not None:
# We assume this is the correct SHA-1, avoids having
# to decompress and calculate the SHA-1
sha1 = real_name
else:
# Assume we are not interested in the parent's SHA-1
sha1 = "ffffffffffffffffffffffffffffffffffffffff"
if sha1 is None:
if f is None:
f = archive.open(path)
while True:
if self.stop_check():
return
data = f.read(65536)
if not data:
break
raw_sha1_obj.update(data)
if filter_name.startswith("Skip("):
filter_sha1_obj.update(data)
filter_size += len(data)
elif filter_name == "ByteSwapWords":
# We don't really expect odd number of bytes when
# byteswapping words, but this handles the cases where we
# have "false positives" that aren't supposed to be
# byteswapped, and this prevents errors.
data_size = len(data)
for i in range(0, len(data), 2):
if data_size - i >= 2:
filter_sha1_obj.update(data[i + 1:i + 2])
filter_sha1_obj.update(data[i:i + 1])
filter_size += 2
else:
filter_sha1_obj.update(data[i:i + 1])
filter_size += 1
sha1 = raw_sha1_obj.hexdigest()
filter_sha1 = filter_sha1_obj.hexdigest()
if ext == ".rom":
try:
filter_data = ROMManager.decrypt_archive_rom(archive, path)
filter_sha1 = filter_data["sha1"]
filter_size = len(filter_data["data"])
except Exception:
import traceback
traceback.print_exc()
filter_sha1 = None
if filter_sha1:
if filter_sha1 != sha1:
print("[Files] Found encrypted rom {0} => {1}".format(
sha1, filter_sha1))
# sha1 is now the decrypted sha1, not the actual sha1 of the
# file itself, a bit ugly, since md5 and crc32 are still
# encrypted hashes, but it works well with the kickstart
# lookup mechanism
# FIXME: Enable use of filter mechanism for Cloanto ROMs
sha1 = filter_sha1
# filter_name = "Cloanto"
else:
# If the ROM was encrypted and could not be decrypted, we
# don't add it to the database. This way, the file will be
# correctly added on later scans if rom.key is added to the
# directory.
return None
if parent:
path = "#/" + path.rsplit("#/", 1)[1]
if parent is not None:
# FIXME: size is incorrect here -- it's the size of the parent
# file currently (not a big problem), setting it to -1 instead
# for now.
size = -1
file_id = database.add_file(path=path,
sha1=sha1,
mtime=mtime,
size=size,
parent=parent)
self.files_added += 1
if parent is None:
self.bytes_added += size
if filter_name:
if parent:
# We want to add to the previous archive path
pass
else:
# Reset path
path = ""
path += "#?Filter=" + filter_name
# If not already in an archive (has a real file parent), set
# parent to the real file
database.add_file(
path=path,
sha1=filter_sha1,
mtime=mtime,
size=filter_size,
parent=(parent or file_id),
)
if ext == ".rom":
if self.on_rom_found:
self.on_rom_found(path, sha1)
return file_id
def prepare_roms(self):
print("LaunchHandler.prepare_roms")
current_task.set_progress(gettext("Preparing kickstart ROMs..."))
amiga_model = self.config.get("amiga_model", "A500")
model_config = Amiga.get_model_config(amiga_model)
roms = [("kickstart_file", model_config["kickstarts"])]
if self.config["kickstart_ext_file"] or model_config["ext_roms"]:
# not all Amigas have extended ROMs
roms.append(("kickstart_ext_file", model_config["ext_roms"]))
if amiga_model.lower() == "cd32/fmv":
roms.append(("fvm_rom", [CD32_FMV_ROM]))
if self.config["graphics_card"].lower().startswith("picasso-iv"):
roms.append(("graphics_card_rom", [PICASSO_IV_74_ROM]))
if self.config["accelerator"].lower() == "cyberstorm-ppc":
roms.append(("accelerator_rom", ["cyberstormppc.rom"]))
if self.config["freezer_cartridge"] == "action-replay-2":
# Ideally, we would want to recognize ROMs based on zeroing the
# first four bytes, but right now we simply recognize a common
# additional version. freezer_cartridge_rom isn't a real option,
# we just want to copy the rom file and let FS-UAE find it
roms.append(
(
"[freezer_cartridge]",
[
ACTION_REPLAY_MK_II_2_14_ROM.sha1,
ACTION_REPLAY_MK_II_2_14_MOD_ROM.sha1,
],
)
)
elif self.config["freezer_cartridge"] == "action-replay-3":
roms.append(
(
"[freezer_cartridge]",
[
ACTION_REPLAY_MK_III_3_17_ROM.sha1,
ACTION_REPLAY_MK_III_3_17_MOD_ROM.sha1,
],
)
)
use_temp_kickstarts_dir = False
for config_key, default_roms in roms:
print("[ROM]", config_key, default_roms)
src = self.config[config_key]
print("[ROM]", src)
if not src:
for sha1 in default_roms:
print("[ROM] Trying", sha1)
if is_sha1(sha1):
rom_src = self.fsgs.file.find_by_sha1(sha1)
if rom_src:
src = rom_src
print("[ROM] Found", rom_src)
break
else:
# roms_dir = FSGSDirectories.get_kickstarts_dir()
# src = os.path.join(roms_dir, sha1)
# if os.path.exists(src):
# break
# loop up file in roms dir instead
src = sha1
elif src == "internal":
continue
elif src:
src = Paths.expand_path(src)
if not src:
raise TaskFailure(
gettext(
"Did not find required Kickstart or "
"ROM for {}. Wanted one of these files: {}".format(
config_key, repr(default_roms)
)
)
)
dest = os.path.join(self.temp_dir, os.path.basename(src))
def lookup_rom_from_src(src):
parts = src.split(":", 1)
if len(parts) == 2 and len(parts[0]) > 1:
# src has a scheme (not a Windows drive letter). Assume
# we can find this file.
return src
archive = Archive(src)
if archive.exists(src):
return src
dirs = [self.fsgs.amiga.get_kickstarts_dir()]
for dir_ in dirs:
path = os.path.join(dir_, src)
print("[ROM] Checking", repr(path))
archive = Archive(path)
if archive.exists(path):
return path
return None
org_src = src
src = lookup_rom_from_src(src)
if not src and org_src == "cyberstormppc.rom":
src = lookup_rom_from_src(
"ralphschmidt-cyberstorm-ppc-4471.rom"
)
if not src:
for (
dir_
) in FSGSDirectories.get_amiga_forever_directories():
path = os.path.join(
dir_,
"Shared",
"rom",
"ralphschmidt-cyberstorm-ppc-4471.rom",
)
if os.path.exists(path):
src = path
print("[ROM] Found", path)
break
else:
print("[ROM] Trying", path)
stream = None
# FIXME: prepare_roms should be rewritten, it's kind of crap.
# Rom patching and decryption should be handled differently. Should
# use file database filters, and decryption via rom.key should only
# be supported when using uncompressed files directly on disk.
if not src or not os.path.exists(src):
try:
stream = self.fsgs.file.open(src)
if stream is None:
raise FileNotFoundError(src)
except FileNotFoundError:
raise TaskFailure(
gettext(
"Cannot find required ROM "
"file: {name}".format(name=repr(org_src))
)
)
with open(dest, "wb") as f:
if stream:
print("[ROM] From stream => {}".format(dest))
rom = {}
rom["data"] = stream.read()
rom["sha1"] = hashlib.sha1(rom["data"]).hexdigest()
ROMManager.patch_rom(rom)
f.write(rom["data"])
else:
archive = Archive(src)
ROMManager.decrypt_archive_rom(archive, src, file=f)
if use_temp_kickstarts_dir:
self.config[config_key] = os.path.basename(src)
else:
self.config[config_key] = dest
if use_temp_kickstarts_dir:
self.config["kickstarts_dir"] = self.temp_dir
def checksum_rom(self, path):
print("[CHECKSUM] ROM:", repr(path))
archive = Archive(path)
return ROMManager.decrypt_archive_rom(archive, path)["sha1"]