def upload_prefix(self, prefix): self.stop_check() result = self.upload_check(prefix) print(len(result)) for k in range(0, len(result), 20): self.stop_check() sha1 = result[k:k + 20] path = fsgs.file.find_by_sha1(bytes_to_hex(sha1)) if not path: continue try: # this is done to properly handle encrypted ROMs archive = Archive(path) # FIXME: Use Archive.open to get support for filter functions # FIXME: Also use stream api, do not buffer entire file data = ROMManager.decrypt_archive_rom(archive, path)["data"] except Exception: traceback.print_exc() uri = "sha1://{0}".format(bytes_to_hex(sha1)) print(uri) try: input_stream = fsgs.file.open(uri) data = input_stream.read() except Exception: continue assert not input_stream.read() print("uploading file of size ", len(data)) # self.progressed(gettext("Verifying {name}").format( # name=bytes_to_hex(sha1))) self.progressed( gettext("Uploading {name}").format(name=bytes_to_hex(sha1))) import hashlib new_hash = hashlib.sha1(data).hexdigest() print(new_hash, "vs", bytes_to_hex(sha1)) if hashlib.sha1(data).hexdigest() != bytes_to_hex(sha1): print("hash mismatch, probably Cloanto ROM...") continue retry_seconds = 1 while True: try: self.client.post("/api/locker-upload-file", data=data) except OGDClient.NonRetryableHTTPError as e: raise e except Exception: traceback.print_exc() self.progressed( gettext("Re-trying in {0} seconds...").format( retry_seconds)) for _ in range(retry_seconds): self.stop_check() time.sleep(1.0) retry_seconds = min(retry_seconds * 2, 60 * 10) else: break
def scan_archive_stream(self, database, archive, path, name, size, mtime, parent=None): self.set_status( gettext("Scanning files ({count} scanned)").format( count=self.scan_count), name, ) f = None sha1 = None raw_sha1_obj = hashlib.sha1() filter_sha1_obj = hashlib.sha1() filter_name = "" filter_size = 0 base_name, ext = os.path.splitext(name) ext = ext.lower() if ext == ".nes": # FIXME: NES header hack. Would be better to add a proper notion of # file filters to the file database. # FIXME: This will confuse some functionality, such as the # Locker uploader or other tools expecting on-disk data to match # the database checksum (this also applies to the Cloanto ROM # hack). Should be done properly. f = archive.open(path) data = f.read(16) if len(data) == 16 and data.startswith(b"NES\x1a"): print("Stripping iNES header for", path) filter_name = "Skip(16)" raw_sha1_obj.update(data) elif ext == ".a78": # FIXME: Check if 128 is a fixed or variable number of bytes f = archive.open(path) data = f.read(128) if len(data) == 128 and data[1:10] == b"ATARI7800": print("Stripping A78 header for", path) filter_name = "Skip(128)" raw_sha1_obj.update(data) elif ext == ".smc": f = archive.open(path) data = f.read(512) if len(data) == 512 and all(map(lambda x: x == 0, data[48:])): print("Stripping SMC header for", path) filter_name = "Skip(512)" raw_sha1_obj.update(data) elif ext in [".v64", ".n64"]: filter_name = "ByteSwapWords" def is_sha1(name): if not len(name) == 40: return False name = name.lower() for c in name: if c not in "0123456789abcdef": return False return True if filter_name: # We have a filter, so we must calculate checksum of filtered contents pass else: # Try to see if we can deduce the checksum of the contained file. # Supports symlinks to compressed files. real_path = os.path.realpath(archive.path) real_name = os.path.basename(real_path) real_name = real_name.lower() real_name, real_ext = os.path.splitext(real_name) if real_ext in [".xz", ".gz"]: if is_sha1(real_name): if parent is not None: # We assume this is the correct SHA-1, avoids having # to decompress and calculate the SHA-1 sha1 = real_name else: # Assume we are not interested in the parent's SHA-1 sha1 = "ffffffffffffffffffffffffffffffffffffffff" if sha1 is None: if f is None: f = archive.open(path) while True: if self.stop_check(): return data = f.read(65536) if not data: break raw_sha1_obj.update(data) if filter_name.startswith("Skip("): filter_sha1_obj.update(data) filter_size += len(data) elif filter_name == "ByteSwapWords": # We don't really expect odd number of bytes when # byteswapping words, but this handles the cases where we # have "false positives" that aren't supposed to be # byteswapped, and this prevents errors. data_size = len(data) for i in range(0, len(data), 2): if data_size - i >= 2: filter_sha1_obj.update(data[i + 1:i + 2]) filter_sha1_obj.update(data[i:i + 1]) filter_size += 2 else: filter_sha1_obj.update(data[i:i + 1]) filter_size += 1 sha1 = raw_sha1_obj.hexdigest() filter_sha1 = filter_sha1_obj.hexdigest() if ext == ".rom": try: filter_data = ROMManager.decrypt_archive_rom(archive, path) filter_sha1 = filter_data["sha1"] filter_size = len(filter_data["data"]) except Exception: import traceback traceback.print_exc() filter_sha1 = None if filter_sha1: if filter_sha1 != sha1: print("[Files] Found encrypted rom {0} => {1}".format( sha1, filter_sha1)) # sha1 is now the decrypted sha1, not the actual sha1 of the # file itself, a bit ugly, since md5 and crc32 are still # encrypted hashes, but it works well with the kickstart # lookup mechanism # FIXME: Enable use of filter mechanism for Cloanto ROMs sha1 = filter_sha1 # filter_name = "Cloanto" else: # If the ROM was encrypted and could not be decrypted, we # don't add it to the database. This way, the file will be # correctly added on later scans if rom.key is added to the # directory. return None if parent: path = "#/" + path.rsplit("#/", 1)[1] if parent is not None: # FIXME: size is incorrect here -- it's the size of the parent # file currently (not a big problem), setting it to -1 instead # for now. size = -1 file_id = database.add_file(path=path, sha1=sha1, mtime=mtime, size=size, parent=parent) self.files_added += 1 if parent is None: self.bytes_added += size if filter_name: if parent: # We want to add to the previous archive path pass else: # Reset path path = "" path += "#?Filter=" + filter_name # If not already in an archive (has a real file parent), set # parent to the real file database.add_file( path=path, sha1=filter_sha1, mtime=mtime, size=filter_size, parent=(parent or file_id), ) if ext == ".rom": if self.on_rom_found: self.on_rom_found(path, sha1) return file_id
def prepare_roms(self): print("LaunchHandler.prepare_roms") current_task.set_progress(gettext("Preparing kickstart ROMs...")) amiga_model = self.config.get("amiga_model", "A500") model_config = Amiga.get_model_config(amiga_model) roms = [("kickstart_file", model_config["kickstarts"])] if self.config["kickstart_ext_file"] or model_config["ext_roms"]: # not all Amigas have extended ROMs roms.append(("kickstart_ext_file", model_config["ext_roms"])) if amiga_model.lower() == "cd32/fmv": roms.append(("fvm_rom", [CD32_FMV_ROM])) if self.config["graphics_card"].lower().startswith("picasso-iv"): roms.append(("graphics_card_rom", [PICASSO_IV_74_ROM])) if self.config["accelerator"].lower() == "cyberstorm-ppc": roms.append(("accelerator_rom", ["cyberstormppc.rom"])) if self.config["freezer_cartridge"] == "action-replay-2": # Ideally, we would want to recognize ROMs based on zeroing the # first four bytes, but right now we simply recognize a common # additional version. freezer_cartridge_rom isn't a real option, # we just want to copy the rom file and let FS-UAE find it roms.append( ( "[freezer_cartridge]", [ ACTION_REPLAY_MK_II_2_14_ROM.sha1, ACTION_REPLAY_MK_II_2_14_MOD_ROM.sha1, ], ) ) elif self.config["freezer_cartridge"] == "action-replay-3": roms.append( ( "[freezer_cartridge]", [ ACTION_REPLAY_MK_III_3_17_ROM.sha1, ACTION_REPLAY_MK_III_3_17_MOD_ROM.sha1, ], ) ) use_temp_kickstarts_dir = False for config_key, default_roms in roms: print("[ROM]", config_key, default_roms) src = self.config[config_key] print("[ROM]", src) if not src: for sha1 in default_roms: print("[ROM] Trying", sha1) if is_sha1(sha1): rom_src = self.fsgs.file.find_by_sha1(sha1) if rom_src: src = rom_src print("[ROM] Found", rom_src) break else: # roms_dir = FSGSDirectories.get_kickstarts_dir() # src = os.path.join(roms_dir, sha1) # if os.path.exists(src): # break # loop up file in roms dir instead src = sha1 elif src == "internal": continue elif src: src = Paths.expand_path(src) if not src: raise TaskFailure( gettext( "Did not find required Kickstart or " "ROM for {}. Wanted one of these files: {}".format( config_key, repr(default_roms) ) ) ) dest = os.path.join(self.temp_dir, os.path.basename(src)) def lookup_rom_from_src(src): parts = src.split(":", 1) if len(parts) == 2 and len(parts[0]) > 1: # src has a scheme (not a Windows drive letter). Assume # we can find this file. return src archive = Archive(src) if archive.exists(src): return src dirs = [self.fsgs.amiga.get_kickstarts_dir()] for dir_ in dirs: path = os.path.join(dir_, src) print("[ROM] Checking", repr(path)) archive = Archive(path) if archive.exists(path): return path return None org_src = src src = lookup_rom_from_src(src) if not src and org_src == "cyberstormppc.rom": src = lookup_rom_from_src( "ralphschmidt-cyberstorm-ppc-4471.rom" ) if not src: for ( dir_ ) in FSGSDirectories.get_amiga_forever_directories(): path = os.path.join( dir_, "Shared", "rom", "ralphschmidt-cyberstorm-ppc-4471.rom", ) if os.path.exists(path): src = path print("[ROM] Found", path) break else: print("[ROM] Trying", path) stream = None # FIXME: prepare_roms should be rewritten, it's kind of crap. # Rom patching and decryption should be handled differently. Should # use file database filters, and decryption via rom.key should only # be supported when using uncompressed files directly on disk. if not src or not os.path.exists(src): try: stream = self.fsgs.file.open(src) if stream is None: raise FileNotFoundError(src) except FileNotFoundError: raise TaskFailure( gettext( "Cannot find required ROM " "file: {name}".format(name=repr(org_src)) ) ) with open(dest, "wb") as f: if stream: print("[ROM] From stream => {}".format(dest)) rom = {} rom["data"] = stream.read() rom["sha1"] = hashlib.sha1(rom["data"]).hexdigest() ROMManager.patch_rom(rom) f.write(rom["data"]) else: archive = Archive(src) ROMManager.decrypt_archive_rom(archive, src, file=f) if use_temp_kickstarts_dir: self.config[config_key] = os.path.basename(src) else: self.config[config_key] = dest if use_temp_kickstarts_dir: self.config["kickstarts_dir"] = self.temp_dir
def checksum_rom(self, path): print("[CHECKSUM] ROM:", repr(path)) archive = Archive(path) return ROMManager.decrypt_archive_rom(archive, path)["sha1"]