def copy_whdload_kickstart(self, base_dir, name, checksums): dest = os.path.join(base_dir, "Devs", "Kickstarts") if not os.path.exists(dest): os.makedirs(dest) dest = os.path.join(dest, name) for checksum in checksums: # print("find kickstart with sha1", checksum) path = self.fsgs.file.find_by_sha1(checksum) if path: # and os.path.exists(path): print("found kickstart for", name, "at", path) archive = Archive(path) if archive.exists(path): with open(dest, "wb") as f: ROMManager.decrypt_archive_rom(archive, path, file=f) print(repr(dest)) break else: stream = self.fsgs.file.open(path) if stream is None: raise Exception("Cannot find kickstart " + repr(path)) with open(dest, "wb") as f: f.write(stream.read()) else: print("did not find kickstart for", name)
def scan_dir_for_kickstarts(scan_dir): file_database = FileDatabase.get_instance() for dir_path, dir_names, file_names in os.walk(scan_dir): for file_name in file_names: if not file_name.endswith(".rom"): continue path = Paths.join(dir_path, file_name) if file_database.find_file(path=path): continue print("[startup] adding kickstart", path) ROMManager.add_rom_to_database(path, file_database)
def kickstart_startup_scan(cls): if cls._kickstart_scanned: return cls._kickstart_scanned = True print("kickstart_startup_scan") kickstarts_dir = FSGSDirectories.get_kickstarts_dir() if LauncherSettings.get("kickstarts_dir_mtime") == \ cls.get_dir_mtime_str(kickstarts_dir): print("... mtime not changed") else: file_database = FileDatabase.get_instance() print("... database.find_local_roms") local_roms = file_database.find_local_roms() print("... walk kickstarts_dir") for dir_path, dir_names, file_names in os.walk(kickstarts_dir): for file_name in file_names: if not file_name.lower().endswith(".rom") and not \ file_name.lower().endswith(".bin"): continue path = Paths.join(dir_path, file_name) if path in local_roms: local_roms[path] = None # already exists in database continue print("[startup] adding kickstart", path) ROMManager.add_rom_to_database(path, file_database) print(local_roms) for path, id in local_roms.items(): if id is not None: print("[startup] removing kickstart", path) file_database.delete_file(id=id) print("... commit") file_database.commit() LauncherSettings.set( "kickstarts_dir_mtime", cls.get_dir_mtime_str(kickstarts_dir)) amiga = Amiga.get_model_config("A500") for sha1 in amiga["kickstarts"]: if fsgs.file.find_by_sha1(sha1=sha1): break else: file_database = FileDatabase.get_instance() cls.amiga_forever_kickstart_scan() file_database.commit()
def kickstart_startup_scan(cls): if cls._kickstart_scanned: return cls._kickstart_scanned = True print("kickstart_startup_scan") kickstarts_dir = FSGSDirectories.get_kickstarts_dir() if LauncherSettings.get("kickstarts_dir_mtime") == \ cls.get_dir_mtime_str(kickstarts_dir): print("... mtime not changed") else: file_database = FileDatabase.get_instance() print("... database.find_local_roms") local_roms = file_database.find_local_roms() print("... walk kickstarts_dir") for dir_path, dir_names, file_names in os.walk(kickstarts_dir): for file_name in file_names: if not file_name.lower().endswith(".rom") and not \ file_name.lower().endswith(".bin"): continue path = Paths.join(dir_path, file_name) if path in local_roms: local_roms[path] = None # already exists in database continue print("[startup] adding kickstart", path) ROMManager.add_rom_to_database(path, file_database) print(local_roms) for path, file_id in local_roms.items(): if file_id is not None: print("[startup] removing kickstart", path) file_database.delete_file(id=file_id) print("... commit") file_database.commit() LauncherSettings.set("kickstarts_dir_mtime", cls.get_dir_mtime_str(kickstarts_dir)) amiga = Amiga.get_model_config("A500") for sha1 in amiga["kickstarts"]: if fsgs.file.find_by_sha1(sha1=sha1): break else: file_database = FileDatabase.get_instance() cls.amiga_forever_kickstart_scan() file_database.commit()
def upload_prefix(self, prefix): self.stop_check() result = self.upload_check(prefix) print(len(result)) for k in range(0, len(result), 20): self.stop_check() sha1 = result[k:k + 20] path = fsgs.file.find_by_sha1(bytes_to_hex(sha1)) if not path: continue try: # this is done to properly handle encrypted ROMs archive = Archive(path) data = ROMManager.decrypt_archive_rom(archive, path)["data"] except Exception: traceback.print_exc() uri = "sha1://{0}".format(bytes_to_hex(sha1)) print(uri) try: input_stream = fsgs.file.open(uri) data = input_stream.read() except Exception: continue assert not input_stream.read() print("uploading file of size ", len(data)) # self.progressed(gettext("Verifying {name}").format( # name=bytes_to_hex(sha1))) self.progressed( gettext("Uploading {name}").format(name=bytes_to_hex(sha1))) import hashlib new_hash = hashlib.sha1(data).hexdigest() print(new_hash, "vs", bytes_to_hex(sha1)) if hashlib.sha1(data).hexdigest() != bytes_to_hex(sha1): print("hash mismatch, probably Cloanto ROM...") continue retry_seconds = 1 while True: try: self.client.post("/api/locker-upload-file", data=data) except OGDClient.NonRetryableHTTPError as e: raise e except Exception: traceback.print_exc() self.progressed( gettext("Re-trying in {0} seconds...").format( retry_seconds)) for _ in range(retry_seconds): self.stop_check() time.sleep(1.0) retry_seconds = min(retry_seconds * 2, 60 * 10) else: break
def upload_prefix(self, prefix): self.stop_check() result = self.upload_check(prefix) print(len(result)) for k in range(0, len(result), 20): self.stop_check() sha1 = result[k:k + 20] path = fsgs.file.find_by_sha1(bytes_to_hex(sha1)) if not path: continue try: # this is done to properly handle encrypted ROMs archive = Archive(path) data = ROMManager.decrypt_archive_rom(archive, path)["data"] except Exception: traceback.print_exc() uri = "sha1://{0}".format(bytes_to_hex(sha1)) print(uri) try: input_stream = fsgs.file.open(uri) data = input_stream.read() except Exception: continue assert not input_stream.read() print("uploading file of size ", len(data)) # self.progressed(gettext("Verifying {name}").format( # name=bytes_to_hex(sha1))) self.progressed(gettext("Uploading {name}").format( name=bytes_to_hex(sha1))) import hashlib new_hash = hashlib.sha1(data).hexdigest() print(new_hash, "vs", bytes_to_hex(sha1)) if hashlib.sha1(data).hexdigest() != bytes_to_hex(sha1): print("hash mismatch, probably Cloanto ROM...") continue retry_seconds = 1 while True: try: self.client.post("/api/locker-upload-file", data=data) except OGDClient.NonRetryableHTTPError as e: raise e except Exception: traceback.print_exc() self.progressed(gettext( "Re-trying in {0} seconds...").format(retry_seconds)) for _ in range(retry_seconds): self.stop_check() time.sleep(1.0) retry_seconds = min(retry_seconds * 2, 60 * 10) else: break
def scan_archive_stream( self, database, archive, path, name, size, mtime, parent=None): self.set_status( gettext("Scanning files ({count} scanned)").format( count=self.scan_count), name) base_name, ext = os.path.splitext(name) ext = ext.lower() f = archive.open(path) s = hashlib.sha1() while True: if self.stop_check(): return data = f.read(65536) if not data: break s.update(data) sha1 = s.hexdigest() if ext == ".rom": try: sha1_dec = ROMManager.decrypt_archive_rom(archive, path)["sha1"] except Exception: import traceback traceback.print_exc() sha1_dec = None if sha1_dec: if sha1_dec != sha1: print("found encrypted rom {0} => {1}".format( sha1, sha1_dec)) # sha1 is now the decrypted sha1, not the actual sha1 of the # file itself, a bit ugly, since md5 and crc32 are still # encrypted hashes, but it works well with the kickstart # lookup mechanism sha1 = sha1_dec else: # If the ROM was encrypted and could not be decrypted, we # don't add it to the database. This way, the file will be # correctly added on later scans if rom.key is added to the # directory. return None if parent: path = "#/" + path.rsplit("#/", 1)[1] file_id = database.add_file( path=path, sha1=sha1, mtime=mtime, size=size, parent=parent) self.files_added += 1 self.bytes_added += size if ext == '.rom': if self.on_rom_found: self.on_rom_found(path, sha1) return file_id
def copy_roms(self, src, dst): count = 0 if not os.path.isdir(src): self.log("{0} is not a directory".format(src)) return count src_file = os.path.join(src, "rom.key") if os.path.exists(src_file): dst_file = os.path.join(dst, "rom.key") self.copy_file(src_file, dst_file) for file_name in os.listdir(src): name, ext = os.path.splitext(file_name) if ext not in [".rom"]: continue src_file = os.path.join(src, file_name) dst_file = os.path.join(dst, file_name) self.copy_file(src_file, dst_file) database = FileDatabase.get_instance() ROMManager.add_rom_to_database(dst_file, database, self.log) database.commit() count += 1 return count
def prepare_roms(self): print("LaunchHandler.prepare_roms") current_task.set_progress(gettext("Preparing kickstart ROMs...")) amiga_model = self.config.get("amiga_model", "A500") model_config = Amiga.get_model_config(amiga_model) roms = [("kickstart_file", model_config["kickstarts"])] if self.config["kickstart_ext_file"] or model_config["ext_roms"]: # not all Amigas have extended ROMs roms.append(("kickstart_ext_file", model_config["ext_roms"])) if amiga_model.lower() == "cd32/fmv": roms.append(("fvm_rom", [CD32_FMV_ROM])) if self.config["graphics_card"].lower().startswith("picasso-iv"): roms.append(("graphics_card_rom", [PICASSO_IV_74_ROM])) if self.config["accelerator"].lower() == "cyberstorm-ppc": roms.append(("accelerator_rom", ["cyberstormppc.rom"])) if self.config["freezer_cartridge"] == "action-replay-2": # Ideally, we would want to recognize ROMs based on zeroing the # first four bytes, but right now we simply recognize a common # additional version. freezer_cartridge_rom isn't a real option, # we just want to copy the rom file and let FS-UAE find it roms.append(("[freezer_cartridge]", [ACTION_REPLAY_MK_II_2_14_ROM.sha1, ACTION_REPLAY_MK_II_2_14_MOD_ROM.sha1])) elif self.config["freezer_cartridge"] == "action-replay-3": roms.append(("[freezer_cartridge]", [ACTION_REPLAY_MK_III_3_17_ROM.sha1, ACTION_REPLAY_MK_III_3_17_MOD_ROM.sha1])) for config_key, default_roms in roms: print("ROM:", config_key, default_roms) src = self.config[config_key] if not src: for sha1 in default_roms: if is_sha1(sha1): rom_src = self.fsgs.file.find_by_sha1(sha1) if rom_src: src = rom_src break else: # roms_dir = FSGSDirectories.get_kickstarts_dir() # src = os.path.join(roms_dir, sha1) # if os.path.exists(src): # break # loop up file in roms dir instead src = sha1 elif src == "internal": continue elif src: src = Paths.expand_path(src) if not src: raise TaskFailure( gettext("Did not find required Kickstart or " "ROM for {}. Wanted one of these files: {}".format( config_key, repr(default_roms)))) use_temp_kickstarts_dir = False dest = os.path.join(self.temp_dir, os.path.basename(src)) archive = Archive(src) stream = None if not archive.exists(src): dirs = [self.fsgs.amiga.get_kickstarts_dir()] for dir in dirs: path = os.path.join(dir, src) print("checking", repr(path)) archive = Archive(path) if archive.exists(path): src = path break else: try: stream = self.fsgs.file.open(src) if stream is None: raise FileNotFoundError(src) except FileNotFoundError: raise TaskFailure(gettext( "Cannot find required ROM " "file: {name}".format(name=repr(src)))) with open(dest, "wb") as f: if stream: f.write(stream.read()) else: ROMManager.decrypt_archive_rom(archive, src, file=f) if use_temp_kickstarts_dir: self.config[config_key] = os.path.basename(src) else: self.config[config_key] = dest if use_temp_kickstarts_dir: self.config["kickstarts_dir"] = self.temp_dir
def checksum_rom(self, path): print("checksum_rom", repr(path)) archive = Archive(path) return ROMManager.decrypt_archive_rom(archive, path)["sha1"]