def get_game_type(self) -> str: """Returns game type from arguments or Papyrus Project""" if self.options.game_type: return self.options.game_type if self.options.game_path: for game_type, game_name in GameName.items(): if endswith(self.options.game_path, game_name, ignorecase=True): ProjectBase.log.info( f'Using game type: {game_name} (determined from game path)' ) return GameType.get(game_type) if self.options.registry_path: game_type = self._get_game_type_from_path( self.options.registry_path) if game_type: ProjectBase.log.info( f'Using game type: {GameName.get(game_type)} (determined from registry path)' ) return game_type if self.import_paths: for import_path in reversed(self.import_paths): game_type = self._get_game_type_from_path(import_path) if game_type: ProjectBase.log.info( f'Using game type: {GameName.get(game_type)} (determined from import paths)' ) return game_type if self.options.flags_path: if endswith(self.options.flags_path, FlagsName.FO4, ignorecase=True): ProjectBase.log.info( f'Using game type: {GameName.FO4} (determined from flags path)' ) return GameType.FO4 if endswith(self.options.flags_path, FlagsName.TES5, ignorecase=True): try: self.get_game_path('sse') except FileNotFoundError: ProjectBase.log.info( f'Using game type: {GameName.TES5} (determined from flags path)' ) return GameType.TES5 else: ProjectBase.log.info( f'Using game type: {GameName.SSE} (determined from flags path)' ) return GameType.SSE raise AssertionError( 'Cannot determine game type from game path, registry path, import paths, or flags path' )
def fetch_contents(self, url: str, output_path: str) -> Generator: """ Downloads files from URL to output path """ parsed_url = urlparse(url) schemeless_url = url.removeprefix(f'{parsed_url.scheme}://') if endswith(parsed_url.netloc, 'github.com', ignorecase=True): if self.config or not self.access_token: self.access_token = self.find_access_token(schemeless_url) if not self.access_token: raise PermissionError( 'Cannot download from GitHub remote without access token' ) github = GitHubRemote(access_token=self.access_token, worker_limit=self.worker_limit, force_overwrite=self.force_overwrite) yield from github.fetch_contents(url, output_path) elif endswith(parsed_url.netloc, 'bitbucket.org', ignorecase=True): bitbucket = BitbucketRemote() yield from bitbucket.fetch_contents(url, output_path) else: if self.config or not self.access_token: self.access_token = self.find_access_token(schemeless_url) if not self.access_token: raise PermissionError( 'Cannot download from Gitea remote without access token' ) gitea = GiteaRemote(access_token=self.access_token, worker_limit=self.worker_limit, force_overwrite=self.force_overwrite) yield from gitea.fetch_contents(url, output_path)
def __setattr__(self, key: str, value: object) -> None: if isinstance(value, str) and endswith(key, 'path'): if os.altsep in value: value = os.path.normpath(value) elif isinstance(value, list) and endswith(key, 'paths'): value = [ os.path.normpath(path) if path != os.curdir else path for path in value ] super(ProjectBase, self).__setattr__(key, value)
def build_commands(self, containing_folder: str, output_path: str) -> str: """ Builds command for creating package with BSArch """ arguments = CommandArguments() arguments.append(self.options.bsarch_path, enquote_value=True) arguments.append('pack') arguments.append(containing_folder, enquote_value=True) arguments.append(output_path, enquote_value=True) if self.options.game_type == GameType.FO4: arguments.append('-fo4') elif self.options.game_type == GameType.SSE: arguments.append('-sse') # SSE has an ctd bug with uncompressed textures in a bsa that # has an Embed Filenames flag on it, so force it to false. has_textures = False for f in glob.iglob(os.path.join(containing_folder, r'**/*'), recursive=True): if not os.path.isfile(f): continue if endswith(f, '.dds', ignorecase=True): has_textures = True break if has_textures: arguments.append('-af:0x3') else: arguments.append('-tes5') return arguments.join()
def merge_implicit_import_paths(implicit_paths: list, import_paths: list) -> None: """Inserts orphan and descendant implicit paths into list of import paths at correct positions""" if not implicit_paths: return implicit_paths.sort() for implicit_path in reversed(PathHelper.uniqify(implicit_paths)): implicit_path = os.path.normpath(implicit_path) # do not add import paths that are already declared if any( endswith(x, implicit_path, ignorecase=True) for x in import_paths): continue # insert implicit path before ancestor import path, if ancestor exists i = PathHelper.find_index_of_ancestor_import_path( implicit_path, import_paths) if i > -1: import_paths.insert(i, implicit_path) continue # insert orphan implicit path at the first position import_paths.insert(0, implicit_path)
def _try_exclude_unmodified_scripts(self) -> dict: psc_paths: dict = {} for object_name, script_path in self.psc_paths.items(): script_name, _ = os.path.splitext(os.path.basename(script_path)) # if pex exists, compare time_t in pex header with psc's last modified timestamp matching_path: str = '' for pex_path in self.pex_paths: if endswith(pex_path, f'{script_name}.pex', ignorecase=True): matching_path = pex_path break if not os.path.isfile(matching_path): continue try: header = PexReader.get_header(matching_path) except ValueError: PapyrusProject.log.error( f'Cannot determine compilation time due to unknown magic: "{matching_path}"' ) sys.exit(1) compiled_time: int = header.compilation_time.value if os.path.getmtime(script_path) < compiled_time: continue if script_path not in psc_paths: psc_paths[object_name] = script_path return psc_paths
def _get_remote_path(self, node: etree.ElementBase) -> str: import_path: str = node.text url_hash = hashlib.sha1(import_path.encode()).hexdigest()[:8] temp_path = os.path.join(self.options.remote_temp_path, url_hash) if self.options.force_overwrite or not os.path.isdir(temp_path): try: for message in self.remote.fetch_contents( import_path, temp_path): if message: if not startswith(message, 'Failed to load'): PapyrusProject.log.info(message) else: PapyrusProject.log.error(message) sys.exit(1) except PermissionError as e: PapyrusProject.log.error(e) sys.exit(1) if endswith(import_path, '.git', ignorecase=True): url_path = self.remote.create_local_path(import_path[:-4]) else: url_path = self.remote.create_local_path(import_path) local_path = os.path.join(temp_path, url_path) matcher = wcmatch.WcMatch(local_path, '*.psc', flags=wcmatch.IGNORECASE | wcmatch.RECURSIVE) for f in matcher.imatch(): return os.path.dirname(f) return local_path
def _find_modified_scripts(self) -> list: pex_paths: list = [] for object_name, script_path in self.ppj.psc_paths.items(): script_name, _ = os.path.splitext(os.path.basename(script_path)) # if pex exists, compare time_t in pex header with psc's last modified timestamp pex_match: list = [pex_path for pex_path in self.ppj.pex_paths if endswith(pex_path, f'{script_name}.pex', ignorecase=True)] if not pex_match: continue pex_path: str = pex_match[0] if not os.path.isfile(pex_path): continue try: header = PexReader.get_header(pex_path) except ValueError: BuildFacade.log.error(f'Cannot determine compilation time due to unknown magic: "{pex_path}"') sys.exit(1) psc_last_modified: float = os.path.getmtime(script_path) pex_last_compiled: float = float(header.compilation_time.value) # if psc is older than the pex if psc_last_modified < pex_last_compiled: pex_paths.append(pex_path) return PathHelper.uniqify(pex_paths)
def __setattr__(self, key: str, value: object) -> None: if value and isinstance(value, str): # sanitize paths if endswith(key, 'path', ignorecase=True) and os.altsep in value: value = os.path.normpath(value) if key in ('game_type', 'zip_compression'): value = value.casefold() super(ProjectOptions, self).__setattr__(key, value)
def find_script_paths_from_folder(folder_path: str, no_recurse: bool) -> Generator: """Yields existing script paths starting from absolute folder path""" search_path: str = os.path.join(folder_path, '*' if no_recurse else r'**\*') for script_path in glob.iglob(search_path, recursive=not no_recurse): if os.path.isfile(script_path) and endswith( script_path, '.psc', ignorecase=True): yield script_path
def create_packages(self) -> None: # clear temporary data if os.path.isdir(self.options.temp_path): shutil.rmtree(self.options.temp_path, ignore_errors=True) # ensure package path exists if not os.path.isdir(self.options.package_path): os.makedirs(self.options.package_path, exist_ok=True) file_names = CaseInsensitiveList() for i, package_node in enumerate( filter(is_package_node, self.ppj.packages_node)): file_name: str = package_node.get('Name') # prevent clobbering files previously created in this session if file_name in file_names: file_name = f'{self.ppj.project_name} ({i})' if file_name not in file_names: file_names.append(file_name) file_name = self._fix_package_extension(file_name) file_path: str = os.path.join(self.options.package_path, file_name) self._check_write_permission(file_path) PackageManager.log.info(f'Creating "{file_name}"...') for source_path in self._generate_include_paths( package_node, package_node.get('RootDir')): PackageManager.log.info(f'+ "{source_path}"') relpath = os.path.relpath(source_path, package_node.get('RootDir')) target_path = os.path.join(self.options.temp_path, relpath) # fix target path if user passes a deeper package root (RootDir) if endswith(source_path, '.pex', ignorecase=True) and not startswith( relpath, 'scripts', ignorecase=True): target_path = os.path.join(self.options.temp_path, 'Scripts', relpath) os.makedirs(os.path.dirname(target_path), exist_ok=True) shutil.copy2(source_path, target_path) # run bsarch command: str = self.build_commands(self.options.temp_path, file_path) ProcessManager.run_bsarch(command) # clear temporary data if os.path.isdir(self.options.temp_path): shutil.rmtree(self.options.temp_path, ignore_errors=True)
def __init__(self, options: ProjectOptions) -> None: self.options = options self.program_path = os.path.dirname(__file__) if endswith(sys.argv[0], ('pyro', '.exe')): self.program_path = os.path.abspath( os.path.join(self.program_path, os.pardir)) self.project_name = os.path.splitext( os.path.basename(self.options.input_path))[0] self.project_path = os.path.dirname(self.options.input_path)
def get_flags_path(self) -> str: """ Returns absolute flags path or flags file name from arguments or game path Used by: BuildFacade """ if self.options.flags_path: if endswith(self.options.flags_path, tuple(self.flag_types.values()), ignorecase=True): return self.options.flags_path if os.path.isabs(self.options.flags_path): return self.options.flags_path return os.path.join(self.project_path, self.options.flags_path) if endswith(self.options.game_path, self.game_names[GameType.FO4], ignorecase=True): return self.flag_types[GameType.FO4] return self.flag_types[GameType.TES5]
def fetch_contents(self, url: str, output_path: str) -> Generator: """ Downloads files from URL to output path """ owner, repo, request_url = self.extract_request_args(url) request = Request(request_url) request.add_header('Authorization', f'token {self.access_token}') response = urlopen(request, timeout=30) if response.status != 200: yield 'Failed to load URL (%s): "%s"' % (response.status, request_url) return payload_objects: list = json.loads(response.read().decode('utf-8')) yield f'Downloading scripts from "{request_url}"... Please wait.' script_count: int = 0 for payload_object in payload_objects: target_path = os.path.normpath( os.path.join(output_path, owner, repo, payload_object['path'])) download_url = payload_object['download_url'] # handle folders if not download_url: yield from self.fetch_contents(payload_object['url'], output_path) continue # we only care about scripts if not endswith(download_url, '.psc', ignorecase=True): continue file_response = urlopen(download_url, timeout=30) if file_response.status != 200: yield f'Failed to download ({file_response.status}): "{payload_object["download_url"]}"' continue os.makedirs(os.path.dirname(target_path), exist_ok=True) with open(target_path, mode='w+b') as f: f.write(file_response.read()) script_count += 1 if script_count > 0: yield f'Downloaded {script_count} scripts from "{request_url}"'
def __init__(self, ppj: PapyrusProject) -> None: self.ppj = ppj self.scripts_count = len(self.ppj.psc_paths) # WARN: if methods are renamed and their respective option names are not, this will break. options: dict = deepcopy(self.ppj.options.__dict__) for key in options: if key in ('args', 'input_path', 'anonymize', 'package', 'zip', 'zip_compression'): continue if startswith(key, ('ignore_', 'no_', 'force_', 'create_', 'resolve_'), ignorecase=True): continue if endswith(key, '_token', ignorecase=True): continue setattr(self.ppj.options, key, getattr(self.ppj, f'get_{key}')())
def anonymize_script(path: str) -> None: """ Obfuscates script path, user name, and computer name in compiled script """ try: header: PexHeader = PexReader.get_header(path) except ValueError: Anonymizer.log.error(f'Cannot anonymize script due to unknown file magic: "{path}"') sys.exit(1) file_path: str = header.script_path.value user_name: str = header.user_name.value computer_name: str = header.computer_name.value if '.' not in file_path: Anonymizer.log.warning(f'Cannot anonymize script again: "{path}"') return if not endswith(file_path, '.psc', ignorecase=True): Anonymizer.log.error(f'Cannot anonymize script due to invalid file extension: "{path}"') sys.exit(1) if not len(file_path) > 0: Anonymizer.log.error(f'Cannot anonymize script due to zero-length file path: "{path}"') sys.exit(1) if not len(user_name) > 0: Anonymizer.log.error(f'Cannot anonymize script due to zero-length user name: "{path}"') sys.exit(1) if not len(computer_name) > 0: Anonymizer.log.error(f'Cannot anonymize script due to zero-length computer name: "{path}"') sys.exit(1) with open(path, mode='r+b') as f: f.seek(header.script_path.offset, os.SEEK_SET) f.write(bytes(Anonymizer._randomize_str(header.script_path_size.value), encoding='ascii')) f.seek(header.user_name.offset, os.SEEK_SET) f.write(bytes(Anonymizer._randomize_str(header.user_name_size.value), encoding='ascii')) f.seek(header.computer_name.offset, os.SEEK_SET) f.write(bytes(Anonymizer._randomize_str(header.computer_name_size.value, True), encoding='ascii')) Anonymizer.log.info(f'Anonymized "{path}"...')
def fetch_contents(self, url: str, output_path: str) -> Generator: """ Downloads files from URL to output path """ owner, repo, request_url = self.extract_request_args(url) yield f'Downloading scripts from "{request_url}"... Please wait.' script_count: int = 0 for payload in self._fetch_payloads(request_url): for payload_object in payload['values']: payload_object_type = payload_object['type'] target_path = os.path.normpath( os.path.join(output_path, owner, repo, payload_object['path'])) download_url = payload_object['links']['self']['href'] if payload_object_type == 'commit_file': # we only care about scripts if not endswith(download_url, '.psc', ignorecase=True): continue file_response = urlopen(download_url, timeout=30) if file_response.status != 200: yield f'Failed to download ({file_response.status}): "{download_url}"' continue os.makedirs(os.path.dirname(target_path), exist_ok=True) with open(target_path, mode='w+b') as f: f.write(file_response.read()) script_count += 1 elif payload_object_type == 'commit_directory': yield from self.fetch_contents(download_url, output_path) if script_count > 0: yield f'Downloaded {script_count} scripts from "{request_url}"'
def create_packages(self) -> None: # clear temporary data if os.path.isdir(self.options.temp_path): shutil.rmtree(self.options.temp_path, ignore_errors=True) # ensure package path exists if not os.path.isdir(self.options.package_path): os.makedirs(self.options.package_path, exist_ok=True) file_names = CaseInsensitiveList() for i, package_node in enumerate( filter(is_package_node, self.ppj.packages_node)): attr_file_name: str = package_node.get(XmlAttributeName.NAME) # noinspection PyProtectedMember root_dir: str = self.ppj._get_path( package_node.get(XmlAttributeName.ROOT_DIR), relative_root_path=self.ppj.project_path, fallback_path=[ self.ppj.project_path, os.path.basename(attr_file_name) ]) # prevent clobbering files previously created in this session if attr_file_name in file_names: attr_file_name = f'{self.ppj.project_name} ({i})' if attr_file_name not in file_names: file_names.append(attr_file_name) attr_file_name = self._fix_package_extension(attr_file_name) file_path: str = os.path.join(self.options.package_path, attr_file_name) self._check_write_permission(file_path) PackageManager.log.info(f'Creating "{attr_file_name}"...') for source_path, attr_path in self._generate_include_paths( package_node, root_dir): if os.path.isabs(source_path): relpath: str = os.path.relpath(source_path, root_dir) else: relpath: str = source_path source_path = os.path.join(self.ppj.project_path, source_path) adj_relpath = os.path.normpath(os.path.join( attr_path, relpath)) PackageManager.log.info(f'+ "{adj_relpath.casefold()}"') target_path: str = os.path.join(self.options.temp_path, adj_relpath) # fix target path if user passes a deeper package root (RootDir) if endswith(source_path, '.pex', ignorecase=True) and not startswith( relpath, 'scripts', ignorecase=True): target_path = os.path.join(self.options.temp_path, 'Scripts', relpath) os.makedirs(os.path.dirname(target_path), exist_ok=True) shutil.copy2(source_path, target_path) self.includes += 1 # run bsarch command: str = self.build_commands(self.options.temp_path, file_path) ProcessManager.run_bsarch(command) # clear temporary data if os.path.isdir(self.options.temp_path): shutil.rmtree(self.options.temp_path, ignore_errors=True)
def _fix_zip_extension(self, zip_name: str) -> str: if not endswith(zip_name, '.zip', ignorecase=True): return f'{zip_name}{self.zip_extension}' return f'{os.path.splitext(zip_name)[0]}{self.zip_extension}'
def fetch_contents(self, url: str, output_path: str) -> Generator: """ Downloads files from URL to output path """ request_url = self.extract_request_args(url) request = Request(request_url.url) request.add_header('Authorization', f'token {self.access_token}') try: response = urlopen(request, timeout=30) except urllib.error.HTTPError as e: status: HTTPStatus = HTTPStatus(e.code) yield 'Failed to load remote: "%s" (%s %s)' % ( request_url.url, e.code, status.phrase) sys.exit(1) if response.status != 200: status: HTTPStatus = HTTPStatus(response.status) yield 'Failed to load remote: "%s" (%s %s)' % ( request_url.url, response.status, status.phrase) sys.exit(1) payload_objects: Union[dict, list] = json.loads( response.read().decode('utf-8')) if 'contents_url' in payload_objects: branch = payload_objects['default_branch'] contents_url = payload_objects['contents_url'].replace( '{+path}', f'?ref={branch}') yield from self.fetch_contents(contents_url, output_path) return scripts: list = [] for payload_object in payload_objects: target_path = os.path.normpath( os.path.join(output_path, request_url.owner, request_url.repo, payload_object['path'])) if not self.force_overwrite and os.path.isfile(target_path): with open(target_path, mode='rb') as f: data = f.read() sha1 = hashlib.sha1(b'blob %s\x00%s' % (len(data), data.decode())) if sha1.hexdigest() == payload_object['sha']: continue download_url = payload_object['download_url'] # handle folders if not download_url: yield from self.fetch_contents(payload_object['url'], output_path) continue # we only care about scripts and flags files if not (payload_object['type'] == 'file' and endswith( payload_object['name'], ('.flg', '.psc'), ignorecase=True)): continue scripts.append((download_url, target_path)) script_count: int = len(scripts) if script_count == 0: return multiprocessing.freeze_support() worker_limit: int = min(script_count, self.worker_limit) with multiprocessing.Pool(processes=worker_limit) as pool: for download_result in pool.imap_unordered(self.download_file, scripts): yield download_result pool.close() pool.join() if script_count > 0: yield f'Downloaded {script_count} scripts from "{request_url.url}"'
def _fix_package_extension(self, package_name: str) -> str: if not endswith(package_name, ('.ba2', '.bsa'), ignorecase=True): return f'{package_name}{self.pak_extension}' return f'{os.path.splitext(package_name)[0]}{self.pak_extension}'
def get_game_type(self) -> GameType: """Returns game type from arguments or Papyrus Project""" if isinstance(self.options.game_type, str) and self.options.game_type: if GameType.has_member(self.options.game_type): return GameType[self.options.game_type] if self.options.game_path: if endswith(self.options.game_path, self.game_names[GameType.FO4], ignorecase=True): ProjectBase.log.warning( f'Using game type: {self.game_names[GameType.FO4]} (determined from game path)' ) return GameType.FO4 if endswith(self.options.game_path, self.game_names[GameType.SSE], ignorecase=True): ProjectBase.log.warning( f'Using game type: {self.game_names[GameType.SSE]} (determined from game path)' ) return GameType.SSE if endswith(self.options.game_path, self.game_names[GameType.TES5], ignorecase=True): ProjectBase.log.warning( f'Using game type: {self.game_names[GameType.TES5]} (determined from game path)' ) return GameType.TES5 if self.options.registry_path: game_type = self._get_game_type_from_path( self.options.registry_path) ProjectBase.log.warning( f'Using game type: {self.game_names[game_type]} (determined from registry path)' ) return game_type if self.import_paths: for import_path in reversed(self.import_paths): game_type = self._get_game_type_from_path(import_path) if game_type: ProjectBase.log.warning( f'Using game type: {self.game_names[game_type]} (determined from import paths)' ) return game_type if self.options.flags_path: if endswith(self.options.flags_path, self.flag_types[GameType.FO4], ignorecase=True): ProjectBase.log.warning( f'Using game type: {self.game_names[GameType.FO4]} (determined from flags path)' ) return GameType.FO4 if endswith(self.options.flags_path, self.flag_types[GameType.TES5], ignorecase=True): try: self.get_game_path(GameType.SSE) except FileNotFoundError: ProjectBase.log.warning( f'Using game type: {self.game_names[GameType.TES5]} (determined from flags path)' ) return GameType.TES5 else: ProjectBase.log.warning( f'Using game type: {self.game_names[GameType.SSE]} (determined from flags path)' ) return GameType.SSE raise AssertionError( 'Cannot return game type from arguments or Papyrus Project')