def _get_script_paths_from_scripts_node(self) -> typing.Generator: """Returns script paths from the Scripts node""" for script_node in filter(is_script_node, self.scripts_node): self.try_fix_namespace_path(script_node) script_path: str = script_node.text if script_path == os.pardir or script_path == os.curdir: PapyrusProject.log.error( f'Script path at line {script_node.sourceline} in project file is not a file path' ) sys.exit(1) # handle . and .. in path if startswith(script_path, os.pardir): script_path = script_path.replace( os.pardir, os.path.normpath(os.path.join(self.project_path, os.pardir)), 1) elif startswith(script_path, os.curdir): script_path = script_path.replace(os.curdir, self.project_path, 1) if os.path.isdir(script_path): PapyrusProject.log.error( f'Script path at line {script_node.sourceline} in project file is not a file path' ) sys.exit(1) yield os.path.normpath(script_path)
def url2pathname(url_path: str) -> str: """Returns normalized unquoted path from URL""" url = urlparse(url_path) netloc: str = url.netloc path: str = url.path if netloc and startswith(netloc, '/'): netloc = netloc[1:] if path and startswith(path, '/'): path = path[1:] return os.path.normpath(unquote_plus(os.path.join(netloc, path)))
def _get_remote_path(self, node: etree.ElementBase) -> str: import_path: str = node.text url_hash = hashlib.sha1(import_path.encode()).hexdigest()[:8] temp_path = os.path.join(self.options.remote_temp_path, url_hash) if self.options.force_overwrite or not os.path.isdir(temp_path): try: for message in self.remote.fetch_contents( import_path, temp_path): if message: if not startswith(message, 'Failed to load'): PapyrusProject.log.info(message) else: PapyrusProject.log.error(message) sys.exit(1) except PermissionError as e: PapyrusProject.log.error(e) sys.exit(1) if endswith(import_path, '.git', ignorecase=True): url_path = self.remote.create_local_path(import_path[:-4]) else: url_path = self.remote.create_local_path(import_path) local_path = os.path.join(temp_path, url_path) matcher = wcmatch.WcMatch(local_path, '*.psc', flags=wcmatch.IGNORECASE | wcmatch.RECURSIVE) for f in matcher.imatch(): return os.path.dirname(f) return local_path
def _can_remove_folder(import_path: str, object_name: str, script_path: str) -> bool: import_path = import_path.casefold() object_name = object_name.casefold() script_path = script_path.casefold() return startswith(script_path, import_path) and os.path.join( import_path, object_name) != script_path
def create_packages(self) -> None: # clear temporary data if os.path.isdir(self.options.temp_path): shutil.rmtree(self.options.temp_path, ignore_errors=True) # ensure package path exists if not os.path.isdir(self.options.package_path): os.makedirs(self.options.package_path, exist_ok=True) file_names = CaseInsensitiveList() for i, package_node in enumerate( filter(is_package_node, self.ppj.packages_node)): file_name: str = package_node.get('Name') # prevent clobbering files previously created in this session if file_name in file_names: file_name = f'{self.ppj.project_name} ({i})' if file_name not in file_names: file_names.append(file_name) file_name = self._fix_package_extension(file_name) file_path: str = os.path.join(self.options.package_path, file_name) self._check_write_permission(file_path) PackageManager.log.info(f'Creating "{file_name}"...') for source_path in self._generate_include_paths( package_node, package_node.get('RootDir')): PackageManager.log.info(f'+ "{source_path}"') relpath = os.path.relpath(source_path, package_node.get('RootDir')) target_path = os.path.join(self.options.temp_path, relpath) # fix target path if user passes a deeper package root (RootDir) if endswith(source_path, '.pex', ignorecase=True) and not startswith( relpath, 'scripts', ignorecase=True): target_path = os.path.join(self.options.temp_path, 'Scripts', relpath) os.makedirs(os.path.dirname(target_path), exist_ok=True) shutil.copy2(source_path, target_path) # run bsarch command: str = self.build_commands(self.options.temp_path, file_path) ProcessManager.run_bsarch(command) # clear temporary data if os.path.isdir(self.options.temp_path): shutil.rmtree(self.options.temp_path, ignore_errors=True)
def remote_paths(self) -> list: """ Collects list of remote paths from Import and Folder nodes """ results: list = [] if self.has_imports_node: results.extend([ node.text for node in filter(is_import_node, self.imports_node) if startswith(node.text, self.remote_schemas, ignorecase=True) ]) if self.has_folders_node: results.extend([ node.text for node in filter(is_folder_node, self.folders_node) if startswith(node.text, self.remote_schemas, ignorecase=True) ]) return results
def _get_import_paths(self) -> list: """Returns absolute import paths from Papyrus Project""" results: list = [] if self.imports_node is None: return [] for import_node in filter(is_import_node, self.imports_node): import_path: str = import_node.text if startswith(import_path, self.remote_schemas, ignorecase=True): local_path = self._get_remote_path(import_node) PapyrusProject.log.info( f'Adding import path from remote: "{local_path}"...') results.append(local_path) continue if import_path == os.pardir or startswith(import_path, os.pardir): import_path = import_path.replace( os.pardir, os.path.normpath(os.path.join(self.project_path, os.pardir)), 1) elif import_path == os.curdir or startswith( import_path, os.curdir): import_path = import_path.replace(os.curdir, self.project_path, 1) # relative import paths should be relative to the project if not os.path.isabs(import_path): import_path = os.path.join(self.project_path, import_path) import_path = os.path.normpath(import_path) if os.path.isdir(import_path): results.append(import_path) else: PapyrusProject.log.error( f'Import path does not exist: "{import_path}"') sys.exit(1) return PathHelper.uniqify(results)
def get_registry_path(self, game_type: GameType = None) -> str: """Returns path to game installed path in Windows Registry from game type""" if not self.options.registry_path: game_type = self.options.game_type if not game_type else game_type if game_type in self.game_names: game_name = self.game_names[game_type] else: raise KeyError('Cannot determine registry path from game type') if startswith(game_name, 'Fallout'): game_name = game_name.replace(' ', '') return rf'HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\Bethesda Softworks\\{game_name}\Installed Path' return self.options.registry_path.replace('/', '\\')
def _get_script_paths_from_folders_node(self) -> typing.Generator: """Returns script paths from the Folders element array""" for folder_node in filter(is_folder_node, self.folders_node): if folder_node.text == os.pardir: self.log.warning( f'Folder paths cannot be equal to "{os.pardir}"') continue no_recurse: bool = folder_node.get('NoRecurse') == 'True' # try to add project path if folder_node.text == os.curdir: yield from PathHelper.find_script_paths_from_folder( self.project_path, no_recurse) continue if startswith(folder_node.text, self.remote_schemas, ignorecase=True): local_path = self._get_remote_path(folder_node) PapyrusProject.log.info( f'Adding import path from remote: "{local_path}"...') self.import_paths.insert(0, local_path) PapyrusProject.log.info( f'Adding folder path from remote: "{local_path}"...') yield from PathHelper.find_script_paths_from_folder( local_path, no_recurse) continue folder_path: str = os.path.normpath(folder_node.text) # try to add absolute path if os.path.isabs(folder_path) and os.path.isdir(folder_path): yield from PathHelper.find_script_paths_from_folder( folder_path, no_recurse) continue # try to add project-relative folder path test_path = os.path.join(self.project_path, folder_path) if os.path.isdir(test_path): yield from PathHelper.find_script_paths_from_folder( test_path, no_recurse) continue # try to add import-relative folder path for import_path in self.import_paths: test_path = os.path.join(import_path, folder_path) if os.path.isdir(test_path): yield from PathHelper.find_script_paths_from_folder( test_path, no_recurse)
def _get_path(path: str, *, relative_root_path: str, fallback_path: Union[str, List]) -> str: """ Returns absolute path from path or fallback path if path empty or unset :param path: A relative or absolute path :param relative_root_path: Absolute path to directory to join with relative path :param fallback_path: Absolute path to return if path empty or unset """ if path or startswith(path, (os.curdir, os.pardir)): return path if os.path.isabs(path) else os.path.normpath( os.path.join(relative_root_path, path)) if isinstance(fallback_path, list): return os.path.abspath(os.path.join(*fallback_path)) return fallback_path
def __init__(self, ppj: PapyrusProject) -> None: self.ppj = ppj self.scripts_count = len(self.ppj.psc_paths) # WARN: if methods are renamed and their respective option names are not, this will break. options: dict = deepcopy(self.ppj.options.__dict__) for key in options: if key in ('args', 'input_path', 'anonymize', 'package', 'zip', 'zip_compression'): continue if startswith(key, ('ignore_', 'no_', 'force_', 'create_', 'resolve_'), ignorecase=True): continue if endswith(key, '_token', ignorecase=True): continue setattr(self.ppj.options, key, getattr(self.ppj, f'get_{key}')())
def _try_fix_input_path(self, input_path: str) -> str: if not input_path: Application.log.error('required argument missing: -i INPUT.ppj') self._print_help_and_exit() if startswith(input_path, 'file:', ignorecase=True): full_path = PathHelper.url2pathname(input_path) input_path = os.path.normpath(full_path) if not os.path.isabs(input_path): cwd = os.getcwd() Application.log.warning(f'Using working directory: "{cwd}"') input_path = os.path.join(cwd, input_path) Application.log.warning(f'Using input path: "{input_path}"') return input_path
def calculate_relative_object_name(script_path: str, import_paths: list) -> str: """Returns import-relative path from absolute path (should be used only for Fallout 4 paths)""" # reverse the list to find the best import path file_name = os.path.basename(script_path) for import_path in reversed(PathHelper.uniqify(import_paths)): if not os.path.isabs(import_path): import_path = os.path.join(os.getcwd(), import_path) import_path = os.path.normpath(import_path) if len(script_path) > len(import_path): if startswith(script_path, import_path, ignorecase=True): file_name = script_path[len(import_path):] if file_name[0] == '\\' or file_name[0] == '/': file_name = file_name[1:] break return file_name
def get_flags_path(self) -> str: """ Returns absolute flags path or flags file name from arguments or game path Used by: BuildFacade """ if self.options.flags_path: if startswith(self.options.flags_path, tuple(FlagsName.values()), ignorecase=True): return self.options.flags_path if os.path.isabs(self.options.flags_path): return self.options.flags_path return os.path.join(self.project_path, self.options.flags_path) if self.options.game_path: if endswith(self.options.game_path, GameName.FO4, ignorecase=True): return FlagsName.FO4 return FlagsName.TES5
def _get_import_paths(self) -> list: """Returns absolute import paths from Papyrus Project""" results: list = [] if not self.has_imports_node: return [] for import_node in filter(is_import_node, self.imports_node): if startswith(import_node.text, self.remote_schemas, ignorecase=True): local_path = self._get_remote_path(import_node) PapyrusProject.log.info( f'Adding import path from remote: "{local_path}"...') results.append(local_path) continue import_path = os.path.normpath(import_node.text) if import_path == os.pardir: self.log.warning( f'Import paths cannot be equal to "{os.pardir}"') continue if import_path == os.curdir: import_path = self.project_path elif not os.path.isabs(import_path): # relative import paths should be relative to the project import_path = os.path.normpath( os.path.join(self.project_path, import_path)) if os.path.isdir(import_path): results.append(import_path) else: self.log.error(f'Import path does not exist: "{import_path}"') sys.exit(1) return PathHelper.uniqify(results)
def _generate_include_paths(includes_node: etree.ElementBase, root_path: str, zip_mode: bool = False) -> typing.Generator: for include_node in filter(is_include_node, includes_node): attr_no_recurse: bool = include_node.get( XmlAttributeName.NO_RECURSE) == 'True' attr_path: str = include_node.get(XmlAttributeName.PATH).strip() search_path: str = include_node.text if not search_path: PackageManager.log.error( f'Include path at line {include_node.sourceline} in project file is empty' ) sys.exit(1) if not zip_mode and startswith(search_path, os.pardir): PackageManager.log.error( f'Include paths cannot start with "{os.pardir}"') sys.exit(1) if startswith(search_path, os.curdir): search_path = search_path.replace(os.curdir, root_path, 1) # fix invalid pattern with leading separator if not zip_mode and startswith(search_path, (os.path.sep, os.path.altsep)): search_path = '**' + search_path if '\\' in search_path: search_path = search_path.replace('\\', '/') # populate files list using glob patterns or relative paths if '*' in search_path: for include_path in glob.iglob( search_path, root_dir=root_path, flags=PackageManager.DEFAULT_GLFLAGS | glob.GLOBSTAR if not attr_no_recurse else 0x0): yield os.path.join(root_path, include_path), attr_path elif not os.path.isabs(search_path): test_path = os.path.normpath( os.path.join(root_path, search_path)) if os.path.isfile(test_path): yield test_path, attr_path elif os.path.isdir(test_path): yield from PackageManager._match( test_path, '*.*', user_path=attr_path, no_recurse=attr_no_recurse) else: for include_path in glob.iglob( search_path, root_dir=root_path, flags=PackageManager.DEFAULT_GLFLAGS | glob.GLOBSTAR if not attr_no_recurse else 0x0): yield os.path.join(root_path, include_path), attr_path # populate files list using absolute paths else: if not zip_mode and root_path not in search_path: PackageManager.log.error( f'Cannot include path outside RootDir: "{search_path}"' ) sys.exit(1) search_path = os.path.abspath(os.path.normpath(search_path)) if os.path.isfile(search_path): yield search_path, attr_path else: yield from PackageManager._match( search_path, '*.*', user_path=attr_path, no_recurse=attr_no_recurse) for match_node in filter(is_match_node, includes_node): attr_in: str = match_node.get(XmlAttributeName.IN).strip() attr_no_recurse: bool = match_node.get( XmlAttributeName.NO_RECURSE) == 'True' attr_exclude: str = match_node.get( XmlAttributeName.EXCLUDE).strip() attr_path: str = match_node.get(XmlAttributeName.PATH).strip() in_path: str = os.path.normpath(attr_in) if in_path == os.pardir or startswith(in_path, os.pardir): in_path = in_path.replace( os.pardir, os.path.normpath(os.path.join(root_path, os.pardir)), 1) elif in_path == os.curdir or startswith(in_path, os.curdir): in_path = in_path.replace(os.curdir, root_path, 1) if not os.path.isabs(in_path): in_path = os.path.join(root_path, in_path) elif zip_mode and root_path not in in_path: PackageManager.log.error( f'Cannot match path outside RootDir: "{in_path}"') sys.exit(1) if not os.path.isdir(in_path): PackageManager.log.error( f'Cannot match path that does not exist or is not a directory: "{in_path}"' ) sys.exit(1) match_text: str = match_node.text if startswith(match_text, '.'): PackageManager.log.error( f'Match pattern at line {match_node.sourceline} in project file is not a valid wildcard pattern' ) sys.exit(1) yield from PackageManager._match(in_path, match_text, exclude_pattern=attr_exclude, user_path=attr_path, no_recurse=attr_no_recurse)
def create_packages(self) -> None: # clear temporary data if os.path.isdir(self.options.temp_path): shutil.rmtree(self.options.temp_path, ignore_errors=True) # ensure package path exists if not os.path.isdir(self.options.package_path): os.makedirs(self.options.package_path, exist_ok=True) file_names = CaseInsensitiveList() for i, package_node in enumerate( filter(is_package_node, self.ppj.packages_node)): attr_file_name: str = package_node.get(XmlAttributeName.NAME) # noinspection PyProtectedMember root_dir: str = self.ppj._get_path( package_node.get(XmlAttributeName.ROOT_DIR), relative_root_path=self.ppj.project_path, fallback_path=[ self.ppj.project_path, os.path.basename(attr_file_name) ]) # prevent clobbering files previously created in this session if attr_file_name in file_names: attr_file_name = f'{self.ppj.project_name} ({i})' if attr_file_name not in file_names: file_names.append(attr_file_name) attr_file_name = self._fix_package_extension(attr_file_name) file_path: str = os.path.join(self.options.package_path, attr_file_name) self._check_write_permission(file_path) PackageManager.log.info(f'Creating "{attr_file_name}"...') for source_path, attr_path in self._generate_include_paths( package_node, root_dir): if os.path.isabs(source_path): relpath: str = os.path.relpath(source_path, root_dir) else: relpath: str = source_path source_path = os.path.join(self.ppj.project_path, source_path) adj_relpath = os.path.normpath(os.path.join( attr_path, relpath)) PackageManager.log.info(f'+ "{adj_relpath.casefold()}"') target_path: str = os.path.join(self.options.temp_path, adj_relpath) # fix target path if user passes a deeper package root (RootDir) if endswith(source_path, '.pex', ignorecase=True) and not startswith( relpath, 'scripts', ignorecase=True): target_path = os.path.join(self.options.temp_path, 'Scripts', relpath) os.makedirs(os.path.dirname(target_path), exist_ok=True) shutil.copy2(source_path, target_path) self.includes += 1 # run bsarch command: str = self.build_commands(self.options.temp_path, file_path) ProcessManager.run_bsarch(command) # clear temporary data if os.path.isdir(self.options.temp_path): shutil.rmtree(self.options.temp_path, ignore_errors=True)
def run_compiler(command: str) -> ProcessState: """ Creates compiler process and logs output to console :param command: Command to execute, including absolute path to executable and its arguments :return: ProcessState (SUCCESS, FAILURE, INTERRUPTED, ERRORS) """ command_size = len(command) if command_size > 32768: ProcessManager.log.error(f'Cannot create process because command exceeds max length: {command_size}') return ProcessState.FAILURE try: process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) except WindowsError as e: ProcessManager.log.error(f'Cannot create process because: {e.strerror}') return ProcessState.FAILURE exclusions = ( 'Assembly', 'Batch', 'Compilation', 'Copyright', 'Failed', 'No output', 'Papyrus', 'Starting' ) line_error = re.compile(r'(.*)(\(-?\d*\.?\d+,-?\d*\.?\d+\)):\s+(.*)') error_count = 0 try: while process.poll() is None: line = process.stdout.readline().strip() if not line or startswith(line, exclusions): continue match = line_error.search(line) if match is not None: path, location, message = match.groups() head, tail = os.path.split(path) ProcessManager.log.error(f'COMPILATION FAILED: ' f'{os.path.basename(head)}\\{tail}{location}: {message}') process.terminate() error_count += 1 elif 'error(s)' not in line: ProcessManager.log.info(line) except KeyboardInterrupt: try: process.terminate() except OSError: ProcessManager.log.error('Process interrupted by user.') return ProcessState.INTERRUPTED return ProcessState.SUCCESS if error_count == 0 else ProcessState.ERRORS
def run_bsarch(command: str) -> ProcessState: """ Creates bsarch process and logs output to console :param command: Command to execute, including absolute path to executable and its arguments :return: ProcessState (SUCCESS, FAILURE, INTERRUPTED, ERRORS) """ try: process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) except WindowsError as e: ProcessManager.log.error(f'Cannot create process because: {e.strerror}') return ProcessState.FAILURE exclusions = ( '*', '[', 'Archive Flags', 'Bit', 'BSArch', 'Compressed', 'Done', 'Embed', 'Files', 'Format', 'Packer', 'Retain', 'Startup', 'Version', 'XBox', 'XMem' ) try: while process.poll() is None: line = process.stdout.readline().strip() if startswith(line, exclusions): continue if startswith(line, 'Packing'): package_path = line.split(':', 1)[1].strip() ProcessManager.log.info(f'Packaging folder "{package_path}"...') continue if startswith(line, 'Archive Name'): archive_path = line.split(':', 1)[1].strip() ProcessManager.log.info(f'Building "{archive_path}"...') continue if line: ProcessManager.log.info(line) except KeyboardInterrupt: try: process.terminate() except OSError: ProcessManager.log.error('Process interrupted by user.') return ProcessState.INTERRUPTED return ProcessState.SUCCESS
def extract_request_args(url: str) -> RemoteUri: """ Extracts (owner, repo, request_url) from URL """ result = RemoteUri() data = urlparse(url) # remove '.git' from clone url, if suffix exists result.data = data._asdict() result.data['path'] = result.data['path'].replace('.git', '') netloc, path, query = result.data['netloc'], result.data[ 'path'], result.data['query'] # store branch if specified query_params = query.split('&') for param in query_params: if startswith(param, 'ref', ignorecase=True): _, branch = param.split('=') result.branch = branch break url_path_parts = path.split('/') url_path_parts.pop( 0) if not url_path_parts[0] else None # pop empty space request_url = '' if netloc == 'github.com': if len(url_path_parts) == 2: request_url = 'https://api.github.com/repos{}'.format(path) elif 'tree/' in path: # example: /fireundubh/LibFire/tree/master result.branch = url_path_parts.pop( 3) # pop 'master' (or any other branch) url_path_parts.pop( 2) if url_path_parts[2] == 'tree' else None # pop 'tree' url_path_parts.insert(2, 'contents') url_path = '/'.join(url_path_parts) request_url = f'https://api.github.com/repos/{url_path}?ref={result.branch}' elif netloc == 'api.github.com': if startswith(path, '/repos'): url_path_parts.pop() if not url_path_parts[ len(url_path_parts) - 1] else None # pop empty space url_path_parts.pop() if url_path_parts[ len(url_path_parts) - 1] == 'contents' else None # pop 'contents' url_path_parts.pop(0) # pop 'repos' request_url = url elif netloc == 'bitbucket.org': request_url = 'https://api.bitbucket.org/2.0/repositories{}{}'.format( path, query) elif netloc == 'api.bitbucket.org': request_url = url else: raise NotImplementedError result.owner = url_path_parts[0] result.repo = url_path_parts[1] result.url = request_url return result
def _get_script_paths_from_folders_node(self) -> typing.Generator: """Returns script paths from the Folders element array""" for folder_node in filter(is_folder_node, self.folders_node): self.try_fix_namespace_path(folder_node) attr_no_recurse: bool = folder_node.get( XmlAttributeName.NO_RECURSE) == 'True' folder_path: str = folder_node.text # handle . and .. in path if folder_path == os.pardir or startswith(folder_path, os.pardir): folder_path = folder_path.replace( os.pardir, os.path.normpath(os.path.join(self.project_path, os.pardir)), 1) yield from PathHelper.find_script_paths_from_folder( folder_path, no_recurse=attr_no_recurse) continue if folder_path == os.curdir or startswith(folder_path, os.curdir): folder_path = folder_path.replace(os.curdir, self.project_path, 1) yield from PathHelper.find_script_paths_from_folder( folder_path, no_recurse=attr_no_recurse) continue if startswith(folder_path, self.remote_schemas, ignorecase=True): local_path = self._get_remote_path(folder_node) PapyrusProject.log.info( f'Adding import path from remote: "{local_path}"...') self.import_paths.insert(0, local_path) PapyrusProject.log.info( f'Adding folder path from remote: "{local_path}"...') yield from PathHelper.find_script_paths_from_folder( local_path, no_recurse=attr_no_recurse) continue folder_path = os.path.normpath(folder_path) # try to add absolute path if os.path.isabs(folder_path) and os.path.isdir(folder_path): yield from PathHelper.find_script_paths_from_folder( folder_path, no_recurse=attr_no_recurse) continue # try to add project-relative folder path test_path = os.path.join(self.project_path, folder_path) if os.path.isdir(test_path): # count scripts to avoid issue where an errant `test_path` may exist and contain no sources # this can be a problem if that folder contains sources but user error is hard to fix test_passed = False user_flags = wcmatch.RECURSIVE if not attr_no_recurse else 0x0 matcher = wcmatch.WcMatch(test_path, '*.psc', flags=wcmatch.IGNORECASE | user_flags) for _ in matcher.imatch(): test_passed = True break if test_passed: yield from PathHelper.find_script_paths_from_folder( test_path, no_recurse=attr_no_recurse, matcher=matcher) continue # try to add import-relative folder path for import_path in self.import_paths: test_path = os.path.join(import_path, folder_path) if os.path.isdir(test_path): yield from PathHelper.find_script_paths_from_folder( test_path, no_recurse=attr_no_recurse)