def _get_script_paths_from_folders_node(self) -> typing.Generator:
"""Returns script paths from the Folders element array"""
for folder_node in filter(is_folder_node, self.folders_node):
if folder_node.text == os.pardir:
self.log.warning(
f'Folder paths cannot be equal to "{os.pardir}"')
continue
no_recurse: bool = folder_node.get('NoRecurse') == 'True'
# try to add project path
if folder_node.text == os.curdir:
yield from PathHelper.find_script_paths_from_folder(
self.project_path, no_recurse)
continue
if startswith(folder_node.text,
self.remote_schemas,
ignorecase=True):
local_path = self._get_remote_path(folder_node)
PapyrusProject.log.info(
f'Adding import path from remote: "{local_path}"...')
self.import_paths.insert(0, local_path)
PapyrusProject.log.info(
f'Adding folder path from remote: "{local_path}"...')
yield from PathHelper.find_script_paths_from_folder(
local_path, no_recurse)
continue
folder_path: str = os.path.normpath(folder_node.text)
# try to add absolute path
if os.path.isabs(folder_path) and os.path.isdir(folder_path):
yield from PathHelper.find_script_paths_from_folder(
folder_path, no_recurse)
continue
# try to add project-relative folder path
test_path = os.path.join(self.project_path, folder_path)
if os.path.isdir(test_path):
yield from PathHelper.find_script_paths_from_folder(
test_path, no_recurse)
continue
# try to add import-relative folder path
for import_path in self.import_paths:
test_path = os.path.join(import_path, folder_path)
if os.path.isdir(test_path):
yield from PathHelper.find_script_paths_from_folder(
test_path, no_recurse)
def build_commands(self) -> list:
commands: list = []
arguments: CommandArguments = CommandArguments()
compiler_path: str = self.options.compiler_path
flags_path: str = self.options.flags_path
output_path: str = self.options.output_path
import_paths: str = ';'.join(self.import_paths)
if self.options.no_incremental_build:
psc_paths = PathHelper.uniqify(self.psc_paths)
else:
psc_paths = self._try_exclude_unmodified_scripts()
# add .psc scripts whose .pex counterparts do not exist
for missing_psc_path in self.missing_scripts:
if missing_psc_path not in psc_paths:
psc_paths.append(missing_psc_path)
# generate list of commands
for psc_path in psc_paths:
if self.options.game_type == 'fo4':
psc_path = PathHelper.calculate_relative_object_name(
psc_path, self.import_paths)
arguments.clear()
arguments.append_quoted(compiler_path)
arguments.append_quoted(psc_path)
arguments.append_quoted(output_path, 'o')
arguments.append_quoted(import_paths, 'i')
arguments.append_quoted(flags_path, 'f')
if self.options.game_type == 'fo4':
# noinspection PyUnboundLocalVariable
if self.release:
arguments.append('-release')
# noinspection PyUnboundLocalVariable
if self.final:
arguments.append('-final')
if self.optimize:
arguments.append('-op')
commands.append(arguments.join())
return commands
def _find_modified_scripts(self) -> list:
pex_paths: list = []
for object_name, script_path in self.ppj.psc_paths.items():
script_name, _ = os.path.splitext(os.path.basename(script_path))
# if pex exists, compare time_t in pex header with psc's last modified timestamp
pex_match: list = [pex_path for pex_path in self.ppj.pex_paths
if endswith(pex_path, f'{script_name}.pex', ignorecase=True)]
if not pex_match:
continue
pex_path: str = pex_match[0]
if not os.path.isfile(pex_path):
continue
try:
header = PexReader.get_header(pex_path)
except ValueError:
BuildFacade.log.error(f'Cannot determine compilation time due to unknown magic: "{pex_path}"')
sys.exit(1)
psc_last_modified: float = os.path.getmtime(script_path)
pex_last_compiled: float = float(header.compilation_time.value)
# if psc is older than the pex
if psc_last_modified < pex_last_compiled:
pex_paths.append(pex_path)
return PathHelper.uniqify(pex_paths)
def _find_modified_scripts(self) -> list:
pex_paths: list = []
for psc_path in self.ppj.psc_paths:
script_name, _ = os.path.splitext(os.path.basename(psc_path))
# if pex exists, compare time_t in pex header with psc's last modified timestamp
pex_match: list = [
pex_path for pex_path in self.ppj.pex_paths
if pex_path.endswith('%s.pex' % script_name)
]
if not pex_match:
continue
pex_path: str = pex_match[0]
if not os.path.exists(pex_path):
continue
try:
header = PexReader.get_header(pex_path)
except ValueError:
BuildFacade.log.warning(
'Cannot determine compilation time from compiled script due to unknown file magic: "%s"'
% pex_path)
continue
compiled_time: int = header.compilation_time.value
# if psc is older than the pex
if os.path.getmtime(psc_path) >= compiled_time:
pex_paths.append(pex_path)
return PathHelper.uniqify(pex_paths)
def _try_exclude_unmodified_scripts(self) -> list:
psc_paths: list = []
for psc_path in self.psc_paths:
script_name, _ = os.path.splitext(os.path.basename(psc_path))
# if pex exists, compare time_t in pex header with psc's last modified timestamp
matching_path: str = ''
for pex_path in self.pex_paths:
if pex_path.endswith('%s.pex' % script_name):
matching_path = pex_path
break
if not os.path.exists(matching_path):
continue
try:
header = PexReader.get_header(matching_path)
except ValueError:
PapyrusProject.log.warning(
'Cannot determine compilation time from compiled script due to unknown file magic: "%s"'
% matching_path)
continue
compiled_time: int = header.compilation_time.value
if os.path.getmtime(psc_path) < compiled_time:
continue
psc_paths.append(psc_path)
return PathHelper.uniqify(psc_paths)
def _get_implicit_folder_imports(self) -> list:
"""Returns absolute implicit import paths from Folder node paths"""
implicit_paths: list = []
folder_nodes = ElementHelper.get(self.root_node, 'Folders')
if folder_nodes is None:
return []
for folder_node in folder_nodes:
if not folder_node.tag.endswith('Folder'):
continue
if not folder_node.text:
continue
folder_path: str = os.path.normpath(self.parse(folder_node.text))
if os.path.isabs(folder_path):
if os.path.exists(folder_path):
implicit_paths.append(folder_path)
else:
test_path = os.path.join(self.project_path, folder_path)
if os.path.exists(test_path):
implicit_paths.append(test_path)
return PathHelper.uniqify(implicit_paths)
def _get_import_paths(self) -> list:
"""Returns absolute import paths from Papyrus Project"""
results: list = []
import_nodes: etree.ElementBase = ElementHelper.get(
self.root_node, 'Imports')
if import_nodes is None:
return []
for import_node in import_nodes:
if not import_node.tag.endswith('Import'):
continue
if not import_node.text:
continue
import_path: str = os.path.normpath(self.parse(import_node.text))
if import_path == os.pardir:
self.log.warning('Import paths cannot be equal to ".."')
continue
if import_path == os.curdir:
import_path = self.project_path
elif not os.path.isabs(import_path):
# relative import paths should be relative to the project
import_path = os.path.normpath(
os.path.join(self.project_path, import_path))
if os.path.exists(import_path):
results.append(import_path)
return PathHelper.uniqify(results)
def _get_script_paths_from_folders_node(self) -> list:
"""Returns script paths from the Folders element array"""
folder_nodes = ElementHelper.get(self.root_node, 'Folders')
if folder_nodes is None:
return []
script_paths: list = []
folder_paths: list = []
for folder_node in folder_nodes:
if not folder_node.tag.endswith('Folder'):
continue
folder_text: str = self.parse(folder_node.text)
if folder_text == os.pardir:
self.log.warning('Folder paths cannot be equal to ".."')
continue
no_recurse: bool = self._get_attr_as_bool(folder_node, 'NoRecurse')
# try to add project path
if folder_text == os.curdir:
folder_paths.append((self.project_path, no_recurse))
continue
folder_path: str = os.path.normpath(folder_text)
# try to add absolute path
if os.path.isabs(folder_path) and os.path.isdir(folder_path):
folder_paths.append((folder_path, no_recurse))
continue
# try to add project-relative folder path
test_path = os.path.join(self.project_path, folder_path)
if os.path.isdir(test_path):
folder_paths.append((test_path, no_recurse))
continue
# try to add import-relative folder path
for import_path in self.import_paths:
test_path = os.path.join(import_path, folder_path)
if os.path.isdir(test_path):
folder_paths.append((test_path, no_recurse))
continue
PapyrusProject.log.warning('Cannot resolve folder path: "%s"' %
folder_node.text)
for folder_path, no_recurse in folder_paths:
search_path: str = os.path.join(folder_path,
'*' if no_recurse else '**\*')
script_paths.extend([
f for f in glob.iglob(search_path, recursive=not no_recurse)
if os.path.isfile(f) and f.casefold().endswith('.psc')
])
return PathHelper.uniqify(script_paths)
def _find_missing_script_paths(self) -> list:
"""Returns list of script paths for compiled scripts that do not exist"""
results: list = []
for psc_path in self.psc_paths:
if self.options.game_type == 'fo4':
object_name = PathHelper.calculate_relative_object_name(
psc_path, self.import_paths)
else:
object_name = os.path.basename(psc_path)
pex_path: str = os.path.join(self.options.output_path,
object_name.replace('.psc', '.pex'))
if os.path.exists(pex_path):
continue
results.append(psc_path)
return PathHelper.uniqify(results)
def _get_pex_paths(self) -> list:
"""
Returns absolute paths to compiled scripts that may not exist yet in output folder
"""
pex_paths: list = []
for psc_path in self.psc_paths:
if self.options.game_type == 'fo4':
object_name = PathHelper.calculate_relative_object_name(
psc_path, self.import_paths)
else:
object_name = os.path.basename(psc_path)
pex_path = os.path.join(self.options.output_path,
object_name.replace('.psc', '.pex'))
pex_paths.append(pex_path)
return PathHelper.uniqify(pex_paths)
def try_populate_imports(self) -> None:
# we need to populate the list of import paths before we try to determine the game type
# because the game type can be determined from import paths
self.import_paths = self._get_import_paths()
if not self.import_paths:
PapyrusProject.log.error('Failed to build list of import paths')
sys.exit(1)
if not self.options.no_implicit_imports:
# ensure that folder paths are implicitly imported
implicit_folder_paths: list = self._get_implicit_folder_imports()
if len(implicit_folder_paths) > 0:
PapyrusProject.log.info(
'Implicitly imported folder paths found:')
for path in implicit_folder_paths:
PapyrusProject.log.info(f'+ "{path}"')
PathHelper.merge_implicit_import_paths(implicit_folder_paths,
self.import_paths)
# we need to populate psc paths after explicit and implicit import paths are populated
# this also needs to be set before we populate implicit import paths from psc paths
# not sure if this must run again after populating implicit import paths from psc paths
self.psc_paths = self._get_psc_paths()
if not self.psc_paths:
PapyrusProject.log.error('Failed to build list of script paths')
sys.exit(1)
if not self.options.no_implicit_imports:
# this adds implicit imports from script paths
implicit_script_paths: list = self._get_implicit_script_imports()
if len(implicit_script_paths) > 0:
PapyrusProject.log.info(
'Implicitly imported script paths found:')
for path in implicit_script_paths:
PapyrusProject.log.info(f'+ "{path}"')
PathHelper.merge_implicit_import_paths(implicit_script_paths,
self.import_paths)
def _get_implicit_script_imports(self) -> list:
"""Returns absolute implicit import paths from Script node paths"""
implicit_paths: list = []
for psc_path in self.psc_paths:
for import_path in self.import_paths:
relpath = os.path.relpath(os.path.dirname(psc_path),
import_path)
test_path = os.path.normpath(os.path.join(
import_path, relpath))
if os.path.exists(test_path):
implicit_paths.append(test_path)
return PathHelper.uniqify(implicit_paths)
def _get_psc_paths(self) -> list:
"""Returns script paths from Folders and Scripts nodes"""
paths: list = []
# try to populate paths with scripts from Folders and Scripts nodes
for tag in ('Folders', 'Scripts'):
node = ElementHelper.get(self.root_node, tag)
if node is None:
continue
node_paths = getattr(
self, '_get_script_paths_from_%s_node' % tag.casefold())()
PapyrusProject.log.info(
'%s script paths discovered from %s nodes.' %
(len(node_paths), tag[:-1]))
if node_paths:
paths.extend(node_paths)
results: list = []
# convert user paths to absolute paths
for path in paths:
# try to add existing absolute paths
if os.path.isabs(path) and os.path.exists(path):
results.append(path)
continue
# try to add existing project-relative paths
test_path = os.path.join(self.project_path, path)
if os.path.exists(test_path):
results.append(test_path)
continue
# try to add existing import-relative paths
for import_path in self.import_paths:
if not os.path.isabs(import_path):
import_path = os.path.join(self.project_path, import_path)
test_path = os.path.join(import_path, path)
if os.path.exists(test_path):
results.append(test_path)
break
results = PathHelper.uniqify(results)
PapyrusProject.log.info(
'%s unique script paths resolved to absolute paths.' %
len(results))
return results
def _get_implicit_folder_imports(self) -> list:
"""Returns absolute implicit import paths from Folder node paths"""
implicit_paths: list = []
if self.folders_node is None:
return []
try_append_path = lambda path: implicit_paths.append(path) \
if os.path.isdir(path) and path not in self.import_paths else None
for folder_node in filter(is_folder_node, self.folders_node):
folder_path: str = os.path.normpath(folder_node.text)
try_append_path(folder_path if os.path.isabs(folder_path) else os.
path.join(self.project_path, folder_path))
return PathHelper.uniqify(implicit_paths)
def _try_fix_input_path(self, input_path: str) -> str:
if not input_path:
Application.log.error('required argument missing: -i INPUT.ppj')
self._print_help_and_exit()
if startswith(input_path, 'file:', ignorecase=True):
full_path = PathHelper.url2pathname(input_path)
input_path = os.path.normpath(full_path)
if not os.path.isabs(input_path):
cwd = os.getcwd()
Application.log.warning(f'Using working directory: "{cwd}"')
input_path = os.path.join(cwd, input_path)
Application.log.warning(f'Using input path: "{input_path}"')
return input_path
def _get_script_paths_from_scripts_node(self) -> list:
"""Returns script paths from the Scripts node"""
paths: list = []
script_nodes = ElementHelper.get(self.root_node, 'Scripts')
if script_nodes is None:
return []
for script_node in script_nodes:
if not script_node.tag.endswith('Script'):
continue
psc_path: str = self.parse(script_node.text)
if ':' in psc_path:
psc_path = psc_path.replace(':', os.sep)
paths.append(os.path.normpath(psc_path))
return PathHelper.uniqify(paths)
def _get_import_paths(self) -> list:
"""Returns absolute import paths from Papyrus Project"""
results: list = []
if self.imports_node is None:
return []
for import_node in filter(is_import_node, self.imports_node):
import_path: str = import_node.text
if startswith(import_path, self.remote_schemas, ignorecase=True):
local_path = self._get_remote_path(import_node)
PapyrusProject.log.info(
f'Adding import path from remote: "{local_path}"...')
results.append(local_path)
continue
if import_path == os.pardir or startswith(import_path, os.pardir):
import_path = import_path.replace(
os.pardir,
os.path.normpath(os.path.join(self.project_path,
os.pardir)), 1)
elif import_path == os.curdir or startswith(
import_path, os.curdir):
import_path = import_path.replace(os.curdir, self.project_path,
1)
# relative import paths should be relative to the project
if not os.path.isabs(import_path):
import_path = os.path.join(self.project_path, import_path)
import_path = os.path.normpath(import_path)
if os.path.isdir(import_path):
results.append(import_path)
else:
PapyrusProject.log.error(
f'Import path does not exist: "{import_path}"')
sys.exit(1)
return PathHelper.uniqify(results)
def _get_implicit_folder_imports(self) -> list:
"""Returns absolute implicit import paths from Folder node paths"""
implicit_paths: list = []
if not self.has_folders_node:
return []
for folder_node in filter(is_folder_node, self.folders_node):
folder_path: str = os.path.normpath(folder_node.text)
if os.path.isabs(folder_path):
if os.path.isdir(
folder_path) and folder_path not in self.import_paths:
implicit_paths.append(folder_path)
else:
test_path = os.path.join(self.project_path, folder_path)
if os.path.isdir(
test_path) and test_path not in self.import_paths:
implicit_paths.append(test_path)
return PathHelper.uniqify(implicit_paths)
def _get_import_paths(self) -> list:
"""Returns absolute import paths from Papyrus Project"""
results: list = []
if not self.has_imports_node:
return []
for import_node in filter(is_import_node, self.imports_node):
if startswith(import_node.text,
self.remote_schemas,
ignorecase=True):
local_path = self._get_remote_path(import_node)
PapyrusProject.log.info(
f'Adding import path from remote: "{local_path}"...')
results.append(local_path)
continue
import_path = os.path.normpath(import_node.text)
if import_path == os.pardir:
self.log.warning(
f'Import paths cannot be equal to "{os.pardir}"')
continue
if import_path == os.curdir:
import_path = self.project_path
elif not os.path.isabs(import_path):
# relative import paths should be relative to the project
import_path = os.path.normpath(
os.path.join(self.project_path, import_path))
if os.path.isdir(import_path):
results.append(import_path)
else:
self.log.error(f'Import path does not exist: "{import_path}"')
sys.exit(1)
return PathHelper.uniqify(results)
def _get_implicit_script_imports(self) -> list:
"""Returns absolute implicit import paths from Script node paths"""
implicit_paths: list = []
for object_name, script_path in self.psc_paths.items():
script_folder_path = os.path.dirname(script_path)
for import_path in self.import_paths:
# TODO: figure out how to handle imports on different drives
try:
relpath = os.path.relpath(script_folder_path, import_path)
except ValueError as e:
PapyrusProject.log.warning(
f'{e} (path: "{script_folder_path}", start: "{import_path}")'
)
continue
test_path = os.path.normpath(os.path.join(
import_path, relpath))
if os.path.isdir(
test_path) and test_path not in self.import_paths:
implicit_paths.append(test_path)
return PathHelper.uniqify(implicit_paths)
def __init__(self, options: ProjectOptions) -> None:
super().__init__(options)
xml_parser: etree.XMLParser = etree.XMLParser(remove_blank_text=True,
remove_comments=True)
# strip comments from raw text because lxml.etree.XMLParser does not remove XML-unsupported comments
# e.g., '<PapyrusProject <!-- xmlns="PapyrusProject.xsd" -->>'
xml_document: io.StringIO = PapyrusProject._strip_xml_comments(
self.options.input_path)
project_xml: etree.ElementTree = etree.parse(xml_document, xml_parser)
self.root_node: etree.ElementBase = project_xml.getroot()
# TODO: validate earlier
schema: etree.XMLSchema = ElementHelper.validate_schema(
self.root_node, self.program_path)
if schema:
try:
if schema.assertValid(project_xml) is None:
PapyrusProject.log.info(
'Successfully validated XML Schema.')
except etree.DocumentInvalid as e:
PapyrusProject.log.error(
'Failed to validate XML Schema.%s\t%s' % (os.linesep, e))
sys.exit(1)
variables_node = ElementHelper.get(self.root_node, 'Variables')
if variables_node is not None:
for variable_node in variables_node:
if not variable_node.tag.endswith('Variable'):
continue
var_key = variable_node.get('Name', default='')
var_value = variable_node.get('Value', default='')
if any([not var_key, not var_value]):
continue
self.variables.update({var_key: var_value})
# we need to parse all PapyrusProject attributes after validating and before we do anything else
# options can be overridden by arguments when the BuildFacade is initialized
self.options.output_path = self.parse(
self.root_node.get('Output', default=self.options.output_path))
self.options.flags_path = self.parse(
self.root_node.get('Flags', default=self.options.flags_path))
self.optimize: bool = PapyrusProject._get_attr_as_bool(
self.root_node, 'Optimize')
self.release: bool = PapyrusProject._get_attr_as_bool(
self.root_node, 'Release')
self.final: bool = PapyrusProject._get_attr_as_bool(
self.root_node, 'Final')
self.options.anonymize = PapyrusProject._get_attr_as_bool(
self.root_node, 'Anonymize')
self.options.bsarch = PapyrusProject._get_attr_as_bool(
self.root_node, 'Package')
self.options.zip = PapyrusProject._get_attr_as_bool(
self.root_node, 'Zip')
self.packages_node = ElementHelper.get(self.root_node, 'Packages')
if self.options.bsarch:
if self.packages_node is None:
PapyrusProject.log.error(
'Package is enabled but the Packages node is undefined. Setting Package to false.'
)
self.options.bsarch = False
else:
self.options.package_path = self.parse(
self.packages_node.get('Output',
default=self.options.package_path))
self.zipfile_node = ElementHelper.get(self.root_node, 'ZipFile')
if self.options.zip:
if self.zipfile_node is None:
PapyrusProject.log.error(
'Zip is enabled but the ZipFile node is undefined. Setting Zip to false.'
)
self.options.zip = False
else:
self._setup_zipfile_options()
# we need to populate the list of import paths before we try to determine the game type
# because the game type can be determined from import paths
self.import_paths: list = self._get_import_paths()
if not self.import_paths:
PapyrusProject.log.error('Failed to build list of import paths')
sys.exit(1)
# ensure that folder paths are implicitly imported
implicit_folder_paths: list = self._get_implicit_folder_imports()
PathHelper.merge_implicit_import_paths(implicit_folder_paths,
self.import_paths)
for path in implicit_folder_paths:
if path in self.import_paths:
PapyrusProject.log.warning(
'Using import path implicitly: "%s"' % path)
self.psc_paths: list = self._get_psc_paths()
if not self.psc_paths:
PapyrusProject.log.error('Failed to build list of script paths')
sys.exit(1)
# this adds implicit imports from script paths
implicit_script_paths: list = self._get_implicit_script_imports()
PathHelper.merge_implicit_import_paths(implicit_script_paths,
self.import_paths)
for path in implicit_script_paths:
if path in self.import_paths:
PapyrusProject.log.warning(
'Using import path implicitly: "%s"' % path)
# we need to set the game type after imports are populated but before pex paths are populated
# allow xml to set game type but defer to passed argument
if not self.options.game_type:
game_type: str = self.parse(self.root_node.get(
'Game', default='')).casefold()
if game_type and game_type in self.game_types:
PapyrusProject.log.warning(
'Using game type: %s (determined from Papyrus Project)' %
self.game_types[game_type])
self.options.game_type = game_type
if not self.options.game_type:
self.options.game_type = self.get_game_type()
if not self.options.game_type:
PapyrusProject.log.error(
'Cannot determine game type from arguments or Papyrus Project')
sys.exit(1)
# get expected pex paths - these paths may not exist and that is okay!
self.pex_paths: list = self._get_pex_paths()
# these are relative paths to psc scripts whose pex counterparts are missing
self.missing_scripts: list = self._find_missing_script_paths()
# game type must be set before we call this
if not self.options.game_path:
self.options.game_path = self.get_game_path()
def __init__(self, options: ProjectOptions) -> None:
super(PapyrusProject, self).__init__(options)
xml_parser: etree.XMLParser = etree.XMLParser(remove_blank_text=True,
remove_comments=True)
# strip comments from raw text because lxml.etree.XMLParser does not remove XML-unsupported comments
# e.g., '<PapyrusProject <!-- xmlns="PapyrusProject.xsd" -->>'
xml_document: io.StringIO = XmlHelper.strip_xml_comments(
self.options.input_path)
project_xml: etree.ElementTree = etree.parse(xml_document, xml_parser)
self.ppj_root = XmlRoot(project_xml)
schema: etree.XMLSchema = XmlHelper.validate_schema(
self.ppj_root.ns, self.program_path)
if schema:
try:
schema.assertValid(project_xml)
except etree.DocumentInvalid as e:
PapyrusProject.log.error(
f'Failed to validate XML Schema.{os.linesep}\t{e}')
sys.exit(1)
else:
PapyrusProject.log.info('Successfully validated XML Schema.')
# variables need to be parsed before nodes are updated
variables_node = self.ppj_root.find('Variables')
if variables_node is not None:
self._parse_variables(variables_node)
# we need to parse all attributes after validating and before we do anything else
# options can be overridden by arguments when the BuildFacade is initialized
self._update_attributes(self.ppj_root.node)
if self.options.resolve_ppj:
xml_output = etree.tostring(self.ppj_root.node,
encoding='utf-8',
xml_declaration=True,
pretty_print=True)
PapyrusProject.log.debug(
f'Resolved PPJ. Text output:{os.linesep * 2}{xml_output.decode()}'
)
sys.exit(1)
self.options.flags_path = self.ppj_root.get('Flags')
self.options.output_path = self.ppj_root.get('Output')
self.optimize = self.ppj_root.get('Optimize') == 'True'
self.release = self.ppj_root.get('Release') == 'True'
self.final = self.ppj_root.get('Final') == 'True'
self.options.anonymize = self.ppj_root.get('Anonymize') == 'True'
self.options.package = self.ppj_root.get('Package') == 'True'
self.options.zip = self.ppj_root.get('Zip') == 'True'
self.imports_node = self.ppj_root.find('Imports')
self.has_imports_node = self.imports_node is not None
self.scripts_node = self.ppj_root.find('Scripts')
self.has_scripts_node = self.scripts_node is not None
self.folders_node = self.ppj_root.find('Folders')
self.has_folders_node = self.folders_node is not None
self.packages_node = self.ppj_root.find('Packages')
self.has_packages_node = self.packages_node is not None
self.zip_files_node = self.ppj_root.find('ZipFiles')
self.has_zip_files_node = self.zip_files_node is not None
self.pre_build_node = self.ppj_root.find('PreBuildEvent')
self.has_pre_build_node = self.pre_build_node is not None
self.post_build_node = self.ppj_root.find('PostBuildEvent')
self.has_post_build_node = self.post_build_node is not None
if self.options.package and self.has_packages_node:
if not self.options.package_path:
self.options.package_path = self.packages_node.get('Output')
if self.options.zip and self.has_zip_files_node:
if not self.options.zip_output_path:
self.options.zip_output_path = self.zip_files_node.get(
'Output')
# initialize remote if needed
if self.remote_paths:
if not self.options.remote_temp_path:
self.options.remote_temp_path = self.get_remote_temp_path()
self.remote = GenericRemote(self.options)
# validate remote paths
for path in self.remote_paths:
if not self.remote.validate_url(path):
PapyrusProject.log.error(
f'Cannot proceed while node contains invalid URL: "{path}"'
)
sys.exit(1)
# we need to populate the list of import paths before we try to determine the game type
# because the game type can be determined from import paths
self.import_paths = self._get_import_paths()
if not self.import_paths:
PapyrusProject.log.error('Failed to build list of import paths')
sys.exit(1)
# ensure that folder paths are implicitly imported
implicit_folder_paths: list = self._get_implicit_folder_imports()
if len(implicit_folder_paths) > 0:
PapyrusProject.log.info('Implicitly imported folder paths found:')
for path in implicit_folder_paths:
PapyrusProject.log.info(f'+ "{path}"')
PathHelper.merge_implicit_import_paths(implicit_folder_paths,
self.import_paths)
# we need to populate psc paths after explicit and implicit import paths are populated
# this also needs to be set before we populate implicit import paths from psc paths
# not sure if this must run again after populating implicit import paths from psc paths
self.psc_paths = self._get_psc_paths()
if not self.psc_paths:
PapyrusProject.log.error('Failed to build list of script paths')
sys.exit(1)
# this adds implicit imports from script paths
implicit_script_paths: list = self._get_implicit_script_imports()
if len(implicit_script_paths) > 0:
PapyrusProject.log.info('Implicitly imported script paths found:')
for path in implicit_script_paths:
PapyrusProject.log.info(f'+ "{path}"')
PathHelper.merge_implicit_import_paths(implicit_script_paths,
self.import_paths)
# we need to set the game type after imports are populated but before pex paths are populated
# allow xml to set game type but defer to passed argument
if not self.options.game_type:
game_type: str = self.ppj_root.get('Game', default='').upper()
if game_type and GameType.has_member(game_type):
valid_game_type: GameType = GameType[game_type]
PapyrusProject.log.warning(
f'Using game type: {self.game_names[valid_game_type]} (determined from Papyrus Project)'
)
self.options.game_type = valid_game_type
if not self.options.game_type:
self.options.game_type = self.get_game_type()
if not self.options.game_type:
PapyrusProject.log.error(
'Cannot determine game type from arguments or Papyrus Project')
sys.exit(1)
# get expected pex paths - these paths may not exist and that is okay!
self.pex_paths = self._get_pex_paths()
# these are relative paths to psc scripts whose pex counterparts are missing
self.missing_scripts: dict = self._find_missing_script_paths()
# game type must be set before we call this
if not self.options.game_path:
self.options.game_path = self.get_game_path(self.options.game_type)
def _calculate_object_name(self, psc_path: str) -> str:
return PathHelper.calculate_relative_object_name(
psc_path, self.import_paths)
def _get_script_paths_from_folders_node(self) -> typing.Generator:
"""Returns script paths from the Folders element array"""
for folder_node in filter(is_folder_node, self.folders_node):
self.try_fix_namespace_path(folder_node)
attr_no_recurse: bool = folder_node.get(
XmlAttributeName.NO_RECURSE) == 'True'
folder_path: str = folder_node.text
# handle . and .. in path
if folder_path == os.pardir or startswith(folder_path, os.pardir):
folder_path = folder_path.replace(
os.pardir,
os.path.normpath(os.path.join(self.project_path,
os.pardir)), 1)
yield from PathHelper.find_script_paths_from_folder(
folder_path, no_recurse=attr_no_recurse)
continue
if folder_path == os.curdir or startswith(folder_path, os.curdir):
folder_path = folder_path.replace(os.curdir, self.project_path,
1)
yield from PathHelper.find_script_paths_from_folder(
folder_path, no_recurse=attr_no_recurse)
continue
if startswith(folder_path, self.remote_schemas, ignorecase=True):
local_path = self._get_remote_path(folder_node)
PapyrusProject.log.info(
f'Adding import path from remote: "{local_path}"...')
self.import_paths.insert(0, local_path)
PapyrusProject.log.info(
f'Adding folder path from remote: "{local_path}"...')
yield from PathHelper.find_script_paths_from_folder(
local_path, no_recurse=attr_no_recurse)
continue
folder_path = os.path.normpath(folder_path)
# try to add absolute path
if os.path.isabs(folder_path) and os.path.isdir(folder_path):
yield from PathHelper.find_script_paths_from_folder(
folder_path, no_recurse=attr_no_recurse)
continue
# try to add project-relative folder path
test_path = os.path.join(self.project_path, folder_path)
if os.path.isdir(test_path):
# count scripts to avoid issue where an errant `test_path` may exist and contain no sources
# this can be a problem if that folder contains sources but user error is hard to fix
test_passed = False
user_flags = wcmatch.RECURSIVE if not attr_no_recurse else 0x0
matcher = wcmatch.WcMatch(test_path,
'*.psc',
flags=wcmatch.IGNORECASE
| user_flags)
for _ in matcher.imatch():
test_passed = True
break
if test_passed:
yield from PathHelper.find_script_paths_from_folder(
test_path, no_recurse=attr_no_recurse, matcher=matcher)
continue
# try to add import-relative folder path
for import_path in self.import_paths:
test_path = os.path.join(import_path, folder_path)
if os.path.isdir(test_path):
yield from PathHelper.find_script_paths_from_folder(
test_path, no_recurse=attr_no_recurse)
def _populate_include_paths(self, parent_node: etree.ElementBase,
root_path: str) -> list:
include_paths: list = []
for include_node in parent_node:
if not include_node.tag.endswith('Include'):
continue
no_recurse: bool = self.ppj._get_attr_as_bool(
include_node, 'NoRecurse')
wildcard_pattern: str = '*' if no_recurse else '**\*'
include_text: str = self.ppj.parse(include_node.text)
if include_text == os.curdir or include_text == os.pardir:
PackageManager.log.warning(
'Include paths cannot be equal to "." or ".."')
continue
if include_text.startswith('.'):
PackageManager.log.warning(
'Include paths cannot start with "."')
continue
# populate files list using simple glob patterns
if '*' in include_text:
search_path: str = os.path.join(root_path, wildcard_pattern)
files: list = [
f
for f in glob.iglob(search_path, recursive=not no_recurse)
if os.path.isfile(f)
]
matches: list = fnmatch.filter(files, include_text)
if not matches:
PackageManager.log.warning(
'No files in "%s" matched glob pattern: %s' %
(search_path, include_text))
include_paths.extend(matches)
continue
include_path: str = os.path.normpath(include_text)
# populate files list using absolute paths
if os.path.isabs(include_path) and os.path.exists(include_path):
if root_path not in include_path:
PackageManager.log.warning(
'Cannot include path outside RootDir: "%s"' %
include_path)
continue
include_paths.append(include_path)
continue
# populate files list using relative file path
test_path = os.path.join(root_path, include_path)
if not os.path.isdir(test_path):
include_paths.append(test_path)
continue
# populate files list using relative folder path
search_path = os.path.join(root_path, include_path,
wildcard_pattern)
include_paths.extend([
f for f in glob.iglob(search_path, recursive=not no_recurse)
if os.path.isfile(f)
])
return PathHelper.uniqify(include_paths)
def _generate_include_paths(includes_node: etree.ElementBase,
root_path: str) -> typing.Generator:
for include_node in filter(is_include_node, includes_node):
no_recurse: bool = include_node.get('NoRecurse') == 'True'
wildcard_pattern: str = '*' if no_recurse else r'**\*'
if include_node.text.startswith(os.pardir):
PackageManager.log.warning(
f'Include paths cannot start with "{os.pardir}"')
continue
if include_node.text == os.curdir or include_node.text.startswith(
os.curdir):
include_node.text = include_node.text.replace(
os.curdir, root_path, 1)
# normalize path
path_or_pattern = os.path.normpath(include_node.text)
# populate files list using simple glob patterns
if '*' in path_or_pattern:
if not os.path.isabs(path_or_pattern):
search_path = os.path.join(root_path, wildcard_pattern)
elif root_path in path_or_pattern:
search_path = path_or_pattern
else:
PackageManager.log.warning(
f'Cannot include path outside RootDir: "{path_or_pattern}"'
)
continue
for include_path in glob.iglob(search_path,
recursive=not no_recurse):
if os.path.isfile(include_path) and fnmatch.fnmatch(
include_path, path_or_pattern):
yield include_path
# populate files list using absolute paths
elif os.path.isabs(path_or_pattern):
if root_path not in path_or_pattern:
PackageManager.log.warning(
f'Cannot include path outside RootDir: "{path_or_pattern}"'
)
continue
if os.path.isfile(path_or_pattern):
yield path_or_pattern
else:
search_path = os.path.join(path_or_pattern,
wildcard_pattern)
yield from PathHelper.find_include_paths(
search_path, no_recurse)
else:
# populate files list using relative file path
test_path = os.path.join(root_path, path_or_pattern)
if not os.path.isdir(test_path):
yield test_path
# populate files list using relative folder path
else:
search_path = os.path.join(root_path, path_or_pattern,
wildcard_pattern)
yield from PathHelper.find_include_paths(
search_path, no_recurse)