def get_game_type(self) -> str:
"""Returns game type from arguments or Papyrus Project"""
if self.options.game_type:
return self.options.game_type
if self.options.game_path:
for game_type, game_name in GameName.items():
if endswith(self.options.game_path, game_name,
ignorecase=True):
ProjectBase.log.info(
f'Using game type: {game_name} (determined from game path)'
)
return GameType.get(game_type)
if self.options.registry_path:
game_type = self._get_game_type_from_path(
self.options.registry_path)
if game_type:
ProjectBase.log.info(
f'Using game type: {GameName.get(game_type)} (determined from registry path)'
)
return game_type
if self.import_paths:
for import_path in reversed(self.import_paths):
game_type = self._get_game_type_from_path(import_path)
if game_type:
ProjectBase.log.info(
f'Using game type: {GameName.get(game_type)} (determined from import paths)'
)
return game_type
if self.options.flags_path:
if endswith(self.options.flags_path,
FlagsName.FO4,
ignorecase=True):
ProjectBase.log.info(
f'Using game type: {GameName.FO4} (determined from flags path)'
)
return GameType.FO4
if endswith(self.options.flags_path,
FlagsName.TES5,
ignorecase=True):
try:
self.get_game_path('sse')
except FileNotFoundError:
ProjectBase.log.info(
f'Using game type: {GameName.TES5} (determined from flags path)'
)
return GameType.TES5
else:
ProjectBase.log.info(
f'Using game type: {GameName.SSE} (determined from flags path)'
)
return GameType.SSE
raise AssertionError(
'Cannot determine game type from game path, registry path, import paths, or flags path'
)
def fetch_contents(self, url: str, output_path: str) -> Generator:
"""
Downloads files from URL to output path
"""
parsed_url = urlparse(url)
schemeless_url = url.removeprefix(f'{parsed_url.scheme}://')
if endswith(parsed_url.netloc, 'github.com', ignorecase=True):
if self.config or not self.access_token:
self.access_token = self.find_access_token(schemeless_url)
if not self.access_token:
raise PermissionError(
'Cannot download from GitHub remote without access token'
)
github = GitHubRemote(access_token=self.access_token,
worker_limit=self.worker_limit,
force_overwrite=self.force_overwrite)
yield from github.fetch_contents(url, output_path)
elif endswith(parsed_url.netloc, 'bitbucket.org', ignorecase=True):
bitbucket = BitbucketRemote()
yield from bitbucket.fetch_contents(url, output_path)
else:
if self.config or not self.access_token:
self.access_token = self.find_access_token(schemeless_url)
if not self.access_token:
raise PermissionError(
'Cannot download from Gitea remote without access token'
)
gitea = GiteaRemote(access_token=self.access_token,
worker_limit=self.worker_limit,
force_overwrite=self.force_overwrite)
yield from gitea.fetch_contents(url, output_path)
def __setattr__(self, key: str, value: object) -> None:
if isinstance(value, str) and endswith(key, 'path'):
if os.altsep in value:
value = os.path.normpath(value)
elif isinstance(value, list) and endswith(key, 'paths'):
value = [
os.path.normpath(path) if path != os.curdir else path
for path in value
]
super(ProjectBase, self).__setattr__(key, value)
def build_commands(self, containing_folder: str, output_path: str) -> str:
"""
Builds command for creating package with BSArch
"""
arguments = CommandArguments()
arguments.append(self.options.bsarch_path, enquote_value=True)
arguments.append('pack')
arguments.append(containing_folder, enquote_value=True)
arguments.append(output_path, enquote_value=True)
if self.options.game_type == GameType.FO4:
arguments.append('-fo4')
elif self.options.game_type == GameType.SSE:
arguments.append('-sse')
# SSE has an ctd bug with uncompressed textures in a bsa that
# has an Embed Filenames flag on it, so force it to false.
has_textures = False
for f in glob.iglob(os.path.join(containing_folder, r'**/*'),
recursive=True):
if not os.path.isfile(f):
continue
if endswith(f, '.dds', ignorecase=True):
has_textures = True
break
if has_textures:
arguments.append('-af:0x3')
else:
arguments.append('-tes5')
return arguments.join()
def merge_implicit_import_paths(implicit_paths: list,
import_paths: list) -> None:
"""Inserts orphan and descendant implicit paths into list of import paths at correct positions"""
if not implicit_paths:
return
implicit_paths.sort()
for implicit_path in reversed(PathHelper.uniqify(implicit_paths)):
implicit_path = os.path.normpath(implicit_path)
# do not add import paths that are already declared
if any(
endswith(x, implicit_path, ignorecase=True)
for x in import_paths):
continue
# insert implicit path before ancestor import path, if ancestor exists
i = PathHelper.find_index_of_ancestor_import_path(
implicit_path, import_paths)
if i > -1:
import_paths.insert(i, implicit_path)
continue
# insert orphan implicit path at the first position
import_paths.insert(0, implicit_path)
def _try_exclude_unmodified_scripts(self) -> dict:
psc_paths: dict = {}
for object_name, script_path in self.psc_paths.items():
script_name, _ = os.path.splitext(os.path.basename(script_path))
# if pex exists, compare time_t in pex header with psc's last modified timestamp
matching_path: str = ''
for pex_path in self.pex_paths:
if endswith(pex_path, f'{script_name}.pex', ignorecase=True):
matching_path = pex_path
break
if not os.path.isfile(matching_path):
continue
try:
header = PexReader.get_header(matching_path)
except ValueError:
PapyrusProject.log.error(
f'Cannot determine compilation time due to unknown magic: "{matching_path}"'
)
sys.exit(1)
compiled_time: int = header.compilation_time.value
if os.path.getmtime(script_path) < compiled_time:
continue
if script_path not in psc_paths:
psc_paths[object_name] = script_path
return psc_paths
def _get_remote_path(self, node: etree.ElementBase) -> str:
import_path: str = node.text
url_hash = hashlib.sha1(import_path.encode()).hexdigest()[:8]
temp_path = os.path.join(self.options.remote_temp_path, url_hash)
if self.options.force_overwrite or not os.path.isdir(temp_path):
try:
for message in self.remote.fetch_contents(
import_path, temp_path):
if message:
if not startswith(message, 'Failed to load'):
PapyrusProject.log.info(message)
else:
PapyrusProject.log.error(message)
sys.exit(1)
except PermissionError as e:
PapyrusProject.log.error(e)
sys.exit(1)
if endswith(import_path, '.git', ignorecase=True):
url_path = self.remote.create_local_path(import_path[:-4])
else:
url_path = self.remote.create_local_path(import_path)
local_path = os.path.join(temp_path, url_path)
matcher = wcmatch.WcMatch(local_path,
'*.psc',
flags=wcmatch.IGNORECASE | wcmatch.RECURSIVE)
for f in matcher.imatch():
return os.path.dirname(f)
return local_path
def _find_modified_scripts(self) -> list:
pex_paths: list = []
for object_name, script_path in self.ppj.psc_paths.items():
script_name, _ = os.path.splitext(os.path.basename(script_path))
# if pex exists, compare time_t in pex header with psc's last modified timestamp
pex_match: list = [pex_path for pex_path in self.ppj.pex_paths
if endswith(pex_path, f'{script_name}.pex', ignorecase=True)]
if not pex_match:
continue
pex_path: str = pex_match[0]
if not os.path.isfile(pex_path):
continue
try:
header = PexReader.get_header(pex_path)
except ValueError:
BuildFacade.log.error(f'Cannot determine compilation time due to unknown magic: "{pex_path}"')
sys.exit(1)
psc_last_modified: float = os.path.getmtime(script_path)
pex_last_compiled: float = float(header.compilation_time.value)
# if psc is older than the pex
if psc_last_modified < pex_last_compiled:
pex_paths.append(pex_path)
return PathHelper.uniqify(pex_paths)
def __setattr__(self, key: str, value: object) -> None:
if value and isinstance(value, str):
# sanitize paths
if endswith(key, 'path', ignorecase=True) and os.altsep in value:
value = os.path.normpath(value)
if key in ('game_type', 'zip_compression'):
value = value.casefold()
super(ProjectOptions, self).__setattr__(key, value)
def find_script_paths_from_folder(folder_path: str,
no_recurse: bool) -> Generator:
"""Yields existing script paths starting from absolute folder path"""
search_path: str = os.path.join(folder_path,
'*' if no_recurse else r'**\*')
for script_path in glob.iglob(search_path, recursive=not no_recurse):
if os.path.isfile(script_path) and endswith(
script_path, '.psc', ignorecase=True):
yield script_path
def create_packages(self) -> None:
# clear temporary data
if os.path.isdir(self.options.temp_path):
shutil.rmtree(self.options.temp_path, ignore_errors=True)
# ensure package path exists
if not os.path.isdir(self.options.package_path):
os.makedirs(self.options.package_path, exist_ok=True)
file_names = CaseInsensitiveList()
for i, package_node in enumerate(
filter(is_package_node, self.ppj.packages_node)):
file_name: str = package_node.get('Name')
# prevent clobbering files previously created in this session
if file_name in file_names:
file_name = f'{self.ppj.project_name} ({i})'
if file_name not in file_names:
file_names.append(file_name)
file_name = self._fix_package_extension(file_name)
file_path: str = os.path.join(self.options.package_path, file_name)
self._check_write_permission(file_path)
PackageManager.log.info(f'Creating "{file_name}"...')
for source_path in self._generate_include_paths(
package_node, package_node.get('RootDir')):
PackageManager.log.info(f'+ "{source_path}"')
relpath = os.path.relpath(source_path,
package_node.get('RootDir'))
target_path = os.path.join(self.options.temp_path, relpath)
# fix target path if user passes a deeper package root (RootDir)
if endswith(source_path, '.pex',
ignorecase=True) and not startswith(
relpath, 'scripts', ignorecase=True):
target_path = os.path.join(self.options.temp_path,
'Scripts', relpath)
os.makedirs(os.path.dirname(target_path), exist_ok=True)
shutil.copy2(source_path, target_path)
# run bsarch
command: str = self.build_commands(self.options.temp_path,
file_path)
ProcessManager.run_bsarch(command)
# clear temporary data
if os.path.isdir(self.options.temp_path):
shutil.rmtree(self.options.temp_path, ignore_errors=True)
def __init__(self, options: ProjectOptions) -> None:
self.options = options
self.program_path = os.path.dirname(__file__)
if endswith(sys.argv[0], ('pyro', '.exe')):
self.program_path = os.path.abspath(
os.path.join(self.program_path, os.pardir))
self.project_name = os.path.splitext(
os.path.basename(self.options.input_path))[0]
self.project_path = os.path.dirname(self.options.input_path)
def get_flags_path(self) -> str:
"""
Returns absolute flags path or flags file name from arguments or game path
Used by: BuildFacade
"""
if self.options.flags_path:
if endswith(self.options.flags_path,
tuple(self.flag_types.values()),
ignorecase=True):
return self.options.flags_path
if os.path.isabs(self.options.flags_path):
return self.options.flags_path
return os.path.join(self.project_path, self.options.flags_path)
if endswith(self.options.game_path,
self.game_names[GameType.FO4],
ignorecase=True):
return self.flag_types[GameType.FO4]
return self.flag_types[GameType.TES5]
def fetch_contents(self, url: str, output_path: str) -> Generator:
"""
Downloads files from URL to output path
"""
owner, repo, request_url = self.extract_request_args(url)
request = Request(request_url)
request.add_header('Authorization', f'token {self.access_token}')
response = urlopen(request, timeout=30)
if response.status != 200:
yield 'Failed to load URL (%s): "%s"' % (response.status,
request_url)
return
payload_objects: list = json.loads(response.read().decode('utf-8'))
yield f'Downloading scripts from "{request_url}"... Please wait.'
script_count: int = 0
for payload_object in payload_objects:
target_path = os.path.normpath(
os.path.join(output_path, owner, repo, payload_object['path']))
download_url = payload_object['download_url']
# handle folders
if not download_url:
yield from self.fetch_contents(payload_object['url'],
output_path)
continue
# we only care about scripts
if not endswith(download_url, '.psc', ignorecase=True):
continue
file_response = urlopen(download_url, timeout=30)
if file_response.status != 200:
yield f'Failed to download ({file_response.status}): "{payload_object["download_url"]}"'
continue
os.makedirs(os.path.dirname(target_path), exist_ok=True)
with open(target_path, mode='w+b') as f:
f.write(file_response.read())
script_count += 1
if script_count > 0:
yield f'Downloaded {script_count} scripts from "{request_url}"'
def __init__(self, ppj: PapyrusProject) -> None:
self.ppj = ppj
self.scripts_count = len(self.ppj.psc_paths)
# WARN: if methods are renamed and their respective option names are not, this will break.
options: dict = deepcopy(self.ppj.options.__dict__)
for key in options:
if key in ('args', 'input_path', 'anonymize', 'package', 'zip', 'zip_compression'):
continue
if startswith(key, ('ignore_', 'no_', 'force_', 'create_', 'resolve_'), ignorecase=True):
continue
if endswith(key, '_token', ignorecase=True):
continue
setattr(self.ppj.options, key, getattr(self.ppj, f'get_{key}')())
def anonymize_script(path: str) -> None:
"""
Obfuscates script path, user name, and computer name in compiled script
"""
try:
header: PexHeader = PexReader.get_header(path)
except ValueError:
Anonymizer.log.error(f'Cannot anonymize script due to unknown file magic: "{path}"')
sys.exit(1)
file_path: str = header.script_path.value
user_name: str = header.user_name.value
computer_name: str = header.computer_name.value
if '.' not in file_path:
Anonymizer.log.warning(f'Cannot anonymize script again: "{path}"')
return
if not endswith(file_path, '.psc', ignorecase=True):
Anonymizer.log.error(f'Cannot anonymize script due to invalid file extension: "{path}"')
sys.exit(1)
if not len(file_path) > 0:
Anonymizer.log.error(f'Cannot anonymize script due to zero-length file path: "{path}"')
sys.exit(1)
if not len(user_name) > 0:
Anonymizer.log.error(f'Cannot anonymize script due to zero-length user name: "{path}"')
sys.exit(1)
if not len(computer_name) > 0:
Anonymizer.log.error(f'Cannot anonymize script due to zero-length computer name: "{path}"')
sys.exit(1)
with open(path, mode='r+b') as f:
f.seek(header.script_path.offset, os.SEEK_SET)
f.write(bytes(Anonymizer._randomize_str(header.script_path_size.value), encoding='ascii'))
f.seek(header.user_name.offset, os.SEEK_SET)
f.write(bytes(Anonymizer._randomize_str(header.user_name_size.value), encoding='ascii'))
f.seek(header.computer_name.offset, os.SEEK_SET)
f.write(bytes(Anonymizer._randomize_str(header.computer_name_size.value, True), encoding='ascii'))
Anonymizer.log.info(f'Anonymized "{path}"...')
def fetch_contents(self, url: str, output_path: str) -> Generator:
"""
Downloads files from URL to output path
"""
owner, repo, request_url = self.extract_request_args(url)
yield f'Downloading scripts from "{request_url}"... Please wait.'
script_count: int = 0
for payload in self._fetch_payloads(request_url):
for payload_object in payload['values']:
payload_object_type = payload_object['type']
target_path = os.path.normpath(
os.path.join(output_path, owner, repo,
payload_object['path']))
download_url = payload_object['links']['self']['href']
if payload_object_type == 'commit_file':
# we only care about scripts
if not endswith(download_url, '.psc', ignorecase=True):
continue
file_response = urlopen(download_url, timeout=30)
if file_response.status != 200:
yield f'Failed to download ({file_response.status}): "{download_url}"'
continue
os.makedirs(os.path.dirname(target_path), exist_ok=True)
with open(target_path, mode='w+b') as f:
f.write(file_response.read())
script_count += 1
elif payload_object_type == 'commit_directory':
yield from self.fetch_contents(download_url, output_path)
if script_count > 0:
yield f'Downloaded {script_count} scripts from "{request_url}"'
def create_packages(self) -> None:
# clear temporary data
if os.path.isdir(self.options.temp_path):
shutil.rmtree(self.options.temp_path, ignore_errors=True)
# ensure package path exists
if not os.path.isdir(self.options.package_path):
os.makedirs(self.options.package_path, exist_ok=True)
file_names = CaseInsensitiveList()
for i, package_node in enumerate(
filter(is_package_node, self.ppj.packages_node)):
attr_file_name: str = package_node.get(XmlAttributeName.NAME)
# noinspection PyProtectedMember
root_dir: str = self.ppj._get_path(
package_node.get(XmlAttributeName.ROOT_DIR),
relative_root_path=self.ppj.project_path,
fallback_path=[
self.ppj.project_path,
os.path.basename(attr_file_name)
])
# prevent clobbering files previously created in this session
if attr_file_name in file_names:
attr_file_name = f'{self.ppj.project_name} ({i})'
if attr_file_name not in file_names:
file_names.append(attr_file_name)
attr_file_name = self._fix_package_extension(attr_file_name)
file_path: str = os.path.join(self.options.package_path,
attr_file_name)
self._check_write_permission(file_path)
PackageManager.log.info(f'Creating "{attr_file_name}"...')
for source_path, attr_path in self._generate_include_paths(
package_node, root_dir):
if os.path.isabs(source_path):
relpath: str = os.path.relpath(source_path, root_dir)
else:
relpath: str = source_path
source_path = os.path.join(self.ppj.project_path,
source_path)
adj_relpath = os.path.normpath(os.path.join(
attr_path, relpath))
PackageManager.log.info(f'+ "{adj_relpath.casefold()}"')
target_path: str = os.path.join(self.options.temp_path,
adj_relpath)
# fix target path if user passes a deeper package root (RootDir)
if endswith(source_path, '.pex',
ignorecase=True) and not startswith(
relpath, 'scripts', ignorecase=True):
target_path = os.path.join(self.options.temp_path,
'Scripts', relpath)
os.makedirs(os.path.dirname(target_path), exist_ok=True)
shutil.copy2(source_path, target_path)
self.includes += 1
# run bsarch
command: str = self.build_commands(self.options.temp_path,
file_path)
ProcessManager.run_bsarch(command)
# clear temporary data
if os.path.isdir(self.options.temp_path):
shutil.rmtree(self.options.temp_path, ignore_errors=True)
def _fix_zip_extension(self, zip_name: str) -> str:
if not endswith(zip_name, '.zip', ignorecase=True):
return f'{zip_name}{self.zip_extension}'
return f'{os.path.splitext(zip_name)[0]}{self.zip_extension}'
def fetch_contents(self, url: str, output_path: str) -> Generator:
"""
Downloads files from URL to output path
"""
request_url = self.extract_request_args(url)
request = Request(request_url.url)
request.add_header('Authorization', f'token {self.access_token}')
try:
response = urlopen(request, timeout=30)
except urllib.error.HTTPError as e:
status: HTTPStatus = HTTPStatus(e.code)
yield 'Failed to load remote: "%s" (%s %s)' % (
request_url.url, e.code, status.phrase)
sys.exit(1)
if response.status != 200:
status: HTTPStatus = HTTPStatus(response.status)
yield 'Failed to load remote: "%s" (%s %s)' % (
request_url.url, response.status, status.phrase)
sys.exit(1)
payload_objects: Union[dict, list] = json.loads(
response.read().decode('utf-8'))
if 'contents_url' in payload_objects:
branch = payload_objects['default_branch']
contents_url = payload_objects['contents_url'].replace(
'{+path}', f'?ref={branch}')
yield from self.fetch_contents(contents_url, output_path)
return
scripts: list = []
for payload_object in payload_objects:
target_path = os.path.normpath(
os.path.join(output_path, request_url.owner, request_url.repo,
payload_object['path']))
if not self.force_overwrite and os.path.isfile(target_path):
with open(target_path, mode='rb') as f:
data = f.read()
sha1 = hashlib.sha1(b'blob %s\x00%s' %
(len(data), data.decode()))
if sha1.hexdigest() == payload_object['sha']:
continue
download_url = payload_object['download_url']
# handle folders
if not download_url:
yield from self.fetch_contents(payload_object['url'],
output_path)
continue
# we only care about scripts and flags files
if not (payload_object['type'] == 'file' and endswith(
payload_object['name'],
('.flg', '.psc'), ignorecase=True)):
continue
scripts.append((download_url, target_path))
script_count: int = len(scripts)
if script_count == 0:
return
multiprocessing.freeze_support()
worker_limit: int = min(script_count, self.worker_limit)
with multiprocessing.Pool(processes=worker_limit) as pool:
for download_result in pool.imap_unordered(self.download_file,
scripts):
yield download_result
pool.close()
pool.join()
if script_count > 0:
yield f'Downloaded {script_count} scripts from "{request_url.url}"'
def _fix_package_extension(self, package_name: str) -> str:
if not endswith(package_name, ('.ba2', '.bsa'), ignorecase=True):
return f'{package_name}{self.pak_extension}'
return f'{os.path.splitext(package_name)[0]}{self.pak_extension}'
def get_game_type(self) -> GameType:
"""Returns game type from arguments or Papyrus Project"""
if isinstance(self.options.game_type, str) and self.options.game_type:
if GameType.has_member(self.options.game_type):
return GameType[self.options.game_type]
if self.options.game_path:
if endswith(self.options.game_path,
self.game_names[GameType.FO4],
ignorecase=True):
ProjectBase.log.warning(
f'Using game type: {self.game_names[GameType.FO4]} (determined from game path)'
)
return GameType.FO4
if endswith(self.options.game_path,
self.game_names[GameType.SSE],
ignorecase=True):
ProjectBase.log.warning(
f'Using game type: {self.game_names[GameType.SSE]} (determined from game path)'
)
return GameType.SSE
if endswith(self.options.game_path,
self.game_names[GameType.TES5],
ignorecase=True):
ProjectBase.log.warning(
f'Using game type: {self.game_names[GameType.TES5]} (determined from game path)'
)
return GameType.TES5
if self.options.registry_path:
game_type = self._get_game_type_from_path(
self.options.registry_path)
ProjectBase.log.warning(
f'Using game type: {self.game_names[game_type]} (determined from registry path)'
)
return game_type
if self.import_paths:
for import_path in reversed(self.import_paths):
game_type = self._get_game_type_from_path(import_path)
if game_type:
ProjectBase.log.warning(
f'Using game type: {self.game_names[game_type]} (determined from import paths)'
)
return game_type
if self.options.flags_path:
if endswith(self.options.flags_path,
self.flag_types[GameType.FO4],
ignorecase=True):
ProjectBase.log.warning(
f'Using game type: {self.game_names[GameType.FO4]} (determined from flags path)'
)
return GameType.FO4
if endswith(self.options.flags_path,
self.flag_types[GameType.TES5],
ignorecase=True):
try:
self.get_game_path(GameType.SSE)
except FileNotFoundError:
ProjectBase.log.warning(
f'Using game type: {self.game_names[GameType.TES5]} (determined from flags path)'
)
return GameType.TES5
else:
ProjectBase.log.warning(
f'Using game type: {self.game_names[GameType.SSE]} (determined from flags path)'
)
return GameType.SSE
raise AssertionError(
'Cannot return game type from arguments or Papyrus Project')
