def make_parent_unless_dir(*pathnames: str) -> None:
'''
Create the parent directories of all passed directories that do *not*
already exist, silently ignoring those that *do* already exist.
Parameters
-----------
pathnames : tuple[str]
Tuple of the absolute or relative pathnames to create the parent
directories of.
See Also
-----------
:func:`make_unless_dir`
Further details.
'''
# Avoid circular import dependencies.
from betse.util.path.pathnames import get_dirname, canonicalize
# Canonicalize each pathname *BEFORE* attempting to get its dirname.
# Relative pathnames do *NOT* have sane dirnames (e.g., the dirname for a
# relative pathname "metatron" is the empty string) and hence *MUST* be
# converted to absolute pathnames first.
for pathname in pathnames:
make_unless_dir(get_dirname(canonicalize(pathname)))
def get_dirname_canonical(module: ModuleOrStrTypes) -> str:
'''
**Absolute canonical dirname** (i.e., absolute dirname after resolving
symbolic links) of the directory providing the passed module or package.
If this module is a non-namespace package, this is the directory containing
this package's top-level ``__init__`` submodule.
Parameters
----------
module : ModuleOrStrTypes
Either:
* The fully-qualified name of this module, in which case this function
dynamically imports this module as a non-optional side effect.
* A previously imported module object.
Returns
----------
str
Absolute canonical dirname of the directory providing this module or
package.
'''
# Avoid circular import dependencies.
from betse.util.path import pathnames
# Return this dirname canonicalized.
return pathnames.canonicalize(get_dirname(module))
def is_subdir(parent_dirname: str, child_dirname: str) -> bool:
'''
``True`` only if the child directory with the passed dirname actually is a
child (i.e., subdirectory) of the parent directory with the passed dirname.
Parameters
-----------
parent_dirname: str
Absolute or relative dirname of the parent directory to be tested.
child_dirname: str
Absolute or relative dirname of the child directory to be tested.
Returns
-----------
bool
``True`` only if this child directory actually is a child (i.e.,
subdirectory) of this parent directory.
See Also
-----------
https://stackoverflow.com/a/37095733/2809027
StackOverflow answer strongly inspiring this implementation.
'''
# Avoid circular import dependencies.
from betse.util.path import pathnames
# Canonicalized child and parent dirnames, conditionally resolving both
# relative dirnames and symbolic links as needed.
#
# Note that the subsequently called os_path.commonpath() function
# explicitly requires these dirnames to be canonicalized, raising a
# "ValueError" exception if this is *NOT* the case.
parent_dirname = pathnames.canonicalize(parent_dirname)
child_dirname = pathnames.canonicalize(child_dirname)
# Longest common dirname shared between these dirnames if any *OR* the root
# directory otherwise (e.g., "/" under Linux).
common_dirname = os_path.commonpath((parent_dirname, child_dirname))
# Return true only if this dirname is this parent's canonicalized dirname.
return parent_dirname == common_dirname
def get_dirname_canonical(module: ModuleOrStrTypes) -> str:
'''
**Absolute canonical dirname** (i.e., absolute dirname after resolving
symbolic links) of the directory providing the passed module or package.
See Also
----------
:func:`get_dirname`
Further details.
'''
# Avoid circular import dependencies.
from betse.util.path import pathnames
# Return this dirname canonicalized.
return pathnames.canonicalize(get_dirname(module))
def _set_conf_filename(self, conf_filename: str) -> None:
'''
Set the absolute path of the YAML-formatted file associated with this
configuration.
Design
----------
To prevent external callers from unsafely setting this path, this
setter is intentionally implemented as an manual setter rather than a
more preferable :meth:`conf_filename` property setter.
'''
# Unique absolute filename of this file assigned *BEFORE* this file's
# parent directory, ensuring the latter is non-empty.
self._conf_filename = pathnames.canonicalize(conf_filename)
# Basename of this file.
self._conf_basename = pathnames.get_basename(self._conf_filename)
# Unique absolute dirname of the parent directory of this file.
self._conf_dirname = pathnames.get_dirname(self._conf_filename)
def canonicalize_and_make_unless_dir(dirname: str) -> str:
'''
Create the directory with the passed absolute or relative path if this
directory does not already exist and return the **canonical form** (i.e.,
unique absolute path) of this path.
This convenience function chains the lower-level
:func:`betse.util.path.pathnames.canonicalize` and
:func:`make_unless_dir` functions.
'''
# Avoid circular import dependencies.
from betse.util.path import pathnames
# Dirname canonicalized from this possibly non-canonical dirname.
dirname = pathnames.canonicalize(dirname)
# Create this directory if needed.
make_unless_dir(dirname)
# Return this dirname.
return dirname
def _init_pyinstaller_command(self) -> None:
'''
Initialize the list of all shell words of the PyInstaller command to be
run.
'''
# Defer heavyweight imports.
from betse.util.io import stderrs
from betse.util.path import dirs, pathnames
from betse.util.os.command import cmds
from betse.util.os.shell import shellstr
# Relative path of the top-level PyInstaller directory.
pyinstaller_dirname = 'freeze'
# Relative path of the PyInstaller spec file converting such
# platform-independent script into a platform-specific executable.
self._pyinstaller_spec_filename = pathnames.join(
pyinstaller_dirname, '.spec')
# If the frozen executable directory was *NOT* explicitly passed on the
# command-line, default to a subdirectory of this top-level directory.
if self.dist_dir is None:
self.dist_dir = pathnames.join(pyinstaller_dirname, 'dist')
# Else, canonicalize the passed directory.
else:
self.dist_dir = pathnames.canonicalize(self.dist_dir)
assert isinstance(self.dist_dir,
str), ('"{}" not a string.'.format(self.dist_dir))
# Relative path of the input hooks subdirectory.
self._pyinstaller_hooks_dirname = pathnames.join(
pyinstaller_dirname, 'hooks')
# Relative path of the intermediate build subdirectory.
pyinstaller_work_dirname = pathnames.join(pyinstaller_dirname, 'build')
# Create such hooks subdirectory if not found, as failing to do so
# will induce fatal PyInstaller errors.
dirs.make_unless_dir(self._pyinstaller_hooks_dirname)
# List of all shell words of the PyInstaller command to be run,
# starting with the basename of this command.
self._pyinstaller_args = []
# Append all PyInstaller command options common to running such command
# for both reuse and regeneration of spec files. (Most such options are
# specific to the latter only and hence omitted.)
self._pyinstaller_args = [
# Overwrite existing output paths under the "dist/" subdirectory
# without confirmation, the default behaviour.
'--noconfirm',
# Non-default PyInstaller directories.
'--workpath=' + shellstr.shell_quote(pyinstaller_work_dirname),
'--distpath=' + shellstr.shell_quote(self.dist_dir),
# Non-default log level.
# '--log-level=DEBUG',
'--log-level=INFO',
]
# Forward all custom boolean options passed by the user to the current
# setuptools command (e.g., "--clean") to the "pyinstaller" command.
if self.clean:
self._pyinstaller_args.append('--clean')
if self.debug:
self._pyinstaller_args.extend((
'--debug',
# UPX-based compression uselessly consumes non-trivial time
# (especially under Windows, where process creation is fairly
# heavyweight) when freezing debug binaries. To optimize and
# simplify debugging, such compression is disabled.
'--noupx',
))
stderrs.output_warning('Enabling bootloader debug messages.')
stderrs.output_warning('Disabling UPX-based compression.')
# If *NOT* debugging and UPX is *NOT* found, print a non-fatal warning.
# While optional, freezing in the absence of UPX produces uncompressed
# and hence considerably larger executables.
elif not cmds.is_cmd('upx'):
stderrs.output_warning(
'UPX not installed or "upx" not in the current ${PATH}.')
stderrs.output_warning('Frozen binaries will *NOT* be compressed.')
def clone_worktree_shallow(
branch_or_tag_name: str,
src_dirname: str,
trg_dirname: str,
) -> None:
'''
**Shallowly clone** (i.e., recursively copy without commit history) from
the passed branch or tag name of the source Git working tree with the
passed dirname into the target non-existing directory.
For space efficiency, the target directory will *not* be a Git working tree
(i.e., will *not* contain a ``.git`` subdirectory). This directory will
only contain the contents of the source Git working tree isolated at
either:
* The ``HEAD`` of the branch with the passed name.
* The commit referenced by the tag with the passed name.
Caveats
----------
Due to long-standing issues in Git itself, the passed branch or tag name
*cannot* be the general-purpose SHA-1 hash of an arbitrary commit. While
performing a shallow clone of an arbitrary commit from such a hash is
technically feasible, doing so requires running a pipeline of multiple
sequential Git subprocesses and hence exceeds the mandate of this function.
Parameters
----------
branch_or_tag_name : str
Name of the branch or tag to clone from this source Git working tree.
src_dirname : str
Relative or absolute pathname of the source Git working tree to clone
from.
trg_dirname : str
Relative or absolute pathname of the target non-existing directory to
clone into.
Raises
----------
BetseGitException
If this source directory is *not* a Git working tree.
BetseDirException
If this target directory already exists.
See Also
----------
https://stackoverflow.com/questions/26135216/why-isnt-there-a-git-clone-specific-commit-option
StackOverflow question entitled "Why Isn't There A Git Clone Specific
Commit Option?", detailing Git's current omission of such
functionality.
'''
# Avoid circular import dependencies.
from betse.util.path import paths, pathnames
from betse.util.os.command import cmdrun
# If this source directory is *NOT* a Git working tree, raise an exception.
die_unless_worktree(src_dirname)
# If this target directory already exists, raise an exception.
paths.die_if_path(trg_dirname)
# Absolute pathname of the source Git working tree to clone from. The
# "git clone" command requires "file:///"-prefixed absolute pathnames.
src_dirname = pathnames.canonicalize(src_dirname)
# Git-specific URI of this source Git working tree.
src_dir_uri = 'file://' + src_dirname
# Tuple of shell words comprising the Git command performing this clone.
git_command = (
'git',
'clone',
# Branch or tag to be cloned.
'--branch',
branch_or_tag_name,
# Perform a shallow clone.
'--depth',
'1',
# From this source to target directory.
src_dir_uri,
trg_dirname,
)
# Shallowly clone this source to target directory. Contrary to expectation,
# "git" redirects non-error or -warning output resembling the following to
# stderr rather than stdout:
#
# Cloning into '/tmp/pytest-of-leycec/pytest-26/cli_sim_backward_compatibility0/betse_old'...
# Note: checking out 'd7d6bf6d61ff2b467f9983bc6395a8ba9d0f234e'.
#
# You are in 'detached HEAD' state. You can look around, make experimental
# changes and commit them, and you can discard any commits you make in this
# state without impacting any branches by performing another checkout.
#
# If you want to create a new branch to retain commits you create, you may
# do so (now or later) by using -b with the checkout command again. Example:
#
# git checkout -b <new-branch-name>
#
# This is distasteful. Clearly, this output should *NOT* be logged as either
# an error or warning. Instead, both stdout and stderr output would ideally
# be logged as informational messages. Unfortunately, the following call
# appears to erroneously squelch rather than redirect most stderr output:
#
# cmdrun.log_output_or_die(
# command_words=git_command,
# stdout_log_level=LogLevel.INFO,
# stderr_log_level=LogLevel.ERROR,
# )
#
# Frankly, we have no idea what is happening here -- and it doesn't
# particularly matter. The current approach, while non-ideal, suffices.
cmdrun.run_or_die(command_words=git_command)
def _is_blas_optimized_posix_symlink() -> BoolOrNoneTypes:
'''
``True`` only if the current platform is POSIX-compliant and hence
supports symbolic links *and* the first item of the ``libraries`` list of
the global :data:`numpy.__config__.blas_opt_info` dictionary is a symbolic
link masquerading as either the unoptimized reference BLAS implementation
but in fact linking to an optimized BLAS implementation.
This function returns ``None`` when unable to deterministically decide this
boolean, in which case a subsequent heuristic will attempt to do so.
'''
# If the current platform is POSIX-incompatible and hence does *NOT*
# support symbolic links, continue to the next heuristic.
if not posix.is_posix():
return None
#FIXME: Generalize to macOS as well once the
#libs.iter_linked_filenames() function supports macOS.
# If the current platform is *NOT* Linux, continue to the next heuristic.
#
# The libs.iter_linked_filenames() function called below currently only
# supports Linux.
if not linux.is_linux():
return None
# First element of the list of uniquely identifying substrings of all BLAS
# library basenames this version of Numpy is linked against.
#
# Note that this list is guaranteed to both exist and be non-empty due to
# the previously called _is_blas_optimized_opt_info_basename() function.
blas_basename_substr = numpy_config.blas_opt_info['libraries'][0]
# If this element appears to be neither the reference BLAS or CBLAS
# implementations (e.g., "blas", "cblas", "refblas", "refcblas"), continue
# to the next heuristic.
if not blas_basename_substr.endswith('blas'):
return None
# Arbitrary Numpy C extension.
#
# Unfortunately, the "numpy.__config__" API fails to specify the absolute
# paths of the libraries it links against. Since there exists no reliable
# means of reverse engineering these paths from this API, these paths must
# be obtained by another means: specifically, by querying the standard
# "numpy.core.multiarray" C extension installed under all supported Numpy
# for the absolute paths of all external shared libraries to which this
# extension links -- exactly one of which is guaranteed to be the absolute
# path of what appears to be a reference BLAS or CBLAS implementation.
numpy_lib = get_c_extension()
# Absolute filename of this C extension.
numpy_lib_filename = pymodule.get_filename(module=numpy_lib)
# For the basename and absolute filename of each shared library linked to
# by this Numpy shared library...
for (numpy_linked_lib_basename, numpy_linked_lib_filename) in (
dlls.iter_linked_filenames(numpy_lib_filename)):
# Basename excluding all suffixing filetypes of this library.
numpy_linked_lib_rootname = pathnames.get_pathname_sans_filetypes(
numpy_linked_lib_basename)
# logs.log_info('rootname: %s; basename: %s; filename: %s', numpy_linked_lib_rootname, numpy_linked_lib_basename, numpy_linked_lib_filename)
# If this appears to be neither the BLAS nor CBLAS reference library,
# continue to the next library.
if not numpy_linked_lib_rootname.endswith('blas'):
continue
# Else, this is either the BLAS or CBLAS reference library.
# Absolute filename of the target library to which this library links
# if this library is a symbolic link *OR* of this library as is
# otherwise (i.e., if this is a library rather than symbolic link).
numpy_linked_lib_target_filename = pathnames.canonicalize(
numpy_linked_lib_filename)
# logs.log_info('target filename: %s', numpy_linked_lib_target_filename)
# If either the basename or dirname of this path corresponds to that of
# an optimized BLAS library, return True.
if regexes.is_match(
text=pathnames.get_basename(numpy_linked_lib_target_filename),
regex=_OPTIMIZED_BLAS_LINKED_LIB_BASENAME_REGEX,
) or regexes.is_match(
text=pathnames.get_dirname(numpy_linked_lib_target_filename),
regex=_OPTIMIZED_BLAS_LINKED_LIB_DIRNAME_REGEX,
):
return True
# Else, Numpy links against an unoptimized BLAS implementation. Halt!
break
# Else, instruct our caller to continue to the next heuristic.
return None
def save(
self,
# Mandatory parameters.
conf_filename: str,
# Optional parameters.
is_conf_file_overwritable: bool = False,
conf_subdir_overwrite_policy: DirOverwritePolicy = (
DirOverwritePolicy.SKIP_WITH_WARNING),
) -> None:
'''
Serialize (i.e., save, write) the low-level mapping or sequence
internally persisted in this wrapper to the YAML-formatted file with
the passed filename, copying *all* external resources internally
referenced by this mapping or sequence into this file's directory.
This method effectively implements the "Save As..." GUI metaphor.
Specifically, this method (in order):
#. Serializes this mapping or sequence to the file with this filename,
optionally overwriting the existing contents of this file depending
on the passed ``is_conf_file_overwritable`` parameter.
#. Recursively copies all relative subdirectories internally referenced
(and hence required) by this file from the directory of the current
file associated with this wrapper into the directory of the passed
file, optionally overwriting the existing contents of these
subdirectories depending on the passed
``conf_subdir_overwrite_policy`` parameter.
#. Associates this wrapper with this filename.
Parameters
----------
conf_filename : str
Absolute or relative filename of the target file to be serialized.
is_conf_file_overwritable : optional[bool]
If this target file already exists *and* this boolean is:
* ``True``, this target file is silently overwritten.
* ``False``, an exception is raised.
Defaults to ``False``.
conf_subdir_overwrite_policy : DirOverwritePolicy
**Subdirectory overwrite policy** (i.e., strategy for handling
existing subdirectories to be overwritten by this save), where the
subdirectories in question are all subdirectories of the directory
of this target file. Defaults to
:attr:`DirOverwritePolicy.SKIP_WITH_WARNING`, ignoring each target
subdirectory that already exists with a non-fatal warning.
Raises
----------
BetseDirException
If the passed ``conf_subdir_overwrite_policy`` parameter is
:attr:`DirOverwritePolicy.HALT_WITH_EXCEPTION` *and* one or more
subdirectories of the target directory already exist that are also
subdirectories of the source directory.
BetseFileException
If the passed ``is_conf_file_overwritable`` parameter is ``False``
*and* this target file already exists.
'''
# Log this save.
logs.log_debug(
'Saving YAML file: %s -> %s',
self._conf_filename, conf_filename)
# Serialize this mapping or sequence to this file.
#
# Note that the die_unless_loaded() method need *NOT* be called here,
# as the "conf" property implicitly does so on our behalf.
yamls.save(
container=self.conf,
filename=conf_filename,
is_overwritable=is_conf_file_overwritable,
)
# Absolute dirnames of the directories containing the current file and
# the passed file, canonicalized to permit comparison below.
src_dirname = pathnames.canonicalize(self.conf_dirname)
trg_dirname = pathnames.canonicalize(
pathnames.get_dirname(conf_filename))
# If these directories differ, recursively copy all relative
# subdirectories internally referenced and hence required by this file.
if src_dirname != trg_dirname:
# For the basename of each such subdirectory...
#
# Note that the ideal solution of recursively copying this source
# directory into the directory of this target file (e.g., via
# "dirs.copy(src_dirname, pathnames.get_dirname(conf_filename))")
# fails for the following subtle reasons:
#
# * This target directory may be already exist, which dirs.copy()
# prohibits even when the directory is empty.
# * This target configuration file basename may differ from that of
# this source configuration file, necessitating a subsequent call
# to file.move().
for conf_subdirname in self._iter_conf_subdir_basenames():
# Absolute dirname of the source subdirectory.
src_subdirname = pathnames.join(src_dirname, conf_subdirname)
# Recursively copy from the old into the new subdirectory.
dirs.copy_dir_into_dir(
src_dirname=src_subdirname,
trg_dirname=trg_dirname,
overwrite_policy=conf_subdir_overwrite_policy,
# Ignore all empty ".gitignore" files in all subdirectories
# of this source subdirectory. These files serve only as
# placeholders instructing Git to track otherwise empty
# subdirectories. Preserving such files only invites end
# user confusion.
ignore_basename_globs=('.gitignore',),
)
# Associate this object with this file *AFTER* successfully copying to
# this file and all external paths required by this file.
self._set_conf_filename(conf_filename)