def reading_chars(filename: str, encoding: str = 'utf-8') -> TextIOWrapper:
'''
Open and return a filehandle suitable for reading the plaintext file with
the passed filename encoded with the passed encoding.
This function returns a :class:`file`-like object suitable for use wherever
the :func:`open` builtin is callable (e.g., in ``with`` statements).
Parameters
----------
filename : str
Relative or absolute path of the plaintext text to be read.
encoding : optional[str]
Name of the encoding to be used. Defaults to UTF-8.
Returns
----------
TextIOWrapper
:class:`file`-like object encapsulating the opened file.
'''
# Avoid circular import dependencies.
from betse.util.path import files
# Log this I/O operation.
logs.log_debug('Reading chars: %s', filename)
# Raise an exception unless this file exists.
files.die_unless_file(filename)
# Open this file.
return open(filename, mode='rt', encoding=encoding)
def __init__(self, filename: str, dirname: str) -> None:
'''
Initialize this tissue picker.
Parameters
----------
filename : str
Absolute or relative filename of this image mask. If relative (i.e.,
*not* prefixed by a directory separator), this filename will be
canonicalized into an absolute filename relative to the passed
absolute dirname.
dirname : str
Absolute dirname of the directory containing this image mask. If the
``filename`` parameter is relative, that filename will be prefixed
by this path to convert that filename into an absolute path; else,
this dirname is ignored.
'''
# If this is a relative path, convert this into an absolute path
# relative to the directory containing the source configuration file.
if pathnames.is_relative(filename):
filename = pathnames.join(dirname, filename)
# If this absolute path is *NOT* an existing file, raise an exception.
files.die_unless_file(filename)
# Classify this parameter.
self.filename = filename
def _set_environment_variables(
self,
script_basename: str,
script_type: str,
entry_point: str,
) -> None:
'''
Set all environment variables used to communicate with the
application-specific PyInstaller specification file run in a separate
process, given the passed arguments yielded by the
:meth:`command_entry_points` generator.
While hardly ideal, PyInstaller appears to provide no other means of
communicating with that file.
'''
# Defer heavyweight imports.
from betse.util.app.meta import appmetaone
from betse.util.os.shell import shellenv
from betse.util.path import files, pathnames
# Absolute path of the entry module.
#
# This module's relative path to the top-level project directory is
# obtained by converting the entry point specifier defined by
# "setup.py" for the current entry point (e.g.,
# "betse.gui.guicli:main") into a platform-specific path. Sadly,
# setuptools provides no cross-platform API for reliably obtaining the
# absolute path of the corresponding script wrapper. Even if it did,
# such path would be of little use under POSIX-incompatible platforms
# (e.g., Windows), where these wrappers are binary blobs rather than
# valid Python scripts.
#
# Instead, we reverse-engineer the desired path via brute-force path
# manipulation. Thus burns out another tawdry piece of my soul.
module_filename = pathnames.join(
appmetaone.get_app_meta().project_dirname,
entry_point.module_name.replace('.', os.path.sep) + '.py')
# Ensure such module exists.
files.die_unless_file(module_filename)
# Such path.
shellenv.set_var('__FREEZE_MODULE_FILENAME', module_filename)
# Whether to freeze in "one-file" or "one-directory" mode.
shellenv.set_var('__FREEZE_MODE', self._get_freeze_mode)
# Whether to freeze a CLI- or GUI-based application.
shellenv.set_var('__FREEZE_INTERFACE_TYPE', script_type)
def read_bytes(filename: str) -> BufferedIOBase:
'''
Open and return a filehandle suitable for reading the binary archive file
with the passed filename.
This function returns a :class:`file`-like object suitable for use wherever
the :func:`open` builtin is callable (e.g., in ``with`` statements).
Parameters
----------
filename : str
Relative or absolute path of the binary archive file to be read, whose
**rightmost filetype** (i.e., substring suffixing the last ``.``
character in this pathname) *must* be that of a supported archive
format.
Returns
----------
BufferedIOBase
:class:`file`-like object encapsulating this opened file.
Raises
----------
BetseArchiveException
If this filename is *not* suffixed by a supported archive filetype.
BetseFileException
If this filename is *not* that of an existing file.
'''
# Avoid circular import dependencies.
from betse.util.path import files, pathnames
# Raise an exception unless this filename has a supported archive filetype
# *AND* is an existing file.
die_unless_filetype(filename)
files.die_unless_file(filename)
# Filetype of this pathname. By the above validation, this filetype is
# guaranteed to both exist and be a supported archive filetype. Hence, no
# additional validation is required.
filetype = pathnames.get_filetype_undotted(filename)
# Callable reading archives of this filetype.
reader = _ARCHIVE_FILETYPE_TO_READER[filetype]
# Open and return a filehandle suitable for reading this archive.
return reader(filename)
def reading_bytes(filename: str) -> BufferedIOBase:
'''
Open and return a filehandle suitable for reading the binary file with the
passed filename, transparently decompressing this file if the filetype of
this filename is that of a supported archive format.
This function returns a :class:`file`-like object suitable for use wherever
the :func:`open` builtin is callable (e.g., in ``with`` statements).
Parameters
----------
filename : str
Relative or absolute path of the binary file to be read. If this
filename is suffixed by a supported archive filetype (i.e., if the
:func:`betse.util.path.archives.is_filetype` function returns ``True``
for this filename), the returned filehandle automatically reads the
decompressed rather than compressed byte contents of this file.
Returns
----------
BufferedIOBase
:class:`file`-like object encapsulating this opened file.
Raises
----------
BetseFileException
If this filename is *not* that of an existing file.
'''
# Avoid circular import dependencies.
from betse.util.path import archives, files
# Log this I/O operation.
logs.log_debug('Reading bytes: %s', filename)
# Raise an exception unless this file exists.
files.die_unless_file(filename)
# If this file is compressed, open and return a file handle reading
# decompressed bytes from this file.
if archives.is_filetype(filename):
return archives.read_bytes(filename)
# Else, this file is uncompressed. Open and return a typical file handle
# reading bytes from this file.
else:
return open(filename, mode='rb')
def _run_pyinstaller_command(
self,
script_basename: str,
script_type: str,
entry_point,
) -> None:
'''
Run the currently configured PyInstaller command for the passed entry
point's script wrapper.
Attributes
----------
script_basename : str
Basename of the executable wrapper script running this entry point.
script_type : str
Type of the executable wrapper script running this entry point,
guaranteed to be either:
* If this script is console-specific, ``console`` .
* Else, ``gui``.
entry_point : EntryPoint
Entry point, whose attributes specify the module to be imported and
function to be run by this script.
'''
# Defer heavyweight imports.
from betse.util.io import stderrs
from betse.util.os.shell import shellstr
from betse.util.path import files, pathnames
# If this spec exists, instruct PyInstaller to reuse rather than
# recreate this file, thus preserving edits to this file.
if files.is_file(self._pyinstaller_spec_filename):
print('Reusing spec file "{}".'.format(
self._pyinstaller_spec_filename))
# Append the relative path of this spec file.
self._pyinstaller_args.append(
shellstr.shell_quote(self._pyinstaller_spec_filename))
# Freeze this script with this spec file.
self._run_pyinstaller_imported()
# Else, instruct PyInstaller to (re)create this spec file.
else:
# Absolute path of the directory containing this files.
pyinstaller_spec_dirname = pathnames.get_dirname(
self._pyinstaller_spec_filename)
# Absolute path of the current script wrapper.
script_filename = pathnames.join(self.install_dir, script_basename)
files.die_unless_file(
script_filename,
'File "{}" not found. {}'.format(script_filename,
freeze.EXCEPTION_ADVICE))
# Inform the user of this action *AFTER* the above validation.
# Since specification files should typically be reused rather
# than regenerated, do so as a non-fatal warning.
stderrs.output_warning('Generating spec file "{}".'.format(
self._pyinstaller_spec_filename))
# Append all options specific to spec file generation.
self._pyinstaller_args.extend([
# If this is a console script, configure standard input and
# output for console handling; else, do *NOT* and, if the
# current operating system is OS X, generate an ".app"-suffixed
# application bundle rather than a customary executable.
'--console' if script_type == 'console' else '--windowed',
# Non-default PyInstaller directories.
'--additional-hooks-dir=' +
shellstr.shell_quote(self._pyinstaller_hooks_dirname),
'--specpath=' + shellstr.shell_quote(pyinstaller_spec_dirname),
])
# Append all subclass-specific options.
self._pyinstaller_args.extend(self._get_pyinstaller_options())
# Append the absolute path of this script.
self._pyinstaller_args.append(
shellstr.shell_quote(script_filename))
# Freeze this script and generate a spec file.
self._run_pyinstaller_imported()
# Absolute path of this file.
script_spec_filename = pathnames.join(pyinstaller_spec_dirname,
script_basename + '.spec')
# Rename this file to have the basename expected by the prior
# conditional on the next invocation of this setuptools command.
#
# Note that "pyinstaller" accepts an option "--name" permitting
# the basename of this file to be specified prior to generating
# this file. Unfortunately, this option *ALSO* specifies the
# basename of the generated executable. While the former is
# reliably renamable, the former is *NOT* (e.g., due to code
# signing). Hence, this file is manually renamed without passing
# this option.
files.move_file(script_spec_filename,
self._pyinstaller_spec_filename)
def load_image(
# Mandatory parameters.
filename: str,
# Optional parameters.
is_signed: bool = True,
mode: ImageModeOrNoneTypes = None,
) -> NumpyArrayType:
'''
Load the raw pixel data from the image with the passed filename into a
multi-dimensional Numpy array and return this array.
This array is guaranteed to be at least three-dimensional. Specifically, if
this image is:
* Greyscale, this array is three-dimensional such that:
* The first dimension indexes each row of this image.
* The second dimension indexes each column of the current row.
* The third dimension indexes the grayscale value of the current pixel.
* RGB or RGBA, this array is four-dimensional such that:
* The first dimension indexes each row of this image.
* The second dimension indexes each column of the current row.
* The third dimension is either a 3-tuple ``(R, G, B)`` or 4-tuple
``(R, G, B, A)`` indexing each color component of the current pixel.
* The fourth dimension indexes the value of the current color component.
Caveats
----------
When attempting to load user-defined images of arbitrary filetype, callers
should pass the following parameters:
* ``mode``.
* ``is_signed`` to ``True``. Since this is (and *always* will be) the
default, *not* passing this parameter satisfies this suggestion.
Failure to do so invites subtle issues in computations falsely assuming the
data type and shape of a returned array to be sane, which is *not* the case
in common edge cases. Thanks to the heterogeneity of image file formats,
returned arrays may exhibit anomalous features if any of these paremeters
are *not* passed as suggested. In particular, the ``is_signed`` parameter
should typically either *not* be passed or be passed as ``True``.
Failure to do so instructs Pillow to produce an array with data type
automatically corresponds to that of the input image. Since most (but *not*
necessarily all) images reside in the :attr:`ImageModeType.COLOR_RGB` and
:attr:`ImageModeType.COLOR_RGBA` colour spaces whose three- and
four-channel pixel data is homogenously constrained onto unsigned bytes,
most arrays returned by this function when explicitly passed an
``is_signed`` parameter of ``False`` will be **unsigned byte arrays**
(i.e., arrays whose data types are :attr:`np.uint8`).
Is that a subtle problem? **It is.**
Python silently coerces scalar types as needed to preserve precision across
operations that change precision. The canonical example is integer
division. In Python, dividing two integers that are *not* simple integer
multiples of one another implicitly expands precision by producing a real
number rather than integer (e.g., ``1 / 2 == 0.5`` rather than
``1 / 2 == 0``).
On the other hand:
* For all **signed Numpy arrays** (i.e., arrays whose data types are
implicitly signed rather than explicitly unsigned), Numpy silently
coerces the data types of these arrays as needed to preserve precision
across precision-modifying operations.
* For all **unsigned Numpy arrays** (i.e., arrays whose data types are
explicitly unsigned rather than implicitly signed), Numpy silently
preserves the unsigned facet of these arrays as needed by wrapping all
numerical results to the integer range of these unsigned data types, thus
discarding precision across precision-modifying operations.
The canonical example is integer addition and substraction applied to
unsigned byte arrays. Since unsigned bytes are confined to the integer
range ``[0, 255]``, attempting to perform even seemingly trivial
computation with unsigned byte arrays silently wraps results exceeding this
range onto this range. The resulting arrays typically contain so-called
"garbage data." As the following example shows, applying integer
subtraction to signed but *not* unsigned Numpy arrays produces expected
results:
>>> import numpy as np
>>> unsigned_garbage = np.array(((1,2), (3,4)), dtype=np.uint8)
>>> unsigned_garbage[:,0] - unsigned_garbage[:,1]
... array([255, 255], dtype=uint8)
>>> signed_nongarbage = np.array(((1,2), (3,4)))
>>> signed_nongarbage[:,0] - signed_nongarbage[:,1]
... array([-1, -1])
Design
----------
This utility function is a thin wrapper around a similar function provided
by some unspecified third-party dependency. This function currently wraps
the :meth:`PIL.Image.open` method but previously wrapped the:
* :func:`imageio.imread` function, which failed to expose support for
colourspace conversion provided by Pillow.
* :func:`scipy.misc.imread` function, which SciPy 1.0.0 formally deprecated
and SciPy 1.2.0 permanently killed. Thus, SciPy 1.2.0 broke backward
compatibility with downstream applications (notably, *this* application)
requiring that API.
This utility function principally exists to mitigate the costs associated
with similar upstream API changes in the future. (We are ready this time.)
Parameters
----------
filename : str
Absolute or relative filename of this image.
is_signed : optional[bool]
``True`` only if converting the possibly unsigned array loaded from
this image into a signed array. Defaults to ``True`` for the reasons
detailed above. Since explicitly setting this to ``False`` invites
errors in computations employing the returned array, callers should do
so *ONLY* where these issues are acknowledged and handled
appropriately.
mode : ImageModeOrNoneTypes
Type and depth of all pixels in the array loaded from this image,
converted from this image's pixel data according to industry-standard
image processing transforms implemented by :mod:`PIL`. Note that this
is *not* the type and depth of all pixels in the input image, which
:mod:`PIL` implicitly detects and hence requires no explicit
designation. Defaults to ``None``, in which case no such conversion is
performed (i.e., this image's pixel data is returned as is).
Returns
----------
ndarray
Numpy array loaded from this image.
'''
# Log this load attempt.
logs.log_debug('Loading image "%s"...', pathnames.get_basename(filename))
# If this image does *NOT* exist, raise an exception.
files.die_unless_file(filename)
# In-memory image object loaded from this on-disk image file.
image = Image.open(filename)
# If the caller requests a mode conversion *AND* this image is not already
# of the required mode, do this conversion.
if mode is not None and mode != image.mode:
image = image.convert(mode.value)
# Numpy array converted from this image to be returned.
image_array = array(image)
# If converting unsigned to signed arrays, do so.
if is_signed:
image_array = nparray.to_signed(image_array)
# Return this array.
return image_array
def is_aqua() -> bool:
'''
``True`` only if the current process has access to the Aqua display server
specific to macOS, implying this process to be headfull and hence support
both CLIs and GUIs.
See Also
----------
https://developer.apple.com/library/content/technotes/tn2083/_index.html#//apple_ref/doc/uid/DTS10003794-CH1-SUBSECTION19
"Security Context" subsection of "Technical Note TN2083: Daemons and
Agents," a psuedo-human-readable discussion of the
``sessionHasGraphicAccess`` bit flag returned by the low-level
``SessionGetInfo()`` C function.
'''
# Avoid circular import dependencies.
from betse.util.path import files
from betse.util.os.command.cmdexit import SUCCESS
# If the current platform is *NOT* macOS, return false.
if not is_macos():
return False
# Else, the current platform is macOS.
# Attempt all of the following in a safe manner catching, logging, and
# converting exceptions into a false return value. This tester is *NOT*
# mission-critical and hence should *NOT* halt the application on
# library-specific failures.
try:
# If the system-wide Macho-O shared library providing the macOS
# security context for the current process does *NOT* exist (after
# following symbolic links), raise an exception.
files.die_unless_file(_SECURITY_FRAMEWORK_DYLIB_FILENAME)
# Dynamically load this library into the address space of this process.
security_framework = CDLL(_SECURITY_FRAMEWORK_DYLIB_FILENAME)
# Possibly non-unique identifier of the security session to request the
# attributes of, signifying that of the current process.
session_id = _SECURITY_SESSION_ID_CURRENT
# Unique identifier of the requested security session, returned
# by reference from the SessionGetInfo() C function called below. This
# identifier is useless for our purposes and hence ignored below.
session_id_real = c_int(0)
# Attributes bit field of the requested security session, returned by
# reference from the SessionGetInfo() C function called below.
session_attributes = c_int(0)
# C-style error integer returned by calling the SessionGetInfo() C
# function exported by this Macho-O shared library, passing:
#
# * The input non-unique session identifier by value.
# * The output unique session identifier by reference.
# * The output session attributes integer by reference.
session_errno = security_framework.SessionGetInfo(
session_id, byref(session_id_real), byref(session_attributes))
# This process has access to the Aqua display server if and only if...
return (
# The above function call succeeded *AND*...
session_errno == SUCCESS and
# The session attributes bit field returned by this call has the
# corresponding bit flag enabled.
session_attributes.value & _SECURITY_SESSION_HAS_GRAPHIC_ACCESS)
# If the above logic fails with any exception...
except Exception as exc:
# Log a non-fatal warning informing users of this failure.
logs.log_warning(
'macOS-specific SessionGetInfo() C function failed: {}'.format(
exc.strerror))
# Assume this process to *NOT* have access to the Aqua display server.
return False
def replace_substrs(
filename_source: str,
filename_target: str,
replacements: SequenceTypes,
**kwargs
) -> None:
'''
Write the passed target non-directory file with the result of replacing all
substrings in the passed source non-directory file matching the passed
regular expressions with the corresponding passed substitutions.
This function implements the equivalent of the ``sed`` line processor in a
pure-Python manner requiring no external commands or additional
dependencies. This is a good thing.
This source file will *not* be changed. This target file will be written in
an atomic manner maximally preserving source metadata (e.g., owner, group,
permissions, times, extended file system attributes). If either the source
file does not exist *or* the target file already exists, an exception is
raised.
These regular expressions may be either strings *or* instances of the
:class:`Pattern` class (i.e., compiled regular expression object), while
these substitutions may be either strings *or* functions. See the
"Arguments" section below for how this function expects these objects to be
passed.
This function accepts the same optional keyword arguments as the standard
:func:`re.sub` function.
Arguments
----------
filename_source : str
Absolute path of the source filename to be read.
filename_target : str
Absolute path of the target filename to be written.
replacements : Sequence
Non-string sequence (e.g., list, tuple) of non-string sequences of
length 2 (i.e., pairs), whose:
* First element is a regular expression matching all substrings in the
source file to be replaced.
* Second element is the substring in the target file to replace all
substrings in the source file matching that regular expression.
'''
# Avoid circular import dependencies.
from betse.util.path import files, temps
# Log this substitution.
if filename_source == filename_target:
logs.log_debug(
'Munging file "%s" in-place.', filename_source)
else:
logs.log_debug(
'Munging file "%s" to "%s".', filename_source, filename_target)
# Raise an exception unless the source file exists.
files.die_unless_file(filename_source)
# Raise an exception if the target file already exists.
files.die_if_file(filename_target)
# For efficiency, replace all passed uncompiled with compiled regular
# expressions via a tuple comprehension.
replacements = tuple(
(re.compile(regex), substitution)
for (regex, substitution) in replacements
)
#FIXME: This functionality is probably quite useful, where at least one
#matching line is absolutely expected. Consider formalizing into a passed
#argument, as lackluster time permits.
# is_line_matches = False
# For atomicity, incrementally write to a temporary file rather than the
# desired target file *BEFORE* moving the former to the latter. This
# obscures such writes from other threads and/or processes, avoiding
# potential race conditions elsewhere.
with temps.write_chars() as file_target_temp:
with reading_chars(filename_source) as file_source:
# For each line of the source file...
for line in file_source:
# For each passed regular expression and corresponding
# substitution, replace all substrings in this line matching
# that regular expression with that substitution.
for (regex, substitution) in replacements:
# if regex.search(line, **kwargs) is not None:
# is_line_matches = True
# loggers.log_info('Line "%s" matches!', line)
line = regex.sub(substitution, line, **kwargs)
# Append such line to the temporary file.
file_target_temp.write(line)
# if not is_line_matches:
# raise BetseFileException('No line matches!')
# Copy all metadata (e.g., permissions) from the source to temporary
# file *BEFORE* moving the latter, avoiding potential race conditions
# and security vulnerabilities elsewhere.
shutil.copystat(filename_source, file_target_temp.name)
# Move the temporary to the target file.
shutil.move(file_target_temp.name, filename_target)