def create_static_lib(self, objects, output_libname,
output_dir=None, debug=False, target_lang=None):
objects, output_dir = self._fix_object_args(objects, output_dir)
output_filename = \
self.library_filename(output_libname, output_dir=output_dir)
if self._need_link(objects, output_filename):
self.mkpath(os.path.dirname(output_filename))
self.spawn(self.archiver +
[output_filename] +
objects + self.objects)
# Not many Unices required ranlib anymore -- SunOS 4.x is, I
# think the only major Unix that does. Maybe we need some
# platform intelligence here to skip ranlib if it's not
# needed -- or maybe Python's configure script took care of
# it for us, hence the check for leading colon.
if self.ranlib:
try:
self.spawn(self.ranlib + [output_filename])
except PackagingExecError as msg:
raise LibError(msg)
else:
logger.debug("skipping %s (up-to-date)", output_filename)
def run(self):
# remove the build/temp.<plat> directory (unless it's already
# gone)
if os.path.exists(self.build_temp):
if self.dry_run:
logger.info('removing %s', self.build_temp)
else:
rmtree(self.build_temp)
else:
logger.debug("'%s' does not exist -- can't clean it",
self.build_temp)
if self.all:
# remove build directories
for directory in (self.build_lib,
self.bdist_base,
self.build_scripts):
if os.path.exists(directory):
if self.dry_run:
logger.info('removing %s', directory)
else:
rmtree(directory)
else:
logger.warning("'%s' does not exist -- can't clean it",
directory)
# just for the heck of it, try to remove the base build directory:
# we might have emptied it right now, but if not we don't care
if not self.dry_run:
try:
os.rmdir(self.build_base)
logger.info("removing '%s'", self.build_base)
except OSError:
pass
def find_vcvarsall(version):
"""Find the vcvarsall.bat file
At first it tries to find the productdir of VS 2008 in the registry. If
that fails it falls back to the VS90COMNTOOLS env var.
"""
vsbase = VS_BASE % version
try:
productdir = Reg.get_value(r"%s\Setup\VC" % vsbase,
"productdir")
except KeyError:
logger.debug("Unable to find productdir in registry")
productdir = None
if not productdir or not os.path.isdir(productdir):
toolskey = "VS%0.f0COMNTOOLS" % version
toolsdir = os.environ.get(toolskey, None)
if toolsdir and os.path.isdir(toolsdir):
productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC")
productdir = os.path.abspath(productdir)
if not os.path.isdir(productdir):
logger.debug("%s is not a valid directory", productdir)
return None
else:
logger.debug("env var %s is not set or invalid", toolskey)
if not productdir:
logger.debug("no productdir found")
return None
vcvarsall = os.path.join(productdir, "vcvarsall.bat")
if os.path.isfile(vcvarsall):
return vcvarsall
logger.debug("unable to find vcvarsall.bat")
return None
def _has_setup_cfg(srcdir):
setup_cfg = os.path.join(srcdir, "setup.cfg")
if os.path.isfile(setup_cfg):
logger.debug("setup.cfg file found.")
return True
logger.debug("No setup.cfg file found.")
return False
def query_vcvarsall(version, arch="x86"):
"""Launch vcvarsall.bat and read the settings from its environment
"""
vcvarsall = find_vcvarsall(version)
interesting = set(("include", "lib", "libpath", "path"))
result = {}
if vcvarsall is None:
raise PackagingPlatformError("Unable to find vcvarsall.bat")
logger.debug("calling 'vcvarsall.bat %s' (version=%s)", arch, version)
popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = popen.communicate()
if popen.wait() != 0:
raise PackagingPlatformError(stderr.decode("mbcs"))
stdout = stdout.decode("mbcs")
for line in stdout.split("\n"):
line = Reg.convert_mbcs(line)
if '=' not in line:
continue
line = line.strip()
key, value = line.split('=', 1)
key = key.lower()
if key in interesting:
if value.endswith(os.pathsep):
value = value[:-1]
result[key] = removeDuplicates(value)
if len(result) != len(interesting):
raise ValueError(str(list(result)))
return result
def make_file(self, infiles, outfile, func, args,
exec_msg=None, skip_msg=None, level=1):
"""Special case of 'execute()' for operations that process one or
more input files and generate one output file. Works just like
'execute()', except the operation is skipped and a different
message printed if 'outfile' already exists and is newer than all
files listed in 'infiles'. If the command defined 'self.force',
and it is true, then the command is unconditionally run -- does no
timestamp checks.
"""
if skip_msg is None:
skip_msg = "skipping %s (inputs unchanged)" % outfile
# Allow 'infiles' to be a single string
if isinstance(infiles, str):
infiles = (infiles,)
elif not isinstance(infiles, (list, tuple)):
raise TypeError(
"'infiles' must be a string, or a list or tuple of strings")
if exec_msg is None:
exec_msg = "generating %s from %s" % (outfile, ', '.join(infiles))
# If 'outfile' must be regenerated (either because it doesn't
# exist, is out-of-date, or the 'force' flag is true) then
# perform the action that presumably regenerates it
if self.force or util.newer_group(infiles, outfile):
self.execute(func, args, exec_msg, level)
# Otherwise, print the "skip" message
else:
logger.debug(skip_msg)
def _has_pkg_info(srcdir):
pkg_info = os.path.join(srcdir, "PKG-INFO")
has_pkg_info = os.path.isfile(pkg_info)
if has_pkg_info:
logger.debug("PKG-INFO file found.")
else:
logger.debug("No PKG-INFO file found.")
return has_pkg_info
def _has_text(setup_py, installer):
installer_pattern = re.compile("import {0}|from {0}".format(installer))
with open(setup_py, "r", encoding="utf-8") as setup:
for line in setup:
if re.search(installer_pattern, line):
logger.debug("Found %s text in setup.py.", installer)
return True
logger.debug("No %s text found in setup.py.", installer)
return False
def _has_egg_info(srcdir):
if os.path.isdir(srcdir):
for item in os.listdir(srcdir):
full_path = os.path.join(srcdir, item)
if item.endswith(".egg-info") and os.path.isdir(full_path):
logger.debug("Found egg-info directory.")
return True
logger.debug("No egg-info directory found.")
return False
def __init__(self, dry_run=False, force=False):
super(CygwinCCompiler, self).__init__(dry_run, force)
status, details = check_config_h()
logger.debug("Python's GCC status: %s (details: %s)", status, details)
if status is not CONFIG_H_OK:
self.warn(
"Python's pyconfig.h doesn't seem to support your compiler. "
"Reason: %s. "
"Compiling may fail because of undefined preprocessor macros."
% details)
self.gcc_version, self.ld_version, self.dllwrap_version = \
get_compiler_versions()
logger.debug(self.name + ": gcc %s, ld %s, dllwrap %s\n",
self.gcc_version,
self.ld_version,
self.dllwrap_version)
# ld_version >= "2.10.90" and < "2.13" should also be able to use
# gcc -mdll instead of dllwrap
# Older dllwraps had own version numbers, newer ones use the
# same as the rest of binutils ( also ld )
# dllwrap 2.10.90 is buggy
if self.ld_version >= "2.10.90":
self.linker_dll = "gcc"
else:
self.linker_dll = "dllwrap"
# ld_version >= "2.13" support -shared so use it instead of
# -mdll -static
if self.ld_version >= "2.13":
shared_option = "-shared"
else:
shared_option = "-mdll -static"
# Hard-code GCC because that's what this is all about.
# XXX optimization, warnings etc. should be customizable.
self.set_executables(compiler='gcc -mcygwin -O -Wall',
compiler_so='gcc -mcygwin -mdll -O -Wall',
compiler_cxx='g++ -mcygwin -O -Wall',
linker_exe='gcc -mcygwin',
linker_so=('%s -mcygwin %s' %
(self.linker_dll, shared_option)))
# cygwin and mingw32 need different sets of libraries
if self.gcc_version == "2.91.57":
# cygwin shouldn't need msvcrt, but without the dlls will crash
# (gcc version 2.91.57) -- perhaps something about initialization
self.dll_libraries=["msvcrt"]
self.warn(
"Consider upgrading to a newer version of gcc")
else:
# Include the appropriate MSVC runtime library if Python was built
# with MSVC 7.0 or later.
self.dll_libraries = get_msvcr()
def link(self, target_desc, objects,
output_filename, output_dir=None, libraries=None,
library_dirs=None, runtime_library_dirs=None,
export_symbols=None, debug=False, extra_preargs=None,
extra_postargs=None, build_temp=None, target_lang=None):
objects, output_dir = self._fix_object_args(objects, output_dir)
libraries, library_dirs, runtime_library_dirs = \
self._fix_lib_args(libraries, library_dirs, runtime_library_dirs)
lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs,
libraries)
if type(output_dir) not in (str, type(None)):
raise TypeError("'output_dir' must be a string or None")
if output_dir is not None:
output_filename = os.path.join(output_dir, output_filename)
if self._need_link(objects, output_filename):
ld_args = (objects + self.objects +
lib_opts + ['-o', output_filename])
if debug:
ld_args[:0] = ['-g']
if extra_preargs:
ld_args[:0] = extra_preargs
if extra_postargs:
ld_args.extend(extra_postargs)
self.mkpath(os.path.dirname(output_filename))
try:
if target_desc == CCompiler.EXECUTABLE:
linker = self.linker_exe[:]
else:
linker = self.linker_so[:]
if target_lang == "c++" and self.compiler_cxx:
# skip over environment variable settings if /usr/bin/env
# is used to set up the linker's environment.
# This is needed on OSX. Note: this assumes that the
# normal and C++ compiler have the same environment
# settings.
i = 0
if os.path.basename(linker[0]) == "env":
i = 1
while '=' in linker[i]:
i = i + 1
linker[i] = self.compiler_cxx[i]
if sys.platform == 'darwin':
linker = _darwin_compiler_fixup(linker, ld_args)
self.spawn(linker + ld_args)
except PackagingExecError as msg:
raise LinkError(msg)
else:
logger.debug("skipping %s (up-to-date)", output_filename)
def dump_dirs(self, msg):
"""Dump the list of user options."""
logger.debug(msg + ":")
for opt in self.user_options:
opt_name = opt[0]
if opt_name[-1] == "=":
opt_name = opt_name[0:-1]
if opt_name in self.negative_opt:
opt_name = self.negative_opt[opt_name]
opt_name = opt_name.replace('-', '_')
val = not getattr(self, opt_name)
else:
opt_name = opt_name.replace('-', '_')
val = getattr(self, opt_name)
logger.debug(" %s: %s", opt_name, val)
def create_static_lib(self, objects, output_libname, output_dir=None,
debug=False, target_lang=None):
objects, output_dir = self._fix_object_args(objects, output_dir)
output_filename = \
self.library_filename(output_libname, output_dir=output_dir)
if self._need_link(objects, output_filename):
lib_args = [output_filename, '/u'] + objects
if debug:
pass # XXX what goes here?
try:
self.spawn([self.lib] + lib_args)
except PackagingExecError as msg:
raise LibError(msg)
else:
logger.debug("skipping %s (up-to-date)", output_filename)
def spawn(cmd, search_path=True, dry_run=False, env=None):
"""Run another program specified as a command list 'cmd' in a new process.
'cmd' is just the argument list for the new process, ie.
cmd[0] is the program to run and cmd[1:] are the rest of its arguments.
There is no way to run a program with a name different from that of its
executable.
If 'search_path' is true (the default), the system's executable
search path will be used to find the program; otherwise, cmd[0]
must be the exact path to the executable. If 'dry_run' is true,
the command will not actually be run.
If 'env' is given, it's a environment dictionary used for the execution
environment.
Raise PackagingExecError if running the program fails in any way; just
return on success.
"""
logger.debug("spawn: running %r", cmd)
if dry_run:
logger.debug("dry run, no process actually spawned")
return
if sys.platform == "darwin":
global _cfg_target, _cfg_target_split
if _cfg_target is None:
_cfg_target = sysconfig.get_config_var("MACOSX_DEPLOYMENT_TARGET") or ""
if _cfg_target:
_cfg_target_split = [int(x) for x in _cfg_target.split(".")]
if _cfg_target:
# ensure that the deployment target of build process is not less
# than that used when the interpreter was built. This ensures
# extension modules are built with correct compatibility values
env = env or os.environ
cur_target = env.get("MACOSX_DEPLOYMENT_TARGET", _cfg_target)
if _cfg_target_split > [int(x) for x in cur_target.split(".")]:
my_msg = "$MACOSX_DEPLOYMENT_TARGET mismatch: " 'now "%s" but "%s" during configure' % (
cur_target,
_cfg_target,
)
raise PackagingPlatformError(my_msg)
env = dict(env, MACOSX_DEPLOYMENT_TARGET=cur_target)
exit_status = subprocess.call(cmd, env=env)
if exit_status != 0:
msg = "command %r failed with exit status %d"
raise PackagingExecError(msg % (cmd, exit_status))
def find_package_modules(self, package, package_dir):
self.check_package(package, package_dir)
module_files = glob(os.path.join(package_dir, "*.py"))
modules = []
if self.distribution.script_name is not None:
setup_script = os.path.abspath(self.distribution.script_name)
else:
setup_script = None
for f in module_files:
abs_f = os.path.abspath(f)
if abs_f != setup_script:
module = os.path.splitext(os.path.basename(f))[0]
modules.append((package, module, f))
else:
logger.debug("excluding %r", setup_script)
return modules
def find_config_files(self):
"""Find as many configuration files as should be processed for this
platform, and return a list of filenames in the order in which they
should be parsed. The filenames returned are guaranteed to exist
(modulo nasty race conditions).
There are three possible config files: packaging.cfg in the
Packaging installation directory (ie. where the top-level
Packaging __inst__.py file lives), a file in the user's home
directory named .pydistutils.cfg on Unix and pydistutils.cfg
on Windows/Mac; and setup.cfg in the current directory.
The file in the user's home directory can be disabled with the
--no-user-cfg option.
"""
files = []
check_environ()
# Where to look for the system-wide Packaging config file
sys_dir = os.path.dirname(sys.modules['packaging'].__file__)
# Look for the system config file
sys_file = os.path.join(sys_dir, "packaging.cfg")
if os.path.isfile(sys_file):
files.append(sys_file)
# What to call the per-user config file
if os.name == 'posix':
user_filename = ".pydistutils.cfg"
else:
user_filename = "pydistutils.cfg"
# And look for the user config file
if self.dist.want_user_cfg:
user_file = os.path.join(os.path.expanduser('~'), user_filename)
if os.path.isfile(user_file):
files.append(user_file)
# All platforms support local setup.cfg
local_file = "setup.cfg"
if os.path.isfile(local_file):
files.append(local_file)
if logger.isEnabledFor(logging.DEBUG):
logger.debug("using config files: %s", ', '.join(files))
return files
def get_releases(self, requirements, prefer_final=None,
force_update=False):
"""Search for releases and return a ReleasesList object containing
the results.
"""
predicate = get_version_predicate(requirements)
if predicate.name.lower() in self._projects and not force_update:
return self._projects.get(predicate.name.lower())
prefer_final = self._get_prefer_final(prefer_final)
logger.debug('Reading info on PyPI about %s', predicate.name)
self._process_index_page(predicate.name)
if predicate.name.lower() not in self._projects:
raise ProjectNotFound
releases = self._projects.get(predicate.name.lower())
releases.sort_releases(prefer_final=prefer_final)
return releases
def run(self):
"""Runs the command."""
# Obviously have to build before we can install
if not self.skip_build:
self.run_command('build')
# If we built for any other platform, we can't install.
build_plat = self.distribution.get_command_obj('build').plat_name
# check warn_dir - it is a clue that the 'install_dist' is happening
# internally, and not to sys.path, so we don't check the platform
# matches what we are running.
if self.warn_dir and build_plat != get_platform():
raise PackagingPlatformError("Can't install when "
"cross-compiling")
# Run all sub-commands (at least those that need to be run)
for cmd_name in self.get_sub_commands():
self.run_command(cmd_name)
if self.path_file:
self.create_path_file()
# write list of installed files, if requested.
if self.record:
outputs = self.get_outputs()
if self.root: # strip any package prefix
root_len = len(self.root)
for counter in range(len(outputs)):
outputs[counter] = outputs[counter][root_len:]
self.execute(write_file,
(self.record, outputs),
"writing list of installed files to '%s'" %
self.record)
normpath, normcase = os.path.normpath, os.path.normcase
sys_path = [normcase(normpath(p)) for p in sys.path]
install_lib = normcase(normpath(self.install_lib))
if (self.warn_dir and
not (self.path_file and self.install_path_file) and
install_lib not in sys_path):
logger.debug(("modules installed to '%s', which is not in "
"Python's module search path (sys.path) -- "
"you'll have to change the search path yourself"),
self.install_lib)
def _set_command_options(self, command_obj, option_dict=None):
"""Set the options for 'command_obj' from 'option_dict'. Basically
this means copying elements of a dictionary ('option_dict') to
attributes of an instance ('command').
'command_obj' must be a Command instance. If 'option_dict' is not
supplied, uses the standard option dictionary for this command
(from 'self.command_options').
"""
command_name = command_obj.get_command_name()
if option_dict is None:
option_dict = self.get_option_dict(command_name)
logger.debug(" setting options for %r command:", command_name)
for option, (source, value) in option_dict.items():
logger.debug(" %s = %s (from %s)", option, value, source)
try:
bool_opts = [x.replace('-', '_')
for x in command_obj.boolean_options]
except AttributeError:
bool_opts = []
try:
neg_opt = command_obj.negative_opt
except AttributeError:
neg_opt = {}
try:
is_string = isinstance(value, str)
if option in neg_opt and is_string:
setattr(command_obj, neg_opt[option], not strtobool(value))
elif option in bool_opts and is_string:
setattr(command_obj, option, strtobool(value))
elif hasattr(command_obj, option):
setattr(command_obj, option, value)
else:
raise PackagingOptionError(
"error in %s: command %r has no such option %r" %
(source, command_name, option))
except ValueError as msg:
raise PackagingOptionError(msg)
def get_command_obj(self, command, create=True):
"""Return the command object for 'command'. Normally this object
is cached on a previous call to 'get_command_obj()'; if no command
object for 'command' is in the cache, then we either create and
return it (if 'create' is true) or return None.
"""
cmd_obj = self.command_obj.get(command)
if not cmd_obj and create:
logger.debug("Distribution.get_command_obj(): "
"creating %r command object", command)
cls = get_command_class(command)
cmd_obj = self.command_obj[command] = cls(self)
self.have_run[command] = 0
# Set any options that were supplied in config files or on the
# command line. (XXX support for error reporting is suboptimal
# here: errors aren't reported until finalize_options is called,
# which means we won't report the source of the error.)
options = self.command_options.get(command)
if options:
self._set_command_options(cmd_obj, options)
return cmd_obj
def link(self, target_desc, objects, output_filename, output_dir=None,
libraries=None, library_dirs=None, runtime_library_dirs=None,
export_symbols=None, debug=False, extra_preargs=None,
extra_postargs=None, build_temp=None, target_lang=None):
if not self.initialized:
self.initialize()
objects, output_dir = self._fix_object_args(objects, output_dir)
fixed_args = self._fix_lib_args(libraries, library_dirs,
runtime_library_dirs)
libraries, library_dirs, runtime_library_dirs = fixed_args
if runtime_library_dirs:
self.warn("don't know what to do with 'runtime_library_dirs': "
+ str(runtime_library_dirs))
lib_opts = gen_lib_options(self,
library_dirs, runtime_library_dirs,
libraries)
if output_dir is not None:
output_filename = os.path.join(output_dir, output_filename)
if self._need_link(objects, output_filename):
if target_desc == CCompiler.EXECUTABLE:
if debug:
ldflags = self.ldflags_shared_debug[1:]
else:
ldflags = self.ldflags_shared[1:]
else:
if debug:
ldflags = self.ldflags_shared_debug
else:
ldflags = self.ldflags_shared
export_opts = []
for sym in (export_symbols or []):
export_opts.append("/EXPORT:" + sym)
ld_args = (ldflags + lib_opts + export_opts +
objects + ['/OUT:' + output_filename])
# The MSVC linker generates .lib and .exp files, which cannot be
# suppressed by any linker switches. The .lib files may even be
# needed! Make sure they are generated in the temporary build
# directory. Since they have different names for debug and release
# builds, they can go into the same directory.
build_temp = os.path.dirname(objects[0])
if export_symbols is not None:
dll_name, dll_ext = os.path.splitext(
os.path.basename(output_filename))
implib_file = os.path.join(
build_temp,
self.library_filename(dll_name))
ld_args.append('/IMPLIB:' + implib_file)
# Embedded manifests are recommended - see MSDN article titled
# "How to: Embed a Manifest Inside a C/C++ Application"
# (currently at http://msdn2.microsoft.com/en-us/library/ms235591(VS.80).aspx)
# Ask the linker to generate the manifest in the temp dir, so
# we can embed it later.
temp_manifest = os.path.join(
build_temp,
os.path.basename(output_filename) + ".manifest")
ld_args.append('/MANIFESTFILE:' + temp_manifest)
if extra_preargs:
ld_args[:0] = extra_preargs
if extra_postargs:
ld_args.extend(extra_postargs)
self.mkpath(os.path.dirname(output_filename))
try:
self.spawn([self.linker] + ld_args)
except PackagingExecError as msg:
raise LinkError(msg)
# embed the manifest
# XXX - this is somewhat fragile - if mt.exe fails, distutils
# will still consider the DLL up-to-date, but it will not have a
# manifest. Maybe we should link to a temp file? OTOH, that
# implies a build environment error that shouldn't go undetected.
if target_desc == CCompiler.EXECUTABLE:
mfid = 1
else:
mfid = 2
self._remove_visual_c_ref(temp_manifest)
out_arg = '-outputresource:%s;%s' % (output_filename, mfid)
if self.__version < 10:
try:
self.spawn(['mt.exe', '-nologo', '-manifest',
temp_manifest, out_arg])
except PackagingExecError as msg:
raise LinkError(msg)
else:
logger.debug("skipping %s (up-to-date)", output_filename)
def byte_compile(py_files, optimize=0, force=False, prefix=None, base_dir=None, dry_run=False, direct=None):
"""Byte-compile a collection of Python source files to either .pyc
or .pyo files in a __pycache__ subdirectory.
'py_files' is a list of files to compile; any files that don't end in
".py" are silently skipped. 'optimize' must be one of the following:
0 - don't optimize (generate .pyc)
1 - normal optimization (like "python -O")
2 - extra optimization (like "python -OO")
This function is independent from the running Python's -O or -B options;
it is fully controlled by the parameters passed in.
If 'force' is true, all files are recompiled regardless of
timestamps.
The source filename encoded in each bytecode file defaults to the
filenames listed in 'py_files'; you can modify these with 'prefix' and
'basedir'. 'prefix' is a string that will be stripped off of each
source filename, and 'base_dir' is a directory name that will be
prepended (after 'prefix' is stripped). You can supply either or both
(or neither) of 'prefix' and 'base_dir', as you wish.
If 'dry_run' is true, doesn't actually do anything that would
affect the filesystem.
Byte-compilation is either done directly in this interpreter process
with the standard py_compile module, or indirectly by writing a
temporary script and executing it. Normally, you should let
'byte_compile()' figure out to use direct compilation or not (see
the source for details). The 'direct' flag is used by the script
generated in indirect mode; unless you know what you're doing, leave
it set to None.
"""
# FIXME use compileall + remove direct/indirect shenanigans
# First, if the caller didn't force us into direct or indirect mode,
# figure out which mode we should be in. We take a conservative
# approach: choose direct mode *only* if the current interpreter is
# in debug mode and optimize is 0. If we're not in debug mode (-O
# or -OO), we don't know which level of optimization this
# interpreter is running with, so we can't do direct
# byte-compilation and be certain that it's the right thing. Thus,
# always compile indirectly if the current interpreter is in either
# optimize mode, or if either optimization level was requested by
# the caller.
if direct is None:
direct = __debug__ and optimize == 0
# "Indirect" byte-compilation: write a temporary script and then
# run it with the appropriate flags.
if not direct:
from tempfile import mkstemp
# XXX use something better than mkstemp
script_fd, script_name = mkstemp(".py")
os.close(script_fd)
script_fd = None
logger.info("writing byte-compilation script '%s'", script_name)
if not dry_run:
if script_fd is not None:
script = os.fdopen(script_fd, "w", encoding="utf-8")
else:
script = open(script_name, "w", encoding="utf-8")
with script:
script.write(
"""\
from packaging.util import byte_compile
files = [
"""
)
# XXX would be nice to write absolute filenames, just for
# safety's sake (script should be more robust in the face of
# chdir'ing before running it). But this requires abspath'ing
# 'prefix' as well, and that breaks the hack in build_lib's
# 'byte_compile()' method that carefully tacks on a trailing
# slash (os.sep really) to make sure the prefix here is "just
# right". This whole prefix business is rather delicate -- the
# problem is that it's really a directory, but I'm treating it
# as a dumb string, so trailing slashes and so forth matter.
# py_files = map(os.path.abspath, py_files)
# if prefix:
# prefix = os.path.abspath(prefix)
script.write(",\n".join(map(repr, py_files)) + "]\n")
script.write(
"""
byte_compile(files, optimize=%r, force=%r,
prefix=%r, base_dir=%r,
dry_run=False,
direct=True)
"""
% (optimize, force, prefix, base_dir)
)
cmd = [sys.executable, script_name]
env = os.environ.copy()
env["PYTHONPATH"] = os.path.pathsep.join(sys.path)
try:
spawn(cmd, env=env)
finally:
execute(os.remove, (script_name,), "removing %s" % script_name, dry_run=dry_run)
# "Direct" byte-compilation: use the py_compile module to compile
# right here, right now. Note that the script generated in indirect
# mode simply calls 'byte_compile()' in direct mode, a weird sort of
# cross-process recursion. Hey, it works!
else:
from py_compile import compile
for file in py_files:
if file[-3:] != ".py":
# This lets us be lazy and not filter filenames in
# the "install_lib" command.
continue
# Terminology from the py_compile module:
# cfile - byte-compiled file
# dfile - purported source filename (same as 'file' by default)
# The second argument to cache_from_source forces the extension to
# be .pyc (if true) or .pyo (if false); without it, the extension
# would depend on the calling Python's -O option
cfile = imp.cache_from_source(file, not optimize)
dfile = file
if prefix:
if file[: len(prefix)] != prefix:
raise ValueError("invalid prefix: filename %r doesn't " "start with %r" % (file, prefix))
dfile = dfile[len(prefix) :]
if base_dir:
dfile = os.path.join(base_dir, dfile)
cfile_base = os.path.basename(cfile)
if direct:
if force or newer(file, cfile):
logger.info("byte-compiling %s to %s", file, cfile_base)
if not dry_run:
compile(file, cfile, dfile)
else:
logger.debug("skipping byte-compilation of %s to %s", file, cfile_base)
def _has_setup_py(srcdir):
setup_py = os.path.join(srcdir, "setup.py")
if os.path.isfile(setup_py):
logger.debug("setup.py file found.")
return True
return False
def run(self):
if (sys.platform != "win32" and
(self.distribution.has_ext_modules() or
self.distribution.has_c_libraries())):
raise PackagingPlatformError \
("distribution contains extensions and/or C libraries; "
"must be compiled on a Windows 32 platform")
if not self.skip_build:
self.run_command('build')
install = self.reinitialize_command('install', reinit_subcommands=True)
install.root = self.bdist_dir
install.skip_build = self.skip_build
install.warn_dir = False
install.plat_name = self.plat_name
install_lib = self.reinitialize_command('install_lib')
# we do not want to include pyc or pyo files
install_lib.compile = False
install_lib.optimize = 0
if self.distribution.has_ext_modules():
# If we are building an installer for a Python version other
# than the one we are currently running, then we need to ensure
# our build_lib reflects the other Python version rather than ours.
# Note that for target_version!=sys.version, we must have skipped the
# build step, so there is no issue with enforcing the build of this
# version.
target_version = self.target_version
if not target_version:
assert self.skip_build, "Should have already checked this"
target_version = '%s.%s' % sys.version_info[:2]
plat_specifier = ".%s-%s" % (self.plat_name, target_version)
build = self.get_finalized_command('build')
build.build_lib = os.path.join(build.build_base,
'lib' + plat_specifier)
# Use a custom scheme for the zip-file, because we have to decide
# at installation time which scheme to use.
for key in ('purelib', 'platlib', 'headers', 'scripts', 'data'):
value = key.upper()
if key == 'headers':
value = value + '/Include/$dist_name'
setattr(install,
'install_' + key,
value)
logger.info("installing to %s", self.bdist_dir)
install.ensure_finalized()
# avoid warning of 'install_lib' about installing
# into a directory not in sys.path
sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB'))
install.run()
del sys.path[0]
# And make an archive relative to the root of the
# pseudo-installation tree.
from tempfile import NamedTemporaryFile
archive_basename = NamedTemporaryFile().name
fullname = self.distribution.get_fullname()
arcname = self.make_archive(archive_basename, "zip",
root_dir=self.bdist_dir)
# create an exe containing the zip-file
self.create_exe(arcname, fullname, self.bitmap)
if self.distribution.has_ext_modules():
pyversion = get_python_version()
else:
pyversion = 'any'
self.distribution.dist_files.append(('bdist_wininst', pyversion,
self.get_installer_filename(fullname)))
# remove the zip-file again
logger.debug("removing temporary file '%s'", arcname)
os.remove(arcname)
if not self.keep_temp:
logger.info('removing %s', self.bdist_dir)
if not self.dry_run:
rmtree(self.bdist_dir)
def build_extension(self, ext):
sources = ext.sources
if sources is None or not isinstance(sources, (list, tuple)):
raise PackagingSetupError(("in 'ext_modules' option (extension '%s'), " +
"'sources' must be present and must be " +
"a list of source filenames") % ext.name)
sources = list(sources)
ext_path = self.get_ext_fullpath(ext.name)
depends = sources + ext.depends
if not (self.force or newer_group(depends, ext_path, 'newer')):
logger.debug("skipping '%s' extension (up-to-date)", ext.name)
return
else:
logger.info("building '%s' extension", ext.name)
# First, scan the sources for SWIG definition files (.i), run
# SWIG on 'em to create .c files, and modify the sources list
# accordingly.
sources = self.swig_sources(sources, ext)
# Next, compile the source code to object files.
# XXX not honouring 'define_macros' or 'undef_macros' -- the
# CCompiler API needs to change to accommodate this, and I
# want to do one thing at a time!
# Two possible sources for extra compiler arguments:
# - 'extra_compile_args' in Extension object
# - CFLAGS environment variable (not particularly
# elegant, but people seem to expect it and I
# guess it's useful)
# The environment variable should take precedence, and
# any sensible compiler will give precedence to later
# command-line args. Hence we combine them in order:
extra_args = ext.extra_compile_args or []
macros = ext.define_macros[:]
for undef in ext.undef_macros:
macros.append((undef,))
objects = self.compiler_obj.compile(sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=ext.include_dirs,
debug=self.debug,
extra_postargs=extra_args,
depends=ext.depends)
# XXX -- this is a Vile HACK!
#
# The setup.py script for Python on Unix needs to be able to
# get this list so it can perform all the clean up needed to
# avoid keeping object files around when cleaning out a failed
# build of an extension module. Since Packaging does not
# track dependencies, we have to get rid of intermediates to
# ensure all the intermediates will be properly re-built.
#
self._built_objects = objects[:]
# Now link the object files together into a "shared object" --
# of course, first we have to figure out all the other things
# that go into the mix.
if ext.extra_objects:
objects.extend(ext.extra_objects)
extra_args = ext.extra_link_args or []
# Detect target language, if not provided
language = ext.language or self.compiler_obj.detect_language(sources)
self.compiler_obj.link_shared_object(
objects, ext_path,
libraries=self.get_libraries(ext),
library_dirs=ext.library_dirs,
runtime_library_dirs=ext.runtime_library_dirs,
extra_postargs=extra_args,
export_symbols=self.get_export_symbols(ext),
debug=self.debug,
build_temp=self.build_temp,
target_lang=language)
def link(self, target_desc, objects, output_filename, output_dir=None,
libraries=None, library_dirs=None, runtime_library_dirs=None,
export_symbols=None, debug=False, extra_preargs=None,
extra_postargs=None, build_temp=None, target_lang=None):
# XXX this ignores 'build_temp'! should follow the lead of
# msvccompiler.py
objects, output_dir = self._fix_object_args(objects, output_dir)
libraries, library_dirs, runtime_library_dirs = \
self._fix_lib_args(libraries, library_dirs, runtime_library_dirs)
if runtime_library_dirs:
logger.warning("don't know what to do with "
"'runtime_library_dirs': %r", runtime_library_dirs)
if output_dir is not None:
output_filename = os.path.join(output_dir, output_filename)
if self._need_link(objects, output_filename):
# Figure out linker args based on type of target.
if target_desc == CCompiler.EXECUTABLE:
startup_obj = 'c0w32'
if debug:
ld_args = self.ldflags_exe_debug[:]
else:
ld_args = self.ldflags_exe[:]
else:
startup_obj = 'c0d32'
if debug:
ld_args = self.ldflags_shared_debug[:]
else:
ld_args = self.ldflags_shared[:]
# Create a temporary exports file for use by the linker
if export_symbols is None:
def_file = ''
else:
head, tail = os.path.split(output_filename)
modname, ext = os.path.splitext(tail)
temp_dir = os.path.dirname(objects[0]) # preserve tree structure
def_file = os.path.join(temp_dir, '%s.def' % modname)
contents = ['EXPORTS']
for sym in (export_symbols or []):
contents.append(' %s=_%s' % (sym, sym))
self.execute(write_file, (def_file, contents),
"writing %s" % def_file)
# Borland C++ has problems with '/' in paths
objects2 = [os.path.normpath(o) for o in objects]
# split objects in .obj and .res files
# Borland C++ needs them at different positions in the command line
objects = [startup_obj]
resources = []
for file in objects2:
base, ext = os.path.splitext(os.path.normcase(file))
if ext == '.res':
resources.append(file)
else:
objects.append(file)
for l in library_dirs:
ld_args.append("/L%s" % os.path.normpath(l))
ld_args.append("/L.") # we sometimes use relative paths
# list of object files
ld_args.extend(objects)
# XXX the command line syntax for Borland C++ is a bit wonky;
# certain filenames are jammed together in one big string, but
# comma-delimited. This doesn't mesh too well with the
# Unix-centric attitude (with a DOS/Windows quoting hack) of
# 'spawn()', so constructing the argument list is a bit
# awkward. Note that doing the obvious thing and jamming all
# the filenames and commas into one argument would be wrong,
# because 'spawn()' would quote any filenames with spaces in
# them. Arghghh!. Apparently it works fine as coded...
# name of dll/exe file
ld_args.extend((',',output_filename))
# no map file and start libraries
ld_args.append(',,')
for lib in libraries:
# see if we find it and if there is a bcpp specific lib
# (xxx_bcpp.lib)
libfile = self.find_library_file(library_dirs, lib, debug)
if libfile is None:
ld_args.append(lib)
# probably a BCPP internal library -- don't warn
else:
# full name which prefers bcpp_xxx.lib over xxx.lib
ld_args.append(libfile)
# some default libraries
ld_args.append('import32')
ld_args.append('cw32mt')
# def file for export symbols
ld_args.extend((',',def_file))
# add resource files
ld_args.append(',')
ld_args.extend(resources)
if extra_preargs:
ld_args[:0] = extra_preargs
if extra_postargs:
ld_args.extend(extra_postargs)
self.mkpath(os.path.dirname(output_filename))
try:
self.spawn([self.linker] + ld_args)
except PackagingExecError as msg:
raise LinkError(msg)
else:
logger.debug("skipping %s (up-to-date)", output_filename)
def copy_scripts(self):
"""Copy each script listed in 'self.scripts'; if it's marked as a
Python script in the Unix way (first line matches 'first_line_re',
ie. starts with "\#!" and contains "python"), then adjust the first
line to refer to the current Python interpreter as we copy.
"""
self.mkpath(self.build_dir)
outfiles = []
for script in self.scripts:
adjust = False
script = convert_path(script)
outfile = os.path.join(self.build_dir, os.path.basename(script))
outfiles.append(outfile)
if not self.force and not newer(script, outfile):
logger.debug("not copying %s (up-to-date)", script)
continue
# Always open the file, but ignore failures in dry-run mode --
# that way, we'll get accurate feedback if we can read the
# script.
try:
f = open(script, "rb")
except IOError:
if not self.dry_run:
raise
f = None
else:
encoding, lines = detect_encoding(f.readline)
f.seek(0)
first_line = f.readline()
if not first_line:
logger.warning('%s: %s is an empty file (skipping)',
self.get_command_name(), script)
continue
match = first_line_re.match(first_line)
if match:
adjust = True
post_interp = match.group(1) or b''
if adjust:
logger.info("copying and adjusting %s -> %s", script,
self.build_dir)
if not self.dry_run:
if not sysconfig.is_python_build():
executable = self.executable
else:
executable = os.path.join(
sysconfig.get_config_var("BINDIR"),
"python%s%s" % (sysconfig.get_config_var("VERSION"),
sysconfig.get_config_var("EXE")))
executable = os.fsencode(executable)
shebang = b"#!" + executable + post_interp + b"\n"
# Python parser starts to read a script using UTF-8 until
# it gets a #coding:xxx cookie. The shebang has to be the
# first line of a file, the #coding:xxx cookie cannot be
# written before. So the shebang has to be decodable from
# UTF-8.
try:
shebang.decode('utf-8')
except UnicodeDecodeError:
raise ValueError(
"The shebang ({!r}) is not decodable "
"from utf-8".format(shebang))
# If the script is encoded to a custom encoding (use a
# #coding:xxx cookie), the shebang has to be decodable from
# the script encoding too.
try:
shebang.decode(encoding)
except UnicodeDecodeError:
raise ValueError(
"The shebang ({!r}) is not decodable "
"from the script encoding ({})"
.format(shebang, encoding))
with open(outfile, "wb") as outf:
outf.write(shebang)
outf.writelines(f.readlines())
if f:
f.close()
else:
if f:
f.close()
self.copy_file(script, outfile)
if os.name == 'posix':
for file in outfiles:
if self.dry_run:
logger.info("changing mode of %s", file)
else:
oldmode = os.stat(file).st_mode & 0o7777
newmode = (oldmode | 0o555) & 0o7777
if newmode != oldmode:
logger.info("changing mode of %s from %o to %o",
file, oldmode, newmode)
os.chmod(file, newmode)
return outfiles
def parse_config_files(self, filenames=None):
if filenames is None:
filenames = self.find_config_files()
logger.debug("Distribution.parse_config_files():")
parser = RawConfigParser()
for filename in filenames:
logger.debug(" reading %s", filename)
parser.read(filename, encoding='utf-8')
if os.path.split(filename)[-1] == 'setup.cfg':
self._read_setup_cfg(parser, filename)
for section in parser.sections():
if section == 'global':
if parser.has_option('global', 'compilers'):
self._load_compilers(parser.get('global', 'compilers'))
if parser.has_option('global', 'commands'):
self._load_commands(parser.get('global', 'commands'))
options = parser.options(section)
opt_dict = self.dist.get_option_dict(section)
for opt in options:
if opt == '__name__':
continue
val = parser.get(section, opt)
opt = opt.replace('-', '_')
if opt == 'sub_commands':
val = split_multiline(val)
if isinstance(val, str):
val = [val]
# Hooks use a suffix system to prevent being overriden
# by a config file processed later (i.e. a hook set in
# the user config file cannot be replaced by a hook
# set in a project config file, unless they have the
# same suffix).
if (opt.startswith("pre_hook.") or
opt.startswith("post_hook.")):
hook_type, alias = opt.split(".")
hook_dict = opt_dict.setdefault(
hook_type, (filename, {}))[1]
hook_dict[alias] = val
else:
opt_dict[opt] = filename, val
# Make the RawConfigParser forget everything (so we retain
# the original filenames that options come from)
parser.__init__()
# If there was a "global" section in the config file, use it
# to set Distribution options.
if 'global' in self.dist.command_options:
for opt, (src, val) in self.dist.command_options['global'].items():
alias = self.dist.negative_opt.get(opt)
try:
if alias:
setattr(self.dist, alias, not strtobool(val))
elif opt == 'dry_run': # FIXME ugh!
setattr(self.dist, opt, strtobool(val))
else:
setattr(self.dist, opt, val)
except ValueError as msg:
raise PackagingOptionError(msg)
def get_infos(requirements, index=None, installed=None, prefer_final=True):
"""Return the informations on what's going to be installed and upgraded.
:param requirements: is a *string* containing the requirements for this
project (for instance "FooBar 1.1" or "BarBaz (<1.2)")
:param index: If an index is specified, use this one, otherwise, use
:class index.ClientWrapper: to get project metadatas.
:param installed: a list of already installed distributions.
:param prefer_final: when picking up the releases, prefer a "final" one
over a beta/alpha/etc one.
The results are returned in a dict, containing all the operations
needed to install the given requirements::
>>> get_install_info("FooBar (<=1.2)")
{'install': [<FooBar 1.1>], 'remove': [], 'conflict': []}
Conflict contains all the conflicting distributions, if there is a
conflict.
"""
# this function does several things:
# 1. get a release specified by the requirements
# 2. gather its metadata, using setuptools compatibility if needed
# 3. compare this tree with what is currently installed on the system,
# return the requirements of what is missing
# 4. do that recursively and merge back the results
# 5. return a dict containing information about what is needed to install
# or remove
if not installed:
logger.debug('Reading installed distributions')
installed = list(get_distributions(use_egg_info=True))
infos = {'install': [], 'remove': [], 'conflict': []}
# Is a compatible version of the project already installed ?
predicate = get_version_predicate(requirements)
found = False
# check that the project isn't already installed
for installed_project in installed:
# is it a compatible project ?
if predicate.name.lower() != installed_project.name.lower():
continue
found = True
logger.info('Found %r %s', installed_project.name,
installed_project.version)
# if we already have something installed, check it matches the
# requirements
if predicate.match(installed_project.version):
return infos
break
if not found:
logger.debug('Project not installed')
if not index:
index = wrapper.ClientWrapper()
if not installed:
installed = get_distributions(use_egg_info=True)
# Get all the releases that match the requirements
try:
release = index.get_release(requirements)
except (ReleaseNotFound, ProjectNotFound):
raise InstallationException('Release not found: %r' % requirements)
if release is None:
logger.info('Could not find a matching project')
return infos
metadata = release.fetch_metadata()
# we need to build setuptools deps if any
if 'requires_dist' not in metadata:
metadata['requires_dist'] = _get_setuptools_deps(release)
# build the dependency graph with local and required dependencies
dists = list(installed)
dists.append(release)
depgraph = generate_graph(dists)
# Get what the missing deps are
dists = depgraph.missing[release]
if dists:
logger.info("Missing dependencies found, retrieving metadata")
# we have missing deps
for dist in dists:
_update_infos(infos, get_infos(dist, index, installed))
# Fill in the infos
existing = [d for d in installed if d.name == release.name]
if existing:
infos['remove'].append(existing[0])
infos['conflict'].extend(depgraph.reverse_list[existing[0]])
infos['install'].append(release)
return infos
def link(
self,
target_desc,
objects,
output_filename,
output_dir=None,
libraries=None,
library_dirs=None,
runtime_library_dirs=None,
export_symbols=None,
debug=False,
extra_preargs=None,
extra_postargs=None,
build_temp=None,
target_lang=None,
):
if not self.initialized:
self.initialize()
objects, output_dir = self._fix_object_args(objects, output_dir)
libraries, library_dirs, runtime_library_dirs = self._fix_lib_args(
libraries, library_dirs, runtime_library_dirs
)
if runtime_library_dirs:
self.warn("don't know what to do with 'runtime_library_dirs': %s" % (runtime_library_dirs,))
lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs, libraries)
if output_dir is not None:
output_filename = os.path.join(output_dir, output_filename)
if self._need_link(objects, output_filename):
if target_desc == CCompiler.EXECUTABLE:
if debug:
ldflags = self.ldflags_shared_debug[1:]
else:
ldflags = self.ldflags_shared[1:]
else:
if debug:
ldflags = self.ldflags_shared_debug
else:
ldflags = self.ldflags_shared
export_opts = []
for sym in export_symbols or []:
export_opts.append("/EXPORT:" + sym)
ld_args = ldflags + lib_opts + export_opts + objects + ["/OUT:" + output_filename]
# The MSVC linker generates .lib and .exp files, which cannot be
# suppressed by any linker switches. The .lib files may even be
# needed! Make sure they are generated in the temporary build
# directory. Since they have different names for debug and release
# builds, they can go into the same directory.
if export_symbols is not None:
dll_name, dll_ext = os.path.splitext(os.path.basename(output_filename))
implib_file = os.path.join(os.path.dirname(objects[0]), self.library_filename(dll_name))
ld_args.append("/IMPLIB:" + implib_file)
if extra_preargs:
ld_args[:0] = extra_preargs
if extra_postargs:
ld_args.extend(extra_postargs)
self.mkpath(os.path.dirname(output_filename))
try:
self.spawn([self.linker] + ld_args)
except PackagingExecError as msg:
raise LinkError(msg)
else:
logger.debug("skipping %s (up-to-date)", output_filename)