def make_distribution(self):
"""Create the source distribution(s). First, we create the release
tree with 'make_release_tree()'; then, we create all required
archive files (according to 'self.formats') from the release tree.
Finally, we clean up by blowing away the release tree (unless
'self.keep_temp' is true). The list of archive files created is
stored so it can be retrieved later by 'get_archive_files()'.
"""
# Don't warn about missing metadata here -- should be (and is!)
# done elsewhere.
base_dir = self.distribution.get_fullname()
base_name = os.path.join(self.dist_dir, base_dir)
self.make_release_tree(base_dir, self.filelist.files)
archive_files = [] # remember names of files we create
# tar archive must be created last to avoid overwrite and remove
if 'tar' in self.formats:
self.formats.append(self.formats.pop(self.formats.index('tar')))
for fmt in self.formats:
file = self.make_archive(base_name, fmt, base_dir=base_dir,
owner=self.owner, group=self.group)
archive_files.append(file)
self.distribution.dist_files.append(('sdist', '', file))
self.archive_files = archive_files
if not self.keep_temp:
if self.dry_run:
logger.info('removing %s', base_dir)
else:
rmtree(base_dir)
def build_libraries(self, libraries):
for lib_name, build_info in libraries:
sources = build_info.get('sources')
if sources is None or not isinstance(sources, (list, tuple)):
raise PackagingSetupError(("in 'libraries' option (library '%s'), " +
"'sources' must be present and must be " +
"a list of source filenames") % lib_name)
sources = list(sources)
logger.info("building '%s' library", lib_name)
# First, compile the source code to object files in the library
# directory. (This should probably change to putting object
# files in a temporary build directory.)
macros = build_info.get('macros')
include_dirs = build_info.get('include_dirs')
objects = self.compiler.compile(sources,
output_dir=self.build_temp,
macros=macros,
include_dirs=include_dirs,
debug=self.debug)
# Now "link" the object files together into a static library.
# (On Unix at least, this isn't really linking -- it just
# builds an archive. Whatever.)
self.compiler.create_static_lib(objects, lib_name,
output_dir=self.build_clib,
debug=self.debug)
def run_command_hooks(self, cmd_obj, hook_kind):
"""Run hooks registered for that command and phase.
*cmd_obj* is a finalized command object; *hook_kind* is either
'pre_hook' or 'post_hook'.
"""
if hook_kind not in ('pre_hook', 'post_hook'):
raise ValueError('invalid hook kind: %r' % hook_kind)
hooks = getattr(cmd_obj, hook_kind, None)
if hooks is None:
return
for hook in hooks.values():
if isinstance(hook, str):
try:
hook_obj = resolve_name(hook)
except ImportError as e:
raise PackagingModuleError(e)
else:
hook_obj = hook
if not callable(hook_obj):
raise PackagingOptionError('hook %r is not callable' % hook)
logger.info('running %s %s for command %s',
hook_kind, hook, cmd_obj.get_command_name())
hook_obj(cmd_obj)
def _metadata(dispatcher, args, **kw):
opts = _parse_args(args[1:], 'f:', [])
if opts['args']:
name = opts['args'][0]
dist = get_distribution(name, use_egg_info=True)
if dist is None:
logger.warning('%r not installed', name)
return 1
elif os.path.isfile('setup.cfg'):
logger.info('searching local dir for metadata')
dist = Distribution() # XXX use config module
dist.parse_config_files()
else:
logger.warning('no argument given and no local setup.cfg found')
return 1
metadata = dist.metadata
if 'f' in opts:
keys = (k for k in opts['f'] if k in metadata)
else:
keys = metadata.keys()
for key in keys:
if key in metadata:
print(metadata._convert_name(key) + ':')
value = metadata[key]
if isinstance(value, list):
for v in value:
print(' ', v)
else:
print(' ', value.replace('\n', '\n '))
def read(self, path):
"""Read the manifest file (named by 'self.manifest') and use it to
fill in 'self.filelist', the list of files to include in the source
distribution.
"""
logger.info("reading manifest file %r", path)
with open(path) as manifest:
for line in manifest.readlines():
self.append(line)
def _clean(self, *filenames):
if not filenames:
filenames = self.temp_files
self.temp_files = []
logger.info("removing: %s", " ".join(filenames))
for filename in filenames:
try:
os.remove(filename)
except OSError:
pass
def run(self):
if not self.skip_build:
self.run_command('build')
install = self.reinitialize_command('install_dist',
reinit_subcommands=True)
install.root = self.bdist_dir
install.skip_build = self.skip_build
install.warn_dir = False
logger.info("installing to %s", self.bdist_dir)
self.run_command('install_dist')
# And make an archive relative to the root of the
# pseudo-installation tree.
archive_basename = "%s.%s" % (self.distribution.get_fullname(),
self.plat_name)
# OS/2 objects to any ":" characters in a filename (such as when
# a timestamp is used in a version) so change them to hyphens.
if os.name == "os2":
archive_basename = archive_basename.replace(":", "-")
pseudoinstall_root = os.path.join(self.dist_dir, archive_basename)
if not self.relative:
archive_root = self.bdist_dir
else:
if (self.distribution.has_ext_modules() and
(install.install_base != install.install_platbase)):
raise PackagingPlatformError(
"can't make a dumb built distribution where base and "
"platbase are different (%r, %r)" %
(install.install_base, install.install_platbase))
else:
archive_root = os.path.join(
self.bdist_dir,
self._ensure_relative(install.install_base))
# Make the archive
filename = self.make_archive(pseudoinstall_root,
self.format, root_dir=archive_root,
owner=self.owner, group=self.group)
if self.distribution.has_ext_modules():
pyversion = get_python_version()
else:
pyversion = 'any'
self.distribution.dist_files.append(('bdist_dumb', pyversion,
filename))
if not self.keep_temp:
if self.dry_run:
logger.info('removing %s', self.bdist_dir)
else:
rmtree(self.bdist_dir)
def mkpath(self, name, mode=0o777):
name = os.path.normpath(name)
if os.path.isdir(name) or name == "":
return
if self.dry_run:
head = ""
for part in name.split(os.sep):
logger.info("created directory %s%s", head, part)
head += part + os.sep
return
os.makedirs(name, mode)
def create_exe(self, arcname, fullname, bitmap=None):
import struct
self.mkpath(self.dist_dir)
cfgdata = self.get_inidata()
installer_name = self.get_installer_filename(fullname)
logger.info("creating %s", installer_name)
if bitmap:
with open(bitmap, "rb") as fp:
bitmapdata = fp.read()
bitmaplen = len(bitmapdata)
else:
bitmaplen = 0
with open(installer_name, "wb") as file:
file.write(self.get_exe_bytes())
if bitmap:
file.write(bitmapdata)
# Convert cfgdata from unicode to ascii, mbcs encoded
if isinstance(cfgdata, str):
cfgdata = cfgdata.encode("mbcs")
# Append the pre-install script
cfgdata = cfgdata + b"\0"
if self.pre_install_script:
# We need to normalize newlines, so we open in text mode and
# convert back to bytes. "latin-1" simply avoids any possible
# failures.
with open(self.pre_install_script, encoding="latin-1") as fp:
script_data = fp.read().encode("latin-1")
cfgdata = cfgdata + script_data + b"\n\0"
else:
# empty pre-install script
cfgdata = cfgdata + b"\0"
file.write(cfgdata)
# The 'magic number' 0x1234567B is used to make sure that the
# binary layout of 'cfgdata' is what the wininst.exe binary
# expects. If the layout changes, increment that number, make
# the corresponding changes to the wininst.exe sources, and
# recompile them.
header = struct.pack("<iii",
0x1234567B, # tag
len(cfgdata), # length
bitmaplen, # number of bytes in bitmap
)
file.write(header)
with open(arcname, "rb") as fp:
file.write(fp.read())
def mkpath(self, name, mode=0o777, dry_run=None):
if dry_run is None:
dry_run = self.dry_run
name = os.path.normpath(name)
if os.path.isdir(name) or name == '':
return
if dry_run:
head = ''
for part in name.split(os.sep):
logger.info("created directory %s%s", head, part)
head += part + os.sep
return
os.makedirs(name, mode)
def egginfo_to_distinfo(record_file, installer=_DEFAULT_INSTALLER, requested=False, remove_egginfo=False):
"""Create files and directories required for PEP 376
:param record_file: path to RECORD file as produced by setup.py --record
:param installer: installer name
:param requested: True if not installed as a dependency
:param remove_egginfo: delete egginfo dir?
"""
distinfo = _parse_record_file(record_file)
distinfo_dir = distinfo["distinfo_dir"]
if os.path.isdir(distinfo_dir) and not os.path.islink(distinfo_dir):
shutil.rmtree(distinfo_dir)
elif os.path.exists(distinfo_dir):
os.unlink(distinfo_dir)
os.makedirs(distinfo_dir)
# copy setuptools extra metadata files
if distinfo["extra_metadata"]:
for path in distinfo["extra_metadata"]:
shutil.copy2(path, distinfo_dir)
new_path = path.replace("egg-info", "dist-info")
distinfo["installed"].append(new_path)
metadata_path = distinfo["metadata_path"]
logger.info("creating %s", metadata_path)
shutil.copy2(distinfo["metadata"], metadata_path)
installer_path = distinfo["installer_path"]
logger.info("creating %s", installer_path)
with open(installer_path, "w") as f:
f.write(installer)
if requested:
requested_path = distinfo["requested_path"]
logger.info("creating %s", requested_path)
open(requested_path, "wb").close()
distinfo["installed"].append(requested_path)
record_path = distinfo["record_path"]
logger.info("creating %s", record_path)
_write_record_file(record_path, distinfo["installed"])
if remove_egginfo:
egginfo = distinfo["egginfo"]
logger.info("removing %s", egginfo)
if os.path.isfile(egginfo):
os.remove(egginfo)
else:
shutil.rmtree(egginfo)
def dump_options(self, header=None, indent=""):
if header is None:
header = "command options for '%s':" % self.get_command_name()
logger.info(indent + header)
indent = indent + " "
negative_opt = getattr(self, 'negative_opt', ())
for option, _, _ in self.user_options:
if option in negative_opt:
continue
option = option.replace('-', '_')
if option[-1] == "=":
option = option[:-1]
value = getattr(self, option)
logger.info(indent + "%s = %s", option, value)
def make_release_tree(self, base_dir, files):
"""Create the directory tree that will become the source
distribution archive. All directories implied by the filenames in
'files' are created under 'base_dir', and then we hard link or copy
(if hard linking is unavailable) those files into place.
Essentially, this duplicates the developer's source tree, but in a
directory named after the distribution, containing only the files
to be distributed.
"""
# Create all the directories under 'base_dir' necessary to
# put 'files' there; the 'mkpath()' is just so we don't die
# if the manifest happens to be empty.
self.mkpath(base_dir)
self.create_tree(base_dir, files, dry_run=self.dry_run)
# And walk over the list of files, either making a hard link (if
# os.link exists) to each one that doesn't already exist in its
# corresponding location under 'base_dir', or copying each file
# that's out-of-date in 'base_dir'. (Usually, all files will be
# out-of-date, because by default we blow away 'base_dir' when
# we're done making the distribution archives.)
if hasattr(os, 'link'): # can make hard links on this system
link = 'hard'
msg = "making hard links in %s..." % base_dir
else: # nope, have to copy
link = None
msg = "copying files to %s..." % base_dir
if not files:
logger.warning("no files to distribute -- empty manifest?")
else:
logger.info(msg)
for file in self.distribution.metadata.requires_files:
if file not in files:
msg = "'%s' must be included explicitly in 'extra_files'" \
% file
raise PackagingFileError(msg)
for file in files:
if not os.path.isfile(file):
logger.warning("'%s' not a regular file -- skipping", file)
else:
dest = os.path.join(base_dir, file)
self.copy_file(file, dest, link=link)
self.distribution.metadata.write(os.path.join(base_dir, 'PKG-INFO'))
def _run_install_from_dir(source_dir):
old_dir = os.getcwd()
os.chdir(source_dir)
install_method = get_install_method(source_dir)
func = install_methods[install_method]
try:
func = install_methods[install_method]
try:
func(source_dir)
return True
except ValueError as err:
# failed to install
logger.info(str(err))
return False
finally:
os.chdir(old_dir)
def try_compile(self, body, headers=None, include_dirs=None, lang="c"):
"""Try to compile a source file built from 'body' and 'headers'.
Return true on success, false otherwise.
"""
from packaging.compiler.ccompiler import CompileError
self._check_compiler()
try:
self._compile(body, headers, include_dirs, lang)
ok = True
except CompileError:
ok = False
logger.info(ok and "success!" or "failure.")
self._clean()
return ok
def swig_sources(self, sources, extension):
"""Walk the list of source files in 'sources', looking for SWIG
interface (.i) files. Run SWIG on all that are found, and
return a modified 'sources' list with SWIG source files replaced
by the generated C (or C++) files.
"""
new_sources = []
swig_sources = []
swig_targets = {}
# XXX this drops generated C/C++ files into the source tree, which
# is fine for developers who want to distribute the generated
# source -- but there should be an option to put SWIG output in
# the temp dir.
if ('-c++' in self.swig_opts or '-c++' in extension.swig_opts):
target_ext = '.cpp'
else:
target_ext = '.c'
for source in sources:
base, ext = os.path.splitext(source)
if ext == ".i": # SWIG interface file
new_sources.append(base + '_wrap' + target_ext)
swig_sources.append(source)
swig_targets[source] = new_sources[-1]
else:
new_sources.append(source)
if not swig_sources:
return new_sources
swig = self.swig or self.find_swig()
swig_cmd = [swig, "-python"]
swig_cmd.extend(self.swig_opts)
# Do not override commandline arguments
if not self.swig_opts:
for o in extension.swig_opts:
swig_cmd.append(o)
for source in swig_sources:
target = swig_targets[source]
logger.info("swigging %s to %s", source, target)
self.spawn(swig_cmd + ["-o", target, source])
return new_sources
def try_link(self, body, headers=None, include_dirs=None, libraries=None, library_dirs=None, lang="c"):
"""Try to compile and link a source file, built from 'body' and
'headers', to executable form. Return true on success, false
otherwise.
"""
from packaging.compiler.ccompiler import CompileError, LinkError
self._check_compiler()
try:
self._link(body, headers, include_dirs, libraries, library_dirs, lang)
ok = True
except (CompileError, LinkError):
ok = False
logger.info(ok and "success!" or "failure.")
self._clean()
return ok
def main(args=None):
old_level = logger.level
old_handlers = list(logger.handlers)
try:
dispatcher = Dispatcher(args)
if dispatcher.action is None:
return
return dispatcher()
except KeyboardInterrupt:
logger.info('interrupted')
return 1
except (IOError, os.error, PackagingError, CCompilerError) as exc:
logger.exception(exc)
return 1
finally:
logger.setLevel(old_level)
logger.handlers[:] = old_handlers
def _mkpath(name, mode=0o777, dry_run=False):
# Detect a common bug -- name is None
if not isinstance(name, str):
raise PackagingInternalError("mkpath: 'name' must be a string (got %r)" % (name,))
# XXX what's the better way to handle verbosity? print as we create
# each directory in the path (the current behaviour), or only announce
# the creation of the whole path? (quite easy to do the latter since
# we're not using a recursive algorithm)
name = os.path.normpath(name)
created_dirs = []
if os.path.isdir(name) or name == "":
return created_dirs
if os.path.abspath(name) in _path_created:
return created_dirs
head, tail = os.path.split(name)
tails = [tail] # stack of lone dirs to create
while head and tail and not os.path.isdir(head):
head, tail = os.path.split(head)
tails.insert(0, tail) # push next higher dir onto stack
# now 'head' contains the deepest directory that already exists
# (that is, the child of 'head' in 'name' is the highest directory
# that does *not* exist)
for d in tails:
head = os.path.join(head, d)
abs_head = os.path.abspath(head)
if abs_head in _path_created:
continue
logger.info("creating %s", head)
if not dry_run:
try:
os.mkdir(head, mode)
except OSError as exc:
if not (exc.errno == errno.EEXIST and os.path.isdir(head)):
raise PackagingFileError("could not create '%s': %s" % (head, exc.args[-1]))
created_dirs.append(head)
_path_created.add(abs_head)
return created_dirs
def try_run(self, body, headers=None, include_dirs=None, libraries=None, library_dirs=None, lang="c"):
"""Try to compile, link to an executable, and run a program
built from 'body' and 'headers'. Return true on success, false
otherwise.
"""
from packaging.compiler.ccompiler import CompileError, LinkError
self._check_compiler()
try:
src, obj, exe = self._link(body, headers, include_dirs, libraries, library_dirs, lang)
self.spawn([exe])
ok = True
except (CompileError, LinkError, PackagingExecError):
ok = False
logger.info(ok and "success!" or "failure.")
self._clean()
return ok
def execute(func, args, msg=None, dry_run=False):
"""Perform some action that affects the outside world.
Some actions (e.g. writing to the filesystem) are special because
they are disabled by the 'dry_run' flag. This method takes care of all
that bureaucracy for you; all you have to do is supply the
function to call and an argument tuple for it (to embody the
"external action" being performed), and an optional message to
print.
"""
if msg is None:
msg = "%s%r" % (func.__name__, args)
if msg[-2:] == ",)": # correct for singleton tuple
msg = msg[0:-2] + ")"
logger.info(msg)
if not dry_run:
func(*args)
def finalize_options(self):
if self.repository is None:
self.repository = DEFAULT_REPOSITORY
if self.realm is None:
self.realm = DEFAULT_REALM
if self.upload_dir is None:
build = self.get_finalized_command('build')
self.upload_dir = os.path.join(build.build_base, "docs")
if not os.path.isdir(self.upload_dir):
self.upload_dir = os.path.join(build.build_base, "doc")
logger.info('Using upload directory %s', self.upload_dir)
self.verify_upload_dir(self.upload_dir)
config = read_pypirc(self.repository, self.realm)
if config != {}:
self.username = config['username']
self.password = config['password']
self.repository = config['repository']
self.realm = config['realm']
def _run_2to3(self, files=[], doctests=[], fixers=[]):
""" Takes a list of files and doctests, and performs conversion
on those.
- First, the files which contain the code(`files`) are converted.
- Second, the doctests in `files` are converted.
- Thirdly, the doctests in `doctests` are converted.
"""
if fixers:
self.fixer_names = fixers
if files:
logger.info('converting Python code and doctests')
_KLASS.run_2to3(self, files)
_KLASS.run_2to3(self, files, doctests_only=True)
if doctests:
logger.info('converting doctests in text files')
_KLASS.run_2to3(self, doctests, doctests_only=True)
def run(self):
name = self.distribution.metadata['Name']
version = self.distribution.metadata['Version']
zip_file = zip_dir(self.upload_dir)
fields = [(':action', 'doc_upload'),
('name', name), ('version', version)]
files = [('content', name, zip_file.getvalue())]
content_type, body = encode_multipart(fields, files)
credentials = self.username + ':' + self.password
# FIXME should use explicit encoding
auth = b"Basic " + base64.encodebytes(credentials.encode()).strip()
logger.info("Submitting documentation to %s", self.repository)
scheme, netloc, url, params, query, fragments = urllib.parse.urlparse(
self.repository)
if scheme == "http":
conn = http.client.HTTPConnection(netloc)
elif scheme == "https":
conn = http.client.HTTPSConnection(netloc)
else:
raise AssertionError("unsupported scheme %r" % scheme)
try:
conn.connect()
conn.putrequest("POST", url)
conn.putheader('Content-type', content_type)
conn.putheader('Content-length', str(len(body)))
conn.putheader('Authorization', auth)
conn.endheaders()
conn.send(body)
except socket.error as e:
logger.error(e)
return
r = conn.getresponse()
if r.status == 200:
logger.info('Server response (%s): %s', r.status, r.reason)
elif r.status == 301:
location = r.getheader('Location')
if location is None:
location = 'http://packages.python.org/%s/' % name
logger.info('Upload successful. Visit %s', location)
else:
logger.error('Upload failed (%s): %s', r.status, r.reason)
if self.show_response and logger.isEnabledFor(logging.INFO):
sep = '-' * 75
logger.info('%s\n%s\n%s', sep, r.read().decode('utf-8'), sep)
def run(self):
if not self.skip_build:
self.run_command('build_scripts')
if not os.path.exists(self.build_dir):
self.outfiles = []
return
self.outfiles = self.copy_tree(self.build_dir, self.install_dir)
if os.name == 'posix':
# Set the executable bits (owner, group, and world) on
# all the scripts we just installed.
for file in self.get_outputs():
if self.dry_run:
logger.info("changing mode of %s", file)
else:
mode = (os.stat(file).st_mode | 0o555) & 0o7777
logger.info("changing mode of %s to %o", file, mode)
os.chmod(file, mode)
def write(self, path):
"""Write the file list in 'self.filelist' (presumably as filled in
by 'add_defaults()' and 'read_template()') to the manifest file
named by 'self.manifest'.
"""
if os.path.isfile(path):
with open(path) as fp:
first_line = fp.readline()
if first_line != '# file GENERATED by packaging, do NOT edit\n':
logger.info("not writing to manually maintained "
"manifest file %r", path)
return
self.sort()
self.remove_duplicates()
content = self.files[:]
content.insert(0, '# file GENERATED by packaging, do NOT edit')
logger.info("writing manifest file %r", path)
write_file(path, content)
def copy_tree(src, dst, preserve_mode=True, preserve_times=True, preserve_symlinks=False, update=False, dry_run=False):
# FIXME use of this function is why we get spurious logging message on
# stdout when tests run; kill and replace by shutil!
from distutils.file_util import copy_file
if not dry_run and not os.path.isdir(src):
raise PackagingFileError("cannot copy tree '%s': not a directory" % src)
try:
names = os.listdir(src)
except os.error as e:
errstr = e[1]
if dry_run:
names = []
else:
raise PackagingFileError("error listing files in '%s': %s" % (src, errstr))
if not dry_run:
_mkpath(dst)
outputs = []
for n in names:
src_name = os.path.join(src, n)
dst_name = os.path.join(dst, n)
if preserve_symlinks and os.path.islink(src_name):
link_dest = os.readlink(src_name)
logger.info("linking %s -> %s", dst_name, link_dest)
if not dry_run:
os.symlink(link_dest, dst_name)
outputs.append(dst_name)
elif os.path.isdir(src_name):
outputs.extend(
copy_tree(src_name, dst_name, preserve_mode, preserve_times, preserve_symlinks, update, dry_run=dry_run)
)
else:
copy_file(src_name, dst_name, preserve_mode, preserve_times, update, dry_run=dry_run)
outputs.append(dst_name)
return outputs
def install_dists(dists, path, paths=None):
"""Install all distributions provided in dists, with the given prefix.
If an error occurs while installing one of the distributions, uninstall all
the installed distribution (in the context if this function).
Return a list of installed dists.
:param dists: distributions to install
:param path: base path to install distribution in
:param paths: list of paths (defaults to sys.path) to look for info
"""
installed_dists = []
for dist in dists:
logger.info('Installing %r %s...', dist.name, dist.version)
try:
_install_dist(dist, path)
installed_dists.append(dist)
except Exception as e:
logger.info('Failed: %s', e)
# reverting
for installed_dist in installed_dists:
logger.info('Reverting %r', installed_dist)
remove(installed_dist.name, paths)
raise e
return installed_dists
def run(self):
# remove the build/temp.<plat> directory (unless it's already
# gone)
if os.path.exists(self.build_temp):
if self.dry_run:
logger.info('removing %s', self.build_temp)
else:
rmtree(self.build_temp)
else:
logger.debug("'%s' does not exist -- can't clean it",
self.build_temp)
if self.all:
# remove build directories
for directory in (self.build_lib,
self.bdist_base,
self.build_scripts):
if os.path.exists(directory):
if self.dry_run:
logger.info('removing %s', directory)
else:
rmtree(directory)
else:
logger.warning("'%s' does not exist -- can't clean it",
directory)
# just for the heck of it, try to remove the base build directory:
# we might have emptied it right now, but if not we don't care
if not self.dry_run:
try:
os.rmdir(self.build_base)
logger.info("removing '%s'", self.build_base)
except OSError:
pass
def run_command(self, command, options=None):
"""Do whatever it takes to run a command (including nothing at all,
if the command has already been run). Specifically: if we have
already created and run the command named by 'command', return
silently without doing anything. If the command named by 'command'
doesn't even have a command object yet, create one. Then invoke
'run()' on that command object (or an existing one).
"""
# Already been here, done that? then return silently.
if self.have_run.get(command):
return
if options is not None:
self.command_options[command] = options
cmd_obj = self.get_command_obj(command)
cmd_obj.ensure_finalized()
self.run_command_hooks(cmd_obj, 'pre_hook')
logger.info("running %s", command)
cmd_obj.run()
self.run_command_hooks(cmd_obj, 'post_hook')
self.have_run[command] = 1
