def test_convert_path(self):
# linux/mac
os.sep = '/'
def _join(path):
return '/'.join(path)
os.path.join = _join
self.assertEqual(convert_path('/home/to/my/stuff'),
'/home/to/my/stuff')
# win
os.sep = '\\'
def _join(*path):
return '\\'.join(path)
os.path.join = _join
self.assertRaises(ValueError, convert_path, '/home/to/my/stuff')
self.assertRaises(ValueError, convert_path, 'home/to/my/stuff/')
self.assertEqual(convert_path('home/to/my/stuff'),
'home\\to\\my\\stuff')
self.assertEqual(convert_path('.'),
os.curdir)
def _parse_template_line(self, line):
words = line.split()
action = words[0]
patterns = dir = dir_pattern = None
if action in ('include', 'exclude',
'global-include', 'global-exclude'):
if len(words) < 2:
raise DistutilsTemplateError(
"'%s' expects <pattern1> <pattern2> ..." % action)
patterns = map(convert_path, words[1:])
elif action in ('recursive-include', 'recursive-exclude'):
if len(words) < 3:
raise DistutilsTemplateError(
"'%s' expects <dir> <pattern1> <pattern2> ..." % action)
dir = convert_path(words[1])
patterns = map(convert_path, words[2:])
elif action in ('graft', 'prune'):
if len(words) != 2:
raise DistutilsTemplateError(
"'%s' expects a single <dir_pattern>" % action)
dir_pattern = convert_path(words[1])
else:
raise DistutilsTemplateError("unknown action '%s'" % action)
return action, patterns, dir, dir_pattern
def run(self):
self.mkpath(self.install_dir)
for f in self.data_files:
if isinstance(f, str):
# it's a simple file, so copy it
f = convert_path(f)
if self.warn_dir:
self.warn("setup script did not provide a directory for "
"'%s' -- installing right in '%s'" %
(f, self.install_dir))
(out, _) = self.copy_file(f, self.install_dir)
self.outfiles.append(out)
else:
# it's a tuple with path to install to and a list of files
dir = convert_path(f[0])
if not os.path.isabs(dir):
dir = os.path.join(self.install_dir, dir)
elif self.root:
dir = change_root(self.root, dir)
self.mkpath(dir)
if f[1] == []:
# If there are no files listed, the user must be
# trying to create an empty directory, so add the
# directory to the list of output files.
self.outfiles.append(dir)
else:
# Copy files, adding them to the list of output files.
for data in f[1]:
data = convert_path(data)
(out, _) = self.copy_file(data, dir)
self.outfiles.append(out)
def get_source_files(self):
sources = []
for item in self.data_files:
if isinstance(item, str): # plain file
item = convert_path(item)
if os.path.isfile(item):
sources.append(item)
else: # a (dirname, filenames) tuple
dirname, filenames = item
for f in filenames:
f = convert_path(f)
if os.path.isfile(f):
sources.append(f)
return sources
def handle_extra_path(self):
"""Set `path_file` and `extra_dirs` using `extra_path`."""
if self.extra_path is None:
self.extra_path = self.distribution.extra_path
if self.extra_path is not None:
if isinstance(self.extra_path, str):
self.extra_path = self.extra_path.split(',')
if len(self.extra_path) == 1:
path_file = extra_dirs = self.extra_path[0]
elif len(self.extra_path) == 2:
path_file, extra_dirs = self.extra_path
else:
raise PackagingOptionError(
"'extra_path' option must be a list, tuple, or "
"comma-separated string with 1 or 2 elements")
# convert to local form in case Unix notation used (as it
# should be in setup scripts)
extra_dirs = convert_path(extra_dirs)
else:
path_file = None
extra_dirs = ''
# XXX should we warn if path_file and not extra_dirs? (in which
# case the path file would be harmless but pointless)
self.path_file = path_file
self.extra_dirs = extra_dirs
def finalize_options(self):
self.set_undefined_options('build',
'use_2to3', 'use_2to3_fixers',
'convert_2to3_doctests', 'build_lib',
'force')
# Get the distribution options that are aliases for build_py
# options -- list of packages and list of modules.
self.packages = self.distribution.packages
self.py_modules = self.distribution.py_modules
self.package_data = self.distribution.package_data
self.package_dir = {}
if self.distribution.package_dir:
for name, path in self.distribution.package_dir.items():
self.package_dir[name] = convert_path(path)
self.data_files = self.get_data_files()
# Ick, copied straight from install_lib.py (fancy_getopt needs a
# type system! Hell, *everything* needs a type system!!!)
if not isinstance(self.optimize, int):
try:
self.optimize = int(self.optimize)
assert 0 <= self.optimize <= 2
except (ValueError, AssertionError):
raise DistutilsOptionError("optimize must be 0, 1, or 2")
def handle_extra_path(self):
"""Set `path_file` and `extra_dirs` using `extra_path`."""
if self.extra_path is None:
self.extra_path = self.distribution.extra_path
if self.extra_path is not None:
if isinstance(self.extra_path, basestring):
self.extra_path = self.extra_path.split(',')
if len(self.extra_path) == 1:
path_file = extra_dirs = self.extra_path[0]
elif len(self.extra_path) == 2:
path_file, extra_dirs = self.extra_path
else:
raise PackagingOptionError(
"'extra_path' option must be a list, tuple, or "
"comma-separated string with 1 or 2 elements")
# convert to local form in case Unix notation used (as it
# should be in setup scripts)
extra_dirs = convert_path(extra_dirs)
else:
path_file = None
extra_dirs = ''
# XXX should we warn if path_file and not extra_dirs? (in which
# case the path file would be harmless but pointless)
self.path_file = path_file
self.extra_dirs = extra_dirs
def create_home_path(self):
"""Create directories under ~."""
if not self.user:
return
home = convert_path(os.path.expanduser("~"))
for name, path in self.config_vars.iteritems():
if path.startswith(home) and not os.path.isdir(path):
os.makedirs(path, 0700)
def find_data_files(self, package, src_dir):
"""Return filenames for package's data files in 'src_dir'"""
globs = (self.package_data.get('', [])
+ self.package_data.get(package, []))
files = []
for pattern in globs:
# Each pattern has to be converted to a platform-specific path
filelist = glob(os.path.join(src_dir, convert_path(pattern)))
# Files that match more than one pattern are only added once
files.extend([fn for fn in filelist if fn not in files])
return files
def _parse_template_line(self, line):
words = line.split()
if len(words) == 1 and words[0] not in (
'include', 'exclude', 'global-include', 'global-exclude',
'recursive-include', 'recursive-exclude', 'graft', 'prune'):
# no action given, let's use the default 'include'
words.insert(0, 'include')
action = words[0]
patterns = dir = dir_pattern = None
if action in ('include', 'exclude',
'global-include', 'global-exclude'):
if len(words) < 2:
raise PackagingTemplateError(
"%r expects <pattern1> <pattern2> ..." % action)
patterns = [convert_path(word) for word in words[1:]]
elif action in ('recursive-include', 'recursive-exclude'):
if len(words) < 3:
raise PackagingTemplateError(
"%r expects <dir> <pattern1> <pattern2> ..." % action)
dir = convert_path(words[1])
patterns = [convert_path(word) for word in words[2:]]
elif action in ('graft', 'prune'):
if len(words) != 2:
raise PackagingTemplateError(
"%r expects a single <dir_pattern>" % action)
dir_pattern = convert_path(words[1])
else:
raise PackagingTemplateError("unknown action %r" % action)
return action, patterns, dir, dir_pattern
def _parse_template_line(self, line):
words = line.split()
if len(words) == 1 and words[0] not in (
'include', 'exclude', 'global-include', 'global-exclude',
'recursive-include', 'recursive-exclude', 'graft', 'prune'):
# no action given, let's use the default 'include'
words.insert(0, 'include')
action = words[0]
patterns = dir = dir_pattern = None
if action in ('include', 'exclude', 'global-include',
'global-exclude'):
if len(words) < 2:
raise PackagingTemplateError(
"%r expects <pattern1> <pattern2> ..." % action)
patterns = [convert_path(word) for word in words[1:]]
elif action in ('recursive-include', 'recursive-exclude'):
if len(words) < 3:
raise PackagingTemplateError(
"%r expects <dir> <pattern1> <pattern2> ..." % action)
dir = convert_path(words[1])
patterns = [convert_path(word) for word in words[2:]]
elif action in ('graft', 'prune'):
if len(words) != 2:
raise PackagingTemplateError(
"%r expects a single <dir_pattern>" % action)
dir_pattern = convert_path(words[1])
else:
raise PackagingTemplateError("unknown action %r" % action)
return action, patterns, dir, dir_pattern
def find_data_files(self, package, src_dir):
"""Return filenames for package's data files in 'src_dir'.
Helper function for get_data_files.
"""
globs = (self.package_data.get('', [])
+ self.package_data.get(package, []))
files = []
for pattern in globs:
# Each pattern has to be converted to a platform-specific path
filelist = glob(os.path.join(src_dir, convert_path(pattern)))
# Files that match more than one pattern are only added once
files.extend(fn for fn in filelist if fn not in files)
return files
def run(self):
self.mkpath(self.install_dir)
for _file in self.data_files.items():
destination = convert_path(self.expand_categories(_file[1]))
dir_dest = os.path.abspath(os.path.dirname(destination))
self.mkpath(dir_dest)
try:
out = self.copy_file(_file[0], dir_dest)[0]
except Error as e:
logger.warning('%s: %s', self.get_command_name(), e)
out = destination
self.outfiles.append(out)
self.data_files_out.append((_file[0], destination))
def find_package_data(where='.',
package='',
exclude=standard_exclude,
exclude_directories=standard_exclude_directories):
out = {}
stack = [(convert_path(where), '', package)]
while stack:
where, prefix, package = stack.pop(0)
for name in os.listdir(where):
fn = os.path.join(where, name)
if os.path.isdir(fn):
bad_name = False
for pattern in exclude_directories:
if (fnmatchcase(name, pattern)
or fn.lower() == pattern.lower()):
bad_name = True
break
if bad_name:
continue
if os.path.isfile(os.path.join(fn, '__init__.py')):
if not package:
new_package = name
else:
new_package = package + '.' + name
stack.append((fn, '', new_package))
else:
stack.append((fn, prefix + name + '/', package))
else:
bad_name = False
for pattern in exclude:
if (fnmatchcase(name, pattern)
or fn.lower() == pattern.lower()):
bad_name = True
break
if bad_name:
continue
out.setdefault(package, []).append(prefix + name)
return out
def finalize_options(self):
self.set_undefined_options(
"build", "use_2to3", "use_2to3_fixers", "convert_2to3_doctests", "build_lib", "force"
)
# Get the distribution options that are aliases for build_py
# options -- list of packages and list of modules.
self.packages = self.distribution.packages
self.py_modules = self.distribution.py_modules
self.package_data = self.distribution.package_data
self.package_dir = None
if self.distribution.package_dir is not None:
self.package_dir = convert_path(self.distribution.package_dir)
self.data_files = self.get_data_files()
# Ick, copied straight from install_lib.py (fancy_getopt needs a
# type system! Hell, *everything* needs a type system!!!)
if not isinstance(self.optimize, int):
try:
self.optimize = int(self.optimize)
assert 0 <= self.optimize <= 2
except (ValueError, AssertionError):
raise PackagingOptionError("optimize must be 0, 1, or 2")
def finalize_options(self):
self.set_undefined_options('build',
'use_2to3', 'use_2to3_fixers',
'convert_2to3_doctests', 'build_lib',
'force')
# Get the distribution options that are aliases for build_py
# options -- list of packages and list of modules.
self.packages = self.distribution.packages
self.py_modules = self.distribution.py_modules
self.package_data = self.distribution.package_data
self.package_dir = None
if self.distribution.package_dir is not None:
self.package_dir = convert_path(self.distribution.package_dir)
self.data_files = self.get_data_files()
# Ick, copied straight from install_lib.py (fancy_getopt needs a
# type system! Hell, *everything* needs a type system!!!)
if not isinstance(self.optimize, int):
try:
self.optimize = int(self.optimize)
assert 0 <= self.optimize <= 2
except (ValueError, AssertionError):
raise PackagingOptionError("optimize must be 0, 1, or 2")
def create_user_dirs(self):
"""Create directories under USERBASE as needed."""
home = convert_path(os.path.expanduser("~"))
for name, path in self.config_vars.items():
if path.startswith(home) and not os.path.isdir(path):
os.makedirs(path, 0700)
def convert_paths(self, *names):
"""Call `convert_path` over `names`."""
for name in names:
attr = "install_" + name
setattr(self, attr, convert_path(getattr(self, attr)))
def add_defaults(self):
"""Add all the default files to self.filelist:
- README or README.txt
- setup.py
- test/test*.py
- all pure Python modules mentioned in setup script
- all files pointed by package_data (build_py)
- all files defined in data_files.
- all files defined as scripts.
- all C sources listed as part of extensions or C libraries
in the setup script (doesn't catch C headers!)
Warns if (README or README.txt) or setup.py are missing; everything
else is optional.
"""
standards = [("README", "README.txt"), self.distribution.script_name]
for fn in standards:
if isinstance(fn, tuple):
alts = fn
got_it = 0
for fn in alts:
if os.path.exists(fn):
got_it = 1
self.filelist.append(fn)
break
if not got_it:
self.warn("standard file not found: should have one of " + string.join(alts, ", "))
else:
if os.path.exists(fn):
self.filelist.append(fn)
else:
self.warn("standard file '%s' not found" % fn)
optional = ["test/test*.py", "setup.cfg"]
for pattern in optional:
files = filter(os.path.isfile, glob(pattern))
if files:
self.filelist.extend(files)
# build_py is used to get:
# - python modules
# - files defined in package_data
build_py = self.get_finalized_command("build_py")
# getting python files
if self.distribution.has_pure_modules():
self.filelist.extend(build_py.get_source_files())
# getting package_data files
# (computed in build_py.data_files by build_py.finalize_options)
for pkg, src_dir, build_dir, filenames in build_py.data_files:
for filename in filenames:
self.filelist.append(os.path.join(src_dir, filename))
# getting distribution.data_files
if self.distribution.has_data_files():
for item in self.distribution.data_files:
if isinstance(item, str): # plain file
item = convert_path(item)
if os.path.isfile(item):
self.filelist.append(item)
else: # a (dirname, filenames) tuple
dirname, filenames = item
for f in filenames:
f = convert_path(f)
if os.path.isfile(f):
self.filelist.append(f)
if self.distribution.has_ext_modules():
build_ext = self.get_finalized_command("build_ext")
self.filelist.extend(build_ext.get_source_files())
if self.distribution.has_c_libraries():
build_clib = self.get_finalized_command("build_clib")
self.filelist.extend(build_clib.get_source_files())
if self.distribution.has_scripts():
build_scripts = self.get_finalized_command("build_scripts")
self.filelist.extend(build_scripts.get_source_files())
def copy_scripts(self):
"""Copy each script listed in 'self.scripts'; if it's marked as a
Python script in the Unix way (first line matches 'first_line_re',
ie. starts with "\#!" and contains "python"), then adjust the first
line to refer to the current Python interpreter as we copy.
"""
# XXX use self.execute(shutil.rmtree, ...)
self.rmpath(self.build_dir)
self.mkpath(self.build_dir)
outfiles = []
for script in self.scripts:
adjust = False
script = convert_path(script)
outfile = os.path.join(self.build_dir, os.path.basename(script))
outfiles.append(outfile)
# Always open the file, but ignore failures in dry-run mode --
# that way, we'll get accurate feedback if we can read the
# script.
try:
f = open(script, "rb")
except IOError:
if not self.dry_run:
raise
f = None
else:
encoding, lines = detect_encoding(f.readline)
f.seek(0)
first_line = f.readline()
if not first_line:
logger.warning('%s: %s is an empty file (skipping)',
self.get_command_name(), script)
continue
match = first_line_re.match(first_line)
if match:
adjust = True
post_interp = match.group(1) or ''
if adjust:
logger.info("copying and adjusting %s -> %s", script,
self.build_dir)
if not self.dry_run:
if not sysconfig.is_python_build():
executable = self.executable
else:
executable = os.path.join(
sysconfig.get_config_var("BINDIR"),
"python%s%s" % (sysconfig.get_config_var("VERSION"),
sysconfig.get_config_var("EXE")))
executable = fsencode(executable)
shebang = "#!" + executable + post_interp + "\n"
# Python parser starts to read a script using UTF-8 until
# it gets a #coding:xxx cookie. The shebang has to be the
# first line of a file, the #coding:xxx cookie cannot be
# written before. So the shebang has to be decodable from
# UTF-8.
try:
shebang.decode('utf-8')
except UnicodeDecodeError:
raise ValueError(
"The shebang (%r) is not decodable "
"from utf-8" % shebang)
# If the script is encoded to a custom encoding (use a
# #coding:xxx cookie), the shebang has to be decodable from
# the script encoding too.
try:
shebang.decode(encoding)
except UnicodeDecodeError:
raise ValueError(
"The shebang (%r) is not decodable "
"from the script encoding (%s)" % (
shebang, encoding))
outf = open(outfile, "wb")
try:
outf.write(shebang)
outf.writelines(f.readlines())
finally:
outf.close()
if f:
f.close()
else:
if f:
f.close()
self.copy_file(script, outfile)
if os.name == 'posix':
for file in outfiles:
if self.dry_run:
logger.info("changing mode of %s", file)
else:
oldmode = os.stat(file).st_mode & 07777
newmode = (oldmode | 0555) & 07777
if newmode != oldmode:
logger.info("changing mode of %s from %o to %o",
file, oldmode, newmode)
os.chmod(file, newmode)
return outfiles
def copy_scripts (self):
"""Copy each script listed in 'self.scripts'; if it's marked as a
Python script in the Unix way (first line matches 'first_line_re',
ie. starts with "\#!" and contains "python"), then adjust the first
line to refer to the current Python interpreter as we copy.
"""
self.mkpath(self.build_dir)
outfiles = []
for script in self.scripts:
adjust = 0
script = convert_path(script)
outfile = os.path.join(self.build_dir, os.path.basename(script))
outfiles.append(outfile)
if not self.force and not newer(script, outfile):
log.debug("not copying %s (up-to-date)", script)
continue
# Always open the file, but ignore failures in dry-run mode --
# that way, we'll get accurate feedback if we can read the
# script.
try:
f = open(script, "r")
except IOError:
if not self.dry_run:
raise
f = None
else:
first_line = f.readline()
if not first_line:
self.warn("%s is an empty file (skipping)" % script)
continue
match = first_line_re.match(first_line)
if match:
adjust = 1
post_interp = match.group(1) or ''
if adjust:
log.info("copying and adjusting %s -> %s", script,
self.build_dir)
if not self.dry_run:
outf = open(outfile, "w")
if not sysconfig.is_python_build():
outf.write("#!%s%s\n" %
(self.executable,
post_interp))
else:
outf.write("#!%s%s\n" %
(os.path.join(
sysconfig.get_config_var("BINDIR"),
"python%s%s" % (sysconfig.get_config_var("VERSION"),
sysconfig.get_config_var("EXE"))),
post_interp))
outf.writelines(f.readlines())
outf.close()
if f:
f.close()
else:
if f:
f.close()
self.copy_file(script, outfile)
if os.name == 'posix':
for file in outfiles:
if self.dry_run:
log.info("changing mode of %s", file)
else:
oldmode = os.stat(file)[ST_MODE] & 07777
newmode = (oldmode | 0555) & 07777
if newmode != oldmode:
log.info("changing mode of %s from %o to %o",
file, oldmode, newmode)
os.chmod(file, newmode)
return outfiles
def convert_paths(self, *names):
"""Call `convert_path` over `names`."""
for name in names:
attr = "install_" + name
setattr(self, attr, convert_path(getattr(self, attr)))
def create_user_dirs(self):
"""Create directories under USERBASE as needed."""
home = convert_path(os.path.expanduser("~"))
for name, path in self.config_vars.items():
if path.startswith(home) and not os.path.isdir(path):
os.makedirs(path, 0o700)
def copy_scripts(self):
"""Copy each script listed in 'self.scripts'; if it's marked as a
Python script in the Unix way (first line matches 'first_line_re',
ie. starts with "\#!" and contains "python"), then adjust the first
line to refer to the current Python interpreter as we copy.
"""
# XXX use self.execute(shutil.rmtree, ...)
self.rmpath(self.build_dir)
self.mkpath(self.build_dir)
outfiles = []
for script in self.scripts:
adjust = False
script = convert_path(script)
outfile = os.path.join(self.build_dir, os.path.basename(script))
outfiles.append(outfile)
# Always open the file, but ignore failures in dry-run mode --
# that way, we'll get accurate feedback if we can read the
# script.
try:
f = open(script, "rb")
except IOError:
if not self.dry_run:
raise
f = None
else:
encoding, lines = detect_encoding(f.readline)
f.seek(0)
first_line = f.readline()
if not first_line:
logger.warning('%s: %s is an empty file (skipping)',
self.get_command_name(), script)
continue
match = first_line_re.match(first_line)
if match:
adjust = True
post_interp = match.group(1) or b''
if adjust:
logger.info("copying and adjusting %s -> %s", script,
self.build_dir)
if not self.dry_run:
if not sysconfig.is_python_build():
executable = self.executable
else:
executable = os.path.join(
sysconfig.get_config_var("BINDIR"), "python%s%s" %
(sysconfig.get_config_var("VERSION"),
sysconfig.get_config_var("EXE")))
executable = fsencode(executable)
shebang = b"#!" + executable + post_interp + b"\n"
# Python parser starts to read a script using UTF-8 until
# it gets a #coding:xxx cookie. The shebang has to be the
# first line of a file, the #coding:xxx cookie cannot be
# written before. So the shebang has to be decodable from
# UTF-8.
try:
shebang.decode('utf-8')
except UnicodeDecodeError:
raise ValueError("The shebang (%r) is not decodable "
"from utf-8" % shebang)
# If the script is encoded to a custom encoding (use a
# #coding:xxx cookie), the shebang has to be decodable from
# the script encoding too.
try:
shebang.decode(encoding)
except UnicodeDecodeError:
raise ValueError("The shebang (%r) is not decodable "
"from the script encoding (%s)" %
(shebang, encoding))
with open(outfile, "wb") as outf:
outf.write(shebang)
outf.writelines(f.readlines())
if f:
f.close()
else:
if f:
f.close()
self.copy_file(script, outfile)
if os.name == 'posix':
for file in outfiles:
if self.dry_run:
logger.info("changing mode of %s", file)
else:
oldmode = os.stat(file).st_mode & 0o7777
newmode = (oldmode | 0o555) & 0o7777
if newmode != oldmode:
logger.info("changing mode of %s from %o to %o", file,
oldmode, newmode)
os.chmod(file, newmode)
return outfiles
