def download_setuptools(version=DEFAULT_VERSION,
download_base=DEFAULT_URL,
to_dir=os.curdir,
delay=15,
downloader_factory=get_best_downloader):
"""Download setuptools from a specified location and return its filename
`version` should be a valid setuptools version number that is available
as an egg for download under the `download_base` URL (which should end
with a '/'). `to_dir` is the directory where the egg will be downloaded.
`delay` is the number of seconds to pause before an actual download
attempt.
``downloader_factory`` should be a function taking no arguments and
returning a function for downloading a URL to a target.
"""
# making sure we use the absolute path
to_dir = os.path.abspath(to_dir)
tgz_name = "setuptools-%s.tar.gz" % version
url = download_base + tgz_name
saveto = os.path.join(to_dir, tgz_name)
if not os.path.exists(saveto): # Avoid repeated downloads
log.warning("Downloading %s", url)
downloader = downloader_factory()
downloader(url, saveto)
return os.path.realpath(saveto)
def run(self):
from distutils import log
from distutils.filelist import FileList
global CLEANUP
distutils.command.clean.clean.run(self)
if self.all:
fl = FileList()
fl.include_pattern("*.pyc", 0)
fl.include_pattern("*.pyd", 0)
fl.include_pattern("*.so", 0)
CLEANUP += fl.files
for f in CLEANUP:
if os.path.isdir(f):
try:
if not self.dry_run and os.path.exists(f):
os.rmdir(f)
log.info("removing '%s'", f)
except IOError:
log.warning("unable to remove '%s'", f)
else:
try:
if not self.dry_run and os.path.exists(f):
os.remove(f)
log.info("removing '%s'", f)
except IOError:
log.warning("unable to remove '%s'", f)
def run(self):
# do the normal cleanup
clean.run(self)
# try to remove '_composite.{so,pyd,...}' extension,
# regardless of the current system's extension name convention
build_ext = self.get_finalized_command('build_ext')
ext_fname = build_ext.get_ext_filename('overviewer_core.c_overviewer')
versionpath = os.path.join("overviewer_core", "overviewer_version.py")
primspath = os.path.join("overviewer_core", "src", "primitives.h")
for fname in [ext_fname, primspath]:
if os.path.exists(fname):
try:
log.info("removing '%s'", fname)
if not self.dry_run:
os.remove(fname)
except OSError:
log.warning(
"'%s' could not be cleaned -- permission denied",
fname)
else:
log.debug("'%s' does not exist -- can't clean it", fname)
# now try to purge all *.pyc files
for root, dirs, files in os.walk(
os.path.join(os.path.dirname(__file__), ".")):
for f in files:
if f.endswith(".pyc"):
if self.dry_run:
log.warning("Would remove %s", os.path.join(root, f))
else:
os.remove(os.path.join(root, f))
def _install(tarball, install_args=()):
# extracting the tarball
tmpdir = tempfile.mkdtemp()
log.warning('Extracting in %s', tmpdir)
old_wd = os.getcwd()
try:
os.chdir(tmpdir)
tar = tarfile.open(tarball)
_extractall(tar)
tar.close()
# going in the directory
subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
os.chdir(subdir)
log.warning('Now working in %s', subdir)
# installing
log.warning('Installing Setuptools')
if not _python_cmd('setup.py', 'install', *install_args):
log.warning('Something went wrong during the installation.')
log.warning('See the error message above.')
# exitcode will be 2
return 2
finally:
os.chdir(old_wd)
shutil.rmtree(tmpdir)
def download_setuptools(
version=DEFAULT_VERSION, download_base=DEFAULT_URL,
to_dir=DEFAULT_SAVE_DIR, delay=15,
downloader_factory=get_best_downloader):
"""
Download setuptools from a specified location and return its filename.
`version` should be a valid setuptools version number that is available
as an sdist for download under the `download_base` URL (which should end
with a '/'). `to_dir` is the directory where the egg will be downloaded.
`delay` is the number of seconds to pause before an actual download
attempt.
``downloader_factory`` should be a function taking no arguments and
returning a function for downloading a URL to a target.
"""
version = _resolve_version(version)
# making sure we use the absolute path
to_dir = os.path.abspath(to_dir)
zip_name = "setuptools-%s.zip" % version
url = download_base + zip_name
saveto = os.path.join(to_dir, zip_name)
if not os.path.exists(saveto): # Avoid repeated downloads
log.warning("Downloading %s", url)
downloader = downloader_factory()
downloader(url, saveto)
return os.path.realpath(saveto)
def _install(tarball, install_args=()):
# extracting the tarball
tmpdir = tempfile.mkdtemp()
log.warning('Extracting in %s', tmpdir)
old_wd = os.getcwd()
try:
os.chdir(tmpdir)
tar = tarfile.open(tarball)
_extractall(tar)
tar.close()
# going in the directory
subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
os.chdir(subdir)
log.warning('Now working in %s', subdir)
# installing
log.warning('Installing Setuptools')
if not _python_cmd('setup.py', 'install', *install_args):
log.warning('Something went wrong during the installation.')
log.warning('See the error message above.')
# exitcode will be 2
return 2
finally:
os.chdir(old_wd)
shutil.rmtree(tmpdir)
def initialize_options(self):
"""Set default values for options."""
# Each user option must be listed here with their default value.
system = platform.system()
if system == 'Linux':
self.cc = 'gcc'
self.cflags = '-DUSE_DOUBLE -finline-functions -funroll-loops' + \
' -O3 -march=native -DOPENMP -fopenmp'
self.libs = '-lm'
elif system == 'Darwin':
self.cc = 'gcc'
self.cflags = '-DUSE_DOUBLE -finline-functions -funroll-loops' + \
' -O3 -march=native -DOPENMP -fopenmp'
self.libs = '-lm'
elif system.endswith('BSD'):
self.cc = 'clang'
self.cflags = '-DUSE_DOUBLE -finline-functions -funroll-loops' + \
' -O3 -march=native'
self.libs = '-lm'
else:
logger.warning(
'Unrecognized system, using conservative default CFLAGS')
self.cc = 'gcc'
self.cflags = '-DUSE_DOUBLE'
self.libs = '-lm'
def run(self):
from distutils import log
from distutils.filelist import FileList
global CLEANUP
distutils.command.clean.clean.run(self)
if self.all:
fl = FileList()
fl.include_pattern("*.pyc", 0)
fl.include_pattern("*.pyd", 0)
fl.include_pattern("*.so", 0)
CLEANUP += fl.files
for f in CLEANUP:
if os.path.isdir(f):
try:
if not self.dry_run and os.path.exists(f):
os.rmdir(f)
log.info("removing '%s'", f)
except IOError:
log.warning("unable to remove '%s'", f)
else:
try:
if not self.dry_run and os.path.exists(f):
os.remove(f)
log.info("removing '%s'", f)
except IOError:
log.warning("unable to remove '%s'", f)
def _build_egg(egg, archive_filename, to_dir):
with archive_context(archive_filename):
# building an egg
log.warning('Building a Setuptools egg in %s', to_dir)
_python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir)
# returning the result
log.warning(egg)
if not os.path.exists(egg):
raise IOError('Could not build the egg.')
def run(self):
for filename in self._get_po_files():
log.info('checking catalog %s', filename)
with open(filename) as f:
catalog = read_po(f, domain=self.domain)
for message in catalog:
for error in self._check_message(catalog, message):
log.warning('%s:%d: %s', filename, message.lineno,
error)
def _check_submodule_no_git(path):
"""
Like ``_check_submodule_using_git``, but simply parses the .gitmodules file
to determine if the supplied path is a git submodule, and does not exec any
subprocesses.
This can only determine if a path is a submodule--it does not perform
updates, etc. This function may need to be updated if the format of the
.gitmodules file is changed between git versions.
"""
gitmodules_path = os.path.abspath('.gitmodules')
if not os.path.isfile(gitmodules_path):
return False
# This is a minimal reader for gitconfig-style files. It handles a few of
# the quirks that make gitconfig files incompatible with ConfigParser-style
# files, but does not support the full gitconfig syntaix (just enough
# needed to read a .gitmodules file).
gitmodules_fileobj = io.StringIO()
# Must use io.open for cross-Python-compatible behavior wrt unicode
with io.open(gitmodules_path) as f:
for line in f:
# gitconfig files are more flexible with leading whitespace; just
# go ahead and remove it
line = line.lstrip()
# comments can start with either # or ;
if line and line[0] in (':', ';'):
continue
gitmodules_fileobj.write(line)
gitmodules_fileobj.seek(0)
cfg = RawConfigParser()
try:
cfg.readfp(gitmodules_fileobj)
except Exception as exc:
log.warning('Malformatted .gitmodules file: {0}\n'
'{1} cannot be assumed to be a git submodule.'.format(
exc, path))
return False
for section in cfg.sections():
if not cfg.has_option(section, 'path'):
continue
submodule_path = cfg.get(section, 'path').rstrip(os.sep)
if submodule_path == path.rstrip(os.sep):
return True
return False
def _check_submodule_no_git(path):
"""
Like ``_check_submodule_using_git``, but simply parses the .gitmodules file
to determine if the supplied path is a git submodule, and does not exec any
subprocesses.
This can only determine if a path is a submodule--it does not perform
updates, etc. This function may need to be updated if the format of the
.gitmodules file is changed between git versions.
"""
gitmodules_path = os.path.abspath('.gitmodules')
if not os.path.isfile(gitmodules_path):
return False
# This is a minimal reader for gitconfig-style files. It handles a few of
# the quirks that make gitconfig files incompatible with ConfigParser-style
# files, but does not support the full gitconfig syntaix (just enough
# needed to read a .gitmodules file).
gitmodules_fileobj = io.StringIO()
# Must use io.open for cross-Python-compatible behavior wrt unicode
with io.open(gitmodules_path) as f:
for line in f:
# gitconfig files are more flexible with leading whitespace; just
# go ahead and remove it
line = line.lstrip()
# comments can start with either # or ;
if line and line[0] in (':', ';'):
continue
gitmodules_fileobj.write(line)
gitmodules_fileobj.seek(0)
cfg = RawConfigParser()
try:
cfg.readfp(gitmodules_fileobj)
except Exception as exc:
log.warning('Malformatted .gitmodules file: {0}\n'
'{1} cannot be assumed to be a git submodule.'.format(
exc, path))
return False
for section in cfg.sections():
if not cfg.has_option(section, 'path'):
continue
submodule_path = cfg.get(section, 'path').rstrip(os.sep)
if submodule_path == path.rstrip(os.sep):
return True
return False
def _build_egg(egg, archive_filename, to_dir):
"""Build Setuptools egg."""
with archive_context(archive_filename):
# building an egg
log.warning('Building a Setuptools egg in %s', to_dir)
_python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir)
# returning the result
log.warning(egg)
if not os.path.exists(egg):
raise IOError('Could not build the egg.')
def _build_egg(egg, archive_filename, to_dir):
"""Build Setuptools egg."""
with archive_context(archive_filename):
# building an egg
log.warning("Building a Setuptools egg in %s", to_dir)
_python_cmd("setup.py", "-q", "bdist_egg", "--dist-dir", to_dir)
# returning the result
log.warning(egg)
if not os.path.exists(egg):
raise IOError("Could not build the egg.")
def _build_install_args(options):
"""
Build the arguments to 'python setup.py install' on the setuptools package
"""
install_args = []
if options.user_install:
if sys.version_info < (2, 6):
log.warning("--user requires Python 2.6 or later")
raise SystemExit(1)
install_args.append('--user')
return install_args
def _build_install_args(options):
"""
Build the arguments to 'python setup.py install' on the setuptools package
"""
install_args = []
if options.user_install:
if sys.version_info < (2, 6):
log.warning("--user requires Python 2.6 or later")
raise SystemExit(1)
install_args.append('--user')
return install_args
def run(self):
option_dict = self.distribution.get_option_dict('pbr')
if git._git_is_installed():
git.write_git_changelog(option_dict=option_dict)
git.generate_authors(option_dict=option_dict)
tree_index = options.get_boolean_option(option_dict,
'autodoc_tree_index_modules',
'AUTODOC_TREE_INDEX_MODULES')
auto_index = options.get_boolean_option(option_dict,
'autodoc_index_modules',
'AUTODOC_INDEX_MODULES')
if not os.getenv('SPHINX_DEBUG'):
# NOTE(afazekas): These options can be used together,
# but they do a very similar thing in a different way
if tree_index:
self._sphinx_tree()
if auto_index:
self.generate_autoindex(
set(
option_dict.get("autodoc_exclude_modules",
[None, ""])[1].split()))
self.finalize_options()
is_multibuilder_sphinx = version.SemanticVersion.from_pip_string(
sphinx.__version__) >= version.SemanticVersion(1, 6)
# TODO(stephenfin): Remove support for Sphinx < 1.6 in 4.0
if not is_multibuilder_sphinx:
log.warning('[pbr] Support for Sphinx < 1.6 will be dropped in '
'pbr 4.0. Upgrade to Sphinx 1.6+')
# TODO(stephenfin): Remove this at the next MAJOR version bump
if self.builders != ['html']:
log.warning("[pbr] Sphinx 1.6 added native support for "
"specifying multiple builders in the "
"'[sphinx_build] builder' configuration option, "
"found in 'setup.cfg'. As a result, the "
"'[sphinx_build] builders' option has been "
"deprecated and will be removed in pbr 4.0. Migrate "
"to the 'builder' configuration option.")
if is_multibuilder_sphinx:
self.builder = self.builders
if is_multibuilder_sphinx:
# Sphinx >= 1.6
return setup_command.BuildDoc.run(self)
# Sphinx < 1.6
for builder in self.builder:
self.builder = builder
self.finalize_options()
self._sphinx_run()
def run(self):
if self.install_dir is None:
log.warning("Can't install ozw_config to None")
return
if not current_template.copy_openzwave_config:
log.info("Don't install ozw_config for template {0}".format(
current_template))
return
dest = os.path.join(self.install_dir, 'python_openzwave', "ozw_config")
if not os.path.isdir(dest):
os.makedirs(dest)
self.copy_tree(os.path.join(current_template.openzwave, 'config'),
dest)
def _build_egg(egg, tarball, to_dir):
# extracting the tarball
tmpdir = tempfile.mkdtemp()
log.warning('Extracting in %s', tmpdir)
old_wd = os.getcwd()
try:
os.chdir(tmpdir)
tar = tarfile.open(tarball)
_extractall(tar)
tar.close()
# going in the directory
subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
os.chdir(subdir)
log.warning('Now working in %s', subdir)
# building an egg
log.warning('Building a Setuptools egg in %s', to_dir)
_python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir)
finally:
os.chdir(old_wd)
shutil.rmtree(tmpdir)
# returning the result
log.warning(egg)
if not os.path.exists(egg):
raise IOError('Could not build the egg.')
def _build_egg(egg, tarball, to_dir):
# extracting the tarball
tmpdir = tempfile.mkdtemp()
log.warning('Extracting in %s', tmpdir)
old_wd = os.getcwd()
try:
os.chdir(tmpdir)
tar = tarfile.open(tarball)
_extractall(tar)
tar.close()
# going in the directory
subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
os.chdir(subdir)
log.warning('Now working in %s', subdir)
# building an egg
log.warning('Building a Setuptools egg in %s', to_dir)
_python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir)
finally:
os.chdir(old_wd)
shutil.rmtree(tmpdir)
# returning the result
log.warning(egg)
if not os.path.exists(egg):
raise IOError('Could not build the egg.')
def archive_context(filename):
# extracting the archive
tmpdir = tempfile.mkdtemp()
log.warning('Extracting in %s', tmpdir)
old_wd = os.getcwd()
try:
os.chdir(tmpdir)
with ContextualZipFile(filename) as archive:
archive.extractall()
# going in the directory
subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
os.chdir(subdir)
log.warning('Now working in %s', subdir)
yield
finally:
os.chdir(old_wd)
shutil.rmtree(tmpdir)
def run(self):
if self.install_dir is None:
log.warning("Can't install ozw_config to None")
return
if not current_template.copy_openzwave_config:
log.info("Don't install ozw_config for template {0}".format(current_template))
return
log.info("Install ozw_config for template {0}".format(current_template))
dest = os.path.join(self.install_dir, 'python_openzwave', "ozw_config")
if os.path.isdir(dest):
#Try to remove old config
try:
import shutil
shutil.rmtree(dest)
except Exception:
log.exception("Can't remove old config directory")
if not os.path.isdir(dest):
os.makedirs(dest)
self.copy_tree(os.path.join(current_template.openzwave,'config'), dest)
def archive_context(filename):
"""
Unzip filename to a temporary directory, set to the cwd.
The unzipped target is cleaned up after.
"""
tmpdir = tempfile.mkdtemp()
log.warning('Extracting in %s', tmpdir)
old_wd = os.getcwd()
try:
os.chdir(tmpdir)
with ContextualZipFile(filename) as archive:
archive.extractall()
# going in the directory
subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
os.chdir(subdir)
log.warning('Now working in %s', subdir)
yield
finally:
os.chdir(old_wd)
shutil.rmtree(tmpdir)
def archive_context(filename):
"""
Unzip filename to a temporary directory, set to the cwd.
The unzipped target is cleaned up after.
"""
tmpdir = tempfile.mkdtemp()
log.warning("Extracting in %s", tmpdir)
old_wd = os.getcwd()
try:
os.chdir(tmpdir)
with ContextualZipFile(filename) as archive:
archive.extractall()
# going in the directory
subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
os.chdir(subdir)
log.warning("Now working in %s", subdir)
yield
finally:
os.chdir(old_wd)
shutil.rmtree(tmpdir)
def _install(archive_filename, install_args=()):
with archive_context(archive_filename):
# installing
log.warning('Installing Setuptools')
if not _python_cmd('setup.py', 'install', *install_args):
log.warning('Something went wrong during the installation.')
log.warning('See the error message above.')
# exitcode will be 2
return 2
def _install(archive_filename, install_args=()):
"""Install Setuptools."""
with archive_context(archive_filename):
# installing
log.warning('Installing Setuptools')
if not _python_cmd('setup.py', 'install', *install_args):
log.warning('Something went wrong during the installation.')
log.warning('See the error message above.')
# exitcode will be 2
return 2
def _install(archive_filename, install_args=()):
"""Install Setuptools."""
with archive_context(archive_filename):
# installing
log.warning("Installing Setuptools")
if not _python_cmd("setup.py", "install", *install_args):
log.warning("Something went wrong during the installation.")
log.warning("See the error message above.")
# exitcode will be 2
return 2
def run(self):
self.distribution.run_command('jsversion')
jsdeps = self.distribution.get_command_obj('jsdeps')
js = self.distribution.get_command_obj('js')
css = self.distribution.get_command_obj('css')
js.force = strict
targets = [jsdeps.bower_dir]
targets.extend(js.targets)
targets.extend(css.targets)
missing = [t for t in targets if not os.path.exists(t)]
if not is_repo and not missing:
# If we're an sdist, we aren't a repo and everything should be present.
# Don't rebuild js/css in that case.
command.run(self)
return
try:
self.distribution.run_command('js')
self.distribution.run_command('css')
except Exception as e:
# refresh missing
missing = [t for t in targets if not os.path.exists(t)]
if strict or missing:
# die if strict or any targets didn't build
prefix = os.path.commonprefix([repo_root + os.sep] +
missing)
missing = [m[len(prefix):] for m in missing]
log.warning(
"rebuilding js and css failed. The following required files are missing: %s"
% missing)
raise e
else:
log.warning("rebuilding js and css failed (not a problem)")
log.warning(str(e))
# check again for missing targets, just in case:
missing = [t for t in targets if not os.path.exists(t)]
if missing:
# command succeeded, but targets still missing (?!)
prefix = os.path.commonprefix([repo_root + os.sep] + missing)
missing = [m[len(prefix):] for m in missing]
raise ValueError(
"The following required files are missing: %s" % missing)
command.run(self)
def run(self):
self.distribution.run_command('jsversion')
jsdeps = self.distribution.get_command_obj('jsdeps')
js = self.distribution.get_command_obj('js')
css = self.distribution.get_command_obj('css')
js.force = strict
targets = [ jsdeps.bower_dir ]
targets.extend(js.targets)
targets.extend(css.targets)
missing = [ t for t in targets if not os.path.exists(t) ]
if not is_repo and not missing:
# If we're an sdist, we aren't a repo and everything should be present.
# Don't rebuild js/css in that case.
command.run(self)
return
try:
self.distribution.run_command('js')
self.distribution.run_command('css')
except Exception as e:
# refresh missing
missing = [ t for t in targets if not os.path.exists(t) ]
if strict or missing:
# die if strict or any targets didn't build
prefix = os.path.commonprefix([repo_root + os.sep] + missing)
missing = [ m[len(prefix):] for m in missing ]
log.warning("rebuilding js and css failed. The following required files are missing: %s" % missing)
raise e
else:
log.warning("rebuilding js and css failed (not a problem)")
log.warning(str(e))
# check again for missing targets, just in case:
missing = [ t for t in targets if not os.path.exists(t) ]
if missing:
# command succeeded, but targets still missing (?!)
prefix = os.path.commonprefix([repo_root + os.sep] + missing)
missing = [ m[len(prefix):] for m in missing ]
raise ValueError("The following required files are missing: %s" % missing)
command.run(self)
raise RuntimeError("Number of build procs must be an integer")
sys.exit(3)
# decide which extension to add to cython sources (pyx or c)
cy_ext = ".c" # or ".cpp"?
if has_cython and use_cython:
cy_ext = ".pyx"
# add extension to cython sources
for i, d in enumerate(cy_defs):
for j, src in enumerate(d[1]):
fname = cy_defs[i][1][j] + cy_ext
if os.path.isfile(fname):
cy_defs[i][1][j] = fname
else:
log.warning("{0} not found. Skipping extension: "
"{1}".format(fname, cy_defs[i][0]))
print("To use this extension, please install cython",
file=sys.stderr)
cy_defs[i] = None
break
def clean_pyc_files(dry_run=False):
"""remove all .pyc / .pyo files"""
cwd = os.getcwd()
if cwd.endswith("Viscid"):
for root, _, files in os.walk(cwd, topdown=False):
for name in files:
if name.endswith('.pyc') or name.endswith('.pyo'):
if os.path.isfile(os.path.join(root, name)):
print('removing: %s' % os.path.join(root, name))
