def fetch_build_egg(self, req):
""" Specialized version of Distribution.fetch_build_egg
that respects respects allow_hosts and index_url. """
from setuptools.command.easy_install import easy_install
dist = Distribution({'script_args': ['easy_install']})
dist.parse_config_files()
opts = dist.get_option_dict('easy_install')
keep = (
'find_links', 'site_dirs', 'index_url', 'optimize',
'site_dirs', 'allow_hosts'
)
for key in list(opts):
if key not in keep:
del opts[key] # don't use any other settings
if self.dependency_links:
links = self.dependency_links[:]
if 'find_links' in opts:
links = opts['find_links'][1].split() + links
opts['find_links'] = ('setup', links)
if self.allow_hosts:
opts['allow_hosts'] = ('test', self.allow_hosts)
if self.index_url:
opts['index_url'] = ('test', self.index_url)
install_dir_func = getattr(self, 'get_egg_cache_dir', _os.getcwd)
install_dir = install_dir_func()
cmd = easy_install(
dist, args=["x"], install_dir=install_dir,
exclude_scripts=True,
always_copy=False, build_directory=None, editable=False,
upgrade=False, multi_version=True, no_report=True, user=False
)
cmd.ensure_finalized()
return cmd.easy_install(req)
def fetch_build_egg(self, req):
""" Specialized version of Distribution.fetch_build_egg
that respects respects allow_hosts and index_url. """
from setuptools.command.easy_install import easy_install
dist = Distribution({'script_args': ['easy_install']})
dist.parse_config_files()
opts = dist.get_option_dict('easy_install')
keep = (
'find_links', 'site_dirs', 'index_url', 'optimize',
'site_dirs', 'allow_hosts'
)
for key in list(opts):
if key not in keep:
del opts[key] # don't use any other settings
if self.dependency_links:
links = self.dependency_links[:]
if 'find_links' in opts:
links = opts['find_links'][1].split() + links
opts['find_links'] = ('setup', links)
if self.allow_hosts:
opts['allow_hosts'] = ('test', self.allow_hosts)
if self.index_url:
opts['index_url'] = ('test', self.index_url)
install_dir_func = getattr(self, 'get_egg_cache_dir', _os.getcwd)
install_dir = install_dir_func()
cmd = easy_install(
dist, args=["x"], install_dir=install_dir,
exclude_scripts=True,
always_copy=False, build_directory=None, editable=False,
upgrade=False, multi_version=True, no_report=True, user=False
)
cmd.ensure_finalized()
return cmd.easy_install(req)
def self_upgrade():
"""Upgrade ourselves with pip."""
# Run pip using the current python executable to accommodate for virtualenvs
command = (
[sys.executable]
+ ["-m", "pip"]
+ ["install", "MozPhab"]
+ ["--upgrade"]
+ ["--no-cache-dir"]
+ ["--disable-pip-version-check"]
)
if config.get_pre_releases:
command += ["--pre"]
# sys.path[0] is the directory containing the script that was used to
# start python. This will be something like:
# "<python environment>/bin" or "<python environment>\Scripts" (Windows)
script_dir = Path(sys.path[0])
# If moz-phab was installed with --user, we need to pass it to pip
# Create "install" setuptools command with --user to find the scripts_path
d = Distribution()
d.parse_config_files()
i = d.get_command_obj("install", create=True)
# Forcing the environment detected by Distribution to the --user one
i.user = True
i.prefix = i.exec_prefix = i.home = i.install_base = i.install_platbase = None
i.finalize_options()
# Checking if the moz-phab script is installed in user's scripts directory
user_dir = Path(i.install_scripts).resolve()
if script_dir == user_dir:
command.append("--user")
if environment.IS_WINDOWS:
# Windows does not allow to remove the exe file of the running process.
# Renaming the `moz-phab.exe` file to allow pip to install a new version.
temp_exe = script_dir / "moz-phab-temp.exe"
try:
temp_exe.unlink()
except FileNotFoundError:
pass
exe = script_dir / "moz-phab.exe"
exe.rename(temp_exe)
try:
check_call(command)
except Exception:
temp_exe.rename(exe)
raise
if not exe.is_file():
# moz-phab.exe is not created - install wasn't needed.
temp_exe.rename(exe)
else:
check_call(command)
def get_setuptools_install_scripts_dir():
dist = Distribution({"cmdclass": {"install": OnlyGetScriptPath}})
dist.dry_run = True # not sure if necessary, but to be safe
dist.parse_config_files()
command = dist.get_command_obj("install")
command.ensure_finalized()
command.run()
return dist.install_scripts
def get_script_dir():
dist = Distribution({'cmdclass': {'install': OnlyGetScriptPath}})
dist.dry_run = True # not sure if necessary, but to be safe
dist.parse_config_files()
command = dist.get_command_obj('install')
command.ensure_finalized()
command.run()
return dist.install_scripts
def get_python_bin_path():
" Get the directory setuptools installs scripts to for current python "
dist = Distribution({'cmdclass': {'install': OnlyGetScriptPath}})
dist.dry_run = True # not sure if necessary
dist.parse_config_files()
command = dist.get_command_obj('install')
command.ensure_finalized()
command.run()
return dist.install_scripts
def get_setuptools_script_dir():
" Get the directory setuptools installs scripts to for current python "
dist = Distribution({'cmdclass': {'install': OnlyGetScriptPath}})
dist.dry_run = True # not sure if necessary
dist.parse_config_files()
command = dist.get_command_obj('install')
command.ensure_finalized()
command.run()
return dist.install_scripts
def get_setuptools_script_dir():
"""
Find where setuptools will have installed the `microservice` entrypoint executable to.
:return: Path to the setuptools script directory.
"""
dist = Distribution({'cmdclass': {'install': OnlyGetScriptPath}})
dist.dry_run = True # not sure if necessary, but to be safe
dist.parse_config_files()
command = dist.get_command_obj('install')
command.ensure_finalized()
command.run()
return dist.install_scripts
def read_setuptools_cfg():
"""
Reads the `setup.cfg` file and extracts the various requirements lists
"""
# see https://stackoverflow.com/a/30679041/7262247
from setuptools import Distribution
dist = Distribution()
dist.parse_config_files()
return SetupCfg(setup_requires=dist.setup_requires,
install_requires=dist.install_requires,
tests_requires=dist.tests_require,
extras_require=dist.extras_require)
def get_setuptools_script_dir():
"""Summary.
Returns:
TYPE: Description
"""
dist = Distribution({'cmdclass': {'install': OnlyGetScriptPath}})
dist.dry_run = True # not sure if necessary
dist.parse_config_files()
command = dist.get_command_obj('install')
command.ensure_finalized()
command.run()
return dist.install_scripts
def get_setuptools_script_dir():
# Run the above class just to get paths
dist = Distribution({'cmdclass': {'install': GetPaths}})
dist.dry_run = True
dist.parse_config_files()
command = dist.get_command_obj('install')
command.ensure_finalized()
command.run()
src_dir = glob(os.path.join(dist.install_libbase, 'pomoxis-*', 'exes'))[0]
for exe in (os.path.join(src_dir, x) for x in os.listdir(src_dir)):
print("Copying", os.path.basename(exe), '->', dist.install_scripts)
shutil.copy(exe, dist.install_scripts)
return dist.install_libbase, dist.install_scripts
def executable(request) -> pathlib.PosixPath:
"""Let setuptools compute the path to the built executable"""
with chdir(request.config.rootdir) as root_dir:
command = "build"
distribution = Distribution({
"script_name": __file__,
"script_args": [command],
})
distribution.parse_config_files()
distribution.parse_command_line()
command = distribution.get_command_obj(command)
command.ensure_finalized()
return (pathlib.PosixPath(root_dir).absolute() /
command.build_platlib / "hades-dhcp-script")
def get_setuptools_script_dir():
# Run the above class just to get paths
dist = Distribution({'cmdclass': {'install': GetPaths}})
dist.dry_run = True
dist.parse_config_files()
command = dist.get_command_obj('install')
command.ensure_finalized()
command.run()
src_dir = glob(os.path.join(dist.install_libbase, 'pomoxis-*', 'exes'))[0]
for exe in (os.path.join(src_dir, x) for x in os.listdir(src_dir)):
print("Copying", os.path.basename(exe), '->', dist.install_scripts)
shutil.copy(exe, dist.install_scripts)
return dist.install_libbase, dist.install_scripts
def fetch_build_egg(self, req):
""" Specialized version of Distribution.fetch_build_egg
that respects respects allow_hosts and index_url. """
from setuptools.command.easy_install import easy_install
dist = Distribution({"script_args": ["easy_install"]})
dist.parse_config_files()
opts = dist.get_option_dict("easy_install")
keep = (
"find_links",
"site_dirs",
"index_url",
"optimize",
"site_dirs",
"allow_hosts",
)
for key in list(opts):
if key not in keep:
del opts[key] # don't use any other settings
if self.dependency_links:
links = self.dependency_links[:]
if "find_links" in opts:
links = opts["find_links"][1].split() + links
opts["find_links"] = ("setup", links)
if self.allow_hosts:
opts["allow_hosts"] = ("test", self.allow_hosts)
if self.index_url:
opts["index_url"] = ("test", self.index_url)
install_dir_func = getattr(self, "get_egg_cache_dir", _os.getcwd)
install_dir = install_dir_func()
cmd = easy_install(
dist,
args=["x"],
install_dir=install_dir,
exclude_scripts=True,
always_copy=False,
build_directory=None,
editable=False,
upgrade=False,
multi_version=True,
no_report=True,
user=False,
)
cmd.ensure_finalized()
return cmd.easy_install(req)
def getprefix(self):
'''Retrieve setup tool calculated prefix
:returns: prefix
:rtype: string
'''
dist = Distribution({'cmdclass': {'install': OnlyGetScriptPath}})
dist.dry_run = True # not sure if necessary, but to be safe
dist.parse_config_files()
try:
dist.parse_command_line()
except (distutils.errors.DistutilsArgError, AttributeError):
pass
command = dist.get_command_obj('install')
command.ensure_finalized()
command.run()
prefix = dist.install_scripts.replace('/bin', '')
return prefix
def read_setuptools_cfg():
"""
Reads the `setup.cfg` file and extracts the various requirements lists
"""
# see https://stackoverflow.com/a/30679041/7262247
from setuptools import Distribution
dist = Distribution()
dist.parse_config_files()
# standard requirements
options_dct = dist.get_option_dict('options')
setup_reqs = options_dct['setup_requires'][1].strip().splitlines()
install_reqs = options_dct['install_requires'][1].strip().splitlines()
tests_reqs = options_dct['tests_require'][1].strip().splitlines()
return SetupCfg(setup_requires=setup_reqs,
install_requires=install_reqs,
tests_requires=tests_reqs)
def extra_link_args():
"""
Add arguments for Data Execution Prevention and ASLR.
This is only relevant for older Python versions.
:return: `list` of arguments
"""
dist = Distribution()
dist.parse_config_files()
try:
dist.parse_command_line()
except (TypeError, DistutilsArgError): # Happens with setup.py --help
pass
build = dist.get_command_obj('build')
build.ensure_finalized()
if new_compiler(compiler=build.compiler).compiler_type == 'msvc':
return ["/NXCOMPAT", "/DYNAMICBASE"]
return []
def test_dist_fetch_build_egg(tmpdir):
"""
Check multiple calls to `Distribution.fetch_build_egg` work as expected.
"""
index = tmpdir.mkdir('index')
index_url = urllib.parse.urljoin(
'file://', urllib.request.pathname2url(str(index)))
def sdist_with_index(distname, version):
dist_dir = index.mkdir(distname)
dist_sdist = '%s-%s.tar.gz' % (distname, version)
make_nspkg_sdist(str(dist_dir.join(dist_sdist)), distname, version)
with dist_dir.join('index.html').open('w') as fp:
fp.write(DALS(
'''
<!DOCTYPE html><html><body>
<a href="{dist_sdist}" rel="internal">{dist_sdist}</a><br/>
</body></html>
'''
).format(dist_sdist=dist_sdist))
sdist_with_index('barbazquux', '3.2.0')
sdist_with_index('barbazquux-runner', '2.11.1')
with tmpdir.join('setup.cfg').open('w') as fp:
fp.write(DALS(
'''
[easy_install]
index_url = {index_url}
'''
).format(index_url=index_url))
reqs = '''
barbazquux-runner
barbazquux
'''.split()
with tmpdir.as_cwd():
dist = Distribution()
dist.parse_config_files()
resolved_dists = [
dist.fetch_build_egg(r)
for r in reqs
]
assert [dist.key for dist in resolved_dists if dist] == reqs
def test_dist_fetch_build_egg(tmpdir):
"""
Check multiple calls to `Distribution.fetch_build_egg` work as expected.
"""
index = tmpdir.mkdir('index')
index_url = urljoin('file://', pathname2url(str(index)))
def sdist_with_index(distname, version):
dist_dir = index.mkdir(distname)
dist_sdist = '%s-%s.tar.gz' % (distname, version)
make_nspkg_sdist(str(dist_dir.join(dist_sdist)), distname, version)
with dist_dir.join('index.html').open('w') as fp:
fp.write(DALS(
'''
<!DOCTYPE html><html><body>
<a href="{dist_sdist}" rel="internal">{dist_sdist}</a><br/>
</body></html>
'''
).format(dist_sdist=dist_sdist))
sdist_with_index('barbazquux', '3.2.0')
sdist_with_index('barbazquux-runner', '2.11.1')
with tmpdir.join('setup.cfg').open('w') as fp:
fp.write(DALS(
'''
[easy_install]
index_url = {index_url}
'''
).format(index_url=index_url))
reqs = '''
barbazquux-runner
barbazquux
'''.split()
with tmpdir.as_cwd():
dist = Distribution()
dist.parse_config_files()
resolved_dists = [
dist.fetch_build_egg(r)
for r in reqs
]
assert [dist.key for dist in resolved_dists if dist] == reqs
def run(self):
"""
Print desired path (according to subclass) to stdout.
Unfortunately, setuptools automatically prints "running scriptdir" on
stdout. So, for example, the output will look like this (for example):
running scriptdir
/usr/local/bin
The shell command `tail` can be used to get only the relevant line:
`python setup.py scriptdir | tail --lines=1`
"""
# pylint: disable=no-self-use
# Create fake install to get a setuptools script directory path.
dist = Distribution({"cmdclass": {"install": _ScriptDirSpy}})
dist.dry_run = True
dist.parse_config_files()
command = dist.get_command_obj("install")
command.ensure_finalized()
command.run()
print(self.get_dir_from_distribution(dist))
def main(argv=None, **kw):
""" Run a test package's tests.
"""
org.setup_global_org_config()
USAGE = """\
usage: %(script)s <package name> [test options]
or: %(script)s --help
""" % {
'script': sys.argv[0] or 'runtests'
}
if argv is None:
argv = sys.argv[1:]
if not argv:
print "Please specify a package name."
print USAGE
sys.exit(1)
pkg_name, argv = argv[0], argv[1:]
test_pkg_name = 'test.%s' % pkg_name
# Find our
real_dist = [i for i in working_set if i.project_name == pkg_name]
if not real_dist:
print "Package %s is not installed" % pkg_name
sys.exit(1)
real_dist = real_dist[0]
test_dist = [i for i in working_set if i.project_name == test_pkg_name]
if not test_dist:
print "Test package %s is not installed" % test_pkg_name
sys.exit(1)
test_dist = test_dist[0]
# Construct a distutils.Distribtion class from the pkg_resources.Distribution
# of the real package so we can pass it into the test command class.
# We have checked that the packages are already installed so we set the install
# requirements to blank.
args = {
'name': real_dist.project_name,
'install_requires': [],
'tests_require': [],
'namespace_packages':
list(real_dist._get_metadata('namespace_packages')),
'packages': [real_dist.project_name],
}
real_cmd_dist = Distribution(args)
cmd = test(real_cmd_dist)
cmd.args = argv
# Read in the test options saved away during egg_info and set the command defaults,
# this would normally be done by the setup() method via the Distribution class
test_options = join(test_dist.location, 'EGG-INFO', 'test_options.txt')
if isfile(test_options):
real_cmd_dist.parse_config_files([test_options])
for k, v in real_cmd_dist.get_option_dict('test').items():
print "Found test option in %s: %s = %s" % (v[0], k, v[1])
setattr(cmd, k, v[1])
# Finalize and run the command, overriding the test root to be inside the test egg
cmd.finalize_options()
cmd.test_root = join(test_dist.location, CONFIG.test_egg_namespace,
real_dist.project_name.replace('.', '/'))
# Pylint is only for regular Jenkins jobs, this in itself should not trigger even if
# running under Jenkins
cmd.no_pylint = True
cmd.run()
def main(argv=None, **kw):
""" Run a test package's tests.
"""
# TODO: allow cmdline override of org config?
config.setup_org_config()
from path import path
USAGE = """\
usage: %(script)s <package name> [test options]
or: %(script)s --help
""" % {'script': sys.argv[0] or 'runtests'}
if argv is None:
argv = sys.argv[1:]
if not argv:
print "Please specify a package name."
print USAGE
sys.exit(1)
pkg_name, argv = argv[0], argv[1:]
test_pkg_name = 'test.%s' % pkg_name
# Find our
real_dist = [i for i in working_set if i.project_name == pkg_name]
if not real_dist:
print "Package %s is not installed" % pkg_name
sys.exit(1)
real_dist = real_dist[0]
test_dist = [i for i in working_set if i.project_name == test_pkg_name]
if not test_dist:
print "Test package %s is not installed" % test_pkg_name
sys.exit(1)
test_dist = test_dist[0]
# Construct a distutils.Distribtion class from the pkg_resources.Distribution
# of the real package so we can pass it into the test command class.
# We have checked that the packages are already installed so we set the install
# requirements to blank.
args = {'name': real_dist.project_name,
'install_requires': [],
'tests_require': [],
'namespace_packages': list(real_dist._get_metadata('namespace_packages')),
'packages': [real_dist.project_name],
}
real_cmd_dist = Distribution(args)
cmd = test(real_cmd_dist)
cmd.args = argv
# Read in the test options saved away during egg_info and set the command defaults,
# this would normally be done by the setup() method via the Distribution class
test_options = path(test_dist.location) / 'EGG-INFO' / 'test_options.txt'
if test_options.isfile():
real_cmd_dist.parse_config_files([test_options])
for k, v in real_cmd_dist.get_option_dict('test').items():
print "Found test option in %s: %s = %s" % (v[0], k, v[1])
setattr(cmd, k, v[1])
# Finalize and run the command, overriding the test root to be inside the test egg
cmd.finalize_options()
cmd.test_root = path(test_dist.location) / CONFIG.test_egg_namespace / \
real_dist.project_name.replace('.', '/')
# Pylint is only for regular Jenkins jobs, this in itself should not trigger even if
# running under Jenkins
cmd.no_pylint = True
cmd.run()
from distutils.dist import Distribution
import os
import sys
# Third-party modules - we depend on numpy for everything
import numpy
# Obtain the numpy include directory. This logic works across numpy versions.
try:
numpy_include = numpy.get_include()
except AttributeError:
numpy_include = numpy.get_numpy_include()
# Get our own instance of Distribution
dist = Distribution()
dist.parse_config_files()
dist.parse_command_line()
# Get prefix from either config file or command line
prefix = dist.get_option_dict('install')['prefix'][1]
print("Prefix is: " + prefix)
pymod_name = "parcels_nodes"
py_version = 'python%d.%d' % (sys.version_info[0], sys.version_info[1])
print(sys.prefix)
PCLS_NODES_HOME = '/git_code/list_set_experiments' # '/vps/otbknox/williams/OTB_2.0'
PCLS_NODES_INCLDIR = [
os.path.join(PCLS_NODES_HOME, pymod_name, 'include'),
os.path.join(sys.prefix, 'include'),
os.path.join(sys.prefix, 'include', py_version),
os.path.join(sys.prefix, 'lib', py_version, 'site-packages', pymod_name,
