def _initialize_build_configuration(
plugin_resolver: PluginResolver,
options_bootstrapper: OptionsBootstrapper,
env: CompleteEnvironment,
) -> BuildConfiguration:
"""Initialize a BuildConfiguration for the given OptionsBootstrapper.
NB: This method:
1. has the side-effect of (idempotently) adding PYTHONPATH entries for this process
2. is expensive to call, because it might resolve plugins from the network
"""
bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
working_set = plugin_resolver.resolve(options_bootstrapper, env)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in bootstrap_options.pythonpath:
if path not in sys.path:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
return load_backends_and_plugins(
bootstrap_options.plugins,
working_set,
bootstrap_options.backend_packages,
)
def _load_plugins(self, working_set, python_paths, plugins, backend_packages):
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in python_paths:
if path not in sys.path:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
return load_backends_and_plugins(plugins, working_set, backend_packages)
def _load_plugins(self, working_set, python_paths, plugins, backend_packages):
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in python_paths:
if path not in sys.path:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
return load_backends_and_plugins(plugins, working_set, backend_packages)
def syspath_prepend(self, path):
""" Prepend ``path`` to ``sys.path`` list of import locations. """
from pkg_resources import fixup_namespace_packages
if self._savesyspath is None:
self._savesyspath = sys.path[:]
sys.path.insert(0, str(path))
# https://github.com/pypa/setuptools/blob/d8b901bc/docs/pkg_resources.txt#L162-L171
fixup_namespace_packages(str(path))
def _load_plugins(self):
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in self._bootstrap_options.pythonpath:
if path not in sys.path:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
return load_backends_and_plugins(
self._bootstrap_options.plugins, self._bootstrap_options.plugins2,
self._working_set, self._bootstrap_options.backend_packages,
self._bootstrap_options.backend_packages2, BuildConfiguration())
def _load_plugins(self, working_set, python_paths, plugins, backend_packages):
"""Load backends and plugins.
:returns: A `BuildConfiguration` object constructed during backend/plugin loading.
"""
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in python_paths:
if path not in sys.path:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
return load_backends_and_plugins(plugins, working_set, backend_packages)
def _setup_options(self, options_bootstrapper, working_set):
# TODO: This inline import is currently necessary to resolve a ~legitimate cycle between
# `GoalRunner`->`EngineInitializer`->`OptionsInitializer`->`GoalRunner`.
from pants.bin.goal_runner import GoalRunner
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
'Version mismatch: Requested version was {}, our version is {}.'
.format(global_bootstrap_options.pants_version,
pants_version()))
# Get logging setup prior to loading backends so that they can log as needed.
if self._init_logging:
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_backends_and_plugins(
plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(GoalRunner.subsystems()
| Goal.subsystems()
| build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options, build_configuration
def _setup_options(self, options_bootstrapper, working_set):
# TODO: This inline import is currently necessary to resolve a ~legitimate cycle between
# `GoalRunner`->`EngineInitializer`->`OptionsInitializer`->`GoalRunner`.
from pants.bin.goal_runner import GoalRunner
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
'Version mismatch: Requested version was {}, our version is {}.'.format(
global_bootstrap_options.pants_version, pants_version()
)
)
# Get logging setup prior to loading backends so that they can log as needed.
if self._init_logging:
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(
GoalRunner.subsystems() | Goal.subsystems() | build_configuration.subsystems()
)
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options, build_configuration
def _setup_options(self, options_bootstrapper, working_set):
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
# The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually
# requested the version on the command line before deciding to print the version and exit.
if global_bootstrap_options.is_flagged('pants_version'):
print(global_bootstrap_options.pants_version)
self._exiter(0)
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(
plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(GoalRunner.subsystems()
| Goal.subsystems()
| build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options, build_configuration
def _setup_options(self, options_bootstrapper, working_set):
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
# The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually
# requested the version on the command line before deciding to print the version and exit.
if global_bootstrap_options.is_flagged('pants_version'):
print(global_bootstrap_options.pants_version)
self._exiter(0)
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(
GoalRunner.subsystems() | Goal.subsystems() | build_configuration.subsystems()
)
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options, build_configuration
def syspathinsert(self, path=None):
"""Prepend a directory to sys.path, defaults to :py:attr:`tmpdir`.
This is undone automatically when this object dies at the end of each
test.
"""
from pkg_resources import fixup_namespace_packages
if path is None:
path = self.tmpdir
dirname = str(path)
sys.path.insert(0, dirname)
fixup_namespace_packages(dirname)
# a call to syspathinsert() usually means that the caller wants to
# import some dynamically created files, thus with python3 we
# invalidate its import caches
self._possibly_invalidate_import_caches()
def syspathinsert(self, path=None):
"""Prepend a directory to sys.path, defaults to :py:attr:`tmpdir`.
This is undone automatically when this object dies at the end of each
test.
"""
from pkg_resources import fixup_namespace_packages
if path is None:
path = self.tmpdir
dirname = str(path)
sys.path.insert(0, dirname)
fixup_namespace_packages(dirname)
# a call to syspathinsert() usually means that the caller wants to
# import some dynamically created files, thus with python3 we
# invalidate its import caches
self._possibly_invalidate_import_caches()
def syspath_prepend(self, path) -> None:
"""Prepend ``path`` to ``sys.path`` list of import locations."""
from pkg_resources import fixup_namespace_packages
if self._savesyspath is None:
self._savesyspath = sys.path[:]
sys.path.insert(0, str(path))
# https://github.com/pypa/setuptools/blob/d8b901bc/docs/pkg_resources.txt#L162-L171
fixup_namespace_packages(str(path))
# A call to syspathinsert() usually means that the caller wants to
# import some dynamically created files, thus with python3 we
# invalidate its import caches.
# This is especially important when any namespace package is in use,
# since then the mtime based FileFinder cache (that gets created in
# this case already) gets not invalidated when writing the new files
# quickly afterwards.
from importlib import invalidate_caches
invalidate_caches()
def PrepareProtobuf():
CHROME_SRC = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..",
".."))
THIRD_PARTY = os.path.join(CHROME_SRC, 'third_party')
sys.path.insert(0, os.path.join(THIRD_PARTY, 'protobuf', 'python'))
sys.path.insert(
1, os.path.join(THIRD_PARTY, 'protobuf', 'third_party', 'six'))
@contextlib.contextmanager
def temp_sys_path():
orig_path = sys.path[:]
try:
yield
finally:
sys.path = orig_path
with temp_sys_path():
sys.path = [THIRD_PARTY]
sys.modules.pop('google', None)
pkg_resources.declare_namespace('google')
pkg_resources.fixup_namespace_packages(THIRD_PARTY)
def syspath_prepend(self, path):
""" Prepend ``path`` to ``sys.path`` list of import locations. """
from pkg_resources import fixup_namespace_packages
if self._savesyspath is None:
self._savesyspath = sys.path[:]
sys.path.insert(0, str(path))
# https://github.com/pypa/setuptools/blob/d8b901bc/docs/pkg_resources.txt#L162-L171
fixup_namespace_packages(str(path))
# A call to syspathinsert() usually means that the caller wants to
# import some dynamically created files, thus with python3 we
# invalidate its import caches.
# This is especially important when any namespace package is in used,
# since then the mtime based FileFinder cache (that gets created in
# this case already) gets not invalidated when writing the new files
# quickly afterwards.
if sys.version_info >= (3, 3):
from importlib import invalidate_caches
invalidate_caches()
# library.
# c) Build our standard first-is-preferred path for the remainder of the
# engine.
import pkg_resources
with temp_sys_path():
# In a temporary environment where "sys.path" consists solely of our
# "third_party" directory, register the google namespace. By restricting the
# options in "sys.path", "pkg_resources" will not cache or prefer system
# resources for this namespace or its derivatives.
sys.path = [THIRD_PARTY]
# Remove module if there is preloaded 'google' module
sys.modules.pop('google', None)
pkg_resources.declare_namespace('google')
pkg_resources.fixup_namespace_packages(THIRD_PARTY)
# From here on out, we're back to normal imports. Let's assert that the we're
# using the correct protobuf package, though.
#
# We use "realpath" here because the importer may resolve the path differently
# based on symlinks, and we want to make sure our calculated path matches the
# impoter's path regardless.
import google.protobuf
assert (os.path.realpath(THIRD_PARTY)
in os.path.realpath(google.protobuf.__path__[0]))
# Prune all VirtualEnv paths out of $PATH. This means that recipe engine
# 'unwraps' vpython VirtualEnv path manipulation. Invocations of `python` from
# recipes should never inherit the recipe engine's own VirtualEnv.
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
bootstrap_options = options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(bootstrap_options.for_global_scope())
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
# TODO: Create a 'Subsystem' abstraction instead of special-casing run-tracker here
# and in register_options().
known_scopes = ['', 'run-tracker']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes)
self.register_options()
self.run_tracker = RunTracker.from_options(self.options)
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir, relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
self._expand_goals_and_specs()
def setup(self, options_bootstrapper, working_set):
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
# The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually
# requested the version on the command line before deciding to print the version and exit.
if global_bootstrap_options.is_flagged('pants_version'):
print(global_bootstrap_options.pants_version)
self._exiter(0)
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems.
subsystems = Subsystem.closure(set(self.subsystems) |
Goal.subsystems() |
build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scope_infos)
self.register_options(subsystems)
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
self.reporting = Reporting.global_instance()
report = self.reporting.initial_reporting(self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
def setUp(self):
"""Set up testcase."""
super(EnhancedTestCase, self).setUp()
# make sure option parser doesn't pick up any cmdline arguments/options
while len(sys.argv) > 1:
sys.argv.pop()
# keep track of log handlers
log = fancylogger.getLogger(fname=False)
self.orig_log_handlers = log.handlers[:]
log.info("setting up test %s" % self.id())
self.orig_tmpdir = tempfile.gettempdir()
# use a subdirectory for this test (which we can clean up easily after the test completes)
self.test_prefix = set_tmpdir()
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
fd, self.logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
os.close(fd)
self.cwd = os.getcwd()
# keep track of original environment to restore
self.orig_environ = copy.deepcopy(os.environ)
# keep track of original environment/Python search path to restore
self.orig_sys_path = sys.path[:]
testdir = os.path.dirname(os.path.abspath(__file__))
self.test_sourcepath = os.path.join(testdir, 'sandbox', 'sources')
os.environ['EASYBUILD_SOURCEPATH'] = self.test_sourcepath
os.environ['EASYBUILD_PREFIX'] = self.test_prefix
self.test_buildpath = tempfile.mkdtemp()
os.environ['EASYBUILD_BUILDPATH'] = self.test_buildpath
self.test_installpath = tempfile.mkdtemp()
os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath
# make sure that the tests only pick up easyconfigs provided with the tests
os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(testdir, 'easyconfigs')
# make sure no deprecated behaviour is being triggered (unless intended by the test)
# trip *all* log.deprecated statements by setting deprecation version ridiculously high
self.orig_current_version = eb_build_log.CURRENT_VERSION
os.environ['EASYBUILD_DEPRECATED'] = '10000000'
init_config()
import easybuild
# try to import easybuild.easyblocks(.generic) packages
# it's OK if it fails here, but important to import first before fiddling with sys.path
try:
import easybuild.easyblocks
import easybuild.easyblocks.generic
except ImportError:
pass
# add sandbox to Python search path, update namespace packages
sys.path.append(os.path.join(testdir, 'sandbox'))
# workaround for bug in recent setuptools version (19.4 and newer, atleast until 20.3.1)
# injecting <prefix>/easybuild is required to avoid a ValueError being thrown by fixup_namespace_packages
# cfr. https://bitbucket.org/pypa/setuptools/issues/520/fixup_namespace_packages-may-trigger
for path in sys.path[:]:
if os.path.exists(os.path.join(path, 'easybuild', 'easyblocks', '__init__.py')):
# keep track of 'easybuild' paths to inject into sys.path later
sys.path.append(os.path.join(path, 'easybuild'))
# required to make sure the 'easybuild' dir in the sandbox is picked up;
# this relates to the other 'reload' statements below
reload(easybuild)
# this is strictly required to make the test modules in the sandbox available, due to declare_namespace
fixup_namespace_packages(os.path.join(testdir, 'sandbox'))
# remove any entries in Python search path that seem to provide easyblocks (except the sandbox)
for path in sys.path[:]:
if os.path.exists(os.path.join(path, 'easybuild', 'easyblocks', '__init__.py')):
if not os.path.samefile(path, os.path.join(testdir, 'sandbox')):
sys.path.remove(path)
# hard inject location to (generic) test easyblocks into Python search path
# only prepending to sys.path is not enough due to 'declare_namespace' in easybuild/easyblocks/__init__.py
import easybuild.easyblocks
reload(easybuild.easyblocks)
test_easyblocks_path = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks')
easybuild.easyblocks.__path__.insert(0, test_easyblocks_path)
import easybuild.easyblocks.generic
reload(easybuild.easyblocks.generic)
test_easyblocks_path = os.path.join(test_easyblocks_path, 'generic')
easybuild.easyblocks.generic.__path__.insert(0, test_easyblocks_path)
# save values of $PATH & $PYTHONPATH, so they can be restored later
# this is important in case EasyBuild was installed as a module, since that module may be unloaded,
# for example due to changes to $MODULEPATH in case EasyBuild was installed in a module hierarchy
# cfr. https://github.com/hpcugent/easybuild-framework/issues/1685
self.env_path = os.environ['PATH']
self.env_pythonpath = os.environ['PYTHONPATH']
self.modtool = modules_tool()
self.reset_modulepath([os.path.join(testdir, 'modules')])
reset_module_caches()
def setup(self):
options_bootstrapper = OptionsBootstrapper()
bootstrap_options = options_bootstrapper.get_bootstrap_options()
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(bootstrap_options.for_global_scope())
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = bootstrap_options.for_global_scope().plugins
backend_packages = bootstrap_options.for_global_scope().backend_packages
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scope_infos = [ScopeInfo.for_global_scope()]
# Add scopes for all needed subsystems.
subsystems = (set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(ScopeInfo(subsystem.options_scope, ScopeInfo.GLOBAL_SUBSYSTEM))
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scope_infos)
self.register_options(subsystems)
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
self.reporting = Reporting.global_instance()
report = self.reporting.initial_reporting(self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
bootstrap_options = options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(
plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
# TODO: Create a 'Subsystem' abstraction instead of special-casing run-tracker here
# and in register_options().
known_scopes = ['', 'run-tracker']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(
known_scopes=known_scopes)
self.register_options()
# TODO(Eric Ayers) We are missing log messages. Set the log level earlier
# Enable standard python logging for code with no handle to a context/work-unit.
self._setup_logging() # NB: self.options are needed for this call.
self.run_tracker = RunTracker.from_options(self.options)
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
self.run_tracker.log(
Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(
build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals',
'bootstrap_buildfiles',
default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir,
relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
self._expand_goals_and_specs()
def ROS_setup_pythonpath(workspace):
# TODO : support python3 using checks like : sys.version_info >= (2, 7)
# for indigo, python3 is better supported via virtual envs to not conflict with system install.
package_paths = [
os.path.join(workspace, 'lib', 'python2.7',
'dist-packages'), # default for catkin
os.path.join(
workspace, 'lib', 'python2.7', 'site-packages'
), # catkin_pip can create this in workspaces and we need to be able to access it
]
# since ROS logic with package path is *incompatible* with python / venv logic
# inspired from python site module (getting rid of known_path)
def readdsitedir(sitedir):
"""Add 'sitedir' argument to sys.path if missing and handle .pth files in
'sitedir'"""
sitedir, sitedircase = site.makepath(sitedir)
print("Re-adding {sitedir} in front of sys.path".format(**locals()))
if sitedir in sys.path:
sys.path.remove(sitedir) # Remove path component
sys.path.insert(1, sitedir) # Insert path component in front
try:
names = os.listdir(sitedir)
except os.error:
return
dotpth = os.extsep + "pth"
names = [name for name in names if name.endswith(dotpth)]
for name in sorted(names):
readdpackage(sitedir, name)
return
# since ROS logic with package path is *incompatible* with python / venv logic
# inspired from python site module (getting rid of known_path)
def readdpackage(sitedir, name):
"""Process a .pth file within the site-packages directory:
For each line in the file, if it doesnt start with 'import ', combine it with sitedir to a path
and remove that from sys.path.
"""
# All this code is inspired from site.addpackage
fullname = os.path.join(sitedir, name)
try:
f = open(fullname, "rU")
except IOError:
return
with f:
for n, line in enumerate(f):
if line.startswith("#") or line.startswith(
("import ", "import\t")):
continue
try:
line = line.rstrip()
dir, dircase = site.makepath(sitedir, line)
# If the path is already there, we remove it (to reinsert it in the proper place)
print("Re-adding {dir} in front of sys.path".format(
**locals()))
if dir in sys.path:
sys.path.remove(dir)
sys.path.insert(1, dir)
except Exception as err:
print("Error processing line {:d} of {}:\n".format(
n + 1, fullname),
file=sys.stderr)
for record in traceback.format_exception(*sys.exc_info()):
for line in record.splitlines():
print(' ' + line, file=sys.stderr)
print("\nRemainder of file ignored", file=sys.stderr)
break
# Note : virtualenvs are a much better solution to this problem,
# but we have to simulate ROS behavior ( to work with ROS workspaces )
# setting python path needed only to find ros shell commands (rosmaster)
pplist = os.environ.get("PYTHONPATH", "").split(":")
for pp in package_paths:
if pp is not None and os.path.exists(pp):
_logger.warning(
"Prepending path {pp} to sys.path".format(**locals()))
readdsitedir(pp)
# similar logic for pythonpath as with readdsitedir
if pp in pplist:
pplist.remove(pp)
pplist.insert(1, pp)
# making sure our logic works for namespace packages as well
pkg_resources.fixup_namespace_packages(
pp) # ensure that message subpackage is included
os.environ["PYTHONPATH"] = ':'.join(pplist)
def setUp(self):
"""Set up testcase."""
super(EnhancedTestCase, self).setUp()
# make sure option parser doesn't pick up any cmdline arguments/options
while len(sys.argv) > 1:
sys.argv.pop()
# keep track of log handlers
log = fancylogger.getLogger(fname=False)
self.orig_log_handlers = log.handlers[:]
log.info("setting up test %s" % self.id())
self.orig_tmpdir = tempfile.gettempdir()
# use a subdirectory for this test (which we can clean up easily after the test completes)
self.test_prefix = set_tmpdir()
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
fd, self.logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
os.close(fd)
self.cwd = os.getcwd()
# keep track of original environment to restore
self.orig_environ = copy.deepcopy(os.environ)
# keep track of original environment/Python search path to restore
self.orig_sys_path = sys.path[:]
testdir = os.path.dirname(os.path.abspath(__file__))
self.test_sourcepath = os.path.join(testdir, 'sandbox', 'sources')
os.environ['EASYBUILD_SOURCEPATH'] = self.test_sourcepath
os.environ['EASYBUILD_PREFIX'] = self.test_prefix
self.test_buildpath = tempfile.mkdtemp()
os.environ['EASYBUILD_BUILDPATH'] = self.test_buildpath
self.test_installpath = tempfile.mkdtemp()
os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath
# make sure that the tests only pick up easyconfigs provided with the tests
os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(
testdir, 'easyconfigs')
# make sure no deprecated behaviour is being triggered (unless intended by the test)
# trip *all* log.deprecated statements by setting deprecation version ridiculously high
self.orig_current_version = eb_build_log.CURRENT_VERSION
os.environ['EASYBUILD_DEPRECATED'] = '10000000'
init_config()
import easybuild
# try to import easybuild.easyblocks(.generic) packages
# it's OK if it fails here, but important to import first before fiddling with sys.path
try:
import easybuild.easyblocks
import easybuild.easyblocks.generic
except ImportError:
pass
# add sandbox to Python search path, update namespace packages
sys.path.append(os.path.join(testdir, 'sandbox'))
# workaround for bug in recent setuptools version (19.4 and newer, atleast until 20.3.1)
# injecting <prefix>/easybuild is required to avoid a ValueError being thrown by fixup_namespace_packages
# cfr. https://bitbucket.org/pypa/setuptools/issues/520/fixup_namespace_packages-may-trigger
for path in sys.path[:]:
if os.path.exists(
os.path.join(path, 'easybuild', 'easyblocks',
'__init__.py')):
# keep track of 'easybuild' paths to inject into sys.path later
sys.path.append(os.path.join(path, 'easybuild'))
# required to make sure the 'easybuild' dir in the sandbox is picked up;
# this relates to the other 'reload' statements below
reload(easybuild)
# this is strictly required to make the test modules in the sandbox available, due to declare_namespace
fixup_namespace_packages(os.path.join(testdir, 'sandbox'))
# remove any entries in Python search path that seem to provide easyblocks (except the sandbox)
for path in sys.path[:]:
if os.path.exists(
os.path.join(path, 'easybuild', 'easyblocks',
'__init__.py')):
if not os.path.samefile(path, os.path.join(testdir,
'sandbox')):
sys.path.remove(path)
# hard inject location to (generic) test easyblocks into Python search path
# only prepending to sys.path is not enough due to 'declare_namespace' in easybuild/easyblocks/__init__.py
import easybuild.easyblocks
reload(easybuild.easyblocks)
test_easyblocks_path = os.path.join(testdir, 'sandbox', 'easybuild',
'easyblocks')
easybuild.easyblocks.__path__.insert(0, test_easyblocks_path)
import easybuild.easyblocks.generic
reload(easybuild.easyblocks.generic)
test_easyblocks_path = os.path.join(test_easyblocks_path, 'generic')
easybuild.easyblocks.generic.__path__.insert(0, test_easyblocks_path)
# save values of $PATH & $PYTHONPATH, so they can be restored later
# this is important in case EasyBuild was installed as a module, since that module may be unloaded,
# for example due to changes to $MODULEPATH in case EasyBuild was installed in a module hierarchy
# cfr. https://github.com/hpcugent/easybuild-framework/issues/1685
self.env_path = os.environ['PATH']
self.env_pythonpath = os.environ['PYTHONPATH']
self.modtool = modules_tool()
self.reset_modulepath([os.path.join(testdir, 'modules')])
reset_module_caches()
def _fix_sys_path(self):
new_sys_path = self._get_new_sys_path()
for new_path in set(new_sys_path) - set(sys.path):
sys.path.insert(0, new_path)
pkg_resources.fixup_namespace_packages(new_path)
