def _isolation(self, all_targets):
run_dir = '_runs'
mode_dir = 'isolated' if self._per_target else 'combined'
batch_dir = str(self._batch_size) if self._batched else 'all'
output_dir = os.path.join(self.workdir, run_dir,
Target.identify(all_targets), mode_dir,
batch_dir)
safe_mkdir(output_dir, clean=False)
if self._html_report:
junit_html_report = JUnitHtmlReport.create(
xml_dir=output_dir,
open_report=self.get_options().open,
logger=self.context.log,
error_on_conflict=True)
else:
junit_html_report = NoJunitHtmlReport()
coverage = CodeCoverage.global_instance().get_coverage_engine(
self, output_dir, all_targets, self.execute_java_for_coverage)
reports = self.Reports(junit_html_report, coverage)
self.context.release_lock()
try:
yield output_dir, reports, coverage
finally:
lock_file = '.file_lock'
preserve = (run_dir, lock_file)
dist_dir = os.path.join(
self.get_options().pants_distdir,
os.path.relpath(self.workdir,
self.get_options().pants_workdir))
with OwnerPrintingInterProcessFileLock(
os.path.join(dist_dir, lock_file)):
self._link_current_reports(report_dir=output_dir,
link_dir=dist_dir,
preserve=preserve)
if self._legacy_report_layout:
deprecated_conditional(
predicate=lambda: True,
entity_description='[test.junit] legacy_report_layout',
stacklevel=3,
removal_version='1.6.0.dev0',
hint_message=
'Reports are now linked into {} by default; so scripts '
'and CI jobs should be pointed there and the option '
'configured to False in pants.ini until such time as '
'the option is removed.'.format(dist_dir))
# NB: Deposit of the "current" test output in the root workdir (.pants.d/test/junit) is a
# defacto public API and so we implement that behavior here to maintain backwards
# compatibility for non-pants report file consumers.
with OwnerPrintingInterProcessFileLock(
os.path.join(self.workdir, lock_file)):
self._link_current_reports(report_dir=output_dir,
link_dir=self.workdir,
preserve=preserve)
def __init__(self,
build_root,
engine_initializer,
options):
"""
:param str build_root: The path of the build root.
:param class engine_initializer: The class representing the EngineInitializer.
"""
self._build_root = build_root
self._engine_initializer = engine_initializer
# The options we register directly.
self._pailgun_host = options.pailgun_host
self._pailgun_port = options.pailgun_port
self._log_dir = options.log_dir
self._fs_event_workers = options.fs_event_workers
# Values derived from global options (which our scoped options inherit).
self._pants_workdir = options.pants_workdir
self._log_level = options.level.upper()
self._pants_ignore_patterns = options.pants_ignore
self._build_ignore_patterns = options.build_ignore
self._exclude_target_regexp = options.exclude_target_regexp
self._subproject_roots = options.subproject_roots
# Native.create() reads global options, which, thanks to inheritance, it can
# read them via our scoped options.
self._native = Native.create(options)
# TODO(kwlzn): Thread filesystem path ignores here to Watchman's subscription registration.
lock_location = os.path.join(self._build_root, '.pantsd.startup')
self._lock = OwnerPrintingInterProcessFileLock(lock_location)
self._logger = logging.getLogger(__name__)
def __init__(self, bootstrap_options, engine_initializer=None):
"""
:param Options bootstrap_options: An Options object containing the bootstrap options.
:param class engine_initializer: The class representing the EngineInitializer. Only required
for startup.
"""
self._bootstrap_options = bootstrap_options
self._engine_initializer = engine_initializer
self._pailgun_host = bootstrap_options.pantsd_pailgun_host
self._pailgun_port = bootstrap_options.pantsd_pailgun_port
self._log_dir = bootstrap_options.pantsd_log_dir
self._fs_event_workers = bootstrap_options.pantsd_fs_event_workers
self._pants_workdir = bootstrap_options.pants_workdir
self._log_level = bootstrap_options.level.upper()
self._pants_ignore_patterns = bootstrap_options.pants_ignore
self._build_ignore_patterns = bootstrap_options.build_ignore
self._exclude_target_regexp = bootstrap_options.exclude_target_regexp
self._subproject_roots = bootstrap_options.subproject_roots
self._metadata_base_dir = bootstrap_options.pants_subprocessdir
# TODO: https://github.com/pantsbuild/pants/issues/3479
self._build_root = get_buildroot()
self._native = Native.create(bootstrap_options)
self._logger = logging.getLogger(__name__)
self._lock = OwnerPrintingInterProcessFileLock(
os.path.join(self._build_root, '.pantsd.startup'))
def __init__(self,
classpath,
ivy_settings=None,
ivy_resolution_cache_dir=None,
extra_jvm_options=None):
"""Configures an ivy wrapper for the ivy distribution at the given classpath.
:param ivy_settings: path to find settings.xml file
:param ivy_resolution_cache_dir: path to store downloaded ivy artifacts
:param extra_jvm_options: list of strings to add to command line when invoking Ivy
"""
self._classpath = ensure_str_list(classpath)
self._ivy_settings = ivy_settings
if self._ivy_settings and not isinstance(self._ivy_settings, str):
raise ValueError(
"ivy_settings must be a string, given {} of type {}".format(
self._ivy_settings, type(self._ivy_settings)))
self._ivy_resolution_cache_dir = ivy_resolution_cache_dir
if not isinstance(self._ivy_resolution_cache_dir, str):
raise ValueError(
"ivy_resolution_cache_dir must be a string, given {} of type {}"
.format(self._ivy_resolution_cache_dir,
type(self._ivy_resolution_cache_dir)))
self._extra_jvm_options = extra_jvm_options or []
self._lock = OwnerPrintingInterProcessFileLock(
os.path.join(self._ivy_resolution_cache_dir,
"pants_ivy.file_lock"))
def __init__(self, options, run_tracker, target_roots,
requested_goals=None, target_base=None, build_graph=None,
build_file_parser=None, build_configuration=None,
address_mapper=None, console_outstream=None, scm=None,
workspace=None, invalidation_report=None, scheduler=None):
self._options = options
# We register a callback that will cause build graph edits to invalidate our caches, and we hold
# a handle directly to the callback function to ensure that it is not GC'd until the context is.
self.build_graph = build_graph
self._clear_target_cache_handle = self._clear_target_cache
self._targets_cache = dict()
self.build_graph.add_invalidation_callback(self._clear_target_cache_handle)
self._build_file_parser = build_file_parser
self.build_configuration = build_configuration
self.address_mapper = address_mapper
self.run_tracker = run_tracker
self._log = run_tracker.logger
self._target_base = target_base or Target
self._products = Products()
self._buildroot = get_buildroot()
self._source_roots = SourceRootConfig.global_instance().get_source_roots()
self._lock = OwnerPrintingInterProcessFileLock(os.path.join(self._buildroot, '.pants.workdir.file_lock'))
self._java_sysprops = None # Computed lazily.
self.requested_goals = requested_goals or []
self._console_outstream = console_outstream or sys.stdout.buffer
self._scm = scm or get_scm()
self._workspace = workspace or (ScmWorkspace(self._scm) if self._scm else None)
self._replace_targets(target_roots)
self._invalidation_report = invalidation_report
self._scheduler = scheduler
def __init__(self,
options,
run_tracker,
target_roots,
requested_goals=None,
target_base=None,
build_graph=None,
build_file_parser=None,
address_mapper=None,
console_outstream=None,
scm=None,
workspace=None,
invalidation_report=None):
self._options = options
self.build_graph = build_graph
self.build_file_parser = build_file_parser
self.address_mapper = address_mapper
self.run_tracker = run_tracker
self._log = self.Log(run_tracker)
self._target_base = target_base or Target
self._products = Products()
self._buildroot = get_buildroot()
self._source_roots = SourceRootConfig.global_instance(
).get_source_roots()
self._lock = OwnerPrintingInterProcessFileLock(
os.path.join(self._buildroot, '.pants.workdir.file_lock'))
self._java_sysprops = None # Computed lazily.
self.requested_goals = requested_goals or []
self._console_outstream = console_outstream or sys.stdout
self._scm = scm or get_scm()
self._workspace = workspace or (ScmWorkspace(self._scm)
if self._scm else None)
self._replace_targets(target_roots)
self._invalidation_report = invalidation_report
def __init__(self, build_root, pants_workdir, engine_initializer, log_dir,
log_level, pailgun_host, pailgun_port, fs_event_enabled,
fs_event_workers, path_ignore_patterns):
"""
:param str build_root: The path of the build root.
:param str pants_workdir: The path of the pants workdir.
:param class engine_initializer: The class representing the EngineInitializer.
:param str log_dir: The path for pantsd logs.
:param str log_level: The log level for pantsd logs (derived from the pants log level).
:param str pailgun_host: The bind address for the Pailgun server.
:param int pailgun_port: The bind port for the Pailgun server.
:param bool fs_event_enabled: Whether or not to enable fs event detection (Watchman) for graph
invalidation.
:param int fs_event_workers: The number of workers to use for processing the fs event queue.
:param list path_ignore_patterns: A list of ignore patterns for filesystem operations.
"""
self._build_root = build_root
self._pants_workdir = pants_workdir
self._engine_initializer = engine_initializer
self._log_dir = log_dir
self._log_level = log_level
self._pailgun_host = pailgun_host
self._pailgun_port = pailgun_port
self._fs_event_enabled = fs_event_enabled
self._fs_event_workers = fs_event_workers
self._path_ignore_patterns = path_ignore_patterns
# TODO(kwlzn): Thread filesystem path ignores here to Watchman's subscription registration.
lock_location = os.path.join(self._build_root, '.pantsd.startup')
self._lock = OwnerPrintingInterProcessFileLock(lock_location)
self._logger = logging.getLogger(__name__)
def _isolation(self, all_targets):
run_dir = '_runs'
output_dir = os.path.join(self.workdir, run_dir,
Target.identify(all_targets))
safe_mkdir(output_dir, clean=True)
coverage = None
options = self.get_options()
if options.coverage or options.is_flagged('coverage_open'):
coverage_processor = options.coverage_processor
if coverage_processor == 'cobertura':
settings = CoberturaTaskSettings.from_task(self,
workdir=output_dir)
coverage = Cobertura(settings)
else:
raise TaskError('unknown coverage processor {0}'.format(
coverage_processor))
self.context.release_lock()
if coverage:
coverage.instrument(
targets=all_targets,
compute_junit_classpath=lambda: self.classpath(all_targets),
execute_java_for_targets=self.execute_java_for_coverage)
def do_report(exc=None):
if coverage:
coverage.report(all_targets,
self.execute_java_for_coverage,
tests_failed_exception=exc)
if self._html_report:
html_file_path = JUnitHtmlReport().report(
output_dir, os.path.join(output_dir, 'reports'))
if self._open:
desktop.ui_open(html_file_path)
try:
yield output_dir, do_report, coverage
finally:
# NB: Deposit of the "current" test output in the root workdir (.pants.d/test/junit) is a
# defacto public API and so we implement that behavior here to maintain backwards
# compatibility for non-pants report file consumers.
# TODO(John Sirois): Deprecate this ~API and provide a stable directory solution for test
# output: https://github.com/pantsbuild/pants/issues/3879
lock_file = '.file_lock'
with OwnerPrintingInterProcessFileLock(
os.path.join(self.workdir, lock_file)):
# Kill everything except the isolated runs/ dir.
for name in os.listdir(self.workdir):
path = os.path.join(self.workdir, name)
if name not in (run_dir, lock_file):
if os.path.isdir(path):
safe_rmtree(path)
else:
os.unlink(path)
# Link all the isolated run/ dir contents back up to the stable workdir
for name in os.listdir(output_dir):
path = os.path.join(output_dir, name)
os.symlink(path, os.path.join(self.workdir, name))
def lifecycle_lock(self):
"""An identity-keyed inter-process lock for safeguarding lifecycle and other operations."""
safe_mkdir(self._metadata_base_dir)
return OwnerPrintingInterProcessFileLock(
# N.B. This lock can't key into the actual named metadata dir (e.g. `.pids/pantsd/lock`
# via `ProcessMetadataManager._get_metadata_dir_by_name()`) because of a need to purge
# the named metadata dir on startup to avoid stale metadata reads.
os.path.join(self._metadata_base_dir,
'.lock.{}'.format(self._name)))
def hold_lock_until_terminate(path, lock_held, terminate):
lock = OwnerPrintingInterProcessFileLock(path)
lock.acquire()
lock_held.set()
# NOTE: We shouldn't ever wait this long, this is just to ensure
# we don't somehow leak child processes.
terminate.wait(60)
lock.release()
lock_held.clear()
def setUp(self):
self.lock_dir = tempfile.mkdtemp()
self.lock_path = os.path.join(self.lock_dir, "lock")
self.lock = OwnerPrintingInterProcessFileLock(self.lock_path)
self.manager = Manager()
self.lock_held = self.manager.Event()
self.terminate = self.manager.Event()
self.lock_process = Process(target=hold_lock_until_terminate,
args=(self.lock_path, self.lock_held,
self.terminate))
def _isolation(self, all_targets):
run_dir = '_runs'
output_dir = os.path.join(self.workdir, run_dir,
Target.identify(all_targets))
safe_mkdir(output_dir, clean=False)
if self._html_report:
junit_html_report = JUnitHtmlReport.create(output_dir,
self.context.log)
else:
junit_html_report = NoJunitHtmlReport()
if self.get_options().coverage or self.get_options().is_flagged(
'coverage_open'):
settings = CoberturaTaskSettings.from_task(self,
workdir=output_dir)
coverage = Cobertura(settings, all_targets,
self.execute_java_for_coverage)
else:
coverage = NoCoverage()
reports = self.Reports(junit_html_report, coverage)
self.context.release_lock()
try:
yield output_dir, reports, coverage
finally:
# NB: Deposit of the "current" test output in the root workdir (.pants.d/test/junit) is a
# defacto public API and so we implement that behavior here to maintain backwards
# compatibility for non-pants report file consumers.
# TODO(John Sirois): Deprecate this ~API and provide a stable directory solution for test
# output: https://github.com/pantsbuild/pants/issues/3879
lock_file = '.file_lock'
with OwnerPrintingInterProcessFileLock(
os.path.join(self.workdir, lock_file)):
# Kill everything except the isolated `_runs/` dir.
for name in os.listdir(self.workdir):
path = os.path.join(self.workdir, name)
if name not in (run_dir, lock_file):
if os.path.isdir(path):
safe_rmtree(path)
else:
os.unlink(path)
# Link all the isolated run/ dir contents back up to the stable workdir
for name in os.listdir(output_dir):
path = os.path.join(output_dir, name)
os.symlink(path, os.path.join(self.workdir, name))
def setup(self, paths=(), filters=()):
"""Sets up a cache of python interpreters.
:param paths: The paths to search for a python interpreter; the system ``PATH`` by default.
:param filters: A sequence of strings that constrain the interpreter compatibility for this
cache, using the Requirement-style format, e.g. ``'CPython>=3', or just ['>=2.7','<3']``
for requirements agnostic to interpreter class.
:returns: A list of cached interpreters
:rtype: list of :class:`pex.interpreter.PythonInterpreter`
"""
# We filter the interpreter cache itself (and not just the interpreters we pull from it)
# because setting up some python versions (e.g., 3<=python<3.3) crashes, and this gives us
# an escape hatch.
filters = filters if any(
filters) else self._python_setup.interpreter_constraints
setup_paths = (paths or self.pex_python_paths()
or self._python_setup.interpreter_search_paths
or os.getenv('PATH').split(os.pathsep))
self._logger(
'Initializing Python interpreter cache matching filters `{}` from paths `{}`'
.format(':'.join(filters), ':'.join(setup_paths)))
def unsatisfied_filters(interpreters):
return [
f for f in filters
if len(list(self._matching(interpreters, [f]))) == 0
]
interpreters = []
with OwnerPrintingInterProcessFileLock(
path=os.path.join(self._cache_dir, '.file_lock')):
interpreters.extend(self._setup_cached(filters=filters))
if unsatisfied_filters(interpreters):
interpreters.extend(
self._setup_paths(setup_paths, filters=filters))
for filt in unsatisfied_filters(interpreters):
self._logger('No valid interpreters found for {}!'.format(filt))
matches = list(self._matching(interpreters, filters=filters))
if len(matches) == 0:
self._logger('Found no valid interpreters!')
self._logger('Initialized Python interpreter cache with {}'.format(
', '.join([x.binary for x in matches])))
return matches
def __init__(self, *args, **kwargs):
super(PantsDaemonLauncher, self).__init__(*args, **kwargs)
self._logger = logging.getLogger(__name__)
self._build_root = get_buildroot()
options = self.get_options()
self._pants_workdir = options.pants_workdir
self._log_dir = options.log_dir
self._log_level = options.level.upper()
self._pailgun_host = options.pailgun_host
self._pailgun_port = options.pailgun_port
self._fs_event_enabled = options.fs_event_detection
self._fs_event_workers = options.fs_event_workers
self._pantsd = None
self._scheduler = None
self._lock = OwnerPrintingInterProcessFileLock(
os.path.join(self._build_root, '.pantsd.startup'))
def setup(self, filters=()):
"""Sets up a cache of python interpreters.
:param filters: A sequence of strings that constrain the interpreter compatibility for this
cache, using the Requirement-style format, e.g. ``'CPython>=3', or just ['>=2.7','<3']``
for requirements agnostic to interpreter class.
:returns: A list of cached interpreters
:rtype: list of :class:`pex.interpreter.PythonInterpreter`
"""
# We filter the interpreter cache itself (and not just the interpreters we pull from it)
# because setting up some python versions (e.g., 3<=python<3.3) crashes, and this gives us
# an escape hatch.
filters = filters if any(
filters) else self.python_setup.interpreter_constraints
setup_paths = self.python_setup.interpreter_search_paths
logger.debug(
f"Initializing Python interpreter cache matching filters `{':'.join(filters)}` "
f"from paths `{':'.join(setup_paths)}`")
interpreters = []
def unsatisfied_filters():
return [
f for f in filters
if len(list(self._matching(interpreters, [f]))) == 0
]
with OwnerPrintingInterProcessFileLock(
path=os.path.join(self._cache_dir, '.file_lock')):
interpreters.extend(self._setup_cached(filters=filters))
if not interpreters or unsatisfied_filters():
interpreters.extend(
self._setup_paths(setup_paths, filters=filters))
for filt in unsatisfied_filters():
logger.debug(f'No valid interpreters found for {filt}!')
matches = list(self._matching(interpreters, filters=filters))
if len(matches) == 0:
logger.debug('Found no valid interpreters!')
logger.debug('Initialized Python interpreter cache with {}'.format(
', '.join([x.binary for x in matches])))
return matches
def _isolation(self, per_target, all_targets):
run_dir = '_runs'
mode_dir = 'isolated' if per_target else 'combined'
batch_dir = str(self._batch_size) if self._batched else 'all'
output_dir = os.path.join(self.workdir, run_dir,
Target.identify(all_targets), mode_dir,
batch_dir)
safe_mkdir(output_dir, clean=False)
if self._html_report:
junit_html_report = JUnitHtmlReport.create(
xml_dir=output_dir,
open_report=self.get_options().open,
logger=self.context.log,
error_on_conflict=self.get_options(
).html_report_error_on_conflict)
else:
junit_html_report = NoJunitHtmlReport()
coverage = CodeCoverage.global_instance().get_coverage_engine(
self, output_dir, all_targets, self.execute_java_for_coverage)
reports = self.Reports(junit_html_report, coverage)
self.context.release_lock()
try:
yield output_dir, reports, coverage
finally:
lock_file = '.file_lock'
preserve = (run_dir, lock_file)
dist_dir = os.path.join(
self.get_options().pants_distdir,
os.path.relpath(self.workdir,
self.get_options().pants_workdir))
with OwnerPrintingInterProcessFileLock(
os.path.join(dist_dir, lock_file)):
self._link_current_reports(report_dir=output_dir,
link_dir=dist_dir,
preserve=preserve)
def __init__(self, build_root, pants_workdir, engine_initializer, log_dir,
log_level, pailgun_host, pailgun_port, fs_event_workers,
pants_ignore_patterns, build_ignore_patterns,
exclude_target_regexp, subproject_roots):
"""
:param str build_root: The path of the build root.
:param str pants_workdir: The path of the pants workdir.
:param class engine_initializer: The class representing the EngineInitializer.
:param str log_dir: The path for pantsd logs.
:param str log_level: The log level for pantsd logs (derived from the pants log level).
:param str pailgun_host: The bind address for the Pailgun server.
:param int pailgun_port: The bind port for the Pailgun server.
:param int fs_event_workers: The number of workers to use for processing the fs event queue.
:param list pants_ignore_patterns: A list of path ignore patterns for filesystem operations.
:param list build_ignore_patterns: A list of path ignore patterns for BUILD file parsing.
:param list exclude_target_regexp: A list of target exclude regexps.
:param list subproject_roots: A list of subproject roots.
"""
self._build_root = build_root
self._pants_workdir = pants_workdir
self._engine_initializer = engine_initializer
self._log_dir = log_dir
self._log_level = log_level
self._pailgun_host = pailgun_host
self._pailgun_port = pailgun_port
self._fs_event_workers = fs_event_workers
self._pants_ignore_patterns = pants_ignore_patterns
self._build_ignore_patterns = build_ignore_patterns
self._exclude_target_regexp = exclude_target_regexp
self._subproject_roots = subproject_roots
self._native = Native.Factory.global_instance().create()
# TODO(kwlzn): Thread filesystem path ignores here to Watchman's subscription registration.
lock_location = os.path.join(self._build_root, '.pantsd.startup')
self._lock = OwnerPrintingInterProcessFileLock(lock_location)
self._logger = logging.getLogger(__name__)
def _lock(self):
"""An identity-keyed inter-process lock around the cookie file."""
lockfile = "{}.lock".format(self._get_cookie_file())
safe_mkdir_for(lockfile)
return OwnerPrintingInterProcessFileLock(lockfile)
