def identify(targets):
targets = list(targets)
if len(targets) == 1 and targets[0].is_jvm and getattr(
targets[0], 'provides', None):
return targets[0].provides.org, targets[0].provides.name
else:
return IvyUtils.INTERNAL_ORG_NAME, Target.maybe_readable_identify(
targets)
def extra_products(self, target):
"""Override extra_products to produce an annotation processor information file."""
ret = []
if isinstance(target, AnnotationProcessor) and target.processors:
root = os.path.join(self._processor_info_dir, Target.maybe_readable_identify([target]))
processor_info_file = os.path.join(root, self._PROCESSOR_INFO_FILE)
self._write_processor_info(processor_info_file, target.processors)
ret.append((root, [processor_info_file]))
return super(AptCompile, self).extra_products(target) + ret
def _maybe_emit_junit_xml(self, targets):
args = []
xml_base = self.get_options().junit_xml_dir
if xml_base and targets:
xml_base = os.path.realpath(xml_base)
xml_path = os.path.join(xml_base, Target.maybe_readable_identify(targets) + '.xml')
safe_mkdir(os.path.dirname(xml_path))
args.append('--junitxml={}'.format(xml_path))
yield args
def _maybe_emit_junit_xml(self, targets):
args = []
xml_base = self.get_options().junit_xml_dir
if xml_base and targets:
xml_base = os.path.realpath(xml_base)
xml_path = os.path.join(xml_base, Target.maybe_readable_identify(targets) + '.xml')
safe_mkdir(os.path.dirname(xml_path))
args.append('--junitxml={}'.format(xml_path))
yield args
def extra_products(self, target):
"""Override extra_products to produce an annotation processor information file."""
ret = []
if isinstance(target, AnnotationProcessor) and target.processors:
root = os.path.join(self._processor_info_dir,
Target.maybe_readable_identify([target]))
processor_info_file = os.path.join(root, self._PROCESSOR_INFO_FILE)
self._write_processor_info(processor_info_file, target.processors)
ret.append((root, [processor_info_file]))
return super(AptCompile, self).extra_products(target) + ret
def expose_results(self, invalid_tgts, partition, workdirs):
external_junit_xml_dir = self.get_options().junit_xml_dir
if external_junit_xml_dir:
# Either we just ran pytest for a set of invalid targets and generated a junit xml file
# specific to that (sub)set or else we hit the cache for the whole partition and skipped
# running pytest, simply retrieving the partition's full junit xml file.
junitxml_path = workdirs.junitxml_path(*(invalid_tgts or partition))
safe_mkdir(external_junit_xml_dir)
shutil.copy2(junitxml_path, external_junit_xml_dir)
if self.get_options().coverage:
coverage_output_dir = self.get_options().coverage_output_dir
if coverage_output_dir:
target_dir = coverage_output_dir
else:
relpath = Target.maybe_readable_identify(partition)
pants_distdir = self.context.options.for_global_scope().pants_distdir
target_dir = os.path.join(pants_distdir, 'coverage', relpath)
mergetree(workdirs.coverage_path, target_dir)
def expose_results(self, invalid_tgts, partition, workdirs):
external_junit_xml_dir = self.get_options().junit_xml_dir
if external_junit_xml_dir:
# Either we just ran pytest for a set of invalid targets and generated a junit xml file
# specific to that (sub)set or else we hit the cache for the whole partition and skipped
# running pytest, simply retrieving the partition's full junit xml file.
junitxml_path = workdirs.junitxml_path(*(invalid_tgts or partition))
safe_mkdir(external_junit_xml_dir)
shutil.copy2(junitxml_path, external_junit_xml_dir)
if self.get_options().coverage:
coverage_output_dir = self.get_options().coverage_output_dir
if coverage_output_dir:
target_dir = coverage_output_dir
else:
relpath = Target.maybe_readable_identify(partition)
pants_distdir = self.context.options.for_global_scope().pants_distdir
target_dir = os.path.join(pants_distdir, 'coverage', relpath)
mergetree(workdirs.coverage_path, target_dir)
def target_set_id(self, *targets):
return Target.maybe_readable_identify(targets or self.partition)
def for_partition(cls, work_dir, partition):
root_dir = os.path.join(work_dir,
Target.maybe_readable_identify(partition))
safe_mkdir(root_dir, clean=False)
return cls(root_dir=root_dir, partition=partition)
def identify(targets):
targets = list(targets)
if len(targets) == 1 and targets[0].is_jvm and getattr(targets[0], 'provides', None):
return targets[0].provides.org, targets[0].provides.name
else:
return IvyUtils.INTERNAL_ORG_NAME, Target.maybe_readable_identify(targets)
def _maybe_emit_coverage_data(self, targets, pex, workunit):
coverage = self.get_options().coverage
if coverage is None:
yield []
return
def read_coverage_list(prefix):
return coverage[len(prefix):].split(',')
coverage_modules = None
if coverage.startswith('modules:'):
# NB: pytest-cov maps these modules to the `[run] sources` config. So for
# `modules:pants.base,pants.util` the config emitted has:
# [run]
# source =
# pants.base
# pants.util
#
# Now even though these are not paths, coverage sees the dots and switches to a module
# prefix-matching mode. Unfortunately, neither wildcards nor top-level module prefixes
# like `pants.` serve to engage this module prefix-matching as one might hope. It
# appears that `pants.` is treated as a path and `pants.*` is treated as a literal
# module prefix name.
coverage_modules = read_coverage_list('modules:')
elif coverage.startswith('paths:'):
coverage_modules = []
pex_src_root = os.path.relpath(
self.context.products.get_data(
GatherSources.PYTHON_SOURCES).path(), get_buildroot())
for path in read_coverage_list('paths:'):
coverage_modules.append(os.path.join(pex_src_root, path))
with self._cov_setup(
targets, pex.path(),
coverage_modules=coverage_modules) as (args, coverage_rc):
try:
yield args
finally:
env = {'PEX_MODULE': 'coverage.cmdline:main'}
def pex_run(args):
return self._pex_run(pex, workunit, args=args, env=env)
# On failures or timeouts, the .coverage file won't be written.
if not os.path.exists('.coverage'):
self.context.log.warn(
'No .coverage file was found! Skipping coverage reporting.'
)
else:
# Normalize .coverage.raw paths using combine and `paths` config in the rc file.
# This swaps the /tmp pex chroot source paths for the local original source paths
# the pex was generated from and which the user understands.
shutil.move('.coverage', '.coverage.raw')
pex_run(args=['combine', '--rcfile', coverage_rc])
pex_run(args=['report', '-i', '--rcfile', coverage_rc])
# TODO(wickman): If coverage is enabled and we are not using fast mode, write an
# intermediate .html that points to each of the coverage reports generated and
# webbrowser.open to that page.
# TODO(John Sirois): Possibly apply the same logic to the console report. In fact,
# consider combining coverage files from all runs in this Tasks's execute and then
# producing just 1 console and 1 html report whether or not the tests are run in fast
# mode.
if self.get_options().coverage_output_dir:
target_dir = self.get_options().coverage_output_dir
else:
relpath = Target.maybe_readable_identify(targets)
pants_distdir = self.context.options.for_global_scope(
).pants_distdir
target_dir = os.path.join(pants_distdir, 'coverage',
relpath)
safe_mkdir(target_dir)
pex_run(args=[
'html', '-i', '--rcfile', coverage_rc, '-d', target_dir
])
coverage_xml = os.path.join(target_dir, 'coverage.xml')
pex_run(args=[
'xml', '-i', '--rcfile', coverage_rc, '-o',
coverage_xml
])
def _get_junit_xml_path(self, targets):
xml_path = os.path.join(
self.workdir, 'junitxml',
'TEST-{}.xml'.format(Target.maybe_readable_identify(targets)))
safe_mkdir_for(xml_path)
return xml_path
def target_set_id(self, *targets):
return Target.maybe_readable_identify(
targets) if targets else self.target.id
def _maybe_emit_coverage_data(self, targets, pex, workunit):
coverage = self.get_options().coverage
if coverage is None:
yield []
return
pex_src_root = os.path.relpath(
self.context.products.get_data(
GatherSources.PYTHON_SOURCES).path(), get_buildroot())
source_mappings = {}
for target in targets:
libs = (
tgt for tgt in target.closure()
if tgt.has_sources('.py') and not isinstance(tgt, PythonTests))
for lib in libs:
source_mappings[lib.target_base] = [pex_src_root]
def ensure_trailing_sep(path):
return path if path.endswith(os.path.sep) else path + os.path.sep
if coverage == 'auto':
def compute_coverage_sources(tgt):
if tgt.coverage:
return tgt.coverage
else:
# This makes the assumption that tests/python/<tgt> will be testing src/python/<tgt>.
# Note in particular that this doesn't work for pants' own tests, as those are under
# the top level package 'pants_tests', rather than just 'pants'.
# TODO(John Sirois): consider failing fast if there is no explicit coverage scheme;
# but also consider supporting configuration of a global scheme whether that be parallel
# dirs/packages or some arbitrary function that can be registered that takes a test target
# and hands back the source packages or paths under test.
return set(
os.path.dirname(s).replace(os.sep, '.')
for s in tgt.sources_relative_to_source_root())
coverage_sources = set(
itertools.chain(
*[compute_coverage_sources(t) for t in targets]))
else:
coverage_sources = []
for source in coverage.split(','):
if os.path.isdir(source):
# The source is a dir, so correct its prefix for the chroot.
# E.g. if source is /path/to/src/python/foo/bar or src/python/foo/bar then
# rel_source is src/python/foo/bar, and ...
rel_source = os.path.relpath(source, get_buildroot())
rel_source = ensure_trailing_sep(rel_source)
found_target_base = False
for target_base in source_mappings:
prefix = ensure_trailing_sep(target_base)
if rel_source.startswith(prefix):
# ... rel_source will match on prefix=src/python/ ...
suffix = rel_source[len(prefix):]
# ... suffix will equal foo/bar ...
coverage_sources.append(
os.path.join(pex_src_root, suffix))
found_target_base = True
# ... and we end up appending <pex_src_root>/foo/bar to the coverage_sources.
break
if not found_target_base:
self.context.log.warn(
'Coverage path {} is not in any target. Skipping.'.
format(source))
else:
# The source is to be interpreted as a package name.
coverage_sources.append(source)
with self._cov_setup(
source_mappings,
coverage_sources=coverage_sources) as (args, coverage_rc):
try:
yield args
finally:
env = {'PEX_MODULE': 'coverage.cmdline:main'}
def pex_run(args):
return self._pex_run(pex, workunit, args=args, env=env)
# On failures or timeouts, the .coverage file won't be written.
if not os.path.exists('.coverage'):
self.context.log.warn(
'No .coverage file was found! Skipping coverage reporting.'
)
else:
# Normalize .coverage.raw paths using combine and `paths` config in the rc file.
# This swaps the /tmp pex chroot source paths for the local original source paths
# the pex was generated from and which the user understands.
shutil.move('.coverage', '.coverage.raw')
pex_run(args=['combine', '--rcfile', coverage_rc])
pex_run(args=['report', '-i', '--rcfile', coverage_rc])
# TODO(wickman): If coverage is enabled and we are not using fast mode, write an
# intermediate .html that points to each of the coverage reports generated and
# webbrowser.open to that page.
# TODO(John Sirois): Possibly apply the same logic to the console report. In fact,
# consider combining coverage files from all runs in this Tasks's execute and then
# producing just 1 console and 1 html report whether or not the tests are run in fast
# mode.
if self.get_options().coverage_output_dir:
target_dir = self.get_options().coverage_output_dir
else:
relpath = Target.maybe_readable_identify(targets)
pants_distdir = self.context.options.for_global_scope(
).pants_distdir
target_dir = os.path.join(pants_distdir, 'coverage',
relpath)
safe_mkdir(target_dir)
pex_run(args=[
'html', '-i', '--rcfile', coverage_rc, '-d', target_dir
])
coverage_xml = os.path.join(target_dir, 'coverage.xml')
pex_run(args=[
'xml', '-i', '--rcfile', coverage_rc, '-o',
coverage_xml
])
def for_targets(cls, work_dir, targets): root_dir = os.path.join(work_dir, Target.maybe_readable_identify(targets)) safe_mkdir(root_dir, clean=False) return cls(root_dir=root_dir)
def for_partition(cls, work_dir, partition): root_dir = os.path.join(work_dir, Target.maybe_readable_identify(partition)) safe_mkdir(root_dir, clean=False) return cls(root_dir=root_dir, partition=partition)
def _maybe_emit_coverage_data(self, targets, pex):
coverage = self.get_options().coverage
if coverage is None:
yield []
return
pex_src_root = os.path.relpath(
self.context.products.get_data(GatherSources.PYTHON_SOURCES).path(), get_buildroot())
source_mappings = {}
for target in targets:
libs = (tgt for tgt in target.closure()
if tgt.has_sources('.py') and not isinstance(tgt, PythonTests))
for lib in libs:
source_mappings[lib.target_base] = [pex_src_root]
def ensure_trailing_sep(path):
return path if path.endswith(os.path.sep) else path + os.path.sep
if coverage == 'auto':
def compute_coverage_sources(tgt):
if tgt.coverage:
return tgt.coverage
else:
# This makes the assumption that tests/python/<tgt> will be testing src/python/<tgt>.
# Note in particular that this doesn't work for pants' own tests, as those are under
# the top level package 'pants_tests', rather than just 'pants'.
# TODO(John Sirois): consider failing fast if there is no explicit coverage scheme;
# but also consider supporting configuration of a global scheme whether that be parallel
# dirs/packages or some arbitrary function that can be registered that takes a test target
# and hands back the source packages or paths under test.
return set(os.path.dirname(s).replace(os.sep, '.')
for s in tgt.sources_relative_to_source_root())
coverage_sources = set(itertools.chain(*[compute_coverage_sources(t) for t in targets]))
else:
coverage_sources = []
for source in coverage.split(','):
if os.path.isdir(source):
# The source is a dir, so correct its prefix for the chroot.
# E.g. if source is /path/to/src/python/foo/bar or src/python/foo/bar then
# rel_source is src/python/foo/bar, and ...
rel_source = os.path.relpath(source, get_buildroot())
rel_source = ensure_trailing_sep(rel_source)
found_target_base = False
for target_base in source_mappings:
prefix = ensure_trailing_sep(target_base)
if rel_source.startswith(prefix):
# ... rel_source will match on prefix=src/python/ ...
suffix = rel_source[len(prefix):]
# ... suffix will equal foo/bar ...
coverage_sources.append(os.path.join(pex_src_root, suffix))
found_target_base = True
# ... and we end up appending <pex_src_root>/foo/bar to the coverage_sources.
break
if not found_target_base:
self.context.log.warn('Coverage path {} is not in any target. Skipping.'.format(source))
else:
# The source is to be interpreted as a package name.
coverage_sources.append(source)
with self._cov_setup(source_mappings,
coverage_sources=coverage_sources) as (args, coverage_rc):
try:
yield args
finally:
env = {
'PEX_MODULE': 'coverage.cmdline:main'
}
def pex_run(arguments):
return self._pex_run(pex, workunit_name='coverage', args=arguments, env=env)
# On failures or timeouts, the .coverage file won't be written.
if not os.path.exists('.coverage'):
self.context.log.warn('No .coverage file was found! Skipping coverage reporting.')
else:
# Normalize .coverage.raw paths using combine and `paths` config in the rc file.
# This swaps the /tmp pex chroot source paths for the local original source paths
# the pex was generated from and which the user understands.
shutil.move('.coverage', '.coverage.raw')
pex_run(['combine', '--rcfile', coverage_rc])
pex_run(['report', '-i', '--rcfile', coverage_rc])
if self.get_options().coverage_output_dir:
target_dir = self.get_options().coverage_output_dir
else:
relpath = Target.maybe_readable_identify(targets)
pants_distdir = self.context.options.for_global_scope().pants_distdir
target_dir = os.path.join(pants_distdir, 'coverage', relpath)
safe_mkdir(target_dir)
pex_run(['html', '-i', '--rcfile', coverage_rc, '-d', target_dir])
coverage_xml = os.path.join(target_dir, 'coverage.xml')
pex_run(['xml', '-i', '--rcfile', coverage_rc, '-o', coverage_xml])
def _get_junit_xml_path(self, targets):
xml_path = os.path.join(self.workdir, 'junitxml',
'TEST-{}.xml'.format(Target.maybe_readable_identify(targets)))
safe_mkdir_for(xml_path)
return xml_path
def _maybe_emit_coverage_data(self, targets, chroot, pex, workunit):
coverage = self.get_options().coverage
if coverage is None:
yield []
return
def read_coverage_list(prefix):
return coverage[len(prefix):].split(',')
coverage_modules = None
if coverage.startswith('modules:'):
# NB: pytest-cov maps these modules to the `[run] sources` config. So for
# `modules:pants.base,pants.util` the config emitted has:
# [run]
# source =
# pants.base
# pants.util
#
# Now even though these are not paths, coverage sees the dots and switches to a module
# prefix-matching mode. Unfortunately, neither wildcards nor top-level module prefixes
# like `pants.` serve to engage this module prefix-matching as one might hope. It
# appears that `pants.` is treated as a path and `pants.*` is treated as a literal
# module prefix name.
coverage_modules = read_coverage_list('modules:')
elif coverage.startswith('paths:'):
coverage_modules = []
for path in read_coverage_list('paths:'):
if not os.path.exists(path) and not os.path.isabs(path):
# Look for the source in the PEX chroot since its not available from CWD.
path = os.path.join(chroot, path)
coverage_modules.append(path)
with self._cov_setup(targets,
chroot,
coverage_modules=coverage_modules) as (args, coverage_rc):
try:
yield args
finally:
with environment_as(PEX_MODULE='coverage.cmdline:main'):
def pex_run(args):
return self._pex_run(pex, workunit, args=args)
# On failures or timeouts, the .coverage file won't be written.
if not os.path.exists('.coverage'):
logger.warning('No .coverage file was found! Skipping coverage reporting.')
else:
# Normalize .coverage.raw paths using combine and `paths` config in the rc file.
# This swaps the /tmp pex chroot source paths for the local original source paths
# the pex was generated from and which the user understands.
shutil.move('.coverage', '.coverage.raw')
pex_run(args=['combine', '--rcfile', coverage_rc])
pex_run(args=['report', '-i', '--rcfile', coverage_rc])
# TODO(wickman): If coverage is enabled and we are not using fast mode, write an
# intermediate .html that points to each of the coverage reports generated and
# webbrowser.open to that page.
# TODO(John Sirois): Possibly apply the same logic to the console report. In fact,
# consider combining coverage files from all runs in this Tasks's execute and then
# producing just 1 console and 1 html report whether or not the tests are run in fast
# mode.
if self.get_options().coverage_output_dir:
target_dir = self.get_options().coverage_output_dir
else:
relpath = Target.maybe_readable_identify(targets)
pants_distdir = self.context.options.for_global_scope().pants_distdir
target_dir = os.path.join(pants_distdir, 'coverage', relpath)
safe_mkdir(target_dir)
pex_run(args=['html', '-i', '--rcfile', coverage_rc, '-d', target_dir])
coverage_xml = os.path.join(target_dir, 'coverage.xml')
pex_run(args=['xml', '-i', '--rcfile', coverage_rc, '-o', coverage_xml])
def for_targets(cls, work_dir, targets): root_dir = os.path.join(work_dir, Target.maybe_readable_identify(targets)) safe_mkdir(root_dir, clean=False) return cls(root_dir=root_dir)
def target_set_id(self, *targets): return Target.maybe_readable_identify(targets or self.partition)
def generate_ivy(cls, targets, jars, excludes, ivyxml, confs, resolve_hash_name=None,
pinned_artifacts=None, jar_dep_manager=None):
if not resolve_hash_name:
resolve_hash_name = Target.maybe_readable_identify(targets)
return cls._generate_resolve_ivy(jars, excludes, ivyxml, confs, resolve_hash_name, pinned_artifacts,
jar_dep_manager)
