def unpack_target(self, unpacked_jars, unpack_dir):
deprecated_conditional(
lambda: True,
removal_version="1.31.0.dev0",
entity_description="The `unpack-jars` goal",
hint_message=
"Contact the Pants team on Slack or [email protected] "
"if you need this functionality.",
)
direct_coords = {
jar.coordinate
for jar in unpacked_jars.all_imported_jar_deps
}
unpack_filter = self.get_unpack_filter(unpacked_jars)
jar_import_products = self.context.products.get_data(JarImportProducts)
for coordinate, jar_path in jar_import_products.imports(unpacked_jars):
if not unpacked_jars.payload.intransitive or coordinate in direct_coords:
self.context.log.info(
"Unpacking jar {coordinate} from {jar_path} to {unpack_dir}."
.format(coordinate=coordinate,
jar_path=jar_path,
unpack_dir=unpack_dir))
ZIP.extract(jar_path, unpack_dir, filter_func=unpack_filter)
def __init__(self,
module_name,
libraries=None,
include_patterns=None,
exclude_patterns=None,
compatibility=None,
within_data_subdir=None,
payload=None,
**kwargs):
"""
:param str module_name: The name of the specific python module containing headers and/or
libraries to extract (e.g. 'tensorflow').
:param list libraries: addresses of python_requirement_library targets that specify the wheels
you want to unpack
:param list include_patterns: fileset patterns to include from the archive
:param list exclude_patterns: fileset patterns to exclude from the archive. Exclude patterns
are processed before include_patterns.
:param compatibility: Python interpreter constraints used to create the pex for the requirement
target. If unset, the default interpreter constraints are used. This
argument is unnecessary unless the native code depends on libpython.
:param bool within_data_subdir: If True, descend into '<name>-<version>.data/' when matching
`include_patterns`. For python wheels which declare any non-code
data, this is usually needed to extract that without manually
specifying the relative path, including the package version. For
example, when `data_files` is used in a setup.py,
`within_data_subdir=True` will allow specifying
`include_patterns` matching exactly what is specified in the
setup.py.
"""
deprecated_conditional(
lambda: type(within_data_subdir) not in (bool, type(None)),
removal_version="1.28.0.dev2",
entity_description="A non-boolean value for `within_data_subdir`",
hint_message=
"The location of the .data subdirectory will be inferred from the module name!",
)
payload = payload or Payload()
payload.add_fields({
"library_specs":
PrimitiveField(libraries or ()),
"module_name":
PrimitiveField(module_name),
"include_patterns":
PrimitiveField(include_patterns or ()),
"exclude_patterns":
PrimitiveField(exclude_patterns or ()),
"compatibility":
PrimitiveField(
ensure_str_list(compatibility or (), allow_single_str=True)),
"within_data_subdir":
PrimitiveField(within_data_subdir),
# TODO: consider supporting transitive deps like UnpackedJars!
# TODO: consider supporting `platforms` as in PythonBinary!
})
super().__init__(payload=payload, **kwargs)
if not libraries:
raise self.ExpectedLibrariesError(
"Expected non-empty libraries attribute for {spec}".format(
spec=self.address.spec))
def test_deprecated_conditional_true():
predicate = lambda: True
with _test_deprecation():
deprecated_conditional(predicate,
FUTURE_VERSION,
"test hint message",
stacklevel=0)
def __init__(
self,
build_file: Optional[BuildFile] = None,
target_name: Optional[str] = None,
rel_path: Optional[str] = None,
) -> None:
"""
:param build_file: The build file that contains the object this address points to.
:param rel_path: The BUILD files' path, relative to the root_dir.
:param target_name: The name of the target within the BUILD file; defaults to the default
target, aka the name of the BUILD file parent dir.
:API: public
"""
deprecated_conditional(
lambda: build_file is not None,
removal_version="1.31.0.dev0",
entity_description=
"using `build_file` as a parameter to `BuildFileAddress`",
hint_message=
"Use the arguments `target_name` and `rel_path` instead.",
)
if rel_path is None:
if build_file is None:
raise ValueError(
"You must either provide `rel_path` or `build_file` to the `BuildFileAddress` "
"constructor. Both values were None.")
rel_path = build_file.relpath
spec_path = os.path.dirname(rel_path)
super().__init__(spec_path=spec_path,
target_name=target_name
or os.path.basename(spec_path))
self.rel_path = rel_path
def test_deprecated_conditional_false():
predicate = lambda: False
with _test_deprecation(deprecation_expected=False):
deprecated_conditional(predicate,
FUTURE_VERSION,
"test hint message",
stacklevel=0)
def console_output(self, unused_method_argument):
deprecated_conditional(
lambda: not self.is_external_only and not self.is_internal_only,
removal_version="1.26.0.dev0",
entity_description="The default dependencies output including external dependencies",
hint_message="Pants will soon default to `--internal-only`, and remove the `--external-only` option. "
"Currently, Pants defaults to include both internal and external dependencies, which means this "
"task returns a mix of both target addresses and requirement strings."
"\n\nTo prepare, you can run this task with the `--internal-only` option. "
"If you need still need support for including external dependencies in the output, please let us "
"know in the #general channel on Slack at https://pantsbuild.slack.com/."
"\nYou can join the pants slack here: https://pantsslack.herokuapp.com/",
)
ordered_closure = OrderedSet()
for target in self.context.target_roots:
if self.act_transitively:
target.walk(ordered_closure.add)
else:
ordered_closure.update(target.dependencies)
for tgt in ordered_closure:
if not self.is_external_only:
yield tgt.address.spec
if not self.is_internal_only:
# TODO(John Sirois): We need an external payload abstraction at which point knowledge
# of jar and requirement payloads can go and this hairball will be untangled.
if isinstance(tgt.payload.get_field('requirements'), PythonRequirementsField):
for requirement in tgt.payload.requirements:
yield str(requirement.requirement)
elif isinstance(tgt.payload.get_field('jars'), JarsField):
for jar in tgt.payload.jars:
data = dict(org=jar.org, name=jar.name, rev=jar.rev)
yield ('{org}:{name}:{rev}' if jar.rev else '{org}:{name}').format(**data)
def field_adaptors(self):
with exception_logging(logger,
'Exception in `field_adaptors` property'):
field_adaptors = super(JvmAppAdaptor, self).field_adaptors
if getattr(self, 'bundles', None) is None:
return field_adaptors
# Construct a field for the `bundles` argument.
filespecs_list = []
path_globs_list = []
excluded_path_globs_list = []
for bundle in self.bundles:
# Short circuit in the case of `bundles=[..., bundle(), ...]`.
if not hasattr(bundle, 'fileset'):
# N.B. This notice is duplicated in jvm_app.py::Bundle.__call__() for the old engine.
deprecated_conditional(
lambda: True, '1.2.0',
'bare bundle() without `fileset=` param',
"Pass a `fileset=` parameter: `bundle(fileset=globs('*.config')`"
)
logger.warn(
'Ignoring `bundle()` without `fileset` parameter.')
continue
base_globs = BaseGlobs.from_sources_field(bundle.fileset)
filespecs_list.append(base_globs.filespecs)
path_globs, excluded_path_globs = base_globs.to_path_globs(
self.address.spec_path)
path_globs_list.append(path_globs)
excluded_path_globs_list.append(excluded_path_globs)
bundles_field = BundlesField(self.address, self.bundles,
filespecs_list, path_globs_list,
excluded_path_globs_list)
return field_adaptors + (bundles_field, )
def scan_addresses(self, root=None, spec_excludes=None):
"""Recursively gathers all addresses visible under `root` of the virtual address space.
:param string root: The absolute path of the root to scan; defaults to the root directory of the
pants project.
:rtype: set of :class:`pants.build_graph.address.Address`
:raises AddressLookupError: if there is a problem parsing a BUILD file
"""
deprecated_conditional(lambda: spec_excludes is not None,
'0.0.75',
'Use build_ignore_patterns constructor parameter instead.')
root_dir = get_buildroot()
base_path = None
if root:
try:
base_path = fast_relpath(root, root_dir)
except ValueError as e:
raise self.InvalidRootError(e)
addresses = set()
try:
for build_file in BuildFile.scan_build_files(self._project_tree,
base_relpath=base_path,
spec_excludes=spec_excludes,
build_ignore_patterns=self._build_ignore_patterns):
for address in self.addresses_in_spec_path(build_file.spec_path):
addresses.add(address)
except BuildFile.BuildFileError as e:
# Handle exception from BuildFile out of paranoia. Currently, there is no way to trigger it.
raise self.BuildFileScanError("{message}\n while scanning BUILD files in '{root}'."
.format(message=e, root=root))
return addresses
def wrap(cls, context_aware_object_factory, *target_types):
"""Wraps an existing context aware object factory into a target macro factory.
:param context_aware_object_factory: The existing context aware object factory.
:param *target_types: One or more target types the context aware object factory creates.
:returns: A new target macro factory.
:rtype: :class:`TargetMacro.Factory`
"""
if not target_types:
raise ValueError('The given `context_aware_object_factory` {} must expand at least 1 '
'produced type; none were registered'.format(context_aware_object_factory))
deprecated_conditional(
lambda: len(target_types) > 1,
'1.12.0.dev0',
'TargetMacro.Factory instances that construct more than one type are no longer supported. '
'Consider using a `context_aware_object_factory, which can construct any number of '
'different objects.'
)
class Factory(cls):
@property
def target_types(self):
return target_types
def macro(self, parse_context):
class Macro(TargetMacro):
def expand(self, *args, **kwargs):
context_aware_object_factory(parse_context, *args, **kwargs)
return Macro()
return Factory()
def check_unknown(self, target, kwargs, payload):
"""
:API: public
"""
ignore_params = set((self.get_options().ignored or {}).get(target.type_alias, ()))
unknown_args = {arg: value for arg, value in kwargs.items() if arg not in ignore_params}
if 'source' in unknown_args and 'sources' in payload.as_dict():
unknown_args.pop('source')
if 'sources' in unknown_args:
if 'sources' in payload.as_dict():
deprecated_conditional(
lambda: True,
'1.11.0.dev0',
('The source argument is deprecated - it gets automatically promoted to sources.'
'Target {} should just use a sources argument. No BUILD files need changing. '
'The source argument will stop being populated -').format(target.type_alias),
)
unknown_args.pop('sources')
kwargs.pop('sources')
ignored_args = {arg: value for arg, value in kwargs.items() if arg in ignore_params}
if ignored_args:
logger.debug('{target} ignoring the unimplemented arguments: {args}'
.format(target=target.address.spec,
args=', '.join('{} = {}'.format(key, val)
for key, val in ignored_args.items())))
if unknown_args:
error_message = '{target_type} received unknown arguments: {args}'
raise self.UnknownArgumentError(target.address.spec, error_message.format(
target_type=type(target).__name__,
args=''.join('\n {} = {}'.format(key, value) for key, value in unknown_args.items())
))
def console_output(self, targets):
if targets:
raise TaskError(
'The login task does not take any target arguments.')
deprecated_conditional(
lambda: self.get_passthru_args(),
removal_version='1.26.0.dev1',
entity_description='Using passthrough args with `./pants login`',
hint_message=
"Instead of passing the provider through `--login-passthrough-args` or the "
"style `./pants login -- prod`, use the option `--login-to`, such as "
"`./pants login --to=prod`.",
)
# TODO: When we have other auth methods (e.g., OAuth2), select one by provider name.
requested_providers = list(
filter(None, [self.get_options().to] + self.get_passthru_args()))
if len(requested_providers) != 1:
raise TaskError('Must specify exactly one provider.')
provider = requested_providers[0]
try:
BasicAuth.global_instance().authenticate(provider)
return ['', 'Logged in successfully using .netrc credentials.']
except Challenged as e:
creds = self._ask_for_creds(provider, e.url, e.realm)
BasicAuth.global_instance().authenticate(provider, creds=creds)
return ['', 'Logged in successfully.']
def __init__(self,
scm,
workspace,
address_mapper,
build_graph,
include_dependees,
fast=False,
changes_since=None,
diffspec=None,
exclude_target_regexp=None,
spec_excludes=None):
deprecated_conditional(lambda: spec_excludes is not None,
'0.0.75',
'Use address_mapper#build_ignore_patterns instead.')
self._scm = scm
self._workspace = workspace
self._address_mapper = address_mapper
self._build_graph = build_graph
self._include_dependees = include_dependees
self._fast = fast
self._changes_since = changes_since
self._diffspec = diffspec
self._exclude_target_regexp = exclude_target_regexp
self._spec_excludes = spec_excludes
self._mapper_cache = None
def check_unknown(self, target, kwargs, payload):
"""
:API: public
"""
ignore_params = set((self.get_options().ignored or {}).get(target.type_alias, ()))
unknown_args = {arg: value for arg, value in kwargs.items() if arg not in ignore_params}
if 'source' in unknown_args and 'sources' in payload.as_dict():
unknown_args.pop('source')
if 'sources' in unknown_args:
if 'sources' in payload.as_dict():
deprecated_conditional(
lambda: True,
'1.11.0.dev0',
('The source argument is deprecated - it gets automatically promoted to sources.'
'Target {} should just use a sources argument. No BUILD files need changing. '
'The source argument will stop being populated -').format(target.type_alias),
)
unknown_args.pop('sources')
kwargs.pop('sources')
ignored_args = {arg: value for arg, value in kwargs.items() if arg in ignore_params}
if ignored_args:
logger.debug('{target} ignoring the unimplemented arguments: {args}'
.format(target=target.address.spec,
args=', '.join('{} = {}'.format(key, val)
for key, val in ignored_args.items())))
if unknown_args:
error_message = '{target_type} received unknown arguments: {args}'
raise self.UnknownArgumentError(target.address.spec, error_message.format(
target_type=type(target).__name__,
args=''.join('\n {} = {}'.format(key, value) for key, value in unknown_args.items())
))
def register_options(cls, register):
super(RewriteBase, cls).register_options(register)
register('--target-types',
default=cls.target_types(),
advanced=True,
type=list,
help='The target types to apply formatting to.')
try:
sideeffecting = cls.sideeffecting
except AttributeError:
deprecated_conditional(
lambda: True,
'1.12.0.dev0',
"RewriteBase's sideeffecting property should be a class property, not an instance property "
"but class {} didn't have a class property.".format(
cls.__name__),
)
sideeffecting = False
if sideeffecting:
register(
'--output-dir',
advanced=True,
type=dir_option,
fingerprint=True,
help=
'Path to output directory. Any updated files will be written here. '
'If not specified, files will be modified in-place.')
def change_calculator(cls,
options,
address_mapper,
build_graph,
scm=None,
workspace=None,
spec_excludes=None):
deprecated_conditional(
lambda: spec_excludes is not None, '0.0.75',
'Use address_mapper#build_ignore_patterns instead.')
scm = scm or get_scm()
if scm is None:
raise TaskError('No SCM available.')
workspace = workspace or ScmWorkspace(scm)
return ChangeCalculator(
scm,
workspace,
address_mapper,
build_graph,
options.include_dependees,
fast=options.fast,
changes_since=options.changes_since,
diffspec=options.diffspec,
# NB: exclude_target_regexp is a global scope option registered
# elsewhere
exclude_target_regexp=options.exclude_target_regexp,
spec_excludes=spec_excludes)
def __init__(self, excludes_path, log):
self.excludes = {}
if excludes_path:
if not os.path.exists(excludes_path):
raise TaskError('Excludes file does not exist: {0}'.format(excludes_path))
with open(excludes_path) as fh:
for line in fh.readlines():
if line and not line.startswith('#') and '::' in line:
pattern, plugins = line.strip().split('::', 2)
plugins = plugins.split()
deprecated_conditional(
lambda: 'pep8' in plugins,
'1.10.0.dev0',
'The pep8 check has been renamed to pycodestyle. '
'Please update your suppression file: "{}". The pep8 option'.format(excludes_path)
)
map(lambda p:p if p != 'pep8' else 'pycodestyle', plugins)
self.excludes[pattern] = {
'regex': re.compile(pattern),
'plugins': plugins
}
log.debug('Exclude pattern: {pattern}'.format(pattern=pattern))
else:
log.debug('No excludes file specified. All python sources will be checked.')
def __init__(self, excludes_path, log):
self.excludes = {}
if excludes_path:
if not os.path.exists(excludes_path):
raise TaskError(
'Excludes file does not exist: {0}'.format(excludes_path))
with open(excludes_path) as fh:
for line in fh.readlines():
if line and not line.startswith('#') and '::' in line:
pattern, plugins = line.strip().split('::', 2)
plugins = plugins.split()
deprecated_conditional(
lambda: 'pep8' in plugins, '1.10.0.dev0',
'The pep8 check has been renamed to pycodestyle. '
'Please update your suppression file: "{}". The pep8 option'
.format(excludes_path))
map(lambda p: p
if p != 'pep8' else 'pycodestyle', plugins)
self.excludes[pattern] = {
'regex': re.compile(pattern),
'plugins': plugins
}
log.debug('Exclude pattern: {pattern}'.format(
pattern=pattern))
else:
log.debug(
'No excludes file specified. All python sources will be checked.'
)
def field_adaptors(self):
with exception_logging(logger, 'Exception in `field_adaptors` property'):
field_adaptors = super(JvmAppAdaptor, self).field_adaptors
if getattr(self, 'bundles', None) is None:
return field_adaptors
# Construct a field for the `bundles` argument.
filespecs_list = []
path_globs_list = []
excluded_path_globs_list = []
for bundle in self.bundles:
# Short circuit in the case of `bundles=[..., bundle(), ...]`.
if not hasattr(bundle, 'fileset'):
# N.B. This notice is duplicated in jvm_app.py::Bundle.__call__() for the old engine.
deprecated_conditional(lambda: True,
'1.2.0',
'bare bundle() without `fileset=` param',
"Pass a `fileset=` parameter: `bundle(fileset=globs('*.config')`")
logger.warn('Ignoring `bundle()` without `fileset` parameter.')
continue
base_globs = BaseGlobs.from_sources_field(bundle.fileset)
filespecs_list.append(base_globs.filespecs)
path_globs, excluded_path_globs = base_globs.to_path_globs(self.address.spec_path)
path_globs_list.append(path_globs)
excluded_path_globs_list.append(excluded_path_globs)
bundles_field = BundlesField(self.address,
self.bundles,
filespecs_list,
path_globs_list,
excluded_path_globs_list)
return field_adaptors + (bundles_field,)
def report_target_info(self, scope, target, keys, val):
"""Add target information to run_info under target_data.
Will Recursively construct a nested dict with the keys provided.
Primitive values can be overwritten with other primitive values,
but a primitive value cannot be overwritten with a dictionary.
For example:
Where the dictionary being updated is {'a': {'b': 16}}, reporting the value
15 with the key list ['a', 'b'] will result in {'a': {'b':15}};
but reporting the value 20 with the key list ['a', 'b', 'c'] will throw
an error.
:param string scope: The scope for which we are reporting the information.
:param Target target: The target for which we want to store information.
:param list of string keys: The keys that will be recursively
nested and pointing to the information being stored.
:param primitive val: The value of the information being stored.
:API: public
"""
if isinstance(target, Target):
target_spec = target.address.spec
else:
deprecated_conditional(
lambda: True, '1.6.0.dev0',
'The `target=` argument to `report_target_info`',
'Should pass a Target instance rather than a string.')
target_spec = target
new_key_list = [target_spec, scope]
new_key_list += keys
self._merge_list_of_keys_into_dict(self._target_to_data, new_key_list,
val, 0)
def console_output(self, unused_method_argument):
opts = self.get_options()
deprecated_conditional(
lambda: opts.is_default("type") and not opts.internal_only and not opts.external_only,
removal_version="1.27.0.dev0",
entity_description="The default dependencies output including external dependencies",
hint_message="Pants will soon default to `--dependencies-type=source`, rather than "
"`--dependencies-type=source-and-3rdparty`. To prepare, run this goal with"
" `--dependencies-type=source`.",
)
ordered_closure = OrderedSet()
for target in self.context.target_roots:
if self.act_transitively:
target.walk(ordered_closure.add)
else:
ordered_closure.update(target.dependencies)
for tgt in ordered_closure:
if self.dependency_type in [DependencyType.SOURCE, DependencyType.SOURCE_AND_THIRD_PARTY]:
yield tgt.address.spec
if self.dependency_type in [DependencyType.THIRD_PARTY, DependencyType.SOURCE_AND_THIRD_PARTY]:
# TODO(John Sirois): We need an external payload abstraction at which point knowledge
# of jar and requirement payloads can go and this hairball will be untangled.
if isinstance(tgt.payload.get_field('requirements'), PythonRequirementsField):
for requirement in tgt.payload.requirements:
yield str(requirement.requirement)
elif isinstance(tgt.payload.get_field('jars'), JarsField):
for jar in tgt.payload.jars:
data = dict(org=jar.org, name=jar.name, rev=jar.rev)
yield ('{org}:{name}:{rev}' if jar.rev else '{org}:{name}').format(**data)
def wrap(cls, context_aware_object_factory, *target_types):
"""Wraps an existing context aware object factory into a target macro factory.
:param context_aware_object_factory: The existing context aware object factory.
:param *target_types: One or more target types the context aware object factory creates.
:returns: A new target macro factory.
:rtype: :class:`TargetMacro.Factory`
"""
if not target_types:
raise ValueError('The given `context_aware_object_factory` {} must expand at least 1 '
'produced type; none were registered'.format(context_aware_object_factory))
deprecated_conditional(
lambda: len(target_types) > 1,
'1.10.0.dev0',
'TargetMacro.Factory instances that construct more than one type are no longer supported. '
'Consider using a `context_aware_object_factory, which can construct any number of '
'different objects.'
)
class Factory(cls):
@property
def target_types(self):
return target_types
def macro(self, parse_context):
class Macro(TargetMacro):
def expand(self, *args, **kwargs):
context_aware_object_factory(parse_context, *args, **kwargs)
return Macro()
return Factory()
def create_sources_field(self, sources, sources_rel_path, key_arg=None):
"""Factory method to create a SourcesField appropriate for the type of the sources object.
Note that this method is called before the call to Target.__init__ so don't expect fields to
be populated!
:API: public
:return: a payload field object representing the sources parameter
:rtype: SourcesField
"""
if not sources:
sources = FilesetWithSpec.empty(sources_rel_path)
elif not isinstance(sources, FilesetWithSpec):
key_arg_section = "'{}' to be ".format(key_arg) if key_arg else ""
raise TargetDefinitionException(
self,
"Expected {}a glob, an address or a list, but was {}".format(
key_arg_section, type(sources)))
elif not isinstance(sources, EagerFilesetWithSpec):
deprecated_conditional(lambda: True, '1.12.0.dev0', (
'FilesetWithSpec sources values are deprecated except for EagerFilesetWithSpec values. '
'Saw value of type {}').format(type(sources)))
return SourcesField(sources=sources)
def execute(self):
"""Resolves the specified confs for the configured targets and returns an iterator over
tuples of (conf, jar path)."""
deprecated_conditional(
lambda: JvmResolveSubsystem.global_instance().get_options(
).resolver == "ivy",
removal_version="1.27.0.dev0",
entity_description="Ivy Resolve",
hint_message=
"resolve.ivy as well as --resolver-resolver=ivy will be removed."
"Please use --resolver-resolver=coursier instead and refer to "
"https://www.pantsbuild.org/jvm_projects.html#coursier for setup.",
)
if JvmResolveSubsystem.global_instance().get_options(
).resolver != "ivy":
return
compile_classpath = self.context.products.get_data(
"compile_classpath",
init_func=ClasspathProducts.init_func(
self.get_options().pants_workdir),
)
targets = self.context.targets()
if all(not isinstance(target, JarLibrary) for target in targets):
if self._report:
self.context.log.info(
"Not generating a report. No resolution performed.")
return
confs = self.get_options().confs
if "sources" not in confs and self.context.products.is_required_data(
"resolve_sources_signal"):
confs.append("sources")
if "javadoc" not in confs and self.context.products.is_required_data(
"resolve_javadocs_signal"):
confs.append("javadoc")
executor = self.create_java_executor()
results = self.resolve(
executor=executor,
targets=targets,
classpath_products=compile_classpath,
confs=confs,
extra_args=self._args,
)
if self._report:
results_with_resolved_artifacts = [
r for r in results if r.has_resolved_artifacts
]
if not results_with_resolved_artifacts:
self.context.log.info(
"Not generating a report. No resolution performed.")
else:
for result in results_with_resolved_artifacts:
self._generate_ivy_report(result)
def create_trie(self) -> "SourceRootTrie":
"""Create a trie of source root patterns from options."""
trie = SourceRootTrie()
options = self.get_options()
legacy_patterns = []
for category in ["source", "test", "thirdparty"]:
# Add patterns.
for pattern in options.get("{}_root_patterns".format(category),
[]):
trie.add_pattern(pattern)
legacy_patterns.append(pattern)
# Add fixed source roots.
for path, langs in options.get("{}_roots".format(category),
{}).items():
trie.add_fixed(path)
legacy_patterns.append(f"^/{path}")
# We need to issue a deprecation warning even if relying on the default values
# of the deprecated options.
deprecated_conditional(
lambda: True,
removal_version="1.30.0.dev0",
entity_description="the *_root_patterns and *_roots options",
hint_message=
"Explicitly list your source roots with the `root_patterns` option in "
"the [source] scope. See https://pants.readme.io/docs/source-roots. "
f"See your current roots with `{self.options.pants_bin_name} roots`.",
)
return trie
def scan_addresses(self, root=None, spec_excludes=None):
"""Recursively gathers all addresses visible under `root` of the virtual address space.
:param string root: The absolute path of the root to scan; defaults to the root directory of the
pants project.
:rtype: set of :class:`pants.build_graph.address.Address`
:raises AddressLookupError: if there is a problem parsing a BUILD file
"""
deprecated_conditional(lambda: spec_excludes is not None,
'0.0.75',
'Use build_ignore_patterns constructor parameter instead.')
root_dir = get_buildroot()
base_path = None
if root:
try:
base_path = fast_relpath(root, root_dir)
except ValueError as e:
raise self.InvalidRootError(e)
addresses = set()
try:
for build_file in BuildFile.scan_build_files(self._project_tree,
base_relpath=base_path,
spec_excludes=spec_excludes,
build_ignore_patterns=self._build_ignore_patterns):
for address in self.addresses_in_spec_path(build_file.spec_path):
addresses.add(address)
except BuildFile.BuildFileError as e:
# Handle exception from BuildFile out of paranoia. Currently, there is no way to trigger it.
raise self.BuildFileScanError("{message}\n while scanning BUILD files in '{root}'."
.format(message=e, root=root))
return addresses
def __init__(self, name, has_python, skip_java, skip_scala, use_source_root, root_dir,
debug_port, context, targets, transitive, target_util, spec_excludes=None, build_ignore_patterns=None):
"""Creates a new, unconfigured, Project based at root_dir and comprised of the sources visible
to the given targets."""
deprecated_conditional(lambda: spec_excludes is not None,
'0.0.75',
'Use build_ignore_patterns instead.')
self.context = context
self.target_util = target_util
self.name = name
self.root_dir = root_dir
self.targets = OrderedSet(targets)
self.transitive = transitive
self.sources = []
self.py_sources = []
self.py_libs = []
self.resource_extensions = set()
self.has_python = has_python
self.skip_java = skip_java
self.skip_scala = skip_scala
self.use_source_root = use_source_root
self.has_scala = False
self.has_tests = False
self.debug_port = debug_port
self.internal_jars = OrderedSet()
self.external_jars = OrderedSet()
self.spec_excludes = spec_excludes
self.build_ignore_patterns = build_ignore_patterns
def __init__(self,
scm,
workspace,
address_mapper,
build_graph,
include_dependees,
fast=False,
changes_since=None,
diffspec=None,
exclude_target_regexp=None,
spec_excludes=None):
deprecated_conditional(
lambda: spec_excludes is not None, '0.0.75',
'Use address_mapper#build_ignore_patterns instead.')
self._scm = scm
self._workspace = workspace
self._address_mapper = address_mapper
self._build_graph = build_graph
self._include_dependees = include_dependees
self._fast = fast
self._changes_since = changes_since
self._diffspec = diffspec
self._exclude_target_regexp = exclude_target_regexp
self._spec_excludes = spec_excludes
self._mapper_cache = None
def __init__(self, options, run_tracker, target_roots,
requested_goals=None, target_base=None, build_graph=None,
build_file_parser=None, address_mapper=None, console_outstream=None, scm=None,
workspace=None, spec_excludes=None, invalidation_report=None):
"""
:API: public
"""
deprecated_conditional(lambda: spec_excludes is not None,
'0.0.75',
'Use address_mapper#build_ignore_patterns instead.')
self._options = options
self.build_graph = build_graph
self.build_file_parser = build_file_parser
self.address_mapper = address_mapper
self.run_tracker = run_tracker
self._log = self.Log(run_tracker)
self._target_base = target_base or Target
self._products = Products()
self._buildroot = get_buildroot()
self._source_roots = SourceRootConfig.global_instance().get_source_roots()
self._lock = OwnerPrintingPIDLockFile(os.path.join(self._buildroot, '.pants.run'))
self._java_sysprops = None # Computed lazily.
self.requested_goals = requested_goals or []
self._console_outstream = console_outstream or sys.stdout
self._scm = scm or get_scm()
self._workspace = workspace or (ScmWorkspace(self._scm) if self._scm else None)
self._spec_excludes = spec_excludes
self._replace_targets(target_roots)
self._invalidation_report = invalidation_report
def _do_create_artifact_cache(self, spec, action):
"""Returns an artifact cache for the specified spec.
spec can be:
- a path to a file-based cache root.
- a URL of a RESTful cache root.
- a URL of a S3 cache.
- a bar-separated list of URLs, where we'll pick the one with the best ping times.
- A list or tuple of two specs, local, then remote, each as described above
"""
compression = self._options.compression_level
if compression not in range(10):
raise ValueError(
'compression_level must be an integer 0-9: {}'.format(compression))
deprecated_conditional(
lambda: compression == 0, '1.4.0.dev0', 'compression==0',
'The artifact cache depends on gzip compression for checksumming: a compression level '
'==0 disables compression, and can prevent detection of corrupted artifacts.'
)
artifact_root = self._options.pants_workdir
def create_local_cache(parent_path):
path = os.path.join(parent_path, self._stable_name)
self._log.debug('{0} {1} local artifact cache at {2}'.format(
self._stable_name, action, path))
return LocalArtifactCache(
artifact_root,
path,
compression,
self._options.max_entries_per_target,
permissions=self._options.write_permissions,
dereference=self._options.dereference_symlinks)
def create_remote_cache(remote_spec, local_cache):
urls = remote_spec.split('|')
if len(urls) == 0:
return None
local_cache = local_cache or TempLocalArtifactCache(
artifact_root, compression)
if _is_s3(urls[0]):
return S3ArtifactCache(artifact_root, urls[0], local_cache)
urls = self.get_available_urls(urls)
if len(urls) > 0:
best_url_selector = BestUrlSelector([
'{}/{}'.format(url.rstrip('/'), self._stable_name)
for url in urls
])
return RESTfulArtifactCache(artifact_root, best_url_selector,
local_cache)
local_cache = create_local_cache(spec.local) if spec.local else None
remote_cache = create_remote_cache(spec.remote,
local_cache) if spec.remote else None
if remote_cache:
return remote_cache
return local_cache
def _isolation(self, all_targets):
run_dir = '_runs'
mode_dir = 'isolated' if self._per_target else 'combined'
batch_dir = str(self._batch_size) if self._batched else 'all'
output_dir = os.path.join(self.workdir, run_dir,
Target.identify(all_targets), mode_dir,
batch_dir)
safe_mkdir(output_dir, clean=False)
if self._html_report:
junit_html_report = JUnitHtmlReport.create(
xml_dir=output_dir,
open_report=self.get_options().open,
logger=self.context.log,
error_on_conflict=True)
else:
junit_html_report = NoJunitHtmlReport()
coverage = CodeCoverage.global_instance().get_coverage_engine(
self, output_dir, all_targets, self.execute_java_for_coverage)
reports = self.Reports(junit_html_report, coverage)
self.context.release_lock()
try:
yield output_dir, reports, coverage
finally:
lock_file = '.file_lock'
preserve = (run_dir, lock_file)
dist_dir = os.path.join(
self.get_options().pants_distdir,
os.path.relpath(self.workdir,
self.get_options().pants_workdir))
with OwnerPrintingInterProcessFileLock(
os.path.join(dist_dir, lock_file)):
self._link_current_reports(report_dir=output_dir,
link_dir=dist_dir,
preserve=preserve)
if self._legacy_report_layout:
deprecated_conditional(
predicate=lambda: True,
entity_description='[test.junit] legacy_report_layout',
stacklevel=3,
removal_version='1.6.0.dev0',
hint_message=
'Reports are now linked into {} by default; so scripts '
'and CI jobs should be pointed there and the option '
'configured to False in pants.ini until such time as '
'the option is removed.'.format(dist_dir))
# NB: Deposit of the "current" test output in the root workdir (.pants.d/test/junit) is a
# defacto public API and so we implement that behavior here to maintain backwards
# compatibility for non-pants report file consumers.
with OwnerPrintingInterProcessFileLock(
os.path.join(self.workdir, lock_file)):
self._link_current_reports(report_dir=output_dir,
link_dir=self.workdir,
preserve=preserve)
def __init__(self,
compiler=None,
language=None,
rpc_style=None,
namespace_map=None,
thrift_linter_strict=None,
default_java_namespace=None,
include_paths=None,
compiler_args=None,
**kwargs):
"""
:API: public
:param compiler: The compiler used to compile the thrift files. The default is defined in
the global options under ``--thrift-default-compiler``.
:param language: The language used to generate the output files. The default is defined in
the global options under ``--thrift-default-language``.
:param rpc_style: An optional rpc style to generate service stubs with. The default is defined
in the global options under ``--thrift-default-rpc-style``.
:param namespace_map: An optional dictionary of namespaces to remap {old: new}
:param thrift_linter_strict: If True, fail if thrift linter produces any warnings.
:param default_java_namespace: The namespace used for Java generated code when a Java
namespace is not explicitly specified in the IDL. The default is defined in the global
options under ``--thrift-default-default-java-namespace``.
:param compiler_args: Extra arguments to the compiler.
"""
super(JavaThriftLibrary, self).__init__(**kwargs)
def check_value_for_arg(arg, value, values):
if value and value not in values:
raise TargetDefinitionException(self, "{} may only be set to {} ('{}' not valid)"
.format(arg, ', or '.join(map(repr, values)), value))
return value
# The following fields are only added to the fingerprint via FingerprintStrategy when their
# values impact the outcome of the task. See JavaThriftLibraryFingerprintStrategy.
self._compiler = check_value_for_arg('compiler', compiler, self._COMPILERS)
self._language = language
deprecated_conditional(
lambda: rpc_style is not None,
'1.6.0.dev0',
'rpc_style',
'''
Deprecated property rpc_style used for {target}, use compiler_args instead.
e.g. [ \'--finagle\'] for \'finagle\'
and [\'--finagle\', \'--ostrich\'] for \'ostrich\'.
If both rpc_style and compiler_args are set then only compiler_args is used
and rpc_style is discarded.
'''.format(target=self.address.spec)
)
self._rpc_style = rpc_style
self.namespace_map = namespace_map
self.thrift_linter_strict = thrift_linter_strict
self._default_java_namespace = default_java_namespace
self._include_paths = include_paths
self._compiler_args = compiler_args
def console_output(self, targets):
deprecated_conditional(lambda: True, '1.5.0.dev0',
'The `pathdeps` goal is deprecated. Please use `filedeps` instead.')
def is_safe(t):
return hasattr(t, 'address') and hasattr(t.address, 'rel_path')
buildroot = get_buildroot()
return set(os.path.normpath(os.path.join(buildroot, os.path.dirname(t.address.rel_path)))
for t in targets if is_safe(t))
def get_fingerprintable_for_scope(
self,
bottom_scope: str,
include_passthru: Optional[bool] = None,
fingerprint_key: str = "fingerprint",
invert: bool = False,
):
"""Returns a list of fingerprintable (option type, option value) pairs for the given scope.
Fingerprintable options are options registered via a "fingerprint=True" kwarg. This flag
can be parameterized with `fingerprint_key` for special cases.
This method also searches enclosing options scopes of `bottom_scope` to determine the set of
fingerprintable pairs.
:param bottom_scope: The scope to gather fingerprintable options for.
:param include_passthru: Whether to include passthru args captured by `bottom_scope` in the
fingerprintable options.
:param fingerprint_key: The option kwarg to match against (defaults to 'fingerprint').
:param invert: Whether or not to invert the boolean check for the fingerprint_key value.
:API: public
"""
deprecated_conditional(
predicate=lambda: include_passthru is not None,
removal_version="1.31.0.dev0",
entity_description="get_fingerprintable_for_scope `include_passthru` arg",
hint_message=(
"passthru arguments are fingerprinted if their associated option value is."
),
)
fingerprint_default = bool(invert)
pairs = []
# Note that we iterate over options registered at `bottom_scope` and at all
# enclosing scopes, since option-using code can read those values indirectly
# via its own OptionValueContainer, so they can affect that code's output.
for registration_scope in all_enclosing_scopes(bottom_scope):
parser = self._parser_hierarchy.get_parser_by_scope(registration_scope)
# Sort the arguments, so that the fingerprint is consistent.
for (_, kwargs) in sorted(parser.option_registrations_iter()):
if kwargs.get("recursive", False) and not kwargs.get("recursive_root", False):
continue # We only need to fprint recursive options once.
if not kwargs.get(fingerprint_key, fingerprint_default):
continue
# Note that we read the value from scope, even if the registration was on an enclosing
# scope, to get the right value for recursive options (and because this mirrors what
# option-using code does).
val = self.for_scope(bottom_scope)[kwargs["dest"]]
# If we have a list then we delegate to the fingerprinting implementation of the members.
if is_list_option(kwargs):
val_type = kwargs.get("member_type", str)
else:
val_type = kwargs.get("type", str)
pairs.append((val_type, val))
return pairs
def register(self, *args, **kwargs):
"""Register an option."""
if self._frozen:
raise FrozenRegistration(self.scope, args[0])
# Prevent further registration in enclosing scopes.
ancestor = self._parent_parser
while ancestor:
ancestor._freeze()
ancestor = ancestor._parent_parser
def check_deprecated_types(kwarg_name):
t = kwargs.get(kwarg_name)
# First check for deprecated direct use of the internal types.
if t == list_option:
deprecated_conditional(lambda: True, '0.0.81',
'list_option is deprecated for option {} in scope {}. '
'Use type=list.'.format(args[0], self.scope))
elif t == dict_option:
deprecated_conditional(lambda: True, '0.0.81',
'dict_option is deprecated for option {} in scope {}. '
'Use type=dict.'.format(args[0], self.scope))
check_deprecated_types('type')
check_deprecated_types('member_type')
# Temporary munging to effectively turn type=list options into list options,
# for uniform handling. From here on, type=list is an error.
# TODO: Remove after type=list deprecation.
if kwargs.get('action') == 'append':
if 'type' in kwargs:
kwargs['member_type'] = kwargs['type']
kwargs['type'] = list
del kwargs['action']
deprecated_conditional(lambda: True, '0.0.81',
"action='append' is deprecated for option {} in scope {}. "
"Use type=list.".format(args[0], self.scope))
# Temporary munging to effectively turn type='target_list_option' options into list options,
# with member type 'target_option', for uniform handling.
# TODO: Remove after target_list_option deprecation.
if kwargs.get('type') == target_list_option:
kwargs['type'] = list
kwargs['member_type'] = target_option
deprecated_conditional(lambda: True, '0.0.81',
'target_list_option is deprecated for option {} in scope {}. '
'Use type=list, member_type=target_option.'.format(
args[0], self.scope
))
# Record the args. We'll do the underlying parsing on-demand.
self._option_registrations.append((args, kwargs))
if self._parent_parser:
for arg in args:
existing_scope = self._parent_parser._existing_scope(arg)
if existing_scope is not None:
raise Shadowing(self.scope, arg, outer_scope=self._scope_str(existing_scope))
self._known_args.update(args)
def validate_scope_name_component(cls, s):
# TODO: turn this deprecation warning into a permanent error after 1.2.0.
deprecated_conditional(
lambda: not cls.is_valid_scope_name_component(s),
"1.2.0",
"options scope {}".format(s),
"Replace in code with new scope name consisting of dash-separated-words, "
"with words consisting only of lower-case letters and digits.",
)
def scan_build_files(self, base_path, spec_excludes=None):
deprecated_conditional(
lambda: spec_excludes is not None, '0.0.75',
'Use build_ignore_patterns consturctor parameter instead.')
return BuildFile.scan_build_files(
self._project_tree,
base_path,
spec_excludes,
build_ignore_patterns=self._build_ignore_patterns)
def test_deprecated_conditional(caplog) -> None:
assert not caplog.records
deprecated_conditional(lambda: True, FUTURE_VERSION, "deprecated entity", None)
assert len(caplog.records) == 1
assert "deprecated entity" in caplog.text
caplog.clear()
deprecated_conditional(lambda: False, FUTURE_VERSION, "deprecated entity", None)
assert not caplog.records
def check_target_name(cls, spec_path, name):
if not name:
raise InvalidTargetName('Spec {spec}:{name} has no name part'
.format(spec=spec_path, name=name))
banned_chars = BANNED_CHARS_IN_TARGET_NAME & set(name)
# raise InvalidateTargetName after deprecation
deprecated_conditional(lambda: len(banned_chars) > 0, '1.4.0dev0',
'banned chars found in target name',
'{banned_chars} not allowed in target name: {name}'
.format(banned_chars=banned_chars, name=name))
def __init__(self, root_dir, address_mapper, spec_excludes=None, exclude_target_regexps=None):
deprecated_conditional(lambda: spec_excludes is not None,
'0.0.75',
'Use build_ignore_patterns in address_mapper instead.')
self._root_dir = os.path.realpath(root_dir)
self._address_mapper = address_mapper
self._spec_excludes = spec_excludes
self._exclude_target_regexps = exclude_target_regexps or []
self._exclude_patterns = [re.compile(pattern) for pattern in self._exclude_target_regexps]
self._excluded_target_map = defaultdict(set) # pattern -> targets (for debugging)
def __getattr__(name):
is_deprecated_name = "Typecheck" in name
deprecated_conditional(
lambda: is_deprecated_name,
"2.10.0.dev0",
f"the {name} class",
f"Typecheck-related classes were renamed: s/Typecheck/Check/ and moved to the {check.__name__} module.",
)
if is_deprecated_name:
return getattr(check, name.replace("Typecheck", "Check"))
raise AttributeError(f"module {__name__} has no attribute {name}")
def check_deprecated_types(kwarg_name):
t = kwargs.get(kwarg_name)
# First check for deprecated direct use of the internal types.
if t == list_option:
deprecated_conditional(lambda: True, '0.0.81',
'list_option is deprecated for option {} in scope {}. '
'Use type=list.'.format(args[0], self.scope))
elif t == dict_option:
deprecated_conditional(lambda: True, '0.0.81',
'dict_option is deprecated for option {} in scope {}. '
'Use type=dict.'.format(args[0], self.scope))
def version(self, context=None):
# The versions reported by node and embedded in distribution package names are 'vX.Y.Z'.
# TODO: After the deprecation cycle is over we'll expect the values of the version option
# to already include the 'v' prefix, so there will be no need to normalize, and we can
# delete this entire method override.
version = super(NodeDistribution, self).version(context)
deprecated_conditional(
lambda: not version.startswith('v'), entity_description='', removal_version='1.7.0.dev0',
hint_message='value of --version in scope {} must be of the form '
'vX.Y.Z'.format(self.options_scope))
return version if version.startswith('v') else 'v' + version
def check_target_name(cls, spec_path, name):
if not name:
raise InvalidTargetName('Spec {spec}:{name} has no name part'
.format(spec=spec_path, name=name))
banned_chars = BANNED_CHARS_IN_TARGET_NAME & set(name)
# raise InvalidateTargetName after deprecation
deprecated_conditional(lambda: len(banned_chars) > 0, '1.4.0dev0',
'banned chars found in target name',
'{banned_chars} not allowed in target name: {name}'
.format(banned_chars=banned_chars, name=name))
def setup(self, init_logging=True):
"""Initializes logging, loads backends/plugins and parses options.
:param bool init_logging: Whether or not to initialize logging as part of setup.
:returns: A tuple of (options, build_configuration).
"""
global_bootstrap_options = self._options_bootstrapper.get_bootstrap_options().for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
'Version mismatch: Requested version was {}, our version is {}.'
.format(global_bootstrap_options.pants_version, pants_version())
)
# Get logging setup prior to loading backends so that they can log as needed.
if init_logging:
self._setup_logging(global_bootstrap_options.quiet,
global_bootstrap_options.level,
global_bootstrap_options.logdir)
# Conditionally load backends/plugins and materialize a `BuildConfiguration` object.
if not self._has_build_configuration():
missing = (set(global_bootstrap_options.default_backend_packages).difference(
global_bootstrap_options.backend_packages))
deprecated_conditional(
lambda: len(missing) > 0,
'1.3.0',
'default_backend_packages option',
'You are relying on the following backends being listed in the deprecated '
'default_backend_packages option: {}.\n '
'This is probably because you are overwriting the value of the backend_packages option '
'in your pants.ini, instead of appending to it.\n To get rid of this message, consider '
'changing backend_packages: [...] to backend_packages: +[...] in your pants.ini. '
'Once you are happy with the state of your backend_packages option, you can set '
'default_backend_packages to [] to silence this warning.'.format(
', '.join(missing))
)
backends = (global_bootstrap_options.default_backend_packages +
global_bootstrap_options.backend_packages)
build_configuration = self._load_plugins(self._working_set,
global_bootstrap_options.pythonpath,
global_bootstrap_options.plugins,
backends)
self._set_build_configuration(build_configuration)
else:
build_configuration = self._get_build_configuration()
# Parse and register options.
options = self._install_options(self._options_bootstrapper, build_configuration)
return options, build_configuration
def __init__(self, requirement, name=None, repository=None, version_filter=None, use_2to3=False,
compatibility=None):
deprecated_conditional(lambda: version_filter is not None, '0.0.79',
'version_filter using lambda function is no longer supported.')
# TODO(wickman) Allow PythonRequirements to be specified using pip-style vcs or url identifiers,
# e.g. git+https or just http://...
self._requirement = Requirement.parse(requirement)
self._repository = repository
self._name = name or self._requirement.project_name
self._use_2to3 = use_2to3
# Temporary workaround to allow pickling before we fully deprecate version_filter.
self._version_filter = version_filter or _always_build
# TODO(wickman) Unify this with PythonTarget .compatibility
self.compatibility = compatibility or ['']
def __init__(self, username=None, password=None, **kwargs):
"""
:param string name: The name of these credentials.
:param username: A constant username value.
:param password: A constant password value.
"""
super(LiteralCredentials, self).__init__(**kwargs)
deprecated_conditional(lambda: callable(username) or callable(password), '0.0.82',
'Passing callable arguments to `credentials` is deprecated: '
'use `netrc_credentials` for target {}'.format(
self.address.spec
))
self._username = username if callable(username) else functools.partial(_ignored_repository, username)
self._password = password if callable(password) else functools.partial(_ignored_repository, password)
def _do_create_artifact_cache(self, spec, action):
"""Returns an artifact cache for the specified spec.
spec can be:
- a path to a file-based cache root.
- a URL of a RESTful cache root.
- a bar-separated list of URLs, where we'll pick the one with the best ping times.
- A list or tuple of two specs, local, then remote, each as described above
"""
compression = self._options.compression_level
if compression not in range(10):
raise ValueError('compression_level must be an integer 0-9: {}'.format(compression))
deprecated_conditional(
lambda: compression == 0,
'1.4.0',
'compression==0',
'The artifact cache depends on gzip compression for checksumming: a compression level '
'==0 disables compression, and can prevent detection of corrupted artifacts.'
)
artifact_root = self._options.pants_workdir
def create_local_cache(parent_path):
path = os.path.join(parent_path, self._stable_name)
self._log.debug('{0} {1} local artifact cache at {2}'
.format(self._stable_name, action, path))
return LocalArtifactCache(artifact_root, path, compression,
self._options.max_entries_per_target,
permissions=self._options.write_permissions,
dereference=self._options.dereference_symlinks)
def create_remote_cache(remote_spec, local_cache):
urls = self.get_available_urls(remote_spec.split('|'))
if len(urls) > 0:
best_url_selector = BestUrlSelector(['{}/{}'.format(url.rstrip('/'), self._stable_name)
for url in urls])
local_cache = local_cache or TempLocalArtifactCache(artifact_root, compression)
return RESTfulArtifactCache(artifact_root, best_url_selector, local_cache)
local_cache = create_local_cache(spec.local) if spec.local else None
remote_cache = create_remote_cache(spec.remote, local_cache) if spec.remote else None
if remote_cache:
return remote_cache
return local_cache
def _symlink_bundles(self, app, bundle_dir):
"""For each bundle in the given app, symlinks relevant matched paths from shortest to longest.
Validates that at least one path was matched by a bundle, and (temporarily: see the
deprecation) symlinks matched directories to recursively include their contents.
"""
for bundle_counter, bundle in enumerate(app.bundles):
file_count = 0
dir_count = 0
# Create in ascending path-length order to ensure that symlinks to directories
# are created before their contents would be. Can remove ordering along with the
# 'recursive inclusion' deprecation (when only files and not directories are
# symlinked).
for path, relpath in sorted(bundle.filemap.items(), key=lambda e: len(e[0])):
bundle_path = os.path.join(bundle_dir, relpath)
if os.path.isfile(path):
file_count += 1
elif os.path.isdir(path):
dir_count += 1
else:
continue
if os.path.exists(bundle_path):
continue
safe_mkdir(os.path.dirname(bundle_path))
os.symlink(path, bundle_path)
if file_count == 0 and dir_count == 0:
raise TargetDefinitionException(app.target,
'Bundle index {} of "bundles" field '
'does not match any files.'.format(bundle_counter))
if dir_count == 1 and file_count == 0:
# When this deprecation finishes, we should remove symlinking of directories into the
# bundle (which implicitly includes their contents), and instead create them using mkdir.
spec = os.path.relpath(bundle.filemap.keys()[0], get_buildroot())
deprecated_conditional(
lambda: True,
'1.5.0.dev0',
'default recursive inclusion of files in directory',
'The bundle filespec `{spec}` corresponds to exactly one directory: if you\'d like to '
'continue to recursively include directory contents in future versions, please switch '
'to a recursive glob like `{fixed_spec}`.'.format(
spec=spec,
fixed_spec=os.path.join(spec, '**', '*'),
)
)
def _transform_sections_to_global(parser, global_subsumed_sections):
"""Transforms section names as needed for options scope deprecation."""
default_keys = parser.defaults().keys()
for subsumed_section, removal_version in global_subsumed_sections:
if parser.has_section(subsumed_section):
deprecated_conditional(
lambda: True,
removal_version,
'The pants.ini options scope `[{}]` is deprecated. Please migrate options '
'in this scope to `[GLOBAL]`.'.format(subsumed_section)
)
if not parser.has_section(GLOBAL_SCOPE_CONFIG_SECTION):
parser.add_section(GLOBAL_SCOPE_CONFIG_SECTION)
for k, v in parser.items(subsumed_section):
if k not in default_keys:
parser.set(GLOBAL_SCOPE_CONFIG_SECTION, '_'.join((subsumed_section, k)), v)
parser.remove_section(subsumed_section)
def __init__(
self,
name,
has_python,
skip_java,
skip_scala,
use_source_root,
root_dir,
debug_port,
context,
targets,
transitive,
target_util,
spec_excludes=None,
build_ignore_patterns=None,
):
"""Creates a new, unconfigured, Project based at root_dir and comprised of the sources visible
to the given targets."""
deprecated_conditional(lambda: spec_excludes is not None, "0.0.75", "Use build_ignore_patterns instead.")
self.context = context
self.target_util = target_util
self.name = name
self.root_dir = root_dir
self.targets = OrderedSet(targets)
self.transitive = transitive
self.sources = []
self.py_sources = []
self.py_libs = []
self.resource_extensions = set()
self.has_python = has_python
self.skip_java = skip_java
self.skip_scala = skip_scala
self.use_source_root = use_source_root
self.has_scala = False
self.has_tests = False
self.debug_port = debug_port
self.internal_jars = OrderedSet()
self.external_jars = OrderedSet()
self.spec_excludes = spec_excludes
self.build_ignore_patterns = build_ignore_patterns
def determine_target_roots(self, goal_name, predicate=None):
"""Helper for tasks that scan for default target roots.
:param string goal_name: The goal name to use for any warning emissions.
:param callable predicate: The predicate to pass to `context.scan().targets(predicate=X)`.
"""
deprecated_conditional(
lambda: not self.context.target_roots,
'1.5.0.dev0',
'`./pants {0}` (with no explicit targets) will soon become an error. Please specify '
'one or more explicit target specs (e.g. `./pants {0} ::`).'.format(goal_name))
if not self.context.target_roots and not self.get_options().enable_v2_engine:
# For the v1 path, continue the behavior of e.g. `./pants list` implies `./pants list ::`.
return self.context.scan().targets(predicate=predicate)
# For the v2 path, e.g. `./pants list` is a functional no-op. This matches the v2 mode behavior
# of e.g. `./pants --changed-parent=HEAD list` (w/ no changes) returning an empty result.
return self.context.target_roots
def execute(self):
if not self.url:
raise TaskError('Unable to proceed publishing to confluence. Please set the url option.')
deprecated_conditional(
lambda: True,
'1.6.0.dev0',
'pants.backend.docgen.tasks.confluence_publish.py',
'Use contrib.confluence.tasks.confluence_publish.py instead'
)
pages = []
targets = self.context.targets()
for target in targets:
if isinstance(target, Page):
for wiki_artifact in target.payload.provides:
pages.append((target, wiki_artifact))
urls = list()
genmap = self.context.products.get('wiki_html')
for page, wiki_artifact in pages:
html_info = genmap.get((wiki_artifact, page))
if len(html_info) > 1:
raise TaskError('Unexpected resources for {}: {}'.format(page, html_info))
basedir, htmls = html_info.items()[0]
if len(htmls) != 1:
raise TaskError('Unexpected resources for {}: {}'.format(page, htmls))
with safe_open(os.path.join(basedir, htmls[0])) as contents:
url = self.publish_page(
page.address,
wiki_artifact.config['space'],
wiki_artifact.config['title'],
contents.read(),
# Default to none if not present in the hash.
parent=wiki_artifact.config.get('parent')
)
if url:
urls.append(url)
self.context.log.info('Published {} to {}'.format(page, url))
if self.open and urls:
try:
desktop.ui_open(*urls)
except desktop.OpenError as e:
raise TaskError(e)
def register_options(cls, register):
super(RewriteBase, cls).register_options(register)
register('--target-types',
default=cls.target_types(),
advanced=True, type=list,
help='The target types to apply formatting to.')
try:
sideeffecting = cls.sideeffecting
except AttributeError:
deprecated_conditional(
lambda: True,
'1.12.0.dev0',
"RewriteBase's sideeffecting property should be a class property, not an instance property "
"but class {} didn't have a class property.".format(cls.__name__),
)
sideeffecting = False
if sideeffecting:
register('--output-dir', advanced=True, type=dir_option, fingerprint=True,
help='Path to output directory. Any updated files will be written here. '
'If not specified, files will be modified in-place.')
def scan_specs(self, specs, fail_fast=True, spec_excludes=None):
"""Execute a collection of `specs.Spec` objects and return an ordered set of Addresses."""
excluded_target_map = defaultdict(set) # pattern -> targets (for debugging)
deprecated_conditional(lambda: spec_excludes is not None,
'0.0.75',
'Use build_ignore_patterns in address_mapper instead.')
def exclude_spec(spec):
for pattern in self._exclude_patterns:
if pattern.search(spec) is not None:
excluded_target_map[pattern.pattern].add(spec)
return True
excluded_target_map[self._UNMATCHED_KEY].add(spec)
return False
def exclude_address(address):
return exclude_spec(address.spec)
addresses = OrderedSet()
for spec in specs:
for address in self._scan_spec(spec, fail_fast, spec_excludes, exclude_spec):
if not exclude_address(address):
addresses.add(address)
# Print debug information about the excluded targets
if logger.getEffectiveLevel() <= logging.DEBUG and excluded_target_map:
logger.debug('excludes:\n {excludes}'
.format(excludes='\n '.join(self._exclude_target_regexps)))
targets = ', '.join(excluded_target_map[self._UNMATCHED_KEY])
logger.debug('Targets after excludes: {targets}'.format(targets=targets))
excluded_count = 0
for pattern, targets in six.iteritems(excluded_target_map):
if pattern != self._UNMATCHED_KEY:
logger.debug('Targets excluded by pattern {pattern}\n {targets}'
.format(pattern=pattern,
targets='\n '.join(targets)))
excluded_count += len(targets)
logger.debug('Excluded {count} target{plural}.'
.format(count=excluded_count,
plural=('s' if excluded_count != 1 else '')))
return addresses
def __call__(self, rel_path=None, mapper=None, relative_to=None, fileset=None):
"""
:param rel_path: Base path of the "source" file paths. By default, path of the
BUILD file. Useful for assets that don't live in the source code repo.
:param mapper: Function that takes a path string and returns a path string. Takes a path in
the source tree, returns a path to use in the resulting bundle. By default, an identity
mapper.
:param string relative_to: Set up a simple mapping from source path to bundle path.
:param fileset: The set of files to include in the bundle. A string filename, or list of
filenames, or a Fileset object (e.g. globs()).
E.g., ``relative_to='common'`` removes that prefix from all files in the application bundle.
"""
deprecated_conditional(lambda: fileset is None,
'1.2.0',
'bare bundle() without `fileset=` param',
"Pass the `fileset=` parameter: `bundle(fileset=globs('*.config')`")
if mapper and relative_to:
raise ValueError("Must specify exactly one of 'mapper' or 'relative_to'")
if rel_path and isinstance(fileset, FilesetWithSpec):
raise ValueError("Must not use a glob for 'fileset' with 'rel_path'."
" Globs are eagerly evaluated and ignore 'rel_path'.")
# A fileset is either a glob, a string or a list of strings.
if isinstance(fileset, FilesetWithSpec):
pass
elif isinstance(fileset, six.string_types):
fileset = [fileset]
else:
fileset = assert_list(fileset, key_arg='fileset')
real_rel_path = rel_path or self._rel_path
if relative_to:
base = os.path.join(get_buildroot(), real_rel_path, relative_to)
mapper = RelativeToMapper(base)
else:
mapper = mapper or RelativeToMapper(os.path.join(get_buildroot(), real_rel_path))
return BundleProps(real_rel_path, mapper, fileset)
def scan_build_files(project_tree, base_relpath, spec_excludes=None, build_ignore_patterns=None):
"""Looks for all BUILD files
:param project_tree: Project tree to scan in.
:type project_tree: :class:`pants.base.project_tree.ProjectTree`
:param base_relpath: Directory under root_dir to scan.
:param spec_excludes: List of paths to exclude from the scan. These can be absolute paths
or paths that are relative to the root_dir.
:param build_ignore_patterns: .gitignore like patterns to exclude from BUILD files scan.
:type build_ignore_patterns: pathspec.pathspec.PathSpec
"""
deprecated_conditional(lambda: spec_excludes is not None,
'0.0.75',
'Use build_ignore_patterns instead.')
if base_relpath and os.path.isabs(base_relpath):
raise BuildFile.BadPathError('base_relpath parameter ({}) should be a relative path.'
.format(base_relpath))
if base_relpath and not project_tree.isdir(base_relpath):
raise BuildFile.BadPathError('Can only scan directories and {0} is not a valid dir.'
.format(base_relpath))
if build_ignore_patterns and not isinstance(build_ignore_patterns, PathSpec):
raise TypeError("build_ignore_patterns should be pathspec.pathspec.PathSpec instance, "
"instead {} was given.".format(type(build_ignore_patterns)))
# Hack, will be removed after spec_excludes removal.
build_ignore_patterns = BuildFile._add_spec_excludes_to_build_ignore_patterns(project_tree.build_root,
build_ignore_patterns,
spec_excludes)
build_files = set()
for root, dirs, files in project_tree.walk(base_relpath or '', topdown=True):
excluded_dirs = list(build_ignore_patterns.match_files('{}/'.format(os.path.join(root, dirname))
for dirname in dirs))
for subdir in excluded_dirs:
# Remove trailing '/' from paths which were added to indicate that paths are paths to directories.
dirs.remove(fast_relpath(subdir, root)[:-1])
for filename in files:
if BuildFile._is_buildfile_name(filename):
build_files.add(os.path.join(root, filename))
return BuildFile._build_files_from_paths(project_tree, build_files, build_ignore_patterns)
def _timeout_for_target(self, target):
timeout = getattr(target, 'timeout', None)
deprecated_conditional(
lambda: timeout == 0,
"0.0.65",
hint_message=dedent("""
Target {target} has parameter: 'timeout=0', which is deprecated.
To use the default timeout remove the 'timeout' parameter from your test target.
""".format(target=target.address.spec)))
timeout_maximum = self.get_options().timeout_maximum
if timeout is not None and timeout_maximum is not None:
if timeout > timeout_maximum:
self.context.log.warn(
"Warning: Timeout for {target} ({timeout}s) exceeds {timeout_maximum}s. Capping.".format(
target=target.address.spec,
timeout=timeout,
timeout_maximum=timeout_maximum))
return timeout_maximum
return timeout
def __init__(self, *patterns, **kwargs):
self._patterns = patterns
self._kwargs = kwargs
raw_spec_path = kwargs.pop('spec_path')
self._file_globs = self.legacy_globs_class.to_filespec(patterns).get('globs', [])
raw_exclude = kwargs.pop('exclude', [])
self._excluded_file_globs = self._filespec_for_exclude(raw_exclude, raw_spec_path).get('globs', [])
self._spec_path = raw_spec_path
# `follow_links=True` is the default behavior for wrapped globs, so we pop the old kwarg
# and warn here to bridge the gap from v1->v2 BUILD files.
if kwargs.pop('follow_links', None) is not None:
deprecated_conditional(
lambda: True,
'1.10.0.dev0',
'Ignoring `follow_links` kwarg on glob in `{}`. Default behavior is to follow all links.'
.format(self._spec_path)
)
if kwargs:
raise ValueError('kwargs not supported for {}. Got: {}'.format(type(self), kwargs))
