def __init__(self, name, binary, bundles, basename=None): """ :param string name: The name of this target, which combined with this build file defines the target :class:`twitter.pants.base.address.Address`. :param binary: The :class:`twitter.pants.targets.jvm_binary.JvmBinary`, or a :class:`twitter.pants.targets.pants_target.Pants` pointer to one. :param bundles: One or more :class:`twitter.pants.targets.jvm_binary.Bundle`'s describing "extra files" that should be included with this app (e.g.: config files, startup scripts). :param string basename: Name of this application, if different from the ``name``. Pants uses this in the ``bundle`` goal to name the distribution artifact. In most cases this parameter is not necessary. """ super(JvmApp, self).__init__(name, dependencies=[]) self._binaries = maybe_list( util.resolve(binary), expected_type=(Pants, JarLibrary, JvmBinary), raise_type=partial(TargetDefinitionException, self)) self._bundles = maybe_list(bundles, expected_type=Bundle, raise_type=partial(TargetDefinitionException, self)) if name == basename: raise TargetDefinitionException(self, 'basename must not equal name.') self.basename = basename or name self._resolved_binary = None self._resolved_bundles = []
def _scrub_args(classpath, main, jvm_options, args, cwd): classpath = maybe_list(classpath) if not isinstance(main, string_types) or not main: raise ValueError('A non-empty main classname is required, given: {}'.format(main)) jvm_options = maybe_list(jvm_options or ()) args = maybe_list(args or ()) return classpath, main, jvm_options, args, cwd
def _scrub_args(classpath, main, jvm_options, args, cwd): classpath = maybe_list(classpath) if not isinstance(main, Compatibility.string) or not main: raise ValueError('A non-empty main classname is required, given: %s' % main) jvm_options = maybe_list(jvm_options or ()) args = maybe_list(args or ()) return classpath, main, jvm_options, args, cwd
def __init__(self, name, sources, dependencies, excludes=None, configurations=None, exclusives=None): """ :param string name: The name of this target, which combined with this build file defines the target :class:`pants.base.address.Address`. :param sources: A list of filenames representing the source code this library is compiled from. :type sources: list of strings :param dependencies: List of :class:`pants.base.target.Target` instances this target depends on. :type dependencies: list of targets :param excludes: One or more :class:`pants.targets.exclude.Exclude` instances to filter this target's transitive dependencies against. :param configurations: One or more ivy configurations to resolve for this target. This parameter is not intended for general use. :type configurations: tuple of strings """ InternalTarget.__init__(self, name, dependencies, exclusives=exclusives) TargetWithSources.__init__(self, name, sources) self.add_labels('jvm') for source in self.sources: rel_path = os.path.join(self.target_base, source) TargetWithSources.register_source(rel_path, self) self.excludes = maybe_list(excludes or [], Exclude) self.configurations = maybe_list(configurations or [])
def __init__(self, name, sources, dependencies, excludes=None, configurations=None, exclusives=None): """ :param string name: The name of this target, which combined with this build file defines the target :class:`twitter.pants.base.address.Address`. :param sources: A list of filenames representing the source code this library is compiled from. :type sources: list of strings :param dependencies: List of :class:`twitter.pants.base.target.Target` instances this target depends on. :type dependencies: list of targets :param excludes: One or more :class:`twitter.pants.targets.exclude.Exclude` instances to filter this target's transitive dependencies against. :param configurations: One or more ivy configurations to resolve for this target. This parameter is not intended for general use. :type configurations: tuple of strings """ InternalTarget.__init__(self, name, dependencies, exclusives=exclusives) TargetWithSources.__init__(self, name, sources) self.add_labels('jvm') for source in self.sources: rel_path = os.path.join(self.target_base, source) TargetWithSources.register_source(rel_path, self) self.excludes = maybe_list(excludes or [], Exclude) self.configurations = maybe_list(configurations or [])
def __init__(self, name, binary, bundles, basename=None): """ :param string name: The name of this target, which combined with this build file defines the target :class:`twitter.pants.base.address.Address`. :param binary: The :class:`twitter.pants.targets.jvm_binary.JvmBinary`, or a :class:`twitter.pants.targets.pants_target.Pants` pointer to one. :param bundles: One or more :class:`twitter.pants.targets.jvm_binary.Bundle`'s describing "extra files" that should be included with this app (e.g.: config files, startup scripts). :param string basename: Name of this application, if different from the ``name``. Pants uses this in the ``bundle`` goal to name the distribution artifact. In most cases this parameter is not necessary. """ super(JvmApp, self).__init__(name, dependencies=[]) self._binaries = maybe_list( util.resolve(binary), expected_type=(Pants, JarLibrary, JvmBinary), raise_type=partial(TargetDefinitionException, self)) self._bundles = maybe_list(bundles, expected_type=Bundle, raise_type=partial( TargetDefinitionException, self)) if name == basename: raise TargetDefinitionException(self, 'basename must not equal name.') self.basename = basename or name self._resolved_binary = None self._resolved_bundles = []
def _scrub_args(classpath, main, jvm_options, args): classpath = maybe_list(classpath) if not isinstance(main, Compatibility.string) or not main: raise ValueError('A non-empty main classname is required, given: %s' % main) jvm_options = maybe_list(jvm_options or ()) args = maybe_list(args or ()) return classpath, main, jvm_options, args
def distribution(self, files=None, executables=None): with temporary_dir() as jdk: for f in maybe_list(files or ()): touch(os.path.join(jdk, f)) for exe in maybe_list(executables or (), expected_type=self.EXE): path = os.path.join(jdk, exe.name) with safe_open(path, 'w') as fp: fp.write(exe.contents or '') chmod_plus_x(path) yield jdk
def distribution(files=None, executables=None, java_home=None): with temporary_dir() as dist_root: for f in maybe_list(files or ()): touch(os.path.join(dist_root, f)) for executable in maybe_list(executables or (), expected_type=EXE): path = os.path.join(dist_root, executable.relpath) with safe_open(path, 'w') as fp: java_home = os.path.join(dist_root, java_home) if java_home else dist_root fp.write(executable.contents(java_home)) chmod_plus_x(path) yield dist_root
def distribution(files=None, executables=None, java_home=None): with subsystem_instance(DistributionLocator): with temporary_dir() as dist_root: with environment_as(DIST_ROOT=os.path.join(dist_root, java_home) if java_home else dist_root): for f in maybe_list(files or ()): touch(os.path.join(dist_root, f)) for executable in maybe_list(executables or (), expected_type=EXE): path = os.path.join(dist_root, executable.relpath) with safe_open(path, 'w') as fp: fp.write(executable.contents or '') chmod_plus_x(path) yield dist_root
def __init__(self, address=None, payload=None, sources=None, compatibility=None, setup_requires=None, **kwargs): """ :param address: The Address that maps to this Target in the BuildGraph. :type address: :class:`pants.build_graph.address.Address` :param payload: The configuration encapsulated by this target. Also in charge of most fingerprinting details. :type payload: :class:`pants.base.payload.Payload` :param sources: Files to "include". Paths are relative to the BUILD file's directory. :type sources: ``Fileset`` or list of strings. Must include setup.py. :param compatibility: either a string or list of strings that represents interpreter compatibility for this target, using the Requirement-style format, e.g. ``'CPython>=3', or just ['>=2.7','<3']`` for requirements agnostic to interpreter class. """ payload = payload or Payload() payload.add_fields({ 'sources': self.create_sources_field(sources, address.spec_path, key_arg='sources'), 'compatibility': PrimitiveField(maybe_list(compatibility or ())), 'setup_requires': PrimitiveField(maybe_list(setup_requires or ())) }) super(PythonDistribution, self).__init__(address=address, payload=payload, **kwargs) if not 'setup.py' in sources: raise TargetDefinitionException( self, 'A setup.py in the top-level directory relative to the target definition is required.' ) # Check that the compatibility requirements are well-formed. for req in self.payload.compatibility: try: PythonIdentity.parse_requirement(req) except ValueError as e: raise TargetDefinitionException(self, str(e))
def distribution(installed_sdks=('18', '19'), installed_build_tools=('19.1.0', '20.0.0'), files=('android.jar',), executables=('aapt', 'zipalign')): """Mock Android SDK Distribution. :param tuple[strings] installed_sdks: SDK versions of the files being mocked. :param tuple[strings] installed_build_tools: Build tools version of any tools. :param tuple[strings] files: The files are to mock non-executables and one will be created for each installed_sdks version. :param tuple[strings] executables: Executables are any required tools and one is created for each installed_build_tools version. """ with temporary_dir() as sdk: for sdk_version in installed_sdks: for android_file in files: touch(os.path.join(sdk, 'platforms', 'android-' + sdk_version, android_file)) for version in installed_build_tools: for exe in maybe_list(executables or ()): path = os.path.join(sdk, 'build-tools', version, exe) touch(path) chmod_plus_x(path) dx_path = os.path.join(sdk, 'build-tools', version, 'lib/dx.jar') touch(dx_path) yield sdk
def skip_signatures_and_duplicates_concat_well_known_metadata(cls, default_dup_action=None, additional_rules=None): """Produces a rule set useful in many deploy jar creation contexts. The rule set skips duplicate entries by default, retaining the 1st encountered. In addition it has the following special handling: - jar signature metadata is dropped - ``java.util.ServiceLoader`` provider-configuration files are concatenated in the order encountered :param default_dup_action: An optional default action to take for duplicates. Defaults to `Duplicate.SKIP` if not specified. :param additional_rules: Optionally one or more jar rules to add to those described above. :returns: :class:`twitter.pants.targets.JarRules` """ default_dup_action = Duplicate.validate_action(default_dup_action or Duplicate.SKIP) additional_rules = maybe_list(additional_rules or [], expected_type=(Duplicate, Skip)) rules = [Skip(r'^META-INF/[^/]+\.SF$'), # signature file Skip(r'^META-INF/[^/]+\.DSA$'), # default signature alg. file Skip(r'^META-INF/[^/]+\.RSA$'), # default signature alg. file Duplicate(r'^META-INF/services/', Duplicate.CONCAT)] # 1 svc fqcn per line return cls(rules=rules + additional_rules, default_dup_action=default_dup_action)
def __init__(self, sources=None, **kwargs): """ :param string name: The name of this target, which combined with this build file defines the :doc:`target address <target_addresses>`. :param sources: Source code files to compile. Paths are relative to the BUILD file's directory. :type sources: ``Fileset`` or list of strings :param provides: The ``artifact`` to publish that represents this target outside the repo. :param dependencies: Other targets that this target depends on. :type dependencies: list of target specs :param excludes: List of :ref:`exclude <bdict_exclude>`\s to filter this target's transitive dependencies against. :param resources: An optional list of ``resources`` targets containing text file resources to place in this module's jar. :param exclusives: An optional map of exclusives tags. See CheckExclusives for details. """ _sources = maybe_list(sources or [], expected_type=Compatibility.string) super(JavaTests, self).__init__(sources=_sources, **kwargs) if not _sources: raise TargetDefinitionException(self, 'JavaTests must include a non-empty set of sources.') # TODO(John Sirois): These could be scala, clojure, etc. 'jvm' and 'tests' are the only truly # applicable labels - fixup the 'java' misnomer. self.add_labels('java', 'tests')
def skip_signatures_and_duplicates_concat_well_known_metadata( cls, default_dup_action=None, additional_rules=None): """Produces a rule set useful in many deploy jar creation contexts. The rule set skips duplicate entries by default, retaining the 1st encountered. In addition it has the following special handling: - jar signature metadata is dropped - ``java.util.ServiceLoader`` provider-configuration files are concatenated in the order encountered :param default_dup_action: An optional default action to take for duplicates. Defaults to `Duplicate.SKIP` if not specified. :param additional_rules: Optionally one or more jar rules to add to those described above. :returns: :class:`twitter.pants.targets.JarRules` """ default_dup_action = Duplicate.validate_action(default_dup_action or Duplicate.SKIP) additional_rules = maybe_list(additional_rules or [], expected_type=(Duplicate, Skip)) rules = [ Skip(r'^META-INF/[^/]+\.SF$'), # signature file Skip(r'^META-INF/[^/]+\.DSA$'), # default signature alg. file Skip(r'^META-INF/[^/]+\.RSA$'), # default signature alg. file Duplicate(r'^META-INF/services/', Duplicate.CONCAT) ] # 1 svc fqcn per line return cls(rules=rules + additional_rules, default_dup_action=default_dup_action)
def __init__(self, address=None, payload=None, sources=None, resources=None, # Old-style resources (file list, Fileset). resource_targets=None, # New-style resources (Resources target specs). provides=None, compatibility=None, **kwargs): """ :param dependencies: Other targets that this target depends on. These dependencies may be ``python_library``-like targets (``python_library``, ``python_thrift_library``, ``python_antlr_library`` and so forth) or ``python_requirement_library`` targets. :type dependencies: List of target specs :param sources: Files to "include". Paths are relative to the BUILD file's directory. :type sources: ``Fileset`` or list of strings :param resources: non-Python resources, e.g. templates, keys, other data (it is recommended that your application uses the pkgutil package to access these resources in a .zip-module friendly way.) :param provides: The `setup_py <#setup_py>`_ to publish that represents this target outside the repo. :param compatibility: either a string or list of strings that represents interpreter compatibility for this target, using the Requirement-style format, e.g. ``'CPython>=3', or just ['>=2.7','<3']`` for requirements agnostic to interpreter class. """ self.address = address payload = payload or Payload() payload.add_fields({ 'sources': self.create_sources_field(sources, address.spec_path, key_arg='sources'), 'resources': self.create_sources_field(resources, address.spec_path, key_arg='resources'), 'provides': provides, 'compatibility': PrimitiveField(maybe_list(compatibility or ())), }) super(PythonTarget, self).__init__(address=address, payload=payload, **kwargs) self._resource_target_specs = resource_targets self.add_labels('python') self._synthetic_resources_target = None if provides and not isinstance(provides, PythonArtifact): raise TargetDefinitionException(self, "Target must provide a valid pants setup_py object. Received a '{}' object instead.".format( provides.__class__.__name__)) self._provides = provides # Check that the compatibility requirements are well-formed. for req in self.payload.compatibility: try: PythonIdentity.parse_requirement(req) except ValueError as e: raise TargetDefinitionException(self, str(e))
def __init__(self, timeout=Amount(2, Time.MINUTES), coverage=None, soft_dependencies=False, entry_point='pytest', **kwargs): """ :param name: See PythonLibrary target :param sources: A list of filenames representing the source code this library is compiled from. :type sources: list of strings :param resources: See PythonLibrary target :param dependencies: List of :class:`pants.base.target.Target` instances this target depends on. :type dependencies: list of targets :param timeout: Amount of time before this test should be considered timed-out. :param coverage: the module(s) whose coverage should be generated, e.g. 'twitter.common.log' or ['twitter.common.log', 'twitter.common.http'] :param soft_dependencies: Whether or not we should ignore dependency resolution errors for this test. :param entry_point: The entry point to use to run the tests. :param dict exclusives: An optional dict of exclusives tags. See CheckExclusives for details. """ self._timeout = timeout self._soft_dependencies = bool(soft_dependencies) self._coverage = maybe_list(coverage) if coverage is not None else [] self._entry_point = entry_point super(PythonTests, self).__init__(**kwargs) self.add_labels('python', 'tests')
def __init__(self, address=None, payload=None, sources=None, setup_requires=None, **kwargs): """ :param address: The Address that maps to this Target in the BuildGraph. :type address: :class:`pants.build_graph.address.Address` :param payload: The configuration encapsulated by this target. Also in charge of most fingerprinting details. :type payload: :class:`pants.base.payload.Payload` :param sources: Files to "include". Paths are relative to the BUILD file's directory. :type sources: :class:`twitter.common.dirutil.Fileset` or list of strings. Must include setup.py. :param list setup_requires: A list of python requirements to provide during the invocation of setup.py. """ if not 'setup.py' in sources: raise TargetDefinitionException( self, 'A file named setup.py must be in the same ' 'directory as the BUILD file containing this target.') payload = payload or Payload() payload.add_fields({ 'setup_requires': PrimitiveField(maybe_list(setup_requires or ())) }) super(PythonDistribution, self).__init__( address=address, payload=payload, sources=sources, **kwargs)
def generate_ivy(cls, targets, jars, excludes, ivyxml, confs): org, name = cls.identify(targets) # As it turns out force is not transitive - it only works for dependencies pants knows about # directly (declared in BUILD files - present in generated ivy.xml). The user-level ivy docs # don't make this clear [1], but the source code docs do (see isForce docs) [2]. I was able to # edit the generated ivy.xml and use the override feature [3] though and that does work # transitively as you'd hope. # # [1] http://ant.apache.org/ivy/history/2.3.0/settings/conflict-managers.html # [2] https://svn.apache.org/repos/asf/ant/ivy/core/branches/2.3.0/ # src/java/org/apache/ivy/core/module/descriptor/DependencyDescriptor.java # [3] http://ant.apache.org/ivy/history/2.3.0/ivyfile/override.html dependencies = [cls._generate_jar_template(jar, confs) for jar in jars] overrides = [cls._generate_override_template(dep) for dep in dependencies if dep.force] excludes = [cls._generate_exclude_template(exclude) for exclude in excludes] template_data = TemplateData( org=org, module=name, version='latest.integration', publications=None, configurations=maybe_list(confs), # Mustache doesn't like sets. dependencies=dependencies, excludes=excludes, overrides=overrides) safe_mkdir(os.path.dirname(ivyxml)) with open(ivyxml, 'w') as output: generator = Generator(pkgutil.get_data(__name__, cls.IVY_TEMPLATE_PATH), root_dir=get_buildroot(), lib=template_data) generator.write(output)
def __init__(self, name, sources, resources=None, dependencies=None, provides=None, compatibility=None, exclusives=None): TargetWithSources.__init__(self, name, sources=sources, exclusives=exclusives) TargetWithDependencies.__init__(self, name, dependencies=dependencies, exclusives=exclusives) self.add_labels('python') self.resources = self._resolve_paths(resources) if resources else OrderedSet() if provides and not isinstance(provides, PythonArtifact): raise TargetDefinitionException(self, "Target must provide a valid pants setup_py object. Received a '%s' object instead." % provides.__class__.__name__) self.provides = provides self.compatibility = maybe_list(compatibility or ()) for req in self.compatibility: try: PythonIdentity.parse_requirement(req) except ValueError as e: raise TargetDefinitionException(self, str(e))
def parse_addresses(self, specs, fail_fast=False): """Process a list of command line specs and perform expansion. This method can expand a list of command line specs. :param list specs: either a single spec string or a list of spec strings. :return: a generator of specs parsed into addresses. :raises: CmdLineSpecParser.BadSpecError if any of the address selectors could not be parsed. """ specs = maybe_list(specs) addresses = OrderedSet() for spec in specs: for address in self._parse_spec(spec, fail_fast): addresses.add(address) results = filter(self._not_excluded_address, addresses) # Print debug information about the excluded targets if logger.getEffectiveLevel() <= logging.DEBUG and self._exclude_patterns: logger.debug('excludes:\n {excludes}' .format(excludes='\n '.join(self._exclude_target_regexps))) targets = ', '.join(self._excluded_target_map[CmdLineSpecParser._UNMATCHED_KEY]) logger.debug('Targets after excludes: {targets}'.format(targets=targets)) excluded_count = 0 for pattern, targets in self._excluded_target_map.iteritems(): if pattern != CmdLineSpecParser._UNMATCHED_KEY: logger.debug('Targets excluded by pattern {pattern}\n {targets}' .format(pattern=pattern, targets='\n '.join(targets))) excluded_count += len(targets) logger.debug('Excluded {count} target{plural}.' .format(count=excluded_count, plural=('s' if excluded_count != 1 else ''))) return results
def create_context_from_options(options, target_roots=None, build_graph=None, build_configuration=None, address_mapper=None, console_outstream=None, workspace=None, scheduler=None): """Creates a ``Context`` with the given options and no targets by default. :param options: An :class:`pants.option.options.Option`-alike object that supports read methods. Other params are as for ``Context``. """ run_tracker = TestContext.DummyRunTracker() target_roots = maybe_list(target_roots, Target) if target_roots else [] return TestContext(options=options, run_tracker=run_tracker, target_roots=target_roots, build_graph=build_graph, build_configuration=build_configuration, address_mapper=address_mapper, console_outstream=console_outstream, workspace=workspace, scheduler=scheduler)
def __init__(self, address=None, sources=None, resources=None, # Old-style resources (file list, Fileset). resource_targets=None, # New-style resources (Resources target specs). provides=None, compatibility=None, **kwargs): payload = PythonPayload(sources_rel_path=address.spec_path, sources=sources or [], resources=resources) super(PythonTarget, self).__init__(address=address, payload=payload, **kwargs) self._resource_target_specs = resource_targets self.add_labels('python') self._synthetic_resources_target = None if provides and not isinstance(provides, PythonArtifact): raise TargetDefinitionException(self, "Target must provide a valid pants setup_py object. Received a '%s' object instead." % provides.__class__.__name__) self._provides = provides self._compatibility = maybe_list(compatibility or ()) # Check that the compatibility requirements are well-formed. for req in self._compatibility: try: PythonIdentity.parse_requirement(req) except ValueError as e: raise TargetDefinitionException(self, str(e))
def distribution(installed_sdks=('18', '19'), installed_build_tools=('19.1.0', '20.0.0'), files=('android.jar', ), executables=('aapt', 'zipalign')): """Mock Android SDK Distribution. :param tuple[strings] installed_sdks: SDK versions of the files being mocked. :param tuple[strings] installed_build_tools: Build tools version of any tools. :param tuple[strings] files: The files are to mock non-executables and one will be created for each installed_sdks version. :param tuple[strings] executables: Executables are any required tools and one is created for each installed_build_tools version. """ with temporary_dir() as sdk: for sdk_version in installed_sdks: for android_file in files: touch( os.path.join(sdk, 'platforms', 'android-' + sdk_version, android_file)) for version in installed_build_tools: for exe in maybe_list(executables or ()): path = os.path.join(sdk, 'build-tools', version, exe) touch(path) chmod_plus_x(path) dx_path = os.path.join(sdk, 'build-tools', version, 'lib/dx.jar') touch(dx_path) yield sdk
def __init__(self, identity, workdir, nailgun_classpath, distribution, ins=None, connect_timeout=10, connect_attempts=5): Executor.__init__(self, distribution=distribution) ProcessManager.__init__(self, name=identity, process_name=self._PROCESS_NAME) if not isinstance(workdir, string_types): raise ValueError( 'Workdir must be a path string, not: {workdir}'.format( workdir=workdir)) self._identity = identity self._workdir = workdir self._ng_stdout = os.path.join(workdir, 'stdout') self._ng_stderr = os.path.join(workdir, 'stderr') self._nailgun_classpath = maybe_list(nailgun_classpath) self._ins = ins self._connect_timeout = connect_timeout self._connect_attempts = connect_attempts
def __init__(self, classpath, ivy_settings=None, ivy_cache_dir=None, extra_jvm_options=None): """Configures an ivy wrapper for the ivy distribution at the given classpath. :param ivy_settings: path to find settings.xml file :param ivy_cache_dir: path to store downloaded ivy artifacts :param extra_jvm_options: list of strings to add to command line when invoking Ivy """ self._classpath = maybe_list(classpath) self._ivy_settings = ivy_settings if self._ivy_settings and not isinstance(self._ivy_settings, string_types): raise ValueError( 'ivy_settings must be a string, given {} of type {}'.format( self._ivy_settings, type(self._ivy_settings))) self._ivy_cache_dir = ivy_cache_dir if self._ivy_cache_dir and not isinstance(self._ivy_cache_dir, string_types): raise ValueError( 'ivy_cache_dir must be a string, given {} of type {}'.format( self._ivy_cache_dir, type(self._ivy_cache_dir))) self._extra_jvm_options = extra_jvm_options or []
def __init__(self, address=None, sources=None, resources=None, # Old-style resources (file list, Fileset). resource_targets=None, # New-style resources (Resources target specs). provides=None, compatibility=None, **kwargs): payload = PythonPayload(sources_rel_path=address.spec_path, sources=sources or [], resources=resources) super(PythonTarget, self).__init__(address=address, payload=payload, **kwargs) self._resource_target_specs = resource_targets self.add_labels('python') self._synthetic_resources_target = None if provides and not isinstance(provides, PythonArtifact): raise TargetDefinitionException(self, "Target must provide a valid pants setup_py object. Received a '%s' object instead." % provides.__class__.__name__) self._provides = provides self.compatibility = maybe_list(compatibility or ()) for req in self.compatibility: try: PythonIdentity.parse_requirement(req) except ValueError as e: raise TargetDefinitionException(self, str(e))
def create_context( options=None, target_roots=None, build_graph=None, build_file_parser=None, address_mapper=None, console_outstream=None, workspace=None, ): """Creates a ``Context`` with no config values, options, or targets by default. :param options: A map of scope -> (map of key to value). Other params are as for ``Context``. """ options = create_options(options or {}) target_roots = maybe_list(target_roots, Target) if target_roots else [] return TestContext( options=options, target_roots=target_roots, build_graph=build_graph, build_file_parser=build_file_parser, address_mapper=address_mapper, console_outstream=console_outstream, workspace=workspace, )
def __init__(self, org, name, repo, description=None): """ :param string org: Organization of this artifact, or groupId in maven parlance. :param string name: Name of the artifact, or artifactId in maven parlance. :param repo: :class:`twitter.pants.targets.repository.Repository` this artifact is published to. :param string description: Description of this artifact. """ if not isinstance(org, Compatibility.string): raise ValueError("org must be %s but was %s" % (Compatibility.string, org)) if not isinstance(name, Compatibility.string): raise ValueError("name must be %s but was %s" % (Compatibility.string, name)) if repo is None: raise ValueError("repo must be supplied") repos = [] for tgt in maybe_list(resolve(repo), expected_type=(Pants, Repository)): repos.extend(tgt.resolve()) if len(repos) != 1: raise ValueError("An artifact must have exactly 1 repo, given: %s" % repos) repo = repos[0] if description is not None and not isinstance(description, Compatibility.string): raise ValueError("description must be None or %s but was %s" % (Compatibility.string, description)) self.org = org self.name = name self.rev = None self.repo = repo self.description = description
def __init__(self, classpath, ivy_settings=None, ivy_cache_dir=None, extra_jvm_options=None): """Configures an ivy wrapper for the ivy distribution at the given classpath. :param ivy_settings: path to find settings.xml file :param ivy_cache_dir: path to store downloaded ivy artifacts :param extra_jvm_options: list of strings to add to command line when invoking Ivy """ self._classpath = maybe_list(classpath) self._ivy_settings = ivy_settings if self._ivy_settings and not isinstance(self._ivy_settings, string_types): raise ValueError( "ivy_settings must be a string, given {} of type {}".format( self._ivy_settings, type(self._ivy_settings) ) ) self._ivy_cache_dir = ivy_cache_dir if not isinstance(self._ivy_cache_dir, string_types): raise ValueError( "ivy_cache_dir must be a string, given {} of type {}".format( self._ivy_cache_dir, type(self._ivy_cache_dir) ) ) self._extra_jvm_options = extra_jvm_options or []
def run_export(self, test_target, workdir, load_libs=False, only_default=False, extra_args=None): """Runs ./pants export ... and returns its json output. :param string|list test_target: spec of the targets to run on. :param string workdir: working directory to run pants with. :param bool load_libs: whether to load external libraries (of any conf). :param bool only_default: if loading libraries, whether to only resolve the default conf, or to additionally resolve sources and javadocs. :param list extra_args: list of extra arguments for the pants invocation. :return: the json output of the console task. :rtype: dict """ export_out_file = os.path.join(workdir, 'export_out.txt') args = ['export', f'--output-file={export_out_file}' ] + maybe_list(test_target) libs_args = ['--no-export-libraries' ] if not load_libs else self._confs_args if load_libs and only_default: libs_args = [] pants_run = self.run_pants_with_workdir( args + libs_args + (extra_args or []), workdir) self.assert_success(pants_run) self.assertTrue( os.path.exists(export_out_file), msg=f'Could not find export output file in {export_out_file}') with open(export_out_file, 'r') as json_file: json_data = json.load(json_file) if not load_libs: self.assertIsNone(json_data.get('libraries')) return json_data
def create_context(options=None, passthru_args=None, target_roots=None, build_graph=None, build_file_parser=None, address_mapper=None, console_outstream=None, workspace=None): """Creates a ``Context`` with no options or targets by default. :param options: A map of scope -> (map of key to value). Other params are as for ``Context``. """ options = create_options(options or {}, passthru_args=passthru_args) run_tracker = TestContext.DummyRunTracker() target_roots = maybe_list(target_roots, Target) if target_roots else [] return TestContext(options=options, run_tracker=run_tracker, target_roots=target_roots, build_graph=build_graph, build_file_parser=build_file_parser, address_mapper=address_mapper, console_outstream=console_outstream, workspace=workspace)
def __init__(self, address=None, payload=None, sources=None, setup_requires=None, **kwargs): """ :param address: The Address that maps to this Target in the BuildGraph. :type address: :class:`pants.build_graph.address.Address` :param payload: The configuration encapsulated by this target. Also in charge of most fingerprinting details. :type payload: :class:`pants.base.payload.Payload` :param sources: Files to "include". Paths are relative to the BUILD file's directory. :type sources: :class:`twitter.common.dirutil.Fileset` or list of strings. Must include setup.py. :param list setup_requires: A list of python requirements to provide during the invocation of setup.py. """ if not 'setup.py' in sources: raise TargetDefinitionException( self, 'A file named setup.py must be in the same ' 'directory as the BUILD file containing this target.') payload = payload or Payload() payload.add_fields({ 'setup_requires': PrimitiveField(maybe_list(setup_requires or ())) }) super(PythonDistribution, self).__init__(address=address, payload=payload, sources=sources, **kwargs)
def product_request(self, product, subjects): """Executes a request for a singular product type from the scheduler for one or more subjects and yields the products. :param class product: A product type for the request. :param list subjects: A list of subjects for the request. :yields: The requested products. """ request = self._scheduler.execution_request([product], subjects) result = self.execute(request) if result.error: raise result.error result_items = self._scheduler.root_entries(request).items() # State validation. unknown_state_types = tuple( type(state) for _, state in result_items if type(state) not in (Throw, Return) ) if unknown_state_types: State.raise_unrecognized(unknown_state_types) # Throw handling. # TODO: See https://github.com/pantsbuild/pants/issues/3912 throw_roots = tuple(root for root, state in result_items if type(state) is Throw) if throw_roots: cumulative_trace = '\n'.join(self._scheduler.trace()) raise ExecutionError('Received unexpected Throw state(s):\n{}'.format(cumulative_trace)) # Return handling. returns = tuple(state.value for _, state in result_items if type(state) is Return) for return_value in returns: for computed_product in maybe_list(return_value, expected_type=product): yield computed_product
def __init__(self, classpath, java_executor=None, ivy_settings=None, ivy_cache_dir=None): """Configures an ivy wrapper for the ivy distribution at the given classpath.""" self._classpath = maybe_list(classpath) self._java = java_executor or SubprocessExecutor() if not isinstance(self._java, Executor): raise ValueError( 'java_executor must be an Executor instance, given %s of type %s' % (self._java, type(self._java))) self._ivy_settings = ivy_settings if self._ivy_settings and not isinstance(self._ivy_settings, Compatibility.string): raise ValueError( 'ivy_settings must be a string, given %s of type %s' % (self._ivy_settings, type(self._ivy_settings))) self._ivy_cache_dir = ivy_cache_dir if self._ivy_cache_dir and not isinstance(self._ivy_cache_dir, Compatibility.string): raise ValueError( 'ivy_cache_dir must be a string, given %s of type %s' % (self._ivy_cache_dir, type(self._ivy_cache_dir)))
def run_export(self, test_target, workdir, load_libs=False, only_default=False, extra_args=None): """Runs ./pants export ... and returns its json output. :param string|list test_target: spec of the targets to run on. :param string workdir: working directory to run pants with. :param bool load_libs: whether to load external libraries (of any conf). :param bool only_default: if loading libraries, whether to only resolve the default conf, or to additionally resolve sources and javadocs. :param list extra_args: list of extra arguments for the pants invocation. :return: the json output of the console task. :rtype: dict """ export_out_file = os.path.join(workdir, 'export_out.txt') args = ['export', '--output-file={out_file}'.format(out_file=export_out_file)] + maybe_list(test_target) libs_args = ['--no-export-libraries'] if not load_libs else self._confs_args if load_libs and only_default: libs_args = [] pants_run = self.run_pants_with_workdir(args + libs_args + (extra_args or []), workdir) self.assert_success(pants_run) self.assertTrue(os.path.exists(export_out_file), msg='Could not find export output file in {out_file}' .format(out_file=export_out_file)) with open(export_out_file) as json_file: json_data = json.load(json_file) if not load_libs: self.assertIsNone(json_data.get('libraries')) return json_data
def parse_addresses(self, specs, fail_fast=False): """Process a list of command line specs and perform expansion. This method can expand a list of command line specs. :param list specs: either a single spec string or a list of spec strings. :return: a generator of specs parsed into addresses. :raises: CmdLineSpecParser.BadSpecError if any of the address selectors could not be parsed. """ specs = maybe_list(specs) addresses = OrderedSet() for spec in specs: for address in self._parse_spec(spec, fail_fast): addresses.add(address) results = filter(self._not_excluded_address, addresses) # Print debug information about the excluded targets if logger.getEffectiveLevel() <= logging.DEBUG and self._exclude_patterns: logger.debug('excludes:\n {excludes}' .format(excludes='\n '.join(self._exclude_target_regexps))) targets = ', '.join(self._excluded_target_map[CmdLineSpecParser._UNMATCHED_KEY]) logger.debug('Targets after excludes: {targets}'.format(targets=targets)) excluded_count = 0 for pattern, targets in six.iteritems(self._excluded_target_map): if pattern != CmdLineSpecParser._UNMATCHED_KEY: logger.debug('Targets excluded by pattern {pattern}\n {targets}' .format(pattern=pattern, targets='\n '.join(targets))) excluded_count += len(targets) logger.debug('Excluded {count} target{plural}.' .format(count=excluded_count, plural=('s' if excluded_count != 1 else ''))) return results
def __init__(self, coverage=None, **kwargs): """ :param coverage: the module(s) whose coverage should be generated, e.g. 'twitter.common.log' or ['twitter.common.log', 'twitter.common.http'] """ self._coverage = maybe_list(coverage) if coverage is not None else [] super(PythonTests, self).__init__(**kwargs) self.add_labels('python', 'tests')
def append_classpath(self, classpath): """Specifies a Class-Path entry for this jar's manifest. If called multiple times, new entry will be appended to the existing classpath. :param list classpath: a list of paths """ self._classpath = self._classpath + maybe_list(classpath)
def default_sources(cls, sources_rel_path): """Provide sources, if they weren't specified explicitly in the BUILD file. By default this globs over self.default_sources_globs (e.g., '*.java') but subclasses can override to provide more nuanced default behavior. In this case, the subclasses must also override supports_default_sources(). """ if cls.default_sources_globs is not None: if cls.default_sources_exclude_globs is not None: exclude = [Globs.create_fileset_with_spec(sources_rel_path, *maybe_list(cls.default_sources_exclude_globs))] else: exclude = [] return Globs.create_fileset_with_spec(sources_rel_path, *maybe_list(cls.default_sources_globs), exclude=exclude) return None
def resolve_deps(self, key, default=[]): deps = OrderedSet() for dep in self.context.config.getlist('protobuf-gen', key, default=maybe_list(default)): if dep: deps.update(self.context.resolve(dep)) return deps
def key_for(self, id, sources): """Get a cache key representing some id and its associated source files. Useful primarily in tests. Normally we use key_for_target(). """ sha = hashlib.sha1() num_sources = self._sources_hash(sha, maybe_list(sources)) return CacheKey(id, sha.hexdigest(), num_sources)
def __init__(self, name=None, main=None, basename=None, source=None, deploy_excludes=None, deploy_jar_rules=None, **kwargs): """ :param string name: The name of this target, which combined with this build file defines the target :class:`pants.base.address.Address`. :param string main: The name of the ``main`` class, e.g., ``'com.pants.examples.hello.main.HelloMain'``. This class may be present as the source of this target or depended-upon library. :param string basename: Base name for the generated ``.jar`` file, e.g., ``'hello'``. (By default, uses ``name`` param) :param string source: Name of one ``.java`` or ``.scala`` file (a good place for a ``main``). :param resources: List of ``resource``\s to include in bundle. :param dependencies: List of targets (probably ``java_library`` and ``scala_library`` targets) to "link" in. :param excludes: List of ``exclude``\s to filter this target's transitive dependencies against. :param deploy_excludes: List of ``excludes`` to apply at deploy time. If you, for example, deploy a java servlet that has one version of ``servlet.jar`` onto a Tomcat environment that provides another version, they might conflict. ``deploy_excludes`` gives you a way to build your code but exclude the conflicting ``jar`` when deploying. :param deploy_jar_rules: Rules for packaging this binary in a deploy jar. :type deploy_jar_rules: A :class:`twitter.pants.targets.JarRules` specification. :param configurations: Ivy configurations to resolve for this target. This parameter is not intended for general use. :type configurations: tuple of strings """ sources = [source] if source else None super(JvmBinary, self).__init__(name=name, sources=sources, **kwargs) if main and not isinstance(main, Compatibility.string): raise TargetDefinitionException( self, 'main must be a fully qualified classname') if source and not isinstance(source, Compatibility.string): raise TargetDefinitionException( self, 'source must be a single relative file path') # Consider an alias mechanism (target) that acts like JarLibrary but points to a single item # and admits any pointee type. Its very likely folks will want to share jar_rules but they # cannot today and it seems heavy-handed to force jar_rules to be a target just to get an # address in the off chance its needed. if deploy_jar_rules and not isinstance(deploy_jar_rules, JarRules): raise TargetDefinitionException( self, 'deploy_jar_rules must be a JarRules specification') self.main = main self.basename = basename or name self.deploy_excludes = maybe_list(deploy_excludes, Exclude) if deploy_excludes else [] self.deploy_jar_rules = deploy_jar_rules or JarRules.default()
def __init__(self, address=None, payload=None, sources=None, resources=None, # Old-style resources (file list, Fileset). resource_targets=None, # New-style resources (Resources target specs). provides=None, compatibility=None, **kwargs): """ :param dependencies: Other targets that this target depends on. These dependencies may be ``python_library``-like targets (``python_library``, ``python_thrift_library``, ``python_antlr_library`` and so forth) or ``python_requirement_library`` targets. :type dependencies: List of target specs :param sources: Files to "include". Paths are relative to the BUILD file's directory. :type sources: ``Fileset`` or list of strings :param resources: non-Python resources, e.g. templates, keys, other data (it is recommended that your application uses the pkgutil package to access these resources in a .zip-module friendly way.) :param provides: The `setup_py <#setup_py>`_ to publish that represents this target outside the repo. :param compatibility: either a string or list of strings that represents interpreter compatibility for this target, using the Requirement-style format, e.g. ``'CPython>=3', or just ['>=2.7','<3']`` for requirements agnostic to interpreter class. """ self.address = address payload = payload or Payload() payload.add_fields({ 'sources': self.create_sources_field(sources, address.spec_path, key_arg='sources'), 'resources': self.create_sources_field(resources, address.spec_path, key_arg='resources'), 'provides': provides, 'compatibility': PrimitiveField(maybe_list(compatibility or ())), }) super(PythonTarget, self).__init__(address=address, payload=payload, **kwargs) self._resource_target_specs = resource_targets self.add_labels('python') if provides and not isinstance(provides, PythonArtifact): raise TargetDefinitionException(self, "Target must provide a valid pants setup_py object. Received a '{}' object instead.".format( provides.__class__.__name__)) self._provides = provides # Check that the compatibility requirements are well-formed. for req in self.payload.compatibility: try: PythonIdentity.parse_requirement(req) except ValueError as e: raise TargetDefinitionException(self, str(e))
def __init__(self, address=None, payload=None, sources=None, provides=None, compatibility=None, **kwargs): """ :param dependencies: The addresses of targets that this target depends on. These dependencies may be ``python_library``-like targets (``python_library``, ``python_thrift_library``, ``python_antlr_library`` and so forth) or ``python_requirement_library`` targets. :type dependencies: list of strings :param sources: Files to "include". Paths are relative to the BUILD file's directory. :type sources: ``Fileset`` or list of strings :param provides: The `setup_py <#setup_py>`_ to publish that represents this target outside the repo. :param compatibility: either a string that represents interpreter compatibility for this target using the Requirement-style format, e.g. ``'CPython>=2.7,<3'`` (Select a CPython interpreter with version ``>=2.7`` AND version ``<3``) or a list of Requirement-style strings which will be OR'ed together. If the compatibility requirement is agnostic to interpreter class, using the example above, a Requirement-style compatibility constraint like '>=2.7,<3' (N.B.: not prefixed with CPython) can be used. """ self.address = address payload = payload or Payload() payload.add_fields({ 'sources': self.create_sources_field(sources, address.spec_path, key_arg='sources'), 'provides': provides, 'compatibility': PrimitiveField(maybe_list(compatibility or ())), }) super(PythonTarget, self).__init__(address=address, payload=payload, **kwargs) if provides and not isinstance(provides, PythonArtifact): raise TargetDefinitionException( self, "Target must provide a valid pants setup_py object. Received a '{}' object instead." .format(provides.__class__.__name__)) self._provides = provides # Check that the compatibility requirements are well-formed. for req in self.payload.compatibility: try: PythonIdentity.parse_requirement(req) except ValueError as e: raise TargetDefinitionException(self, str(e))
def __init__(self, name=None, main=None, basename=None, source=None, deploy_excludes=None, deploy_jar_rules=None, **kwargs): """ :param string name: The name of this target, which combined with this build file defines the :doc:`target address <target_addresses>`. :param string main: The name of the ``main`` class, e.g., ``'com.pants.examples.hello.main.HelloMain'``. This class may be present as the source of this target or depended-upon library. :param string basename: Base name for the generated ``.jar`` file, e.g., ``'hello'``. (By default, uses ``name`` param) :param string source: Name of one ``.java`` or ``.scala`` file (a good place for a ``main``). :param resources: List of ``resource``\s to include in bundle. :param dependencies: Targets (probably ``java_library`` and ``scala_library`` targets) to "link" in. :type dependencies: list of target specs :param excludes: List of :ref:`exclude <bdict_exclude>`\s to filter this target's transitive dependencies against. :param deploy_excludes: List of :ref:`exclude <bdict_exclude>`\s to apply at deploy time. If you, for example, deploy a java servlet that has one version of ``servlet.jar`` onto a Tomcat environment that provides another version, they might conflict. ``deploy_excludes`` gives you a way to build your code but exclude the conflicting ``jar`` when deploying. :param deploy_jar_rules: :ref:`Jar rules <bdict_jar_rules>` for packaging this binary in a deploy jar. :param configurations: Ivy configurations to resolve for this target. This parameter is not intended for general use. :type configurations: tuple of strings """ sources = [source] if source else None super(JvmBinary, self).__init__(name=name, sources=sources, **kwargs) if main and not isinstance(main, Compatibility.string): raise TargetDefinitionException(self, 'main must be a fully qualified classname') if source and not isinstance(source, Compatibility.string): raise TargetDefinitionException(self, 'source must be a single relative file path') # Consider an alias mechanism (target) that acts like JarLibrary but points to a single item # and admits any pointee type. Its very likely folks will want to share jar_rules but they # cannot today and it seems heavy-handed to force jar_rules to be a target just to get an # address in the off chance its needed. if deploy_jar_rules and not isinstance(deploy_jar_rules, JarRules): raise TargetDefinitionException(self, 'deploy_jar_rules must be a JarRules specification') self.main = main self.basename = basename or name self.deploy_excludes = maybe_list(deploy_excludes, Exclude) if deploy_excludes else [] self.deploy_jar_rules = deploy_jar_rules or JarRules.default()
def resolve_deps(self, key, default=None): default = default or [] deps = OrderedSet() for dep in self.context.config.getlist('protobuf-gen', key, default=maybe_list(default)): try: deps.update(self.context.resolve(dep)) except AddressLookupError as e: raise self.DepLookupError("{message}\n referenced from [{section}] key: {key} in pants.ini" .format(message=e, section='protobuf-gen', key=key)) return deps
def __init__(self, coverage=None, timeout=None, **kwargs): """ :param coverage: the module(s) whose coverage should be generated, e.g. 'twitter.common.log' or ['twitter.common.log', 'twitter.common.http'] :param int timeout: A timeout (in seconds) which covers the total runtime of all tests in this target. Only applied if `--test-pytest-timeouts` is set to True. """ self._coverage = maybe_list(coverage) if coverage is not None else [] self._timeout = timeout super(PythonTests, self).__init__(**kwargs)
def assertDeps(self, target, expected_deps=None): """Check that actual and expected dependencies of the given target match. :param target: :class:`pants.base.target.Target` to check dependencies of. :param expected_deps: :class:`pants.base.target.Target` or list of ``Target`` instances that are expected dependencies of ``target``. """ expected_deps_list = maybe_list(expected_deps or [], expected_type=Target) self.assertEquals(set(expected_deps_list), set(target.dependencies))