def test_collection_index() -> None:
c1 = Collection([0, 1, 2])
assert c1[0] == 0
assert c1[-1] == 2
assert c1[:] == c1
assert c1[1:] == Collection([1, 2])
assert c1[:1] == Collection([0])
def test_element_typechecking(self):
IntColl = Collection.of(int)
with self.assertRaisesRegexp(TypeCheckError, re.escape("""\
field 'dependencies' was invalid: in wrapped constraint TypedCollection(Exactly(int)) matching iterable object [3, {u}'hello']: value {u}'hello' (with type '{string_type}') must satisfy this type constraint: Exactly(int)."""
.format(u='' if PY3 else 'u',
string_type='str' if PY3 else 'unicode'))):
IntColl([3, "hello"])
IntOrStringColl = Collection.of(int, text_type)
self.assertEqual([3, "hello"], [x for x in IntOrStringColl([3, "hello"])])
with self.assertRaisesRegexp(TypeCheckError, re.escape("""\
field 'dependencies' was invalid: in wrapped constraint TypedCollection(Exactly(int or {string_type})) matching iterable object [()]: value () (with type 'tuple') must satisfy this type constraint: Exactly(int or {string_type})."""
.format(string_type='str' if PY3 else 'unicode'))):
IntOrStringColl([()])
def test_element_typechecking(self):
IntColl = Collection.of(int)
with self.assertRaisesRegexp(
TypeCheckError,
re.escape("""\
field 'dependencies' was invalid: in wrapped constraint TypedCollection(Exactly(int)) matching iterable object [3, {u}'hello']: value {u}'hello' (with type '{string_type}') must satisfy this type constraint: Exactly(int)."""
.format(u='' if PY3 else 'u',
string_type='str' if PY3 else 'unicode'))):
IntColl([3, "hello"])
IntOrStringColl = Collection.of(int, text_type)
self.assertEqual([3, "hello"],
[x for x in IntOrStringColl([3, "hello"])])
with self.assertRaisesRegexp(
TypeCheckError,
re.escape("""\
field 'dependencies' was invalid: in wrapped constraint TypedCollection(Exactly(int or {string_type})) matching iterable object [()]: value () (with type 'tuple') must satisfy this type constraint: Exactly(int or {string_type})."""
.format(string_type='str' if PY3 else 'unicode'))):
IntOrStringColl([()])
def test_element_typechecking(self):
IntColl = Collection.of(int)
with self.assertRaisesRegexp(
TypeCheckError,
re.escape(
"field 'dependencies' was invalid: in wrapped constraint TypedCollection(Exactly(int)) "
"matching iterable object [3, 'hello']: value 'hello' (with type 'str') must satisfy this "
"type constraint: Exactly(int).")):
IntColl([3, "hello"])
IntOrStringColl = Collection.of(int, str)
self.assertEqual([3, "hello"],
[x for x in IntOrStringColl([3, "hello"])])
with self.assertRaisesRegexp(
TypeCheckError,
re.escape(
"field 'dependencies' was invalid: in wrapped constraint TypedCollection(Exactly(int or "
"str)) matching iterable object [()]: value () (with type 'tuple') must satisfy this type "
"constraint: Exactly(int or str)."
"")):
IntOrStringColl([()])
def test_collection_equality() -> None:
assert Collection([]) == Collection([])
c1 = Collection([1, 2, 3])
assert c1 == Collection([1, 2, 3])
assert c1 != Collection([3, 2, 1])
assert c1 != Collection([])
assert c1 != Collection([1, 2])
e1 = Examples([1, 2, 3])
assert e1 == Examples([1, 2, 3])
assert e1 != Examples2([1, 2, 3])
def test_collection_contains() -> None:
c1 = Collection([1, 2])
assert 1 in c1
assert 2 in c1
assert 200 not in c1
assert "bad" not in c1 # type: ignore[comparison-overlap]
def test_collection_iteration() -> None:
c1 = Collection([1, 2])
assert list(iter(c1)) == [1, 2]
assert [x for x in c1] == [1, 2]
from __future__ import absolute_import, division, print_function, unicode_literals
import inspect
from builtins import object
from functools import update_wrapper
from future.utils import string_types
from pants.build_graph.address import Address, BuildFileAddress
from pants.engine.objects import Collection, Resolvable, Serializable
from pants.util.collections_abc_backport import MutableMapping, MutableSequence
from pants.util.objects import TypeConstraintError
Addresses = Collection.of(Address)
class BuildFileAddresses(Collection.of(BuildFileAddress)):
@property
def addresses(self):
"""Converts the BuildFileAddress objects in this collection to Address objects."""
return [bfa.to_address() for bfa in self]
class NotSerializableError(TypeError):
"""Indicates an addressable descriptor is illegally installed in a non-Serializable type."""
class MutationError(AttributeError):
"""Indicates an illegal attempt to mutate an addressable attribute that already has a value."""
def test_collection_iteration(self) -> None:
self.assertEqual([1, 2], [x for x in Collection([1, 2])])
# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
import inspect
from collections.abc import MutableMapping, MutableSequence
from dataclasses import dataclass
from functools import update_wrapper
from pants.base.specs import Spec
from pants.build_graph.address import Address, BuildFileAddress
from pants.engine.objects import Collection, Resolvable, Serializable
from pants.util.objects import TypeConstraintError
Addresses = Collection.of(Address)
@dataclass(frozen=True)
class ProvenancedBuildFileAddress:
"""A BuildFileAddress along with the cmd-line spec it was generated from."""
build_file_address: BuildFileAddress
provenance: Spec
class BuildFileAddresses(Collection.of(BuildFileAddress)):
@property
def addresses(self):
"""Converts the BuildFileAddress objects in this collection to Address objects."""
return [bfa.to_address() for bfa in self]
ProvenancedBuildFileAddresses = Collection.of(ProvenancedBuildFileAddress)
class BuildFileAddresses(Collection.of(BuildFileAddress)):
@property
def addresses(self):
"""Converts the BuildFileAddress objects in this collection to Address objects."""
return [bfa.to_address() for bfa in self]
def test_collection_hash() -> None:
assert hash(Collection([])) == hash(Collection([]))
c1 = Collection([1, 2, 3])
assert hash(c1) == hash(Collection([1, 2, 3]))
assert hash(c1) == hash(Examples([1, 2, 3]))
def test_collection_reversed() -> None:
assert list(reversed(Collection([1, 2, 3]))) == [3, 2, 1]
def test_mismatching_paths(self):
with self.assertRaises(DifferingFamiliesError):
AddressFamily.create('one',
[AddressMap('/dev/null/one/0', {}),
AddressMap('/dev/null/two/0', {})])
def test_duplicate_names(self):
with self.assertRaises(DuplicateNameError):
AddressFamily.create('name/space',
[AddressMap('name/space/0',
{'one': Thing(name='one', age=42)}),
AddressMap('name/space/1',
{'one': Thing(name='one', age=37)})])
HydratedStructs = Collection.of(HydratedStruct)
@rule(HydratedStructs, [BuildFileAddresses])
def unhydrated_structs(build_file_addresses):
tacs = yield [Get(HydratedStruct, Address, a) for a in build_file_addresses.addresses]
yield HydratedStructs(tacs)
class AddressMapperTest(unittest.TestCase, SchedulerTestBase):
def setUp(self):
# Set up a scheduler that supports address mapping.
address_mapper = AddressMapper(parser=JsonParser(TARGET_TABLE),
build_patterns=('*.BUILD.json',))
# We add the `unhydrated_structs` rule because it is otherwise not used in the core engine.
def test_collection_iteration(self):
self.assertEqual([1, 2], [x for x in Collection.of(int)([1, 2])])
def test_mismatching_paths(self):
with self.assertRaises(DifferingFamiliesError):
AddressFamily.create('one',
[AddressMap('/dev/null/one/0', {}),
AddressMap('/dev/null/two/0', {})])
def test_duplicate_names(self):
with self.assertRaises(DuplicateNameError):
AddressFamily.create('name/space',
[AddressMap('name/space/0',
{'one': Thing(name='one', age=42)}),
AddressMap('name/space/1',
{'one': Thing(name='one', age=37)})])
HydratedStructs = Collection.of(HydratedStruct)
@rule(HydratedStructs, [BuildFileAddresses])
def unhydrated_structs(build_file_addresses):
tacs = yield [Get(HydratedStruct, Address, a) for a in build_file_addresses.addresses]
yield HydratedStructs(tacs)
class AddressMapperTest(unittest.TestCase, SchedulerTestBase):
def setUp(self):
# Set up a scheduler that supports address mapping.
address_mapper = AddressMapper(parser=JsonParser(TARGET_TABLE),
build_patterns=('*.BUILD.json',))
# We add the `unhydrated_structs` rule because it is otherwise not used in the core engine.
def test_collection_bool(self):
self.assertTrue(bool(Collection.of(int)([0])))
self.assertFalse(bool(Collection.of(int)([])))
def test_mismatching_paths(self):
with self.assertRaises(DifferingFamiliesError):
AddressFamily.create('one', [
AddressMap('/dev/null/one/0', {}),
AddressMap('/dev/null/two/0', {})
])
def test_duplicate_names(self):
with self.assertRaises(DuplicateNameError):
AddressFamily.create('name/space', [
AddressMap('name/space/0', {'one': Thing(name='one', age=42)}),
AddressMap('name/space/1', {'one': Thing(name='one', age=37)})
])
UnhydratedStructs = Collection.of(UnhydratedStruct)
@rule(UnhydratedStructs, [Select(BuildFileAddresses)])
def unhydrated_structs(build_file_addresses):
uhs = yield [
Get(UnhydratedStruct, Address, a)
for a in build_file_addresses.addresses
]
yield UnhydratedStructs(uhs)
class AddressMapperTest(unittest.TestCase, SchedulerTestBase):
def setUp(self):
# Set up a scheduler that supports address mapping.
symbol_table = TargetTable()
def test_collection_length() -> None:
assert len(Collection([])) == 0
assert len(Collection([1, 2])) == 2
def test_collection_repr() -> None:
assert repr(Collection([])) == "Collection([])"
assert repr(Examples([])) == "Examples([])"
assert repr(Collection([1, 2, 3])) == "Collection([1, 2, 3])"
assert repr(Examples([1, 2, 3])) == "Examples([1, 2, 3])"
return self._raise_or_return(result)
def lease_files_in_graph(self):
self._native.lib.lease_files_in_graph(self._scheduler)
def garbage_collect_store(self):
self._native.lib.garbage_collect_store(self._scheduler)
def new_session(self, zipkin_trace_v2, v2_ui=False):
"""Creates a new SchedulerSession for this Scheduler."""
return SchedulerSession(self, self._native.new_session(
self._scheduler, zipkin_trace_v2, v2_ui, multiprocessing.cpu_count())
)
_PathGlobsAndRootCollection = Collection.of(PathGlobsAndRoot)
_DirectoryDigests = Collection.of(Digest)
_DirectoriesToMaterialize = Collection.of(DirectoryToMaterialize)
class SchedulerSession(object):
"""A handle to a shared underlying Scheduler and a unique Session.
Generally a Session corresponds to a single run of pants: some metrics are specific to
a Session.
"""
def lease_files_in_graph(self):
self._native.lib.lease_files_in_graph(self._scheduler)
def garbage_collect_store(self):
self._native.lib.garbage_collect_store(self._scheduler)
def new_session(self, zipkin_trace_v2, v2_ui=False):
"""Creates a new SchedulerSession for this Scheduler."""
return SchedulerSession(
self,
self._native.new_session(self._scheduler, zipkin_trace_v2, v2_ui,
multiprocessing.cpu_count()))
_PathGlobsAndRootCollection = Collection.of(PathGlobsAndRoot)
_DirectoryDigests = Collection.of(Digest)
_DirectoriesToMaterialize = Collection.of(DirectoryToMaterialize)
class SchedulerSession(object):
"""A handle to a shared underlying Scheduler and a unique Session.
Generally a Session corresponds to a single run of pants: some metrics are specific to
a Session.
"""
execution_error_type = ExecutionError
register('--config', type=dict, fromfile=True,
# TODO: Replace "See documentation" with actual URL, once we have some.
help='Source file regex matching config. See documentation for config schema.')
@memoized_method
def get_multi_matcher(self):
return MultiMatcher(self.get_options().config)
class RegexMatchResult(datatype([
('path', text_type), ('matching', tuple), ('nonmatching', tuple)
])):
"""The result of running regex matches on a source file."""
RegexMatchResults = Collection.of(RegexMatchResult)
class Matcher(object):
"""Class to match a single (possibly inverted) regex.
Matches are allowed anywhere in the string (so really a "search" in the Python regex parlance).
To anchor a match at the beginning of a string, use the ^ anchor. To anchor at the beginning
of any line, use the ^ anchor along with the MULTILINE directive (?m). See test for examples.
"""
def __init__(self, pattern, inverted=False):
self.compiled_regex = re.compile(pattern)
self.inverted = inverted
def matches(self, s):
@dataclass(frozen=True)
class FileContent:
"""The content of a file."""
path: str
content: bytes
is_executable: bool
def __repr__(self):
return 'FileContent(path={}, content=(len:{}), is_executable={})'.format(
self.path,
len(self.content),
self.is_executable,
)
FilesContent = Collection.of(FileContent)
class InputFilesContent(FilesContent):
"""A newtype wrapper for FilesContent.
TODO(7710): This class is currently necessary because the engine
otherwise finds a cycle between FilesContent <=> DirectoryDigest.
"""
@frozen_after_init
@dataclass(unsafe_hash=True)
class PathGlobs:
"""A wrapper around sets of filespecs to include and exclude.
The syntax supported is roughly git's glob syntax.
def test_collection_bool() -> None:
assert bool(Collection([0])) is True
assert bool(Collection([])) is False
class FileContent(
datatype([('path', str), ('content', bytes),
('is_executable', bool)])):
"""The content of a file."""
def __repr__(self):
return 'FileContent(path={}, content=(len:{}), is_executable={})'.format(
self.path,
len(self.content),
self.is_executable,
)
def __str__(self):
return repr(self)
FilesContent = Collection.of(FileContent)
class InputFilesContent(FilesContent):
"""A newtype wrapper for FilesContent.
TODO(7710): This class is currently necessary because the engine
otherwise finds a cycle between FilesContent <=> DirectoryDigest.
"""
class PathGlobs(
datatype([
'include',
'exclude',
('glob_match_error_behavior', GlobMatchErrorBehavior),
('conjunction', GlobExpansionConjunction),
)
@memoized_method
def get_multi_matcher(self):
return MultiMatcher(self.get_options().config)
@dataclass(frozen=True)
class RegexMatchResult:
"""The result of running regex matches on a source file."""
path: str
matching: Tuple
nonmatching: Tuple
RegexMatchResults = Collection.of(RegexMatchResult)
class Matcher:
"""Class to match a single (possibly inverted) regex.
Matches are allowed anywhere in the string (so really a "search" in the Python regex parlance).
To anchor a match at the beginning of a string, use the ^ anchor. To anchor at the beginning
of any line, use the ^ anchor along with the MULTILINE directive (?m). See test for examples.
"""
def __init__(self, pattern, inverted=False):
self.compiled_regex = re.compile(pattern)
self.inverted = inverted
def matches(self, s):
"""Whether the pattern matches anywhere in the string s."""
class AllSourceRoots(Collection.of(SourceRoot)):
pass
# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
import inspect
from collections.abc import MutableMapping, MutableSequence
from functools import update_wrapper
from pants.build_graph.address import Address, BuildFileAddress
from pants.engine.objects import Collection, Resolvable, Serializable
from pants.util.objects import TypeConstraintError
Addresses = Collection.of(Address)
class BuildFileAddresses(Collection.of(BuildFileAddress)):
@property
def addresses(self):
"""Converts the BuildFileAddress objects in this collection to Address objects."""
return [bfa.to_address() for bfa in self]
class NotSerializableError(TypeError):
"""Indicates an addressable descriptor is illegally installed in a non-Serializable type."""
class MutationError(AttributeError):
"""Indicates an illegal attempt to mutate an addressable attribute that already has a value."""
class AddressableTypeValidationError(TypeConstraintError):
class DirectoryWithPrefixToStrip(datatype([('directory_digest', Digest), ('prefix', text_type)])):
pass
class DirectoryToMaterialize(datatype([('path', text_type), ('directory_digest', Digest)])):
"""A request to materialize the contents of a directory digest at the provided path."""
pass
class UrlToFetch(datatype([('url', text_type), ('digest', Digest)])):
pass
FilesContent = Collection.of(FileContent)
# TODO: don't recreate this in python, get this from fs::EMPTY_DIGEST somehow.
_EMPTY_FINGERPRINT = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
EMPTY_DIRECTORY_DIGEST = Digest(
fingerprint=text_type(_EMPTY_FINGERPRINT),
serialized_bytes_length=0
)
EMPTY_SNAPSHOT = Snapshot(
directory_digest=EMPTY_DIRECTORY_DIGEST,
files=(),
dirs=()
def test_collection_iteration(self): self.assertEqual([1, 2], [x for x in Collection.of(int)([1, 2])])
class HydratedTargets(Collection.of(HydratedTarget)): """An intransitive set of HydratedTarget objects."""
def test_collection_bool(self): self.assertTrue(bool(Collection.of(int)([0]))) self.assertFalse(bool(Collection.of(int)([])))
def test_collection_bool(self) -> None:
self.assertTrue(bool(Collection([0])))
self.assertFalse(bool(Collection([])))
