def __init__(self,
python_setup,
python_repos,
ivy_bootstrapper,
thrift_binary_factory,
interpreter,
builder,
targets,
platforms,
extra_requirements=None):
self._python_setup = python_setup
self._python_repos = python_repos
self._ivy_bootstrapper = ivy_bootstrapper
self._thrift_binary_factory = thrift_binary_factory
self._interpreter = interpreter
self._builder = builder
self._targets = targets
self._platforms = platforms
self._extra_requirements = list(
extra_requirements) if extra_requirements else []
# Note: unrelated to the general pants artifact cache.
self._artifact_cache_root = os.path.join(
self._python_setup.artifact_cache_dir,
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator(self._artifact_cache_root)
def __init__(self,
target,
root_dir,
extra_targets=None,
extra_requirements=None,
builder=None,
platforms=None,
interpreter=None,
conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._extra_targets = list(
extra_targets) if extra_targets is not None else []
self._extra_requirements = list(
extra_requirements) if extra_requirements is not None else []
self._builder = builder or PEXBuilder(tempfile.mkdtemp(),
interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
PythonSetup(self._config).scratch_dir('artifact_cache',
default_name='artifacts'),
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator(self._egg_cache_root)
def __init__(self, context, workdir):
"""Subclass __init__ methods, if defined, *must* follow this idiom:
class MyTask(Task):
def __init__(self, *args, **kwargs):
super(MyTask, self).__init__(*args, **kwargs)
...
This allows us to change Task.__init__()'s arguments without changing every subclass.
"""
self.context = context
self._workdir = workdir
# TODO: It would be nice to use self.get_options().cache_key_gen_version here, because then
# we could have a separate value for each scope if we really wanted to. However we can't
# access per-task options in Task.__init__ because GroupTask.__init__ calls it with the
# group task's scope, which isn't currently in the known scopes we generate options for.
self._cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version)
self._read_artifact_cache_spec = None
self._write_artifact_cache_spec = None
self._artifact_cache = None
self._artifact_cache_setup_lock = threading.Lock()
self._cache_key_errors = set()
self._build_invalidator_dir = os.path.join(
self.context.options.for_global_scope().pants_workdir,
'build_invalidator', self.__class__.__name__)
def __init__(self,
python_setup,
python_repos,
ivy_bootstrapper,
thrift_binary_factory,
interpreter,
builder,
targets,
platforms,
extra_requirements=None,
log=None):
self._python_setup = python_setup
self._python_repos = python_repos
self._ivy_bootstrapper = ivy_bootstrapper
self._thrift_binary_factory = thrift_binary_factory
self._interpreter = interpreter
self._builder = builder
self._targets = targets
self._platforms = platforms
self._extra_requirements = list(extra_requirements) if extra_requirements else []
self._logger = log or logger
# Note: unrelated to the general pants artifact cache.
self._artifact_cache_root = os.path.join(
self._python_setup.artifact_cache_dir, str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator(self._artifact_cache_root)
def __init__(self, context, workdir):
"""Subclass __init__ methods, if defined, *must* follow this idiom:
class MyTask(Task):
def __init__(self, *args, **kwargs):
super(MyTask, self).__init__(*args, **kwargs)
...
This allows us to change Task.__init__()'s arguments without changing every subclass.
"""
self.context = context
self._workdir = workdir
self._cache_key_generator = CacheKeyGenerator(
context.config.getdefault('cache_key_gen_version', default='200'))
self._read_artifact_cache_spec = None
self._write_artifact_cache_spec = None
self._artifact_cache = None
self._artifact_cache_setup_lock = threading.Lock()
self._cache_key_errors = set()
default_invalidator_root = os.path.join(
self.context.options.for_global_scope().pants_workdir, 'build_invalidator')
suffix_type = self.__class__.__name__
self._build_invalidator_dir = os.path.join(
context.config.get('tasks', 'build_invalidator', default=default_invalidator_root),
suffix_type)
def __init__(self,
target,
root_dir,
extra_targets=None,
extra_requirements=None,
builder=None,
platforms=None,
interpreter=None,
conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._extra_targets = list(extra_targets) if extra_targets is not None else []
self._extra_requirements = list(extra_requirements) if extra_requirements is not None else []
self._builder = builder or PEXBuilder(tempfile.mkdtemp(), interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
PythonSetup(self._config).scratch_dir('artifact_cache', default_name='artifacts'),
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator( self._egg_cache_root)
def __init__(self, context, workdir):
"""Subclass __init__ methods, if defined, *must* follow this idiom:
class MyTask(Task):
def __init__(self, *args, **kwargs):
super(MyTask, self).__init__(*args, **kwargs)
...
This allows us to change Task.__init__()'s arguments without
changing every subclass. If the subclass does not need its own
initialization, this method can (and should) be omitted entirely.
"""
super(TaskBase, self).__init__()
self.context = context
self._workdir = workdir
# TODO: It would be nice to use self.get_options().cache_key_gen_version here, because then
# we could have a separate value for each scope if we really wanted to. However we can't
# access per-task options in Task.__init__ because GroupTask.__init__ calls it with the
# group task's scope, which isn't currently in the known scopes we generate options for.
self._cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version)
self._cache_key_errors = set()
self._build_invalidator_dir = os.path.join(
self.context.options.for_global_scope().pants_workdir,
'build_invalidator',
self.stable_name())
self._cache_factory = CacheSetup.create_cache_factory_for_task(self)
self._options_fingerprinter = OptionsFingerprinter(self.context.build_graph)
self._fingerprint = None
def __init__(self,
context,
targets,
extra_requirements=None,
builder=None,
platforms=None,
interpreter=None):
self.context = context
# TODO: These should come from the caller, and we should not know about config.
self._python_setup = PythonSetup(self.context.config)
self._python_repos = PythonRepos(self.context.config)
self._targets = targets
self._extra_requirements = list(extra_requirements) if extra_requirements else []
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._builder = builder or PEXBuilder(os.path.realpath(tempfile.mkdtemp()),
interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
self._python_setup.scratch_dir, 'artifacts', str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator( self._egg_cache_root)
def __init__(self,
context,
python_setup,
python_repos,
targets,
extra_requirements=None,
builder=None,
platforms=None,
interpreter=None):
self.context = context
self._python_setup = python_setup
self._python_repos = python_repos
self._targets = targets
self._extra_requirements = list(extra_requirements) if extra_requirements else []
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._builder = builder or PEXBuilder(os.path.realpath(tempfile.mkdtemp()),
interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
self._python_setup.scratch_dir, 'artifacts', str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator( self._egg_cache_root)
def __init__(self, cache_manager, versioned_targets):
self._cache_manager = cache_manager
self.versioned_targets = versioned_targets
self.targets = [vt.target for vt in versioned_targets]
# The following line is a no-op if cache_key was set in the VersionedTarget __init__ method.
self.cache_key = CacheKeyGenerator.combine_cache_keys([vt.cache_key
for vt in versioned_targets])
self.num_chunking_units = self.cache_key.num_chunking_units
self.valid = not cache_manager.needs_update(self.cache_key)
def __init__(self, cache_manager, versioned_targets):
self._cache_manager = cache_manager
self.versioned_targets = versioned_targets
self.targets = [vt.target for vt in versioned_targets]
# The following line is a no-op if cache_key was set in the VersionedTarget __init__ method.
self.cache_key = CacheKeyGenerator.combine_cache_keys(
[vt.cache_key for vt in versioned_targets])
self.num_chunking_units = self.cache_key.num_chunking_units
self.valid = not cache_manager.needs_update(self.cache_key)
def __init__(self, context, workdir):
self.context = context
self._workdir = workdir
self._cache_key_generator = CacheKeyGenerator(
context.config.getdefault('cache_key_gen_version', default=None))
self._read_artifact_cache_spec = None
self._write_artifact_cache_spec = None
self._artifact_cache = None
self._artifact_cache_setup_lock = threading.Lock()
default_invalidator_root = os.path.join(self.context.config.getdefault('pants_workdir'),
'build_invalidator')
self._build_invalidator_dir = os.path.join(
context.config.get('tasks', 'build_invalidator', default=default_invalidator_root),
self.product_type())
def __init__(self,
context,
python_setup,
python_repos,
interpreter,
builder,
targets,
platforms,
extra_requirements=None):
self.context = context
self._python_setup = python_setup
self._python_repos = python_repos
self._interpreter = interpreter
self._builder = builder
self._targets = targets
self._platforms = platforms
self._extra_requirements = list(extra_requirements) if extra_requirements else []
# Note: unrelated to the general pants artifact cache.
self._artifact_cache_root = os.path.join(
self._python_setup.artifact_cache_dir, str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator(self._artifact_cache_root)
class PythonChroot(object):
_VALID_DEPENDENCIES = {
PrepCommand: 'prep',
PythonLibrary: 'libraries',
PythonRequirementLibrary: 'reqs',
PythonBinary: 'binaries',
PythonThriftLibrary: 'thrifts',
PythonAntlrLibrary: 'antlrs',
PythonTests: 'tests'
}
MEMOIZED_THRIFTS = {}
class InvalidDependencyException(Exception):
def __init__(self, target):
Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target)
def __init__(self,
context,
targets,
extra_requirements=None,
builder=None,
platforms=None,
interpreter=None):
self.context = context
self._targets = targets
self._extra_requirements = list(extra_requirements) if extra_requirements else []
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._builder = builder or PEXBuilder(os.path.realpath(tempfile.mkdtemp()),
interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
PythonSetup(self.context.config).scratch_dir('artifact_cache', default_name='artifacts'),
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator( self._egg_cache_root)
def delete(self):
"""Deletes this chroot from disk if it has been dumped."""
safe_rmtree(self.path())
def __del__(self):
if os.getenv('PANTS_LEAVE_CHROOT') is None:
self.delete()
else:
self.debug('Left chroot at %s' % self.path())
@property
def builder(self):
return self._builder
def debug(self, msg, indent=0):
if os.getenv('PANTS_VERBOSE') is not None:
print('%s%s' % (' ' * indent, msg))
def path(self):
return os.path.realpath(self._builder.path())
def _dump_library(self, library):
def copy_to_chroot(base, path, add_function):
src = os.path.join(get_buildroot(), base, path)
add_function(src, path)
self.debug(' Dumping library: %s' % library)
for relpath in library.sources_relative_to_source_root():
try:
copy_to_chroot(library.target_base, relpath, self._builder.add_source)
except OSError as e:
logger.error("Failed to copy {path} for library {library}"
.format(path=os.path.join(library.target_base, relpath),
library=library))
raise
for resources_tgt in library.resources:
for resource_file_from_source_root in resources_tgt.sources_relative_to_source_root():
try:
copy_to_chroot(resources_tgt.target_base, resource_file_from_source_root,
self._builder.add_resource)
except OSError as e:
logger.error("Failed to copy {path} for resource {resource}"
.format(path=os.path.join(resources_tgt.target_base,
resource_file_from_source_root),
resource=resources_tgt.address.spec))
raise
def _dump_requirement(self, req):
self.debug(' Dumping requirement: %s' % req)
self._builder.add_requirement(req)
def _dump_distribution(self, dist):
self.debug(' Dumping distribution: .../%s' % os.path.basename(dist.location))
self._builder.add_distribution(dist)
def _generate_requirement(self, library, builder_cls):
library_key = self._key_generator.key_for_target(library)
builder = builder_cls(library, get_buildroot(),
self.context.options, '-' + library_key.hash[:8])
cache_dir = os.path.join(self._egg_cache_root, library_key.id)
if self._build_invalidator.needs_update(library_key):
sdist = builder.build(interpreter=self._interpreter)
safe_mkdir(cache_dir)
shutil.copy(sdist, os.path.join(cache_dir, os.path.basename(sdist)))
self._build_invalidator.update(library_key)
return PythonRequirement(builder.requirement_string(), repository=cache_dir, use_2to3=True)
def _generate_thrift_requirement(self, library):
return self._generate_requirement(library, PythonThriftBuilder)
def _generate_antlr_requirement(self, library):
return self._generate_requirement(library, PythonAntlrBuilder)
def resolve(self, targets):
children = defaultdict(OrderedSet)
def add_dep(trg):
for target_type, target_key in self._VALID_DEPENDENCIES.items():
if isinstance(trg, target_type):
children[target_key].add(trg)
return
elif isinstance(trg, Dependencies):
return
raise self.InvalidDependencyException(trg)
for target in targets:
target.walk(add_dep)
return children
def dump(self):
self.debug('Building chroot for %s:' % self._targets)
targets = self.resolve(self._targets)
for lib in targets['libraries'] | targets['binaries']:
self._dump_library(lib)
generated_reqs = OrderedSet()
if targets['thrifts']:
for thr in set(targets['thrifts']):
if thr not in self.MEMOIZED_THRIFTS:
self.MEMOIZED_THRIFTS[thr] = self._generate_thrift_requirement(thr)
generated_reqs.add(self.MEMOIZED_THRIFTS[thr])
generated_reqs.add(PythonRequirement('thrift', use_2to3=True))
for antlr in targets['antlrs']:
generated_reqs.add(self._generate_antlr_requirement(antlr))
reqs_from_libraries = OrderedSet()
for req_lib in targets['reqs']:
for req in req_lib.payload.requirements:
reqs_from_libraries.add(req)
reqs_to_build = OrderedSet()
find_links = []
for req in reqs_from_libraries | generated_reqs | self._extra_requirements:
if not req.should_build(self._interpreter.python, Platform.current()):
self.debug('Skipping %s based upon version filter' % req)
continue
reqs_to_build.add(req)
self._dump_requirement(req.requirement)
if req.repository:
find_links.append(req.repository)
distributions = resolve_multi(
self.context.config,
reqs_to_build,
interpreter=self._interpreter,
platforms=self._platforms,
ttl=self.context.options.for_global_scope().python_chroot_requirements_ttl,
find_links=find_links)
locations = set()
for platform, dist_set in distributions.items():
for dist in dist_set:
if dist.location not in locations:
self._dump_distribution(dist)
locations.add(dist.location)
if len(targets['binaries']) > 1:
print('WARNING: Target has multiple python_binary targets!', file=sys.stderr)
return self._builder
class PythonChroot(object):
_VALID_DEPENDENCIES = {
PythonLibrary: 'libraries',
PythonRequirement: 'reqs',
PythonBinary: 'binaries',
PythonThriftLibrary: 'thrifts',
PythonAntlrLibrary: 'antlrs',
PythonTests: 'tests'
}
MEMOIZED_THRIFTS = {}
class InvalidDependencyException(Exception):
def __init__(self, target):
Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target)
def __init__(self,
target,
root_dir,
extra_targets=None,
builder=None,
platforms=None,
interpreter=None,
conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._extra_targets = list(extra_targets) if extra_targets is not None else []
self._builder = builder or PEXBuilder(tempfile.mkdtemp(), interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
PythonSetup(self._config).scratch_dir('artifact_cache', default_name='artifacts'),
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator( self._egg_cache_root)
def __del__(self):
if os.getenv('PANTS_LEAVE_CHROOT') is None:
safe_rmtree(self.path())
else:
self.debug('Left chroot at %s' % self.path())
@property
def builder(self):
return self._builder
def debug(self, msg, indent=0):
if os.getenv('PANTS_VERBOSE') is not None:
print('%s%s' % (' ' * indent, msg))
def path(self):
return self._builder.path()
def _dump_library(self, library):
def copy_to_chroot(base, path, add_function):
src = os.path.join(self._root, base, path)
add_function(src, path)
self.debug(' Dumping library: %s' % library)
for filename in library.sources:
copy_to_chroot(library.target_base, filename, self._builder.add_source)
for filename in library.resources:
copy_to_chroot(library.target_base, filename, self._builder.add_resource)
def _dump_requirement(self, req, dynamic, repo):
self.debug(' Dumping requirement: %s%s%s' % (str(req),
' (dynamic)' if dynamic else '', ' (repo: %s)' % repo if repo else ''))
self._builder.add_requirement(req, dynamic, repo)
def _dump_distribution(self, dist):
self.debug(' Dumping distribution: .../%s' % os.path.basename(dist.location))
self._builder.add_distribution(dist)
def _generate_requirement(self, library, builder_cls):
library_key = self._key_generator.key_for_target(library)
builder = builder_cls(library, self._root, self._config, '-' + library_key.hash[:8])
cache_dir = os.path.join(self._egg_cache_root, library_key.id)
if self._build_invalidator.needs_update(library_key):
sdist = builder.build(interpreter=self._interpreter)
safe_mkdir(cache_dir)
shutil.copy(sdist, os.path.join(cache_dir, os.path.basename(sdist)))
self._build_invalidator.update(library_key)
with ParseContext.temp():
return PythonRequirement(builder.requirement_string(), repository=cache_dir, use_2to3=True)
def _generate_thrift_requirement(self, library):
return self._generate_requirement(library, PythonThriftBuilder)
def _generate_antlr_requirement(self, library):
return self._generate_requirement(library, PythonAntlrBuilder)
def resolve(self, targets):
children = defaultdict(OrderedSet)
def add_dep(trg):
for target_type, target_key in self._VALID_DEPENDENCIES.items():
if isinstance(trg, target_type):
children[target_key].add(trg)
return
raise self.InvalidDependencyException(trg)
for target in targets:
target.walk(add_dep)
return children
def dump(self):
self.debug('Building PythonBinary %s:' % self._target)
targets = self.resolve([self._target] + self._extra_targets)
for lib in targets['libraries'] | targets['binaries']:
self._dump_library(lib)
generated_reqs = OrderedSet()
if targets['thrifts']:
for thr in set(targets['thrifts']):
if thr not in self.MEMOIZED_THRIFTS:
self.MEMOIZED_THRIFTS[thr] = self._generate_thrift_requirement(thr)
generated_reqs.add(self.MEMOIZED_THRIFTS[thr])
with ParseContext.temp():
# trick pants into letting us add this python requirement, otherwise we get
# TargetDefinitionException: Error in target BUILD.temp:thrift: duplicate to
# PythonRequirement(thrift)
#
# TODO(wickman) Instead of just blindly adding a PythonRequirement for thrift, we
# should first detect if any explicit thrift requirements have been added and use
# those. Only if they have not been supplied should we auto-inject it.
generated_reqs.add(PythonRequirement('thrift', use_2to3=True,
name='thrift-' + ''.join(random.sample('0123456789abcdef' * 8, 8))))
for antlr in targets['antlrs']:
generated_reqs.add(self._generate_antlr_requirement(antlr))
targets['reqs'] |= generated_reqs
reqs_to_build = OrderedSet()
for req in targets['reqs']:
if not req.should_build(self._interpreter.python, Platform.current()):
self.debug('Skipping %s based upon version filter' % req)
continue
reqs_to_build.add(req)
self._dump_requirement(req._requirement, False, req._repository)
platforms = self._platforms
if isinstance(self._target, PythonBinary):
platforms = self._target.platforms
distributions = resolve_multi(
self._config,
reqs_to_build,
interpreter=self._interpreter,
platforms=platforms)
locations = set()
for platform, dist_set in distributions.items():
for dist in dist_set:
if dist.location not in locations:
self._dump_distribution(dist)
locations.add(dist.location)
if len(targets['binaries']) > 1:
print('WARNING: Target has multiple python_binary targets!', file=sys.stderr)
return self._builder
class PythonChroot(object):
_VALID_DEPENDENCIES = {
PythonLibrary: 'libraries',
PythonRequirementLibrary: 'reqs',
PythonBinary: 'binaries',
PythonThriftLibrary: 'thrifts',
PythonAntlrLibrary: 'antlrs',
PythonTests: 'tests'
}
MEMOIZED_THRIFTS = {}
class InvalidDependencyException(Exception):
def __init__(self, target):
Exception.__init__(
self, "Not a valid Python dependency! Found: %s" % target)
def __init__(self,
target,
root_dir,
extra_targets=None,
extra_requirements=None,
builder=None,
platforms=None,
interpreter=None,
conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._extra_targets = list(
extra_targets) if extra_targets is not None else []
self._extra_requirements = list(
extra_requirements) if extra_requirements is not None else []
self._builder = builder or PEXBuilder(tempfile.mkdtemp(),
interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
PythonSetup(self._config).scratch_dir('artifact_cache',
default_name='artifacts'),
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator(self._egg_cache_root)
def __del__(self):
if os.getenv('PANTS_LEAVE_CHROOT') is None:
safe_rmtree(self.path())
else:
self.debug('Left chroot at %s' % self.path())
@property
def builder(self):
return self._builder
def debug(self, msg, indent=0):
if os.getenv('PANTS_VERBOSE') is not None:
print('%s%s' % (' ' * indent, msg))
def path(self):
return self._builder.path()
def _dump_library(self, library):
def copy_to_chroot(base, path, add_function):
src = os.path.join(self._root, base, path)
add_function(src, path)
self.debug(' Dumping library: %s' % library)
for relpath in library.sources_relative_to_source_root():
copy_to_chroot(library.target_base, relpath,
self._builder.add_source)
for resources_tgt in library.resources:
for resource_file_from_source_root in resources_tgt.sources_relative_to_source_root(
):
copy_to_chroot(resources_tgt.target_base,
resource_file_from_source_root,
self._builder.add_resource)
def _dump_requirement(self, req, dynamic, repo):
self.debug(' Dumping requirement: %s%s%s' %
(str(req), ' (dynamic)' if dynamic else '',
' (repo: %s)' % repo if repo else ''))
self._builder.add_requirement(req, dynamic, repo)
def _dump_distribution(self, dist):
self.debug(' Dumping distribution: .../%s' %
os.path.basename(dist.location))
self._builder.add_distribution(dist)
def _generate_requirement(self, library, builder_cls):
library_key = self._key_generator.key_for_target(library)
builder = builder_cls(library, self._root, self._config,
'-' + library_key.hash[:8])
cache_dir = os.path.join(self._egg_cache_root, library_key.id)
if self._build_invalidator.needs_update(library_key):
sdist = builder.build(interpreter=self._interpreter)
safe_mkdir(cache_dir)
shutil.copy(sdist, os.path.join(cache_dir,
os.path.basename(sdist)))
self._build_invalidator.update(library_key)
return PythonRequirement(builder.requirement_string(),
repository=cache_dir,
use_2to3=True)
def _generate_thrift_requirement(self, library):
return self._generate_requirement(library, PythonThriftBuilder)
def _generate_antlr_requirement(self, library):
return self._generate_requirement(library, PythonAntlrBuilder)
def resolve(self, targets):
children = defaultdict(OrderedSet)
def add_dep(trg):
for target_type, target_key in self._VALID_DEPENDENCIES.items():
if isinstance(trg, target_type):
children[target_key].add(trg)
return
elif isinstance(trg, Dependencies):
return
raise self.InvalidDependencyException(trg)
for target in targets:
target.walk(add_dep)
return children
def dump(self):
self.debug('Building PythonBinary %s:' % self._target)
targets = self.resolve([self._target] + self._extra_targets)
for lib in targets['libraries'] | targets['binaries']:
self._dump_library(lib)
generated_reqs = OrderedSet()
if targets['thrifts']:
for thr in set(targets['thrifts']):
if thr not in self.MEMOIZED_THRIFTS:
self.MEMOIZED_THRIFTS[
thr] = self._generate_thrift_requirement(thr)
generated_reqs.add(self.MEMOIZED_THRIFTS[thr])
generated_reqs.add(PythonRequirement('thrift', use_2to3=True))
for antlr in targets['antlrs']:
generated_reqs.add(self._generate_antlr_requirement(antlr))
reqs_from_libraries = OrderedSet()
for req_lib in targets['reqs']:
for req in req_lib.payload.requirements:
reqs_from_libraries.add(req)
reqs_to_build = OrderedSet()
for req in reqs_from_libraries | generated_reqs | self._extra_requirements:
if not req.should_build(self._interpreter.python,
Platform.current()):
self.debug('Skipping %s based upon version filter' % req)
continue
reqs_to_build.add(req)
self._dump_requirement(req._requirement, False, req._repository)
platforms = self._platforms
if isinstance(self._target, PythonBinary):
platforms = self._target.platforms
distributions = resolve_multi(self._config,
reqs_to_build,
interpreter=self._interpreter,
platforms=platforms)
locations = set()
for platform, dist_set in distributions.items():
for dist in dist_set:
if dist.location not in locations:
self._dump_distribution(dist)
locations.add(dist.location)
if len(targets['binaries']) > 1:
print('WARNING: Target has multiple python_binary targets!',
file=sys.stderr)
return self._builder
def test_env(content=TEST_CONTENT):
with temporary_dir() as d:
with tempfile.NamedTemporaryFile() as f:
f.write(content)
f.flush()
yield f, CacheKeyGenerator(), BuildInvalidator(d)
class PythonChroot(object):
_VALID_DEPENDENCIES = {
PrepCommand: 'prep',
PythonLibrary: 'libraries',
PythonRequirementLibrary: 'reqs',
PythonBinary: 'binaries',
PythonThriftLibrary: 'thrifts',
PythonAntlrLibrary: 'antlrs',
PythonTests: 'tests'
}
MEMOIZED_THRIFTS = {}
class InvalidDependencyException(Exception):
def __init__(self, target):
Exception.__init__(
self,
"Not a valid Python dependency! Found: {}".format(target))
@staticmethod
def get_platforms(platform_list):
return tuple({
Platform.current() if p == 'current' else p
for p in platform_list
})
def __init__(self,
python_setup,
python_repos,
ivy_bootstrapper,
thrift_binary_factory,
interpreter,
builder,
targets,
platforms,
extra_requirements=None):
self._python_setup = python_setup
self._python_repos = python_repos
self._ivy_bootstrapper = ivy_bootstrapper
self._thrift_binary_factory = thrift_binary_factory
self._interpreter = interpreter
self._builder = builder
self._targets = targets
self._platforms = platforms
self._extra_requirements = list(
extra_requirements) if extra_requirements else []
# Note: unrelated to the general pants artifact cache.
self._artifact_cache_root = os.path.join(
self._python_setup.artifact_cache_dir,
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator(self._artifact_cache_root)
def delete(self):
"""Deletes this chroot from disk if it has been dumped."""
safe_rmtree(self.path())
def debug(self, msg, indent=0):
if os.getenv('PANTS_VERBOSE') is not None:
print('{}{}'.format(' ' * indent, msg))
def path(self):
return os.path.realpath(self._builder.path())
def pex(self):
return PEX(self.path(), interpreter=self._interpreter)
def package_pex(self, filename):
"""Package into a PEX zipfile.
:param filename: The filename where the PEX should be stored.
"""
self._builder.build(filename)
def _dump_library(self, library):
def copy_to_chroot(base, path, add_function):
src = os.path.join(get_buildroot(), base, path)
add_function(src, path)
self.debug(' Dumping library: {}'.format(library))
for relpath in library.sources_relative_to_source_root():
try:
copy_to_chroot(library.target_base, relpath,
self._builder.add_source)
except OSError:
logger.error(
"Failed to copy {path} for library {library}".format(
path=os.path.join(library.target_base, relpath),
library=library))
raise
for resources_tgt in library.resources:
for resource_file_from_source_root in resources_tgt.sources_relative_to_source_root(
):
try:
copy_to_chroot(resources_tgt.target_base,
resource_file_from_source_root,
self._builder.add_resource)
except OSError:
logger.error(
"Failed to copy {path} for resource {resource}".format(
path=os.path.join(resources_tgt.target_base,
resource_file_from_source_root),
resource=resources_tgt.address.spec))
raise
def _dump_requirement(self, req):
self.debug(' Dumping requirement: {}'.format(req))
self._builder.add_requirement(req)
def _dump_distribution(self, dist):
self.debug(' Dumping distribution: .../{}'.format(
os.path.basename(dist.location)))
self._builder.add_distribution(dist)
def _generate_requirement(self, library, builder_cls):
library_key = self._key_generator.key_for_target(library)
builder = builder_cls(target=library,
root_dir=get_buildroot(),
target_suffix='-' + library_key.hash[:8])
cache_dir = os.path.join(self._artifact_cache_root, library_key.id)
if self._build_invalidator.needs_update(library_key):
sdist = builder.build(interpreter=self._interpreter)
safe_mkdir(cache_dir)
shutil.copy(sdist, os.path.join(cache_dir,
os.path.basename(sdist)))
self._build_invalidator.update(library_key)
return PythonRequirement(builder.requirement_string(),
repository=cache_dir,
use_2to3=True)
def _generate_thrift_requirement(self, library):
thrift_builder = functools.partial(
PythonThriftBuilder,
thrift_binary_factory=self._thrift_binary_factory,
workdir=safe_mkdtemp(dir=self.path(), prefix='thrift.'))
return self._generate_requirement(library, thrift_builder)
def _generate_antlr_requirement(self, library):
antlr_builder = functools.partial(
PythonAntlrBuilder,
ivy_bootstrapper=self._ivy_bootstrapper,
workdir=safe_mkdtemp(dir=self.path(), prefix='antlr.'))
return self._generate_requirement(library, antlr_builder)
def resolve(self, targets):
children = defaultdict(OrderedSet)
def add_dep(trg):
# Currently we handle all of our code generation, so we don't want to operate over any
# synthetic targets injected upstream.
# TODO(John Sirois): Revisit this when building a proper python product pipeline.
if trg.is_synthetic:
return
for target_type, target_key in self._VALID_DEPENDENCIES.items():
if isinstance(trg, target_type):
children[target_key].add(trg)
return
elif isinstance(trg, Dependencies):
return
raise self.InvalidDependencyException(trg)
for target in targets:
target.walk(add_dep)
return children
def dump(self):
self.debug('Building chroot for {}:'.format(self._targets))
targets = self.resolve(self._targets)
for lib in targets['libraries'] | targets['binaries']:
self._dump_library(lib)
generated_reqs = OrderedSet()
if targets['thrifts']:
for thr in set(targets['thrifts']):
if thr not in self.MEMOIZED_THRIFTS:
self.MEMOIZED_THRIFTS[
thr] = self._generate_thrift_requirement(thr)
generated_reqs.add(self.MEMOIZED_THRIFTS[thr])
generated_reqs.add(PythonRequirement('thrift', use_2to3=True))
for antlr in targets['antlrs']:
generated_reqs.add(self._generate_antlr_requirement(antlr))
reqs_from_libraries = OrderedSet()
for req_lib in targets['reqs']:
for req in req_lib.payload.requirements:
reqs_from_libraries.add(req)
reqs_to_build = OrderedSet()
find_links = OrderedSet()
for req in reqs_from_libraries | generated_reqs | self._extra_requirements:
if not req.should_build(self._interpreter.python,
Platform.current()):
self.debug('Skipping {} based upon version filter'.format(req))
continue
reqs_to_build.add(req)
self._dump_requirement(req.requirement)
if req.repository:
find_links.add(req.repository)
distributions = self._resolve_multi(reqs_to_build, find_links)
locations = set()
for platform, dist_set in distributions.items():
for dist in dist_set:
if dist.location not in locations:
self._dump_distribution(dist)
locations.add(dist.location)
if len(targets['binaries']) > 1:
print('WARNING: Target has multiple python_binary targets!',
file=sys.stderr)
return self._builder
def _resolve_multi(self, requirements, find_links):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
that must be included in order to satisfy them. That may involve distributions for
multiple platforms.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param find_links: Additional paths to search for source packages during resolution.
"""
distributions = dict()
platforms = self.get_platforms(self._platforms
or self._python_setup.platforms)
fetchers = self._python_repos.get_fetchers()
fetchers.extend(Fetcher([path]) for path in find_links)
context = self._python_repos.get_network_context()
for platform in platforms:
distributions[platform] = resolve(
requirements=[req.requirement for req in requirements],
interpreter=self._interpreter,
fetchers=fetchers,
platform=platform,
context=context,
cache=self._python_setup.resolver_cache_dir,
cache_ttl=self._python_setup.resolver_cache_ttl)
return distributions
class PythonChroot(object):
_VALID_DEPENDENCIES = {
PrepCommand: 'prep',
PythonLibrary: 'libraries',
PythonRequirementLibrary: 'reqs',
PythonBinary: 'binaries',
PythonThriftLibrary: 'thrifts',
PythonAntlrLibrary: 'antlrs',
PythonTests: 'tests'
}
class InvalidDependencyException(Exception):
def __init__(self, target):
Exception.__init__(self, "Not a valid Python dependency! Found: {}".format(target))
@staticmethod
def get_platforms(platform_list):
return tuple({Platform.current() if p == 'current' else p for p in platform_list})
def __init__(self,
python_setup,
python_repos,
ivy_bootstrapper,
thrift_binary_factory,
interpreter,
builder,
targets,
platforms,
extra_requirements=None,
log=None):
self._python_setup = python_setup
self._python_repos = python_repos
self._ivy_bootstrapper = ivy_bootstrapper
self._thrift_binary_factory = thrift_binary_factory
self._interpreter = interpreter
self._builder = builder
self._targets = targets
self._platforms = platforms
self._extra_requirements = list(extra_requirements) if extra_requirements else []
self._logger = log or logger
# Note: unrelated to the general pants artifact cache.
self._artifact_cache_root = os.path.join(
self._python_setup.artifact_cache_dir, str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator(self._artifact_cache_root)
def delete(self):
"""Deletes this chroot from disk if it has been dumped."""
safe_rmtree(self.path())
def debug(self, msg):
self._logger.debug(msg)
def path(self):
return os.path.realpath(self._builder.path())
def pex(self):
return PEX(self.path(), interpreter=self._interpreter)
def package_pex(self, filename):
"""Package into a PEX zipfile.
:param filename: The filename where the PEX should be stored.
"""
self._builder.build(filename)
def _dump_library(self, library):
def copy_to_chroot(base, path, add_function):
src = os.path.join(get_buildroot(), base, path)
add_function(src, path)
self.debug(' Dumping library: {}'.format(library))
for relpath in library.sources_relative_to_source_root():
try:
copy_to_chroot(library.target_base, relpath, self._builder.add_source)
except OSError:
logger.error("Failed to copy {path} for library {library}"
.format(path=os.path.join(library.target_base, relpath),
library=library))
raise
for resources_tgt in library.resources:
for resource_file_from_source_root in resources_tgt.sources_relative_to_source_root():
try:
copy_to_chroot(resources_tgt.target_base, resource_file_from_source_root,
self._builder.add_resource)
except OSError:
logger.error("Failed to copy {path} for resource {resource}"
.format(path=os.path.join(resources_tgt.target_base,
resource_file_from_source_root),
resource=resources_tgt.address.spec))
raise
def _dump_requirement(self, req):
self.debug(' Dumping requirement: {}'.format(req))
self._builder.add_requirement(req)
def _dump_distribution(self, dist):
self.debug(' Dumping distribution: .../{}'.format(os.path.basename(dist.location)))
self._builder.add_distribution(dist)
def _generate_requirement(self, library, builder_cls):
library_key = self._key_generator.key_for_target(library)
builder = builder_cls(target=library,
root_dir=get_buildroot(),
target_suffix='-' + library_key.hash[:8])
cache_dir = os.path.join(self._artifact_cache_root, library_key.id)
if self._build_invalidator.needs_update(library_key):
sdist = builder.build(interpreter=self._interpreter)
safe_mkdir(cache_dir)
shutil.copy(sdist, os.path.join(cache_dir, os.path.basename(sdist)))
self._build_invalidator.update(library_key)
return PythonRequirement(builder.requirement_string(), repository=cache_dir, use_2to3=True)
def _generate_thrift_requirement(self, library):
thrift_builder = functools.partial(PythonThriftBuilder,
thrift_binary_factory=self._thrift_binary_factory,
workdir=safe_mkdtemp(dir=self.path(), prefix='thrift.'))
return self._generate_requirement(library, thrift_builder)
def _generate_antlr_requirement(self, library):
antlr_builder = functools.partial(PythonAntlrBuilder,
ivy_bootstrapper=self._ivy_bootstrapper,
workdir=safe_mkdtemp(dir=self.path(), prefix='antlr.'))
return self._generate_requirement(library, antlr_builder)
def resolve(self, targets):
children = defaultdict(OrderedSet)
def add_dep(trg):
# Currently we handle all of our code generation, so we don't want to operate over any
# synthetic targets injected upstream.
# TODO(John Sirois): Revisit this when building a proper python product pipeline.
if trg.is_synthetic:
return
for target_type, target_key in self._VALID_DEPENDENCIES.items():
if isinstance(trg, target_type):
children[target_key].add(trg)
return
elif isinstance(trg, Dependencies):
return
raise self.InvalidDependencyException(trg)
for target in targets:
target.walk(add_dep)
return children
def dump(self):
self.debug('Building chroot for {}:'.format(self._targets))
targets = self.resolve(self._targets)
for lib in targets['libraries'] | targets['binaries']:
self._dump_library(lib)
generated_reqs = OrderedSet()
if targets['thrifts']:
for thr in targets['thrifts']:
generated_reqs.add(self._generate_thrift_requirement(thr))
generated_reqs.add(PythonRequirement('thrift', use_2to3=True))
for antlr in targets['antlrs']:
generated_reqs.add(self._generate_antlr_requirement(antlr))
reqs_from_libraries = OrderedSet()
for req_lib in targets['reqs']:
for req in req_lib.payload.requirements:
reqs_from_libraries.add(req)
reqs_to_build = OrderedSet()
find_links = OrderedSet()
for req in reqs_from_libraries | generated_reqs | self._extra_requirements:
if not req.should_build(self._interpreter.python, Platform.current()):
self.debug('Skipping {} based upon version filter'.format(req))
continue
reqs_to_build.add(req)
self._dump_requirement(req.requirement)
if req.repository:
find_links.add(req.repository)
distributions = self._resolve_multi(reqs_to_build, find_links)
locations = set()
for platform, dist_set in distributions.items():
for dist in dist_set:
if dist.location not in locations:
self._dump_distribution(dist)
locations.add(dist.location)
if len(targets['binaries']) > 1:
print('WARNING: Target has multiple python_binary targets!', file=sys.stderr)
return self._builder
def _resolve_multi(self, requirements, find_links):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
that must be included in order to satisfy them. That may involve distributions for
multiple platforms.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param find_links: Additional paths to search for source packages during resolution.
"""
distributions = dict()
platforms = self.get_platforms(self._platforms or self._python_setup.platforms)
fetchers = self._python_repos.get_fetchers()
fetchers.extend(Fetcher([path]) for path in find_links)
context = self._python_repos.get_network_context()
for platform in platforms:
requirements_cache_dir = os.path.join(self._python_setup.resolver_cache_dir, str(self._interpreter.identity))
distributions[platform] = resolve(
requirements=[req.requirement for req in requirements],
interpreter=self._interpreter,
fetchers=fetchers,
platform=platform,
context=context,
cache=requirements_cache_dir,
cache_ttl=self._python_setup.resolver_cache_ttl)
return distributions