def __init__(self,
target,
root_dir,
extra_targets=None,
builder=None,
platforms=None,
interpreter=None,
conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._extra_targets = list(
extra_targets) if extra_targets is not None else []
self._builder = builder or PEXBuilder(tempfile.mkdtemp(),
interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
PythonSetup(self._config).scratch_dir('artifact_cache',
default_name='artifacts'),
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator(self._egg_cache_root)
def __init__(self,
target,
root_dir,
extra_targets=None,
builder=None,
interpreter=None,
conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._interpreter = interpreter or PythonInterpreter.get()
self._extra_targets = list(
extra_targets) if extra_targets is not None else []
self._resolver = MultiResolver(self._config,
target,
conn_timeout=conn_timeout)
self._builder = builder or PEXBuilder(tempfile.mkdtemp(),
interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
self._config.get('python-setup', 'artifact_cache'),
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator(self._egg_cache_root)
def __init__(self, cache_manager, targets, per_target_cache_keys):
self._cache_manager = cache_manager
self.per_target_cache_keys = per_target_cache_keys
self.targets = targets
self.cache_key = CacheKeyGenerator.combine_cache_keys(per_target_cache_keys)
self.valid = not cache_manager.needs_update(self.cache_key)
def __init__(self, context):
self.context = context
self.dry_run = self.can_dry_run() and self.context.options.dry_run
self._cache_key_generator = CacheKeyGenerator()
self._artifact_cache = None
self._build_invalidator_dir = os.path.join(context.config.get('tasks', 'build_invalidator'),
self.product_type())
def combine_versioned_targets(vts):
targets = []
for vt in vts:
targets.extend(vt.targets)
cache_key = CacheKeyGenerator.combine_cache_keys([vt.cache_key for vt in vts])
valid = all([vt.valid for vt in vts])
return VersionedTargetSet(targets, cache_key, valid)
def combine_versioned_targets(vts):
targets = []
for vt in vts:
targets.extend(vt.targets)
cache_key = CacheKeyGenerator.combine_cache_keys([vt.cache_key for vt in vts])
valid = all([vt.valid for vt in vts])
return VersionedTargetSet(targets, cache_key, valid)
def __init__(self, cache_manager, targets, per_target_cache_keys):
self._cache_manager = cache_manager
self.per_target_cache_keys = per_target_cache_keys
self.targets = targets
self.cache_key = CacheKeyGenerator.combine_cache_keys(
per_target_cache_keys)
self.valid = not cache_manager.needs_update(self.cache_key)
def __init__(self, cache_manager, versioned_targets):
self._cache_manager = cache_manager
self.versioned_targets = versioned_targets
self.targets = [vt.target for vt in versioned_targets]
# The following line is a no-op if cache_key was set in the VersionedTarget __init__ method.
self.cache_key = CacheKeyGenerator.combine_cache_keys([vt.cache_key for vt in versioned_targets])
self.num_sources = self.cache_key.num_sources
self.valid = not cache_manager.needs_update(self.cache_key)
def __init__(self, context):
self.context = context
self._cache_key_generator = CacheKeyGenerator()
# TODO: Shared, remote build cache.
self._artifact_cache = ArtifactCache(
context.config.get('tasks', 'artifact_cache'))
self._build_invalidator_dir = os.path.join(
context.config.get('tasks', 'build_invalidator'),
self.product_type())
def __init__(self, cache_manager, versioned_targets):
self._cache_manager = cache_manager
self.versioned_targets = versioned_targets
self.targets = [vt.target for vt in versioned_targets]
# The following line is a no-op if cache_key was set in the VersionedTarget __init__ method.
self.cache_key = CacheKeyGenerator.combine_cache_keys(
[vt.cache_key for vt in versioned_targets])
self.num_sources = self.cache_key.num_sources
self.valid = not cache_manager.needs_update(self.cache_key)
def __init__(self, target, root_dir, extra_targets=None, builder=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._key_generator = CacheKeyGenerator()
artifact_cache_root = \
os.path.join(self._config.get('python-setup', 'artifact_cache'), '%s' % PythonIdentity.get())
self._artifact_cache = ArtifactCache(artifact_cache_root)
self._extra_targets = list(
extra_targets) if extra_targets is not None else []
self._resolver = PythonResolver([self._target] + self._extra_targets)
self._builder = builder or PEXBuilder(tempfile.mkdtemp())
self._platforms = (Platform.current(), )
self._pythons = (sys.version[:3], )
# TODO(wickman) Should this be in the binary builder?
if isinstance(self._target, PythonBinary):
self._platforms = self._target._platforms
self._pythons = self._target._interpreters
def __init__(self, context):
self.context = context
self.dry_run = self.can_dry_run() and context.options.dry_run
self._cache_key_generator = CacheKeyGenerator(
context.config.getdefault('cache_key_gen_version', default=None))
self._artifact_cache_spec = None
self._artifact_cache = None
self._build_invalidator_dir = os.path.join(
context.config.get('tasks', 'build_invalidator'),
self.product_type())
self._bootstrap_utils = BootstrapUtils(self.context.products)
def __init__(self, target, root_dir, extra_targets=None, builder=None, conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._key_generator = CacheKeyGenerator()
self._extra_targets = list(extra_targets) if extra_targets is not None else []
self._resolver = MultiResolver.from_target(self._config, target, conn_timeout=conn_timeout)
self._builder = builder or PEXBuilder(tempfile.mkdtemp())
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(self._config.get('python-setup', 'artifact_cache'),
'%s' % PythonIdentity.get())
def __init__(self, target, root_dir, extra_targets=None, builder=None, conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._key_generator = CacheKeyGenerator()
self._extra_targets = list(extra_targets) if extra_targets is not None else []
self._resolver = MultiResolver.from_target(self._config, target, conn_timeout=conn_timeout)
self._builder = builder or PEXBuilder(tempfile.mkdtemp())
artifact_cache_root = os.path.join(self._config.get('python-setup', 'artifact_cache'),
'%s' % PythonIdentity.get())
self._artifact_cache = FileBasedArtifactCache(None, self._root, artifact_cache_root,
self._builder.add_dependency_file)
def __init__(self,
target,
root_dir,
extra_targets=None,
builder=None,
conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._key_generator = CacheKeyGenerator()
self._extra_targets = list(
extra_targets) if extra_targets is not None else []
self._resolver = MultiResolver.from_target(self._config,
target,
conn_timeout=conn_timeout)
self._builder = builder or PEXBuilder(tempfile.mkdtemp())
artifact_cache_root = os.path.join(
self._config.get('python-setup', 'artifact_cache'),
'%s' % PythonIdentity.get())
self._artifact_cache = FileBasedArtifactCache(
None, self._root, artifact_cache_root,
self._builder.add_dependency_file)
def __init__(self, context):
self.context = context
self.dry_run = self.can_dry_run() and context.options.dry_run
self._pants_workdir = self.context.config.getdefault('pants_workdir')
self._cache_key_generator = CacheKeyGenerator(
context.config.getdefault('cache_key_gen_version', default=None))
self._read_artifact_cache_spec = None
self._write_artifact_cache_spec = None
self._artifact_cache = None
self._artifact_cache_setup_lock = threading.Lock()
default_invalidator_root = os.path.join(self.context.config.getdefault('pants_workdir'),
'build_invalidator')
self._build_invalidator_dir = os.path.join(
context.config.get('tasks', 'build_invalidator', default=default_invalidator_root),
self.product_type())
self._jvm_tool_bootstrapper = JvmToolBootstrapper(self.context.products)
def __init__(
self, target, root_dir, extra_targets=None, builder=None, platforms=None, interpreter=None, conn_timeout=None
):
self._config = Config.load()
self._target = target
self._root = root_dir
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._extra_targets = list(extra_targets) if extra_targets is not None else []
self._builder = builder or PEXBuilder(tempfile.mkdtemp(), interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
PythonSetup(self._config).scratch_dir("artifact_cache", default_name="artifacts"),
str(self._interpreter.identity),
)
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator(self._egg_cache_root)
def __init__(self, target, root_dir, extra_targets=None, builder=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._key_generator = CacheKeyGenerator()
artifact_cache_root = \
os.path.join(self._config.get('python-setup', 'artifact_cache'), '%s' % PythonIdentity.get())
self._artifact_cache = ArtifactCache(artifact_cache_root)
self._extra_targets = list(extra_targets) if extra_targets is not None else []
self._resolver = PythonResolver([self._target] + self._extra_targets)
self._builder = builder or PEXBuilder(tempfile.mkdtemp())
self._platforms = (Platform.current(),)
self._pythons = (sys.version[:3],)
# TODO(wickman) Should this be in the binary builder?
if isinstance(self._target, PythonBinary):
self._platforms = self._target._platforms
self._pythons = self._target._interpreters
class PythonChroot(object):
_VALID_DEPENDENCIES = {
PythonLibrary: 'libraries',
PythonRequirement: 'reqs',
PythonBinary: 'binaries',
PythonThriftLibrary: 'thrifts',
PythonAntlrLibrary: 'antlrs',
PythonTests: 'tests'
}
class InvalidDependencyException(Exception):
def __init__(self, target):
Exception.__init__(
self, "Not a valid Python dependency! Found: %s" % target)
class BuildFailureException(Exception):
def __init__(self, msg):
Exception.__init__(self, msg)
def __init__(self,
target,
root_dir,
extra_targets=None,
builder=None,
conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._key_generator = CacheKeyGenerator()
self._extra_targets = list(
extra_targets) if extra_targets is not None else []
self._resolver = MultiResolver.from_target(self._config,
target,
conn_timeout=conn_timeout)
self._builder = builder or PEXBuilder(tempfile.mkdtemp())
artifact_cache_root = os.path.join(
self._config.get('python-setup', 'artifact_cache'),
'%s' % PythonIdentity.get())
self._artifact_cache = FileBasedArtifactCache(
None, self._root, artifact_cache_root,
self._builder.add_dependency_file)
def __del__(self):
if os.getenv('PANTS_LEAVE_CHROOT') is None:
safe_rmtree(self.path())
@property
def builder(self):
return self._builder
def debug(self, msg, indent=0):
if os.getenv('PANTS_VERBOSE') is not None:
print('%s%s' % (' ' * indent, msg))
def path(self):
return self._builder.path()
def _dump_library(self, library):
def translate_module(module):
if module is None:
module = ''
return module.replace('.', os.path.sep)
def copy_to_chroot(base, path, relative_to, add_function):
src = os.path.join(self._root, base, path)
dst = os.path.join(translate_module(relative_to), path)
add_function(src, dst)
self.debug(' Dumping library: %s [relative module: %s]' %
(library, library.module))
for filename in library.sources:
copy_to_chroot(library.target_base, filename, library.module,
self._builder.add_source)
for filename in library.resources:
copy_to_chroot(library.target_base, filename, library.module,
self._builder.add_resource)
def _dump_requirement(self, req, dynamic, repo):
self.debug(' Dumping requirement: %s%s%s' %
(str(req), ' (dynamic)' if dynamic else '',
' (repo: %s)' if repo else ''))
self._builder.add_requirement(req, dynamic, repo)
def _dump_distribution(self, dist):
self.debug(' Dumping distribution: .../%s' %
os.path.basename(dist.location))
self._builder.add_distribution(dist)
def _dump_bin(self, binary_name, base):
src = os.path.join(self._root, base, binary_name)
self.debug(' Dumping binary: %s' % binary_name)
self._builder.set_executable(src, os.path.basename(src))
def _dump_thrift_library(self, library):
self._dump_built_library(
library, PythonThriftBuilder(library, self._root, self._config))
def _dump_antlr_library(self, library):
self._dump_built_library(library,
PythonAntlrBuilder(library, self._root))
def _dump_built_library(self, library, builder):
# TODO(wickman): Port this over to the Installer+Distiller and stop using ArtifactCache.
absolute_sources = library.expand_files()
absolute_sources.sort()
cache_key = self._key_generator.key_for(library.id, absolute_sources)
if self._artifact_cache.has(cache_key):
self.debug(' Generating (cached) %s...' % library)
self._artifact_cache.use_cached_files(cache_key)
else:
self.debug(' Generating %s...' % library)
egg_file = builder.build_egg()
if not egg_file:
raise PythonChroot.BuildFailureException(
"Failed to build %s!" % library)
src_egg_file = egg_file
dst_egg_file = os.path.join(
os.path.dirname(egg_file),
cache_key.hash + '_' + os.path.basename(egg_file))
self.debug(' %s => %s' % (src_egg_file, dst_egg_file))
os.rename(src_egg_file, dst_egg_file)
self._artifact_cache.insert(cache_key, [dst_egg_file])
self._builder.add_egg(dst_egg_file)
def resolve(self, targets):
children = defaultdict(OrderedSet)
def add_dep(trg):
if is_concrete(trg):
for target_type, target_key in self._VALID_DEPENDENCIES.items(
):
if isinstance(trg, target_type):
children[target_key].add(trg)
return
raise self.InvalidDependencyException(trg)
for target in targets:
target.walk(add_dep)
return children
def dump(self):
self.debug('Building PythonBinary %s:' % self._target)
targets = self.resolve([self._target] + self._extra_targets)
for lib in targets['libraries']:
self._dump_library(lib)
for req in targets['reqs']:
if not req.should_build():
self.debug('Skipping %s based upon version filter' % req)
continue
self._dump_requirement(req._requirement, req._dynamic,
req._repository)
for dist in self._resolver.resolve(req._requirement
for req in targets['reqs']
if req.should_build()):
self._dump_distribution(dist)
if targets['thrifts']:
default_thrift_version = self._config.get('python-thrift',
'default-version',
default='0.9')
thrift_versions = set()
for thr in targets['thrifts']:
self._dump_thrift_library(thr)
thrift_version = thr.thrift_version or default_thrift_version
thrift_versions.add(thrift_version)
if len(thrift_versions) > 1:
print('WARNING: Target has multiple thrift versions!')
for version in thrift_versions:
self._builder.add_requirement('thrift==%s' % version)
for dist in self._resolver.resolve(
'thrift==%s' % version for version in thrift_versions):
self._dump_distribution(dist)
for antlr in targets['antlrs']:
self._dump_antlr_library(antlr)
if len(targets['binaries']) > 1:
print('WARNING: Target has multiple python_binary targets!',
file=sys.stderr)
for binary in targets['binaries']:
if len(binary.sources) > 0:
self._dump_bin(binary.sources[0], binary.target_base)
return self._builder
class PythonChroot(object):
_VALID_DEPENDENCIES = {
PythonLibrary: 'libraries',
PythonRequirement: 'reqs',
PythonBinary: 'binaries',
PythonThriftLibrary: 'thrifts',
PythonAntlrLibrary: 'antlrs',
PythonTests: 'tests'
}
MEMOIZED_THRIFTS = {}
class InvalidDependencyException(Exception):
def __init__(self, target):
Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target)
def __init__(self, target, root_dir, extra_targets=None, builder=None, interpreter=None,
conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._interpreter = interpreter or PythonInterpreter.get()
self._extra_targets = list(extra_targets) if extra_targets is not None else []
self._resolver = MultiResolver(self._config, target, conn_timeout=conn_timeout)
self._builder = builder or PEXBuilder(tempfile.mkdtemp(), interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(self._config.get('python-setup', 'artifact_cache'),
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator( self._egg_cache_root)
def __del__(self):
if os.getenv('PANTS_LEAVE_CHROOT') is None:
safe_rmtree(self.path())
else:
self.debug('Left chroot at %s' % self.path())
@property
def builder(self):
return self._builder
def debug(self, msg, indent=0):
if os.getenv('PANTS_VERBOSE') is not None:
print('%s%s' % (' ' * indent, msg))
def path(self):
return self._builder.path()
def _dump_library(self, library):
def copy_to_chroot(base, path, add_function):
src = os.path.join(self._root, base, path)
add_function(src, path)
self.debug(' Dumping library: %s' % library)
for filename in library.sources:
copy_to_chroot(library.target_base, filename, self._builder.add_source)
for filename in library.resources:
copy_to_chroot(library.target_base, filename, self._builder.add_resource)
def _dump_requirement(self, req, dynamic, repo):
self.debug(' Dumping requirement: %s%s%s' % (str(req),
' (dynamic)' if dynamic else '', ' (repo: %s)' % repo if repo else ''))
self._builder.add_requirement(req, dynamic, repo)
def _dump_distribution(self, dist):
self.debug(' Dumping distribution: .../%s' % os.path.basename(dist.location))
self._builder.add_distribution(dist)
def _generate_requirement(self, library, builder_cls):
library_key = self._key_generator.key_for_target(library)
builder = builder_cls(library, self._root, self._config, '-' + library_key.hash[:8])
cache_dir = os.path.join(self._egg_cache_root, library_key.id)
if self._build_invalidator.needs_update(library_key):
sdist = builder.build(interpreter=self._interpreter)
safe_mkdir(cache_dir)
shutil.copy(sdist, os.path.join(cache_dir, os.path.basename(sdist)))
self._build_invalidator.update(library_key)
with ParseContext.temp():
return PythonRequirement(builder.requirement_string(), repository=cache_dir, use_2to3=True)
def _generate_thrift_requirement(self, library):
return self._generate_requirement(library, PythonThriftBuilder)
def _generate_antlr_requirement(self, library):
return self._generate_requirement(library, PythonAntlrBuilder)
def resolve(self, targets):
children = defaultdict(OrderedSet)
def add_dep(trg):
if trg.is_concrete:
for target_type, target_key in self._VALID_DEPENDENCIES.items():
if isinstance(trg, target_type):
children[target_key].add(trg)
return
raise self.InvalidDependencyException(trg)
for target in targets:
target.walk(add_dep)
return children
def dump(self):
self.debug('Building PythonBinary %s:' % self._target)
targets = self.resolve([self._target] + self._extra_targets)
for lib in targets['libraries'] | targets['binaries']:
self._dump_library(lib)
generated_reqs = OrderedSet()
if targets['thrifts']:
for thr in set(targets['thrifts']):
if thr not in self.MEMOIZED_THRIFTS:
self.MEMOIZED_THRIFTS[thr] = self._generate_thrift_requirement(thr)
generated_reqs.add(self.MEMOIZED_THRIFTS[thr])
with ParseContext.temp():
# trick pants into letting us add this python requirement, otherwise we get
# TargetDefinitionException: Error in target BUILD.temp:thrift: duplicate to
# PythonRequirement(thrift)
#
# TODO(wickman) Instead of just blindly adding a PythonRequirement for thrift, we
# should first detect if any explicit thrift requirements have been added and use
# those. Only if they have not been supplied should we auto-inject it.
generated_reqs.add(PythonRequirement('thrift', use_2to3=True,
name='thrift-' + ''.join(random.sample('0123456789abcdef' * 8, 8))))
for antlr in targets['antlrs']:
generated_reqs.add(self._generate_antlr_requirement(antlr))
targets['reqs'] |= generated_reqs
for req in targets['reqs']:
if not req.should_build(self._interpreter.python, Platform.current()):
self.debug('Skipping %s based upon version filter' % req)
continue
self._dump_requirement(req._requirement, False, req._repository)
reqs_to_build = (req for req in targets['reqs']
if req.should_build(self._interpreter.python, Platform.current()))
for dist in self._resolver.resolve(reqs_to_build, interpreter=self._interpreter):
self._dump_distribution(dist)
if len(targets['binaries']) > 1:
print('WARNING: Target has multiple python_binary targets!', file=sys.stderr)
return self._builder
class PythonChroot(object):
_VALID_DEPENDENCIES = {
PythonLibrary: 'libraries',
PythonRequirement: 'reqs',
PythonBinary: 'binaries',
PythonThriftLibrary: 'thrifts',
PythonAntlrLibrary: 'antlrs',
PythonTests: 'tests'
}
class InvalidDependencyException(Exception):
def __init__(self, target):
Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target)
class BuildFailureException(Exception):
def __init__(self, msg):
Exception.__init__(self, msg)
def __init__(self, target, root_dir, extra_targets=None, builder=None, conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._key_generator = CacheKeyGenerator()
self._extra_targets = list(extra_targets) if extra_targets is not None else []
self._resolver = MultiResolver.from_target(self._config, target, conn_timeout=conn_timeout)
self._builder = builder or PEXBuilder(tempfile.mkdtemp())
artifact_cache_root = os.path.join(self._config.get('python-setup', 'artifact_cache'),
'%s' % PythonIdentity.get())
self._artifact_cache = FileBasedArtifactCache(None, self._root, artifact_cache_root,
self._builder.add_dependency_file)
def __del__(self):
if os.getenv('PANTS_LEAVE_CHROOT') is None:
safe_rmtree(self.path())
@property
def builder(self):
return self._builder
def debug(self, msg, indent=0):
if os.getenv('PANTS_VERBOSE') is not None:
print('%s%s' % (' ' * indent, msg))
def path(self):
return self._builder.path()
def _dump_library(self, library):
def translate_module(module):
if module is None:
module = ''
return module.replace('.', os.path.sep)
def copy_to_chroot(base, path, relative_to, add_function):
src = os.path.join(self._root, base, path)
dst = os.path.join(translate_module(relative_to), path)
add_function(src, dst)
self.debug(' Dumping library: %s [relative module: %s]' % (library, library.module))
for filename in library.sources:
copy_to_chroot(library.target_base, filename, library.module, self._builder.add_source)
for filename in library.resources:
copy_to_chroot(library.target_base, filename, library.module, self._builder.add_resource)
def _dump_requirement(self, req, dynamic, repo):
self.debug(' Dumping requirement: %s%s%s' % (str(req),
' (dynamic)' if dynamic else '', ' (repo: %s)' if repo else ''))
self._builder.add_requirement(req, dynamic, repo)
def _dump_distribution(self, dist):
self.debug(' Dumping distribution: .../%s' % os.path.basename(dist.location))
self._builder.add_distribution(dist)
def _dump_bin(self, binary_name, base):
src = os.path.join(self._root, base, binary_name)
self.debug(' Dumping binary: %s' % binary_name)
self._builder.set_executable(src, os.path.basename(src))
def _dump_thrift_library(self, library):
self._dump_built_library(library, PythonThriftBuilder(library, self._root, self._config))
def _dump_antlr_library(self, library):
self._dump_built_library(library, PythonAntlrBuilder(library, self._root))
def _dump_built_library(self, library, builder):
# TODO(wickman): Port this over to the Installer+Distiller and stop using ArtifactCache.
absolute_sources = library.expand_files()
absolute_sources.sort()
cache_key = self._key_generator.key_for(library.id, absolute_sources)
if self._artifact_cache.has(cache_key):
self.debug(' Generating (cached) %s...' % library)
self._artifact_cache.use_cached_files(cache_key)
else:
self.debug(' Generating %s...' % library)
egg_file = builder.build_egg()
if not egg_file:
raise PythonChroot.BuildFailureException("Failed to build %s!" % library)
src_egg_file = egg_file
dst_egg_file = os.path.join(os.path.dirname(egg_file),
cache_key.hash + '_' + os.path.basename(egg_file))
self.debug(' %s => %s' % (src_egg_file, dst_egg_file))
os.rename(src_egg_file, dst_egg_file)
self._artifact_cache.insert(cache_key, [dst_egg_file])
self._builder.add_egg(dst_egg_file)
def resolve(self, targets):
children = defaultdict(OrderedSet)
def add_dep(trg):
if is_concrete(trg):
for target_type, target_key in self._VALID_DEPENDENCIES.items():
if isinstance(trg, target_type):
children[target_key].add(trg)
return
raise self.InvalidDependencyException(trg)
for target in targets:
target.walk(add_dep)
return children
def dump(self):
self.debug('Building PythonBinary %s:' % self._target)
targets = self.resolve([self._target] + self._extra_targets)
for lib in targets['libraries']:
self._dump_library(lib)
for req in targets['reqs']:
if not req.should_build():
self.debug('Skipping %s based upon version filter' % req)
continue
self._dump_requirement(req._requirement, req._dynamic, req._repository)
for dist in self._resolver.resolve(
req._requirement for req in targets['reqs'] if req.should_build()):
self._dump_distribution(dist)
if targets['thrifts']:
default_thrift_version = self._config.get('python-thrift', 'default-version', default='0.9')
thrift_versions = set()
for thr in targets['thrifts']:
self._dump_thrift_library(thr)
thrift_version = thr.thrift_version or default_thrift_version
thrift_versions.add(thrift_version)
if len(thrift_versions) > 1:
print('WARNING: Target has multiple thrift versions!')
for version in thrift_versions:
self._builder.add_requirement('thrift==%s' % version)
for dist in self._resolver.resolve('thrift==%s' % version for version in thrift_versions):
self._dump_distribution(dist)
for antlr in targets['antlrs']:
self._dump_antlr_library(antlr)
if len(targets['binaries']) > 1:
print('WARNING: Target has multiple python_binary targets!', file=sys.stderr)
for binary in targets['binaries']:
if len(binary.sources) > 0:
self._dump_bin(binary.sources[0], binary.target_base)
return self._builder
class PythonChroot(object):
_VALID_DEPENDENCIES = {
PythonLibrary: 'libraries',
PythonRequirement: 'reqs',
PythonBinary: 'binaries',
PythonThriftLibrary: 'thrifts',
PythonAntlrLibrary: 'antlrs',
PythonTests: 'tests'
}
MEMOIZED_THRIFTS = {}
class InvalidDependencyException(Exception):
def __init__(self, target):
Exception.__init__(
self, "Not a valid Python dependency! Found: %s" % target)
def __init__(self,
target,
root_dir,
extra_targets=None,
builder=None,
interpreter=None,
conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._interpreter = interpreter or PythonInterpreter.get()
self._extra_targets = list(
extra_targets) if extra_targets is not None else []
self._resolver = MultiResolver(self._config,
target,
conn_timeout=conn_timeout)
self._builder = builder or PEXBuilder(tempfile.mkdtemp(),
interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
self._config.get('python-setup', 'artifact_cache'),
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator(self._egg_cache_root)
def __del__(self):
if os.getenv('PANTS_LEAVE_CHROOT') is None:
safe_rmtree(self.path())
else:
self.debug('Left chroot at %s' % self.path())
@property
def builder(self):
return self._builder
def debug(self, msg, indent=0):
if os.getenv('PANTS_VERBOSE') is not None:
print('%s%s' % (' ' * indent, msg))
def path(self):
return self._builder.path()
def _dump_library(self, library):
def copy_to_chroot(base, path, add_function):
src = os.path.join(self._root, base, path)
add_function(src, path)
self.debug(' Dumping library: %s' % library)
for filename in library.sources:
copy_to_chroot(library.target_base, filename,
self._builder.add_source)
for filename in library.resources:
copy_to_chroot(library.target_base, filename,
self._builder.add_resource)
def _dump_requirement(self, req, dynamic, repo):
self.debug(' Dumping requirement: %s%s%s' %
(str(req), ' (dynamic)' if dynamic else '',
' (repo: %s)' % repo if repo else ''))
self._builder.add_requirement(req, dynamic, repo)
def _dump_distribution(self, dist):
self.debug(' Dumping distribution: .../%s' %
os.path.basename(dist.location))
self._builder.add_distribution(dist)
def _generate_requirement(self, library, builder_cls):
library_key = self._key_generator.key_for_target(library)
builder = builder_cls(library, self._root, self._config,
'-' + library_key.hash[:8])
cache_dir = os.path.join(self._egg_cache_root, library_key.id)
if self._build_invalidator.needs_update(library_key):
sdist = builder.build(interpreter=self._interpreter)
safe_mkdir(cache_dir)
shutil.copy(sdist, os.path.join(cache_dir,
os.path.basename(sdist)))
self._build_invalidator.update(library_key)
with ParseContext.temp():
return PythonRequirement(builder.requirement_string(),
repository=cache_dir,
use_2to3=True)
def _generate_thrift_requirement(self, library):
return self._generate_requirement(library, PythonThriftBuilder)
def _generate_antlr_requirement(self, library):
return self._generate_requirement(library, PythonAntlrBuilder)
def resolve(self, targets):
children = defaultdict(OrderedSet)
def add_dep(trg):
if trg.is_concrete:
for target_type, target_key in self._VALID_DEPENDENCIES.items(
):
if isinstance(trg, target_type):
children[target_key].add(trg)
return
raise self.InvalidDependencyException(trg)
for target in targets:
target.walk(add_dep)
return children
def dump(self):
self.debug('Building PythonBinary %s:' % self._target)
targets = self.resolve([self._target] + self._extra_targets)
for lib in targets['libraries'] | targets['binaries']:
self._dump_library(lib)
generated_reqs = OrderedSet()
if targets['thrifts']:
for thr in set(targets['thrifts']):
if thr not in self.MEMOIZED_THRIFTS:
self.MEMOIZED_THRIFTS[
thr] = self._generate_thrift_requirement(thr)
generated_reqs.add(self.MEMOIZED_THRIFTS[thr])
with ParseContext.temp():
# trick pants into letting us add this python requirement, otherwise we get
# TargetDefinitionException: Error in target BUILD.temp:thrift: duplicate to
# PythonRequirement(thrift)
#
# TODO(wickman) Instead of just blindly adding a PythonRequirement for thrift, we
# should first detect if any explicit thrift requirements have been added and use
# those. Only if they have not been supplied should we auto-inject it.
generated_reqs.add(
PythonRequirement(
'thrift',
use_2to3=True,
name='thrift-' +
''.join(random.sample('0123456789abcdef' * 8, 8))))
for antlr in targets['antlrs']:
generated_reqs.add(self._generate_antlr_requirement(antlr))
targets['reqs'] |= generated_reqs
for req in targets['reqs']:
if not req.should_build(self._interpreter.python,
Platform.current()):
self.debug('Skipping %s based upon version filter' % req)
continue
self._dump_requirement(req._requirement, False, req._repository)
reqs_to_build = (
req for req in targets['reqs']
if req.should_build(self._interpreter.python, Platform.current()))
for dist in self._resolver.resolve(reqs_to_build,
interpreter=self._interpreter):
self._dump_distribution(dist)
if len(targets['binaries']) > 1:
print('WARNING: Target has multiple python_binary targets!',
file=sys.stderr)
return self._builder
class PythonChroot(object):
class BuildFailureException(Exception):
def __init__(self, msg):
Exception.__init__(self, msg)
def __init__(self, target, root_dir, extra_targets=None, builder=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._key_generator = CacheKeyGenerator()
self._extra_targets = list(
extra_targets) if extra_targets is not None else []
self._resolver = PythonResolver([self._target] + self._extra_targets)
self._builder = builder or PEXBuilder(tempfile.mkdtemp())
self._platforms = (Platform.current(), )
self._pythons = (sys.version[:3], )
artifact_cache_root = \
os.path.join(self._config.get('python-setup', 'artifact_cache'), '%s' % PythonIdentity.get())
self._artifact_cache = FileBasedArtifactCache(
None, self._root, artifact_cache_root,
self._builder.add_dependency_file)
# TODO(wickman) Should this be in the binary builder?
if isinstance(self._target, PythonBinary):
self._platforms = self._target._platforms
self._pythons = self._target._interpreters
def __del__(self):
if os.getenv('PANTS_LEAVE_CHROOT') is None:
safe_rmtree(self.path())
def debug(self, msg, indent=0):
if os.getenv('PANTS_VERBOSE') is not None:
print('%s%s' % (' ' * indent, msg))
def path(self):
return self._builder.path()
def _dump_library(self, library):
def translate_module(module):
if module is None:
module = ''
return module.replace('.', os.path.sep)
def copy_to_chroot(base, path, relative_to, add_function):
src = os.path.join(self._root, base, path)
dst = os.path.join(translate_module(relative_to), path)
add_function(src, dst)
self.debug(' Dumping library: %s [relative module: %s]' %
(library, library.module))
for filename in library.sources:
copy_to_chroot(library.target_base, filename, library.module,
self._builder.add_source)
for filename in library.resources:
copy_to_chroot(library.target_base, filename, library.module,
self._builder.add_resource)
def _dump_requirement(self, req, dynamic, repo):
self.debug(' Dumping requirement: %s%s%s' %
(str(req), ' (dynamic)' if dynamic else '',
' (repo: %s)' if repo else ''))
self._builder.add_requirement(req, dynamic, repo)
def _dump_distribution(self, dist):
self.debug(' Dumping distribution: .../%s' %
os.path.basename(dist.location))
self._builder.add_distribution(dist)
def _dump_bin(self, binary_name, base):
src = os.path.join(self._root, base, binary_name)
self.debug(' Dumping binary: %s' % binary_name)
self._builder.set_executable(src, os.path.basename(src))
def _dump_thrift_library(self, library):
self._dump_built_library(library,
PythonThriftBuilder(library, self._root))
def _dump_antlr_library(self, library):
self._dump_built_library(library,
PythonAntlrBuilder(library, self._root))
def _dump_built_library(self, library, builder):
# TODO(wickman): Port this over to the Installer+Distiller and stop using FileBasedArtifactCache.
absolute_sources = library.expand_files()
absolute_sources.sort()
cache_key = self._key_generator.key_for(library.id, absolute_sources)
if self._artifact_cache.has(cache_key):
self.debug(' Generating (cached) %s...' % library)
self._artifact_cache.use_cached_files(cache_key)
else:
self.debug(' Generating %s...' % library)
egg_file = builder.build_egg()
if not egg_file:
raise PythonChroot.BuildFailureException(
"Failed to build %s!" % library)
src_egg_file = egg_file
dst_egg_file = os.path.join(
os.path.dirname(egg_file),
cache_key.hash + '_' + os.path.basename(egg_file))
self.debug(' %s => %s' % (src_egg_file, dst_egg_file))
os.rename(src_egg_file, dst_egg_file)
self._artifact_cache.insert(cache_key, [dst_egg_file])
self._builder.add_egg(dst_egg_file)
def dump(self):
self.debug('Building PythonBinary %s:' % self._target)
targets = self._resolver.resolve()
for lib in targets['libraries']:
self._dump_library(lib)
for req in targets['reqs']:
if not req.should_build():
self.debug('Skipping %s based upon version filter' % req)
continue
self._dump_requirement(req._requirement, req._dynamic,
req._repository)
for dist in ReqResolver.resolve(
(req._requirement
for req in targets['reqs'] if req.should_build()),
self._config,
self._platforms,
self._pythons,
ignore_errors=self._builder.info().ignore_errors):
self._dump_distribution(dist)
for thr in targets['thrifts']:
self._dump_thrift_library(thr)
for antlr in targets['antlrs']:
self._dump_antlr_library(antlr)
if len(targets['binaries']) > 1:
print('WARNING: Target has multiple python_binary targets!',
file=sys.stderr)
for binary in targets['binaries']:
if len(binary.sources) > 0:
self._dump_bin(binary.sources[0], binary.target_base)
return self._builder
def test_env(content=TEST_CONTENT):
with temporary_dir() as d:
with tempfile.NamedTemporaryFile() as f:
f.write(content)
f.flush()
yield f, CacheKeyGenerator(), BuildInvalidator(d)
class PythonChroot(object):
class BuildFailureException(Exception):
def __init__(self, msg):
Exception.__init__(self, msg)
def __init__(self, target, root_dir, extra_targets=None, builder=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._key_generator = CacheKeyGenerator()
artifact_cache_root = \
os.path.join(self._config.get('python-setup', 'artifact_cache'), '%s' % PythonIdentity.get())
self._artifact_cache = ArtifactCache(artifact_cache_root)
self._extra_targets = list(extra_targets) if extra_targets is not None else []
self._resolver = PythonResolver([self._target] + self._extra_targets)
self._builder = builder or PEXBuilder(tempfile.mkdtemp())
self._platforms = (Platform.current(),)
self._pythons = (sys.version[:3],)
# TODO(wickman) Should this be in the binary builder?
if isinstance(self._target, PythonBinary):
self._platforms = self._target._platforms
self._pythons = self._target._interpreters
def __del__(self):
if os.getenv('PANTS_LEAVE_CHROOT') is None:
safe_rmtree(self.path())
def debug(self, msg, indent=0):
if os.getenv('PANTS_VERBOSE') is not None:
print('%s%s' % (' ' * indent, msg))
def path(self):
return self._builder.path()
def _dump_library(self, library):
def translate_module(module):
if module is None:
module = ''
return module.replace('.', os.path.sep)
def copy_to_chroot(base, path, relative_to, add_function):
src = os.path.join(self._root, base, path)
dst = os.path.join(translate_module(relative_to), path)
add_function(src, dst)
self.debug(' Dumping library: %s [relative module: %s]' % (library, library.module))
for filename in library.sources:
copy_to_chroot(library.target_base, filename, library.module, self._builder.add_source)
for filename in library.resources:
copy_to_chroot(library.target_base, filename, library.module, self._builder.add_resource)
def _dump_requirement(self, req, dynamic, repo):
self.debug(' Dumping requirement: %s%s%s' % (str(req),
' (dynamic)' if dynamic else '', ' (repo: %s)' if repo else ''))
self._builder.add_requirement(req, dynamic, repo)
def _dump_distribution(self, dist):
self.debug(' Dumping distribution: .../%s' % os.path.basename(dist.location))
self._builder.add_distribution(dist)
def _dump_bin(self, binary_name, base):
src = os.path.join(self._root, base, binary_name)
self.debug(' Dumping binary: %s' % binary_name)
self._builder.set_executable(src, os.path.basename(src))
def _dump_thrift_library(self, library):
self._dump_built_library(library, PythonThriftBuilder(library, self._root))
def _dump_antlr_library(self, library):
self._dump_built_library(library, PythonAntlrBuilder(library, self._root))
def _dump_built_library(self, library, builder):
# TODO(wickman): Port this over to the Installer+Distiller and stop using ArtifactCache.
absolute_sources = library.expand_files()
absolute_sources.sort()
cache_key = self._key_generator.key_for(library.id, absolute_sources)
if self._artifact_cache.has(cache_key):
self.debug(' Generating (cached) %s...' % library)
self._artifact_cache.use_cached_files(cache_key, self._builder.add_dependency_file)
else:
self.debug(' Generating %s...' % library)
egg_file = builder.build_egg()
if not egg_file:
raise PythonChroot.BuildFailureException("Failed to build %s!" % library)
src_egg_file = egg_file
dst_egg_file = os.path.join(os.path.dirname(egg_file),
cache_key.hash + '_' + os.path.basename(egg_file))
self.debug(' %s => %s' % (src_egg_file, dst_egg_file))
os.rename(src_egg_file, dst_egg_file)
self._artifact_cache.insert(cache_key, [dst_egg_file])
self._builder.add_egg(dst_egg_file)
def dump(self):
self.debug('Building PythonBinary %s:' % self._target)
targets = self._resolver.resolve()
for lib in targets['libraries']:
self._dump_library(lib)
for req in targets['reqs']:
if not req.should_build():
self.debug('Skipping %s based upon version filter' % req)
continue
self._dump_requirement(req._requirement, req._dynamic, req._repository)
for dist in ReqResolver.resolve(
(req._requirement for req in targets['reqs'] if req.should_build()),
self._config, self._platforms, self._pythons,
ignore_errors=self._builder.info().ignore_errors):
self._dump_distribution(dist)
for thr in targets['thrifts']:
self._dump_thrift_library(thr)
for antlr in targets['antlrs']:
self._dump_antlr_library(antlr)
if len(targets['binaries']) > 1:
print('WARNING: Target has multiple python_binary targets!', file=sys.stderr)
for binary in targets['binaries']:
if len(binary.sources) > 0:
self._dump_bin(binary.sources[0], binary.target_base)
return self._builder