def __init__(self, target, root_dir, extra_targets=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._cache = BuildCache(
os.path.join(self._config.get("python-setup", "artifact_cache"), "%s" % PythonIdentity.get())
)
self._extra_targets = list(extra_targets) if extra_targets is not None else []
self._extra_targets.append(self._get_common_python())
cachedir = self._config.get("python-setup", "cache")
safe_mkdir(cachedir)
self._eggcache = cachedir
local_repo = "file://%s" % os.path.realpath(cachedir)
self._repos = [local_repo] + self._config.getlist("python-setup", "repos")
self._fetcher = ReqFetcher(repos=self._repos, cache=cachedir)
self._index = None
for index in self._config.getlist("python-setup", "indices"):
if PythonChroot.can_contact_index(index):
self._index = index
break
self._additional_reqs = set()
distdir = self._config.getdefault("pants_distdir")
distpath = tempfile.mktemp(dir=distdir, prefix=target.name)
self.env = PythonEnvironment(distpath)
def __init__(self, target, root_dir): self._target = target self._root = root_dir self._cache = BuildCache(os.path.join(root_dir, '.pants.d', 'py_artifact_cache')) distdir = os.path.join(root_dir, 'dist') distpath = tempfile.mktemp(dir=distdir, prefix=target.name) self.env = PythonEnvironment(distpath)
def __init__(self, target, root_dir, extra_targets=None, builder=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._cache = BuildCache(
os.path.join(self._config.get('python-setup', 'artifact_cache'),
'%s' % PythonIdentity.get()))
self._extra_targets = list(
extra_targets) if extra_targets is not None else []
self._resolver = PythonResolver([self._target] + self._extra_targets)
self._builder = builder or PEXBuilder(tempfile.mkdtemp())
self._platforms = (Platform.current(), )
self._pythons = (sys.version[:3], )
# TODO(wickman) Should this be in the binary builder?
if isinstance(self._target, PythonBinary):
self._platforms = self._target._platforms
self._pythons = self._target._interpreters
def __init__(self, target, root_dir, extra_targets=None, builder=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._cache = BuildCache(os.path.join(self._config.get('python-setup', 'artifact_cache'),
'%s' % PythonIdentity.get()))
self._extra_targets = list(extra_targets) if extra_targets is not None else []
self._resolver = PythonResolver([self._target] + self._extra_targets)
self._builder = builder or PEXBuilder(tempfile.mkdtemp())
self._platforms = (Platform.current(),)
self._pythons = (sys.version[:3],)
# TODO(wickman) Should this be in the binary builder?
if isinstance(self._target, PythonBinary):
self._platforms = self._target._platforms
self._pythons = self._target._interpreters
def changed(self, targets, only_buildfiles=False, invalidate_dependants=False):
"""
Yields an iterable over the targets that have changed since the last check to a with block.
If no exceptions are thrown by work in the block, the cache is updated for the targets,
otherwise if a TargetError is thrown by the work in the block all targets except those in the
TargetError are cached.
:targets The targets to check for changes.
:only_buildfiles If True, then just the target's BUILD files are checked for changes.
:invalidate_dependants If True then any targets depending on changed targets are invalidated
:returns: the subset of targets that have changed
"""
safe_mkdir(self._basedir)
cache_manager = Task.CacheManager(BuildCache(self._basedir), targets, only_buildfiles)
check = self.invalidate_for()
if check is not None:
with safe_open(self._extradata, 'w') as pickled:
pickle.dump(check, pickled)
cache_key = cache_manager.check_content(Task.EXTRA_DATA, [self._extradata])
if cache_key:
self.context.log.debug('invalidating all targets for %s' % self.__class__.__name__)
for target in targets:
cache_manager.invalidate(target, cache_key)
for target in targets:
cache_manager.check(target)
if invalidate_dependants and cache_manager.changed:
for target in (self.context.dependants(lambda t: t in cache_manager.changed.keys())).keys():
cache_manager.invalidate(target)
if invalidate_dependants:
if cache_manager.foreign_invalidated_targets:
self.context.log.info('Invalidated %d dependant targets '
'for the next round' % cache_manager.foreign_invalidated_targets)
if cache_manager.changed_files:
msg = 'Operating on %d files in %d changed targets' % (
cache_manager.changed_files,
len(cache_manager.changed)
)
if cache_manager.invalidated_files:
msg += ' and %d files in %d invalidated dependant targets' % (
cache_manager.invalidated_files,
cache_manager.invalidated_targets
)
self.context.log.info(msg)
elif cache_manager.changed_files:
self.context.log.info('Operating on %d files in %d changed targets' % (
cache_manager.changed_files,
len(cache_manager.changed)
))
try:
yield cache_manager.changed.keys()
for cache_keys in cache_manager.changed.values():
for cache_key in cache_keys:
cache_manager.update(cache_key)
except TargetError as e:
for target, cache_keys in cache_manager.changed.items():
if target not in e.targets:
for cache_key in cache_keys:
cache_manager.update(cache_key)
class PythonChroot(object):
def __init__(self, target, root_dir, extra_targets=None, builder=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._cache = BuildCache(os.path.join(self._config.get('python-setup', 'artifact_cache'),
'%s' % PythonIdentity.get()))
self._extra_targets = list(extra_targets) if extra_targets is not None else []
self._resolver = PythonResolver([self._target] + self._extra_targets)
self._builder = builder or PEXBuilder(tempfile.mkdtemp())
self._platforms = (Platform.current(),)
self._pythons = (sys.version[:3],)
# TODO(wickman) Should this be in the binary builder?
if isinstance(self._target, PythonBinary):
self._platforms = self._target._platforms
self._pythons = self._target._interpreters
def __del__(self):
if os.getenv('PANTS_LEAVE_CHROOT') is None:
safe_rmtree(self.path())
def debug(self, msg, indent=0):
if os.getenv('PANTS_VERBOSE') is not None:
print('%s%s' % (' ' * indent, msg))
def path(self):
return self._builder.path()
def _dump_library(self, library):
def translate_module(module):
if module is None:
module = ''
return module.replace('.', os.path.sep)
def copy_to_chroot(base, path, relative_to, add_function):
src = os.path.join(self._root, base, path)
dst = os.path.join(translate_module(relative_to), path)
add_function(src, dst)
self.debug(' Dumping library: %s [relative module: %s]' % (library, library.module))
for filename in library.sources:
copy_to_chroot(library.target_base, filename, library.module, self._builder.add_source)
for filename in library.resources:
copy_to_chroot(library.target_base, filename, library.module, self._builder.add_resource)
def _dump_requirement(self, req, dynamic, repo):
self.debug(' Dumping requirement: %s%s%s' % (str(req),
' (dynamic)' if dynamic else '', ' (repo: %s)' if repo else ''))
self._builder.add_requirement(req, dynamic, repo)
def _dump_distribution(self, dist):
self.debug(' Dumping distribution: .../%s' % os.path.basename(dist.location))
self._builder.add_distribution(dist)
def _dump_bin(self, binary_name, base):
src = os.path.join(self._root, base, binary_name)
self.debug(' Dumping binary: %s' % binary_name)
self._builder.set_executable(src, os.path.basename(src))
def _dump_thrift_library(self, library):
self._dump_built_library(library, PythonThriftBuilder(library, self._root))
def _dump_antlr_library(self, library):
self._dump_built_library(library, PythonAntlrBuilder(library, self._root))
def _dump_built_library(self, library, builder):
# TODO(wickman) Port this over to the Installer+Distiller and stop using
# BuildCache.
absolute_sources = library.expand_files()
absolute_sources.sort()
cache_key = self._cache.key_for(library.id, absolute_sources)
if not self._cache.needs_update(cache_key):
self.debug(' Generating (cached) %s...' % library)
self._cache.use_cached_files(cache_key, self._builder.add_dependency_file)
else:
self.debug(' Generating %s...' % library)
egg_file = builder.build_egg()
if not egg_file:
raise PythonChroot.BuildFailureException("Failed to build %s!" % library)
src_egg_file = egg_file
dst_egg_file = os.path.join(os.path.dirname(egg_file),
cache_key.hash + '_' + os.path.basename(egg_file))
self.debug(' %s => %s' % (src_egg_file, dst_egg_file))
os.rename(src_egg_file, dst_egg_file)
self._cache.update(cache_key, [dst_egg_file])
self._builder.add_egg(dst_egg_file)
def dump(self):
self.debug('Building PythonBinary %s:' % self._target)
targets = self._resolver.resolve()
for lib in targets['libraries']:
self._dump_library(lib)
for req in targets['reqs']:
if not req.should_build():
self.debug('Skipping %s based upon version filter' % req)
continue
self._dump_requirement(req._requirement, req._dynamic, req._repository)
for dist in ReqResolver.resolve(
(req._requirement for req in targets['reqs'] if req.should_build()),
self._config, self._platforms, self._pythons,
ignore_errors=self._builder.info().ignore_errors):
self._dump_distribution(dist)
for thr in targets['thrifts']:
self._dump_thrift_library(thr)
for antlr in targets['antlrs']:
self._dump_antlr_library(antlr)
if len(targets['binaries']) > 1:
print('WARNING: Target has multiple python_binary targets!', file=sys.stderr)
for binary in targets['binaries']:
if len(binary.sources) > 0:
self._dump_bin(binary.sources[0], binary.target_base)
return self._builder
class PythonChroot(object):
class InvalidDependencyException(Exception):
def __init__(self, target):
Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target)
class BuildFailureException(Exception):
def __init__(self, target):
Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target)
def __init__(self, target, root_dir):
self._target = target
self._root = root_dir
self._cache = BuildCache(os.path.join(root_dir, '.pants.d', 'py_artifact_cache'))
distdir = os.path.join(root_dir, 'dist')
distpath = tempfile.mktemp(dir=distdir, prefix=target.name)
self.env = PythonEnvironment(distpath)
def __del__(self):
if os.getenv('PANTS_LEAVE_CHROOT') is None:
try:
shutil.rmtree(self.path())
except OSError as e:
if e.errno != errno.ENOENT:
raise
def path(self):
return self.env.path()
def _dump_library(self, library):
def translate_module(module):
if module is None:
module = ''
return module.replace('.', os.path.sep)
def copy_to_chroot(base, path, relative_to, add_function):
src = os.path.join(self._root, base, path)
dst = os.path.join(translate_module(relative_to), path)
add_function(src, dst)
template = library._create_template_data()
print ' Dumping library: %s [relative module: %s]' % (library, template.module)
for filename in template.sources:
copy_to_chroot(template.template_base, filename, template.module, self.env.add_source)
for filename in template.resources:
copy_to_chroot(template.template_base, filename, template.module, self.env.add_resource)
def _dump_egg(self, egg):
target_name = os.path.pathsep.join(sorted(os.path.basename(e) for e in egg.eggs))
cache_key = self._cache.key_for(target_name, egg.eggs)
if self._cache.needs_update(cache_key):
print ' Dumping egg: %s' % egg
prefixes, all_added_files = set(), set()
for egg_path in egg.eggs:
egg_dep = PythonDependency.from_eggs(egg_path)
prefix, added_files = self.env.add_dependency(egg_dep)
all_added_files.update(added_files)
prefixes.add(prefix)
assert len(prefixes) == 1, 'Ambiguous egg environment!'
self._cache.update(cache_key, all_added_files, artifact_root=prefixes.pop())
else:
print ' Dumping (cached) egg: %s' % egg
self._cache.use_cached_files(cache_key, self.env.add_dependency_file)
def _dump_setuptools(self):
SETUPTOOLS = 'distribute-0.6.21-py2.6.egg'
print ' Dumping setuptools: %s' % SETUPTOOLS
data = pkgutil.get_data(__name__, os.path.join('bootstrap', SETUPTOOLS))
with temporary_dir() as tempdir:
egg_path = os.path.join(tempdir, SETUPTOOLS)
with open(egg_path, 'w') as fp:
fp.write(data)
egg_dep = PythonDependency.from_eggs(egg_path)
self.env.add_dependency(egg_dep)
# TODO(wickman) Just add write() to self.env and do this with pkg_resources or
# just build an egg for twitter.common.python.
def _get_common_python(self):
return Target.get(Address.parse(self._root, 'src/python/twitter/common/python'))
def _dump_bin(self, binary_name, base):
src = os.path.join(self._root, base, binary_name)
print ' Dumping binary: %s' % binary_name
self.env.set_executable(src, os.path.basename(src))
def _dump_thrift_library(self, library):
self._dump_built_library(library, PythonThriftBuilder(library, self._root))
def _dump_antlr_library(self, library):
self._dump_built_library(library, PythonAntlrBuilder(library, self._root))
def _dump_built_library(self, library, builder):
absolute_sources = library.expand_files()
absolute_sources.sort()
cache_key = self._cache.key_for(library._create_id(), absolute_sources)
if not self._cache.needs_update(cache_key):
print ' Generating (cached) %s...' % library
self._cache.use_cached_files(cache_key, self.env.add_dependency_file)
else:
print ' Generating %s...' % library
egg_file = builder.build_egg()
if egg_file:
src_egg_file = egg_file
dst_egg_file = os.path.join(os.path.dirname(egg_file),
cache_key.hash + '_' + os.path.basename(egg_file))
os.rename(src_egg_file, dst_egg_file)
self._cache.update(cache_key, [dst_egg_file])
egg_dep = PythonDependency.from_eggs(dst_egg_file)
for pkg in builder.packages():
print ' found namespace: %s' % pkg
print ' copying...',
self.env.add_dependency(egg_dep)
print 'done.'
else:
print ' Failed!'
raise PythonChroot.BuildFailureException("Failed to build %s!" % library)
def build_dep_tree(self, input_target):
target = copy.deepcopy(input_target)
common_python = self._get_common_python()
if common_python not in target.dependencies:
target.dependencies.add(common_python)
libraries = set()
eggs = set()
binaries = set()
thrifts = set()
antlrs = set()
def add_dep(trg):
if isinstance(trg, PythonLibrary):
if trg.sources:
libraries.add(trg)
for egg in [dep for dep in trg.dependencies if isinstance(dep, PythonEgg)]:
eggs.add(egg)
elif isinstance(trg, PythonEgg):
eggs.add(trg)
elif isinstance(trg, PythonBinary):
binaries.add(trg)
elif isinstance(trg, PythonThriftLibrary):
thrifts.add(trg)
elif isinstance(trg, PythonAntlrLibrary):
antlrs.add(trg)
elif isinstance(trg, PythonTests):
# do not dump test sources/resources, but dump their
# dependencies.
pass
else:
raise PythonChroot.InvalidDependencyException(trg)
return [dep for dep in trg.dependencies if not isinstance(dep, PythonEgg)]
target.walk(lambda t: add_dep(t), lambda typ: not isinstance(typ, PythonEgg))
return libraries, eggs, binaries, thrifts, antlrs
def dump(self):
print 'Building PythonBinary %s:' % self._target
libraries, eggs, binaries, thrifts, antlrs = self.build_dep_tree(self._target)
for lib in libraries:
self._dump_library(lib)
self._dump_setuptools()
for egg in eggs:
self._dump_egg(egg)
for thr in thrifts:
self._dump_thrift_library(thr)
for antlr in antlrs:
self._dump_antlr_library(antlr)
if len(binaries) > 1:
print >> sys.stderr, 'WARNING: Target has multiple python_binary targets!'
for binary in binaries:
self._dump_bin(binary.sources[0], binary.target_base)
self.env.freeze()
return self.env
class PythonChroot(object):
class InvalidDependencyException(Exception):
def __init__(self, target):
Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target)
class BuildFailureException(Exception):
def __init__(self, target):
Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target)
@staticmethod
def can_contact_index(url, timeout=Amount(5, Time.SECONDS)):
try:
response = urlopen(url, timeout=timeout.as_(Time.SECONDS))
return response.code == 200
except URLError:
return False
def __init__(self, target, root_dir, extra_targets=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._cache = BuildCache(
os.path.join(self._config.get("python-setup", "artifact_cache"), "%s" % PythonIdentity.get())
)
self._extra_targets = list(extra_targets) if extra_targets is not None else []
self._extra_targets.append(self._get_common_python())
cachedir = self._config.get("python-setup", "cache")
safe_mkdir(cachedir)
self._eggcache = cachedir
local_repo = "file://%s" % os.path.realpath(cachedir)
self._repos = [local_repo] + self._config.getlist("python-setup", "repos")
self._fetcher = ReqFetcher(repos=self._repos, cache=cachedir)
self._index = None
for index in self._config.getlist("python-setup", "indices"):
if PythonChroot.can_contact_index(index):
self._index = index
break
self._additional_reqs = set()
distdir = self._config.getdefault("pants_distdir")
distpath = tempfile.mktemp(dir=distdir, prefix=target.name)
self.env = PythonEnvironment(distpath)
def __del__(self):
if os.getenv("PANTS_LEAVE_CHROOT") is None:
try:
shutil.rmtree(self.path())
except OSError as e:
if e.errno != errno.ENOENT:
raise
def debug(self, msg, indent=0):
if os.getenv("PANTS_VERBOSE") is not None:
print("%s%s" % (" " * indent, msg))
def path(self):
return self.env.path()
def _dump_library(self, library):
def translate_module(module):
if module is None:
module = ""
return module.replace(".", os.path.sep)
def copy_to_chroot(base, path, relative_to, add_function):
src = os.path.join(self._root, base, path)
dst = os.path.join(translate_module(relative_to), path)
add_function(src, dst)
self.debug(" Dumping library: %s [relative module: %s]" % (library, library.module))
for filename in library.sources:
copy_to_chroot(library.target_base, filename, library.module, self.env.add_source)
for filename in library.resources:
copy_to_chroot(library.target_base, filename, library.module, self.env.add_resource)
def _dump_egg(self, egg):
if isinstance(egg, PythonEgg):
target_name = os.path.pathsep.join(sorted(os.path.basename(e) for e in egg.eggs))
cache_key = self._cache.key_for(target_name, egg.eggs)
if self._cache.needs_update(cache_key):
self.debug(" Dumping egg: %s" % egg)
prefixes, all_added_files = set(), set()
for egg_path in egg.eggs:
egg_dep = PythonDependency.from_eggs(egg_path)
prefix, added_files = self.env.add_dependency(egg_dep)
all_added_files.update(added_files)
prefixes.add(prefix)
assert len(prefixes) == 1, "Ambiguous egg environment!"
self._cache.update(cache_key, all_added_files, artifact_root=prefixes.pop())
else:
self.debug(" Dumping (cached) egg: %s" % egg)
self._cache.use_cached_files(cache_key, self.env.add_dependency_file)
elif isinstance(egg, PythonDependency):
self.debug(" Dumping PythonDependency: %s" % egg)
self.env.add_dependency(egg)
else:
raise PythonChroot.InvalidDependencyException("Unknown egg dependency %s" % egg)
def _dump_distribution(self, dist):
self.debug(" Dumping distribution: .../%s" % os.path.basename(dist.location))
egg_dep = PythonDependency.from_distributions(dist)
self.env.add_dependency(egg_dep)
# TODO(wickman) Just add write() to self.env and do this with pkg_resources or
# just build an egg for twitter.common.python.
def _get_common_python(self):
return Target.get(Address.parse(self._root, "src/python/twitter/common/python"))
def _dump_bin(self, binary_name, base):
src = os.path.join(self._root, base, binary_name)
self.debug(" Dumping binary: %s" % binary_name)
self.env.set_executable(src, os.path.basename(src))
def _dump_thrift_library(self, library):
self._dump_built_library(library, PythonThriftBuilder(library, self._root))
def _dump_antlr_library(self, library):
self._dump_built_library(library, PythonAntlrBuilder(library, self._root))
def _dump_built_library(self, library, builder):
absolute_sources = library.expand_files()
absolute_sources.sort()
cache_key = self._cache.key_for(library._create_id(), absolute_sources)
if not self._cache.needs_update(cache_key):
self.debug(" Generating (cached) %s..." % library)
self._cache.use_cached_files(cache_key, self.env.add_dependency_file)
else:
self.debug(" Generating %s..." % library)
egg_file = builder.build_egg()
if egg_file:
src_egg_file = egg_file
dst_egg_file = os.path.join(
os.path.dirname(egg_file), cache_key.hash + "_" + os.path.basename(egg_file)
)
os.rename(src_egg_file, dst_egg_file)
self._cache.update(cache_key, [dst_egg_file])
egg_dep = PythonDependency.from_eggs(dst_egg_file)
for pkg in builder.packages():
self.debug(" found namespace: %s" % pkg)
self.debug(" copying...")
self.env.add_dependency(egg_dep)
self.debug("done.")
else:
self.debug(" Failed!")
raise PythonChroot.BuildFailureException("Failed to build %s!" % library)
def aggregate_targets(self, targets):
libraries, eggs, reqs, binaries, thrifts, antlrs = set(), set(), set(), set(), set(), set()
for target in targets:
(addl_libraries, addl_eggs, addl_reqs, addl_binaries, addl_thrifts, addl_antlrs) = self.build_dep_tree(
target
)
libraries.update(addl_libraries)
eggs.update(addl_eggs)
reqs.update(addl_reqs)
binaries.update(addl_binaries)
thrifts.update(addl_thrifts)
antlrs.update(addl_antlrs)
return libraries, eggs, reqs, binaries, thrifts, antlrs
def build_dep_tree(self, input_target):
target = copy.deepcopy(input_target)
common_python = self._get_common_python()
if common_python not in target.dependencies:
target.dependencies.add(common_python)
libraries, eggs, reqs, binaries, thrifts, antlrs = set(), set(), set(), set(), set(), set()
def add_dep(trg):
if isinstance(trg, PythonLibrary):
if trg.sources or trg.resources:
libraries.add(trg)
for dep in trg.dependencies:
if isinstance(dep, PythonEgg):
eggs.add(dep)
elif isinstance(trg, PythonEgg):
eggs.add(trg)
elif isinstance(trg, PythonRequirement):
reqs.add(trg)
elif isinstance(trg, PythonBinary):
binaries.add(trg)
elif isinstance(trg, PythonThriftLibrary):
thrifts.add(trg)
elif isinstance(trg, PythonAntlrLibrary):
antlrs.add(trg)
elif isinstance(trg, PythonTests):
pass
else:
raise PythonChroot.InvalidDependencyException(trg)
return trg.dependencies if hasattr(trg, "dependencies") else []
target.walk(lambda t: add_dep(t))
return libraries, eggs, reqs, binaries, thrifts, antlrs
def cache_egg_if_necessary(self, egg):
target_file = os.path.join(self._eggcache, os.path.basename(egg))
if not os.path.exists(target_file):
if os.path.isdir(egg):
shutil.copytree(egg, target_file)
else:
shutil.copyfile(egg, target_file)
def _dump_req(self, req):
self.debug(" - Fetching requirement: %s" % req)
fetched_egg = self._fetcher.fetch(req)
if fetched_egg is None:
print("ERROR: Could not find %s!" % req, file=sys.stderr)
else:
if fetched_egg.endswith(".egg"):
self._dump_egg(PythonDependency.from_eggs(fetched_egg))
self.cache_egg_if_necessary(fetched_egg)
else:
if self._index is None:
self.debug("ERROR: Could not contact any indices! Your application may not " "work properly.")
return
self.debug(" => Building %s" % fetched_egg)
distributions = ReqBuilder.install_requirement(fetched_egg, index=self._index, repositories=self._repos)
if not distributions:
print("WARNING: Unable to build a working distribution from %s!" % fetched_egg, file=sys.stderr)
print("Your application may not work properly.", file=sys.stderr)
else:
for distribution in distributions:
if distribution.location.endswith(".egg"):
self.cache_egg_if_necessary(distribution.location)
self._dump_distribution(distribution)
def _dump_python_req(self, python_req):
self.debug(" Dumping Requirement(%s)" % python_req.name)
self._dump_req(python_req._requirement)
def add_req(self, req):
assert isinstance(req, pkg_resources.Requirement)
self._additional_reqs.add(req)
def dump(self):
self.debug("Building PythonBinary %s:" % self._target)
libraries, eggs, reqs, binaries, thrifts, antlrs = self.aggregate_targets([self._target] + self._extra_targets)
bare_reqs = set([pkg_resources.Requirement.parse("distribute")])
bare_reqs.update(self._additional_reqs)
for lib in libraries:
self._dump_library(lib)
for req in bare_reqs:
self._dump_req(req)
for egg in eggs:
self._dump_egg(egg)
for req in reqs:
self._dump_python_req(req)
for thr in thrifts:
self._dump_thrift_library(thr)
for antlr in antlrs:
self._dump_antlr_library(antlr)
if len(binaries) > 1:
print("WARNING: Target has multiple python_binary targets!", file=sys.stderr)
for binary in binaries:
self._dump_bin(binary.sources[0], binary.target_base)
self.env.freeze()
return self.env
class PythonChroot(object):
def __init__(self, target, root_dir, extra_targets=None, builder=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._cache = BuildCache(
os.path.join(self._config.get('python-setup', 'artifact_cache'),
'%s' % PythonIdentity.get()))
self._extra_targets = list(
extra_targets) if extra_targets is not None else []
self._resolver = PythonResolver([self._target] + self._extra_targets)
self._builder = builder or PEXBuilder(tempfile.mkdtemp())
self._platforms = (Platform.current(), )
self._pythons = (sys.version[:3], )
# TODO(wickman) Should this be in the binary builder?
if isinstance(self._target, PythonBinary):
self._platforms = self._target._platforms
self._pythons = self._target._interpreters
def __del__(self):
if os.getenv('PANTS_LEAVE_CHROOT') is None:
safe_rmtree(self.path())
def debug(self, msg, indent=0):
if os.getenv('PANTS_VERBOSE') is not None:
print('%s%s' % (' ' * indent, msg))
def path(self):
return self._builder.path()
def _dump_library(self, library):
def translate_module(module):
if module is None:
module = ''
return module.replace('.', os.path.sep)
def copy_to_chroot(base, path, relative_to, add_function):
src = os.path.join(self._root, base, path)
dst = os.path.join(translate_module(relative_to), path)
add_function(src, dst)
self.debug(' Dumping library: %s [relative module: %s]' %
(library, library.module))
for filename in library.sources:
copy_to_chroot(library.target_base, filename, library.module,
self._builder.add_source)
for filename in library.resources:
copy_to_chroot(library.target_base, filename, library.module,
self._builder.add_resource)
def _dump_requirement(self, req, dynamic, repo):
self.debug(' Dumping requirement: %s%s%s' %
(str(req), ' (dynamic)' if dynamic else '',
' (repo: %s)' if repo else ''))
self._builder.add_requirement(req, dynamic, repo)
def _dump_distribution(self, dist):
self.debug(' Dumping distribution: .../%s' %
os.path.basename(dist.location))
self._builder.add_distribution(dist)
def _dump_bin(self, binary_name, base):
src = os.path.join(self._root, base, binary_name)
self.debug(' Dumping binary: %s' % binary_name)
self._builder.set_executable(src, os.path.basename(src))
def _dump_thrift_library(self, library):
self._dump_built_library(library,
PythonThriftBuilder(library, self._root))
def _dump_antlr_library(self, library):
self._dump_built_library(library,
PythonAntlrBuilder(library, self._root))
def _dump_built_library(self, library, builder):
# TODO(wickman) Port this over to the Installer+Distiller and stop using
# BuildCache.
absolute_sources = library.expand_files()
absolute_sources.sort()
cache_key = self._cache.key_for(library.id, absolute_sources)
if not self._cache.needs_update(cache_key):
self.debug(' Generating (cached) %s...' % library)
self._cache.use_cached_files(cache_key,
self._builder.add_dependency_file)
else:
self.debug(' Generating %s...' % library)
egg_file = builder.build_egg()
if not egg_file:
raise PythonChroot.BuildFailureException(
"Failed to build %s!" % library)
src_egg_file = egg_file
dst_egg_file = os.path.join(
os.path.dirname(egg_file),
cache_key.hash + '_' + os.path.basename(egg_file))
self.debug(' %s => %s' % (src_egg_file, dst_egg_file))
os.rename(src_egg_file, dst_egg_file)
self._cache.update(cache_key, [dst_egg_file])
self._builder.add_egg(dst_egg_file)
def dump(self):
self.debug('Building PythonBinary %s:' % self._target)
targets = self._resolver.resolve()
for lib in targets['libraries']:
self._dump_library(lib)
for req in targets['reqs']:
if not req.should_build():
self.debug('Skipping %s based upon version filter' % req)
continue
self._dump_requirement(req._requirement, req._dynamic,
req._repository)
for dist in ReqResolver.resolve(
(req._requirement
for req in targets['reqs'] if req.should_build()),
self._config,
self._platforms,
self._pythons,
ignore_errors=self._builder.info().ignore_errors):
self._dump_distribution(dist)
for thr in targets['thrifts']:
self._dump_thrift_library(thr)
for antlr in targets['antlrs']:
self._dump_antlr_library(antlr)
if len(targets['binaries']) > 1:
print('WARNING: Target has multiple python_binary targets!',
file=sys.stderr)
for binary in targets['binaries']:
if len(binary.sources) > 0:
self._dump_bin(binary.sources[0], binary.target_base)
return self._builder
def test_env(content=TEST_CONTENT):
with temporary_dir() as d:
with tempfile.NamedTemporaryFile() as f:
f.write(content)
f.flush()
yield f, BuildCache(d)
