def __init__(self, target, root_dir, extra_targets=None): self._config = Config.load() self._target = target self._root = root_dir self._cache = BuildCache( os.path.join(self._config.get("python-setup", "artifact_cache"), "%s" % PythonIdentity.get()) ) self._extra_targets = list(extra_targets) if extra_targets is not None else [] self._extra_targets.append(self._get_common_python()) cachedir = self._config.get("python-setup", "cache") safe_mkdir(cachedir) self._eggcache = cachedir local_repo = "file://%s" % os.path.realpath(cachedir) self._repos = [local_repo] + self._config.getlist("python-setup", "repos") self._fetcher = ReqFetcher(repos=self._repos, cache=cachedir) self._index = None for index in self._config.getlist("python-setup", "indices"): if PythonChroot.can_contact_index(index): self._index = index break self._additional_reqs = set() distdir = self._config.getdefault("pants_distdir") distpath = tempfile.mktemp(dir=distdir, prefix=target.name) self.env = PythonEnvironment(distpath)
def __init__(self, target, root_dir): self._target = target self._root = root_dir self._cache = BuildCache(os.path.join(root_dir, '.pants.d', 'py_artifact_cache')) distdir = os.path.join(root_dir, 'dist') distpath = tempfile.mktemp(dir=distdir, prefix=target.name) self.env = PythonEnvironment(distpath)
def __init__(self, target, root_dir, extra_targets=None, builder=None): self._config = Config.load() self._target = target self._root = root_dir self._cache = BuildCache( os.path.join(self._config.get('python-setup', 'artifact_cache'), '%s' % PythonIdentity.get())) self._extra_targets = list( extra_targets) if extra_targets is not None else [] self._resolver = PythonResolver([self._target] + self._extra_targets) self._builder = builder or PEXBuilder(tempfile.mkdtemp()) self._platforms = (Platform.current(), ) self._pythons = (sys.version[:3], ) # TODO(wickman) Should this be in the binary builder? if isinstance(self._target, PythonBinary): self._platforms = self._target._platforms self._pythons = self._target._interpreters
def __init__(self, target, root_dir, extra_targets=None, builder=None): self._config = Config.load() self._target = target self._root = root_dir self._cache = BuildCache(os.path.join(self._config.get('python-setup', 'artifact_cache'), '%s' % PythonIdentity.get())) self._extra_targets = list(extra_targets) if extra_targets is not None else [] self._resolver = PythonResolver([self._target] + self._extra_targets) self._builder = builder or PEXBuilder(tempfile.mkdtemp()) self._platforms = (Platform.current(),) self._pythons = (sys.version[:3],) # TODO(wickman) Should this be in the binary builder? if isinstance(self._target, PythonBinary): self._platforms = self._target._platforms self._pythons = self._target._interpreters
def changed(self, targets, only_buildfiles=False, invalidate_dependants=False): """ Yields an iterable over the targets that have changed since the last check to a with block. If no exceptions are thrown by work in the block, the cache is updated for the targets, otherwise if a TargetError is thrown by the work in the block all targets except those in the TargetError are cached. :targets The targets to check for changes. :only_buildfiles If True, then just the target's BUILD files are checked for changes. :invalidate_dependants If True then any targets depending on changed targets are invalidated :returns: the subset of targets that have changed """ safe_mkdir(self._basedir) cache_manager = Task.CacheManager(BuildCache(self._basedir), targets, only_buildfiles) check = self.invalidate_for() if check is not None: with safe_open(self._extradata, 'w') as pickled: pickle.dump(check, pickled) cache_key = cache_manager.check_content(Task.EXTRA_DATA, [self._extradata]) if cache_key: self.context.log.debug('invalidating all targets for %s' % self.__class__.__name__) for target in targets: cache_manager.invalidate(target, cache_key) for target in targets: cache_manager.check(target) if invalidate_dependants and cache_manager.changed: for target in (self.context.dependants(lambda t: t in cache_manager.changed.keys())).keys(): cache_manager.invalidate(target) if invalidate_dependants: if cache_manager.foreign_invalidated_targets: self.context.log.info('Invalidated %d dependant targets ' 'for the next round' % cache_manager.foreign_invalidated_targets) if cache_manager.changed_files: msg = 'Operating on %d files in %d changed targets' % ( cache_manager.changed_files, len(cache_manager.changed) ) if cache_manager.invalidated_files: msg += ' and %d files in %d invalidated dependant targets' % ( cache_manager.invalidated_files, cache_manager.invalidated_targets ) self.context.log.info(msg) elif cache_manager.changed_files: self.context.log.info('Operating on %d files in %d changed targets' % ( cache_manager.changed_files, len(cache_manager.changed) )) try: yield cache_manager.changed.keys() for cache_keys in cache_manager.changed.values(): for cache_key in cache_keys: cache_manager.update(cache_key) except TargetError as e: for target, cache_keys in cache_manager.changed.items(): if target not in e.targets: for cache_key in cache_keys: cache_manager.update(cache_key)
class PythonChroot(object): def __init__(self, target, root_dir, extra_targets=None, builder=None): self._config = Config.load() self._target = target self._root = root_dir self._cache = BuildCache(os.path.join(self._config.get('python-setup', 'artifact_cache'), '%s' % PythonIdentity.get())) self._extra_targets = list(extra_targets) if extra_targets is not None else [] self._resolver = PythonResolver([self._target] + self._extra_targets) self._builder = builder or PEXBuilder(tempfile.mkdtemp()) self._platforms = (Platform.current(),) self._pythons = (sys.version[:3],) # TODO(wickman) Should this be in the binary builder? if isinstance(self._target, PythonBinary): self._platforms = self._target._platforms self._pythons = self._target._interpreters def __del__(self): if os.getenv('PANTS_LEAVE_CHROOT') is None: safe_rmtree(self.path()) def debug(self, msg, indent=0): if os.getenv('PANTS_VERBOSE') is not None: print('%s%s' % (' ' * indent, msg)) def path(self): return self._builder.path() def _dump_library(self, library): def translate_module(module): if module is None: module = '' return module.replace('.', os.path.sep) def copy_to_chroot(base, path, relative_to, add_function): src = os.path.join(self._root, base, path) dst = os.path.join(translate_module(relative_to), path) add_function(src, dst) self.debug(' Dumping library: %s [relative module: %s]' % (library, library.module)) for filename in library.sources: copy_to_chroot(library.target_base, filename, library.module, self._builder.add_source) for filename in library.resources: copy_to_chroot(library.target_base, filename, library.module, self._builder.add_resource) def _dump_requirement(self, req, dynamic, repo): self.debug(' Dumping requirement: %s%s%s' % (str(req), ' (dynamic)' if dynamic else '', ' (repo: %s)' if repo else '')) self._builder.add_requirement(req, dynamic, repo) def _dump_distribution(self, dist): self.debug(' Dumping distribution: .../%s' % os.path.basename(dist.location)) self._builder.add_distribution(dist) def _dump_bin(self, binary_name, base): src = os.path.join(self._root, base, binary_name) self.debug(' Dumping binary: %s' % binary_name) self._builder.set_executable(src, os.path.basename(src)) def _dump_thrift_library(self, library): self._dump_built_library(library, PythonThriftBuilder(library, self._root)) def _dump_antlr_library(self, library): self._dump_built_library(library, PythonAntlrBuilder(library, self._root)) def _dump_built_library(self, library, builder): # TODO(wickman) Port this over to the Installer+Distiller and stop using # BuildCache. absolute_sources = library.expand_files() absolute_sources.sort() cache_key = self._cache.key_for(library.id, absolute_sources) if not self._cache.needs_update(cache_key): self.debug(' Generating (cached) %s...' % library) self._cache.use_cached_files(cache_key, self._builder.add_dependency_file) else: self.debug(' Generating %s...' % library) egg_file = builder.build_egg() if not egg_file: raise PythonChroot.BuildFailureException("Failed to build %s!" % library) src_egg_file = egg_file dst_egg_file = os.path.join(os.path.dirname(egg_file), cache_key.hash + '_' + os.path.basename(egg_file)) self.debug(' %s => %s' % (src_egg_file, dst_egg_file)) os.rename(src_egg_file, dst_egg_file) self._cache.update(cache_key, [dst_egg_file]) self._builder.add_egg(dst_egg_file) def dump(self): self.debug('Building PythonBinary %s:' % self._target) targets = self._resolver.resolve() for lib in targets['libraries']: self._dump_library(lib) for req in targets['reqs']: if not req.should_build(): self.debug('Skipping %s based upon version filter' % req) continue self._dump_requirement(req._requirement, req._dynamic, req._repository) for dist in ReqResolver.resolve( (req._requirement for req in targets['reqs'] if req.should_build()), self._config, self._platforms, self._pythons, ignore_errors=self._builder.info().ignore_errors): self._dump_distribution(dist) for thr in targets['thrifts']: self._dump_thrift_library(thr) for antlr in targets['antlrs']: self._dump_antlr_library(antlr) if len(targets['binaries']) > 1: print('WARNING: Target has multiple python_binary targets!', file=sys.stderr) for binary in targets['binaries']: if len(binary.sources) > 0: self._dump_bin(binary.sources[0], binary.target_base) return self._builder
class PythonChroot(object): class InvalidDependencyException(Exception): def __init__(self, target): Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target) class BuildFailureException(Exception): def __init__(self, target): Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target) def __init__(self, target, root_dir): self._target = target self._root = root_dir self._cache = BuildCache(os.path.join(root_dir, '.pants.d', 'py_artifact_cache')) distdir = os.path.join(root_dir, 'dist') distpath = tempfile.mktemp(dir=distdir, prefix=target.name) self.env = PythonEnvironment(distpath) def __del__(self): if os.getenv('PANTS_LEAVE_CHROOT') is None: try: shutil.rmtree(self.path()) except OSError as e: if e.errno != errno.ENOENT: raise def path(self): return self.env.path() def _dump_library(self, library): def translate_module(module): if module is None: module = '' return module.replace('.', os.path.sep) def copy_to_chroot(base, path, relative_to, add_function): src = os.path.join(self._root, base, path) dst = os.path.join(translate_module(relative_to), path) add_function(src, dst) template = library._create_template_data() print ' Dumping library: %s [relative module: %s]' % (library, template.module) for filename in template.sources: copy_to_chroot(template.template_base, filename, template.module, self.env.add_source) for filename in template.resources: copy_to_chroot(template.template_base, filename, template.module, self.env.add_resource) def _dump_egg(self, egg): target_name = os.path.pathsep.join(sorted(os.path.basename(e) for e in egg.eggs)) cache_key = self._cache.key_for(target_name, egg.eggs) if self._cache.needs_update(cache_key): print ' Dumping egg: %s' % egg prefixes, all_added_files = set(), set() for egg_path in egg.eggs: egg_dep = PythonDependency.from_eggs(egg_path) prefix, added_files = self.env.add_dependency(egg_dep) all_added_files.update(added_files) prefixes.add(prefix) assert len(prefixes) == 1, 'Ambiguous egg environment!' self._cache.update(cache_key, all_added_files, artifact_root=prefixes.pop()) else: print ' Dumping (cached) egg: %s' % egg self._cache.use_cached_files(cache_key, self.env.add_dependency_file) def _dump_setuptools(self): SETUPTOOLS = 'distribute-0.6.21-py2.6.egg' print ' Dumping setuptools: %s' % SETUPTOOLS data = pkgutil.get_data(__name__, os.path.join('bootstrap', SETUPTOOLS)) with temporary_dir() as tempdir: egg_path = os.path.join(tempdir, SETUPTOOLS) with open(egg_path, 'w') as fp: fp.write(data) egg_dep = PythonDependency.from_eggs(egg_path) self.env.add_dependency(egg_dep) # TODO(wickman) Just add write() to self.env and do this with pkg_resources or # just build an egg for twitter.common.python. def _get_common_python(self): return Target.get(Address.parse(self._root, 'src/python/twitter/common/python')) def _dump_bin(self, binary_name, base): src = os.path.join(self._root, base, binary_name) print ' Dumping binary: %s' % binary_name self.env.set_executable(src, os.path.basename(src)) def _dump_thrift_library(self, library): self._dump_built_library(library, PythonThriftBuilder(library, self._root)) def _dump_antlr_library(self, library): self._dump_built_library(library, PythonAntlrBuilder(library, self._root)) def _dump_built_library(self, library, builder): absolute_sources = library.expand_files() absolute_sources.sort() cache_key = self._cache.key_for(library._create_id(), absolute_sources) if not self._cache.needs_update(cache_key): print ' Generating (cached) %s...' % library self._cache.use_cached_files(cache_key, self.env.add_dependency_file) else: print ' Generating %s...' % library egg_file = builder.build_egg() if egg_file: src_egg_file = egg_file dst_egg_file = os.path.join(os.path.dirname(egg_file), cache_key.hash + '_' + os.path.basename(egg_file)) os.rename(src_egg_file, dst_egg_file) self._cache.update(cache_key, [dst_egg_file]) egg_dep = PythonDependency.from_eggs(dst_egg_file) for pkg in builder.packages(): print ' found namespace: %s' % pkg print ' copying...', self.env.add_dependency(egg_dep) print 'done.' else: print ' Failed!' raise PythonChroot.BuildFailureException("Failed to build %s!" % library) def build_dep_tree(self, input_target): target = copy.deepcopy(input_target) common_python = self._get_common_python() if common_python not in target.dependencies: target.dependencies.add(common_python) libraries = set() eggs = set() binaries = set() thrifts = set() antlrs = set() def add_dep(trg): if isinstance(trg, PythonLibrary): if trg.sources: libraries.add(trg) for egg in [dep for dep in trg.dependencies if isinstance(dep, PythonEgg)]: eggs.add(egg) elif isinstance(trg, PythonEgg): eggs.add(trg) elif isinstance(trg, PythonBinary): binaries.add(trg) elif isinstance(trg, PythonThriftLibrary): thrifts.add(trg) elif isinstance(trg, PythonAntlrLibrary): antlrs.add(trg) elif isinstance(trg, PythonTests): # do not dump test sources/resources, but dump their # dependencies. pass else: raise PythonChroot.InvalidDependencyException(trg) return [dep for dep in trg.dependencies if not isinstance(dep, PythonEgg)] target.walk(lambda t: add_dep(t), lambda typ: not isinstance(typ, PythonEgg)) return libraries, eggs, binaries, thrifts, antlrs def dump(self): print 'Building PythonBinary %s:' % self._target libraries, eggs, binaries, thrifts, antlrs = self.build_dep_tree(self._target) for lib in libraries: self._dump_library(lib) self._dump_setuptools() for egg in eggs: self._dump_egg(egg) for thr in thrifts: self._dump_thrift_library(thr) for antlr in antlrs: self._dump_antlr_library(antlr) if len(binaries) > 1: print >> sys.stderr, 'WARNING: Target has multiple python_binary targets!' for binary in binaries: self._dump_bin(binary.sources[0], binary.target_base) self.env.freeze() return self.env
class PythonChroot(object): class InvalidDependencyException(Exception): def __init__(self, target): Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target) class BuildFailureException(Exception): def __init__(self, target): Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target) @staticmethod def can_contact_index(url, timeout=Amount(5, Time.SECONDS)): try: response = urlopen(url, timeout=timeout.as_(Time.SECONDS)) return response.code == 200 except URLError: return False def __init__(self, target, root_dir, extra_targets=None): self._config = Config.load() self._target = target self._root = root_dir self._cache = BuildCache( os.path.join(self._config.get("python-setup", "artifact_cache"), "%s" % PythonIdentity.get()) ) self._extra_targets = list(extra_targets) if extra_targets is not None else [] self._extra_targets.append(self._get_common_python()) cachedir = self._config.get("python-setup", "cache") safe_mkdir(cachedir) self._eggcache = cachedir local_repo = "file://%s" % os.path.realpath(cachedir) self._repos = [local_repo] + self._config.getlist("python-setup", "repos") self._fetcher = ReqFetcher(repos=self._repos, cache=cachedir) self._index = None for index in self._config.getlist("python-setup", "indices"): if PythonChroot.can_contact_index(index): self._index = index break self._additional_reqs = set() distdir = self._config.getdefault("pants_distdir") distpath = tempfile.mktemp(dir=distdir, prefix=target.name) self.env = PythonEnvironment(distpath) def __del__(self): if os.getenv("PANTS_LEAVE_CHROOT") is None: try: shutil.rmtree(self.path()) except OSError as e: if e.errno != errno.ENOENT: raise def debug(self, msg, indent=0): if os.getenv("PANTS_VERBOSE") is not None: print("%s%s" % (" " * indent, msg)) def path(self): return self.env.path() def _dump_library(self, library): def translate_module(module): if module is None: module = "" return module.replace(".", os.path.sep) def copy_to_chroot(base, path, relative_to, add_function): src = os.path.join(self._root, base, path) dst = os.path.join(translate_module(relative_to), path) add_function(src, dst) self.debug(" Dumping library: %s [relative module: %s]" % (library, library.module)) for filename in library.sources: copy_to_chroot(library.target_base, filename, library.module, self.env.add_source) for filename in library.resources: copy_to_chroot(library.target_base, filename, library.module, self.env.add_resource) def _dump_egg(self, egg): if isinstance(egg, PythonEgg): target_name = os.path.pathsep.join(sorted(os.path.basename(e) for e in egg.eggs)) cache_key = self._cache.key_for(target_name, egg.eggs) if self._cache.needs_update(cache_key): self.debug(" Dumping egg: %s" % egg) prefixes, all_added_files = set(), set() for egg_path in egg.eggs: egg_dep = PythonDependency.from_eggs(egg_path) prefix, added_files = self.env.add_dependency(egg_dep) all_added_files.update(added_files) prefixes.add(prefix) assert len(prefixes) == 1, "Ambiguous egg environment!" self._cache.update(cache_key, all_added_files, artifact_root=prefixes.pop()) else: self.debug(" Dumping (cached) egg: %s" % egg) self._cache.use_cached_files(cache_key, self.env.add_dependency_file) elif isinstance(egg, PythonDependency): self.debug(" Dumping PythonDependency: %s" % egg) self.env.add_dependency(egg) else: raise PythonChroot.InvalidDependencyException("Unknown egg dependency %s" % egg) def _dump_distribution(self, dist): self.debug(" Dumping distribution: .../%s" % os.path.basename(dist.location)) egg_dep = PythonDependency.from_distributions(dist) self.env.add_dependency(egg_dep) # TODO(wickman) Just add write() to self.env and do this with pkg_resources or # just build an egg for twitter.common.python. def _get_common_python(self): return Target.get(Address.parse(self._root, "src/python/twitter/common/python")) def _dump_bin(self, binary_name, base): src = os.path.join(self._root, base, binary_name) self.debug(" Dumping binary: %s" % binary_name) self.env.set_executable(src, os.path.basename(src)) def _dump_thrift_library(self, library): self._dump_built_library(library, PythonThriftBuilder(library, self._root)) def _dump_antlr_library(self, library): self._dump_built_library(library, PythonAntlrBuilder(library, self._root)) def _dump_built_library(self, library, builder): absolute_sources = library.expand_files() absolute_sources.sort() cache_key = self._cache.key_for(library._create_id(), absolute_sources) if not self._cache.needs_update(cache_key): self.debug(" Generating (cached) %s..." % library) self._cache.use_cached_files(cache_key, self.env.add_dependency_file) else: self.debug(" Generating %s..." % library) egg_file = builder.build_egg() if egg_file: src_egg_file = egg_file dst_egg_file = os.path.join( os.path.dirname(egg_file), cache_key.hash + "_" + os.path.basename(egg_file) ) os.rename(src_egg_file, dst_egg_file) self._cache.update(cache_key, [dst_egg_file]) egg_dep = PythonDependency.from_eggs(dst_egg_file) for pkg in builder.packages(): self.debug(" found namespace: %s" % pkg) self.debug(" copying...") self.env.add_dependency(egg_dep) self.debug("done.") else: self.debug(" Failed!") raise PythonChroot.BuildFailureException("Failed to build %s!" % library) def aggregate_targets(self, targets): libraries, eggs, reqs, binaries, thrifts, antlrs = set(), set(), set(), set(), set(), set() for target in targets: (addl_libraries, addl_eggs, addl_reqs, addl_binaries, addl_thrifts, addl_antlrs) = self.build_dep_tree( target ) libraries.update(addl_libraries) eggs.update(addl_eggs) reqs.update(addl_reqs) binaries.update(addl_binaries) thrifts.update(addl_thrifts) antlrs.update(addl_antlrs) return libraries, eggs, reqs, binaries, thrifts, antlrs def build_dep_tree(self, input_target): target = copy.deepcopy(input_target) common_python = self._get_common_python() if common_python not in target.dependencies: target.dependencies.add(common_python) libraries, eggs, reqs, binaries, thrifts, antlrs = set(), set(), set(), set(), set(), set() def add_dep(trg): if isinstance(trg, PythonLibrary): if trg.sources or trg.resources: libraries.add(trg) for dep in trg.dependencies: if isinstance(dep, PythonEgg): eggs.add(dep) elif isinstance(trg, PythonEgg): eggs.add(trg) elif isinstance(trg, PythonRequirement): reqs.add(trg) elif isinstance(trg, PythonBinary): binaries.add(trg) elif isinstance(trg, PythonThriftLibrary): thrifts.add(trg) elif isinstance(trg, PythonAntlrLibrary): antlrs.add(trg) elif isinstance(trg, PythonTests): pass else: raise PythonChroot.InvalidDependencyException(trg) return trg.dependencies if hasattr(trg, "dependencies") else [] target.walk(lambda t: add_dep(t)) return libraries, eggs, reqs, binaries, thrifts, antlrs def cache_egg_if_necessary(self, egg): target_file = os.path.join(self._eggcache, os.path.basename(egg)) if not os.path.exists(target_file): if os.path.isdir(egg): shutil.copytree(egg, target_file) else: shutil.copyfile(egg, target_file) def _dump_req(self, req): self.debug(" - Fetching requirement: %s" % req) fetched_egg = self._fetcher.fetch(req) if fetched_egg is None: print("ERROR: Could not find %s!" % req, file=sys.stderr) else: if fetched_egg.endswith(".egg"): self._dump_egg(PythonDependency.from_eggs(fetched_egg)) self.cache_egg_if_necessary(fetched_egg) else: if self._index is None: self.debug("ERROR: Could not contact any indices! Your application may not " "work properly.") return self.debug(" => Building %s" % fetched_egg) distributions = ReqBuilder.install_requirement(fetched_egg, index=self._index, repositories=self._repos) if not distributions: print("WARNING: Unable to build a working distribution from %s!" % fetched_egg, file=sys.stderr) print("Your application may not work properly.", file=sys.stderr) else: for distribution in distributions: if distribution.location.endswith(".egg"): self.cache_egg_if_necessary(distribution.location) self._dump_distribution(distribution) def _dump_python_req(self, python_req): self.debug(" Dumping Requirement(%s)" % python_req.name) self._dump_req(python_req._requirement) def add_req(self, req): assert isinstance(req, pkg_resources.Requirement) self._additional_reqs.add(req) def dump(self): self.debug("Building PythonBinary %s:" % self._target) libraries, eggs, reqs, binaries, thrifts, antlrs = self.aggregate_targets([self._target] + self._extra_targets) bare_reqs = set([pkg_resources.Requirement.parse("distribute")]) bare_reqs.update(self._additional_reqs) for lib in libraries: self._dump_library(lib) for req in bare_reqs: self._dump_req(req) for egg in eggs: self._dump_egg(egg) for req in reqs: self._dump_python_req(req) for thr in thrifts: self._dump_thrift_library(thr) for antlr in antlrs: self._dump_antlr_library(antlr) if len(binaries) > 1: print("WARNING: Target has multiple python_binary targets!", file=sys.stderr) for binary in binaries: self._dump_bin(binary.sources[0], binary.target_base) self.env.freeze() return self.env
class PythonChroot(object): def __init__(self, target, root_dir, extra_targets=None, builder=None): self._config = Config.load() self._target = target self._root = root_dir self._cache = BuildCache( os.path.join(self._config.get('python-setup', 'artifact_cache'), '%s' % PythonIdentity.get())) self._extra_targets = list( extra_targets) if extra_targets is not None else [] self._resolver = PythonResolver([self._target] + self._extra_targets) self._builder = builder or PEXBuilder(tempfile.mkdtemp()) self._platforms = (Platform.current(), ) self._pythons = (sys.version[:3], ) # TODO(wickman) Should this be in the binary builder? if isinstance(self._target, PythonBinary): self._platforms = self._target._platforms self._pythons = self._target._interpreters def __del__(self): if os.getenv('PANTS_LEAVE_CHROOT') is None: safe_rmtree(self.path()) def debug(self, msg, indent=0): if os.getenv('PANTS_VERBOSE') is not None: print('%s%s' % (' ' * indent, msg)) def path(self): return self._builder.path() def _dump_library(self, library): def translate_module(module): if module is None: module = '' return module.replace('.', os.path.sep) def copy_to_chroot(base, path, relative_to, add_function): src = os.path.join(self._root, base, path) dst = os.path.join(translate_module(relative_to), path) add_function(src, dst) self.debug(' Dumping library: %s [relative module: %s]' % (library, library.module)) for filename in library.sources: copy_to_chroot(library.target_base, filename, library.module, self._builder.add_source) for filename in library.resources: copy_to_chroot(library.target_base, filename, library.module, self._builder.add_resource) def _dump_requirement(self, req, dynamic, repo): self.debug(' Dumping requirement: %s%s%s' % (str(req), ' (dynamic)' if dynamic else '', ' (repo: %s)' if repo else '')) self._builder.add_requirement(req, dynamic, repo) def _dump_distribution(self, dist): self.debug(' Dumping distribution: .../%s' % os.path.basename(dist.location)) self._builder.add_distribution(dist) def _dump_bin(self, binary_name, base): src = os.path.join(self._root, base, binary_name) self.debug(' Dumping binary: %s' % binary_name) self._builder.set_executable(src, os.path.basename(src)) def _dump_thrift_library(self, library): self._dump_built_library(library, PythonThriftBuilder(library, self._root)) def _dump_antlr_library(self, library): self._dump_built_library(library, PythonAntlrBuilder(library, self._root)) def _dump_built_library(self, library, builder): # TODO(wickman) Port this over to the Installer+Distiller and stop using # BuildCache. absolute_sources = library.expand_files() absolute_sources.sort() cache_key = self._cache.key_for(library.id, absolute_sources) if not self._cache.needs_update(cache_key): self.debug(' Generating (cached) %s...' % library) self._cache.use_cached_files(cache_key, self._builder.add_dependency_file) else: self.debug(' Generating %s...' % library) egg_file = builder.build_egg() if not egg_file: raise PythonChroot.BuildFailureException( "Failed to build %s!" % library) src_egg_file = egg_file dst_egg_file = os.path.join( os.path.dirname(egg_file), cache_key.hash + '_' + os.path.basename(egg_file)) self.debug(' %s => %s' % (src_egg_file, dst_egg_file)) os.rename(src_egg_file, dst_egg_file) self._cache.update(cache_key, [dst_egg_file]) self._builder.add_egg(dst_egg_file) def dump(self): self.debug('Building PythonBinary %s:' % self._target) targets = self._resolver.resolve() for lib in targets['libraries']: self._dump_library(lib) for req in targets['reqs']: if not req.should_build(): self.debug('Skipping %s based upon version filter' % req) continue self._dump_requirement(req._requirement, req._dynamic, req._repository) for dist in ReqResolver.resolve( (req._requirement for req in targets['reqs'] if req.should_build()), self._config, self._platforms, self._pythons, ignore_errors=self._builder.info().ignore_errors): self._dump_distribution(dist) for thr in targets['thrifts']: self._dump_thrift_library(thr) for antlr in targets['antlrs']: self._dump_antlr_library(antlr) if len(targets['binaries']) > 1: print('WARNING: Target has multiple python_binary targets!', file=sys.stderr) for binary in targets['binaries']: if len(binary.sources) > 0: self._dump_bin(binary.sources[0], binary.target_base) return self._builder
def test_env(content=TEST_CONTENT): with temporary_dir() as d: with tempfile.NamedTemporaryFile() as f: f.write(content) f.flush() yield f, BuildCache(d)