def __init__(self, python_setup, python_repos, ivy_bootstrapper, thrift_binary_factory, interpreter, builder, targets, platforms, extra_requirements=None): self._python_setup = python_setup self._python_repos = python_repos self._ivy_bootstrapper = ivy_bootstrapper self._thrift_binary_factory = thrift_binary_factory self._interpreter = interpreter self._builder = builder self._targets = targets self._platforms = platforms self._extra_requirements = list( extra_requirements) if extra_requirements else [] # Note: unrelated to the general pants artifact cache. self._artifact_cache_root = os.path.join( self._python_setup.artifact_cache_dir, str(self._interpreter.identity)) self._key_generator = CacheKeyGenerator() self._build_invalidator = BuildInvalidator(self._artifact_cache_root)
def __init__(self, target, root_dir, extra_targets=None, extra_requirements=None, builder=None, platforms=None, interpreter=None, conn_timeout=None): self._config = Config.load() self._target = target self._root = root_dir self._platforms = platforms self._interpreter = interpreter or PythonInterpreter.get() self._extra_targets = list( extra_targets) if extra_targets is not None else [] self._extra_requirements = list( extra_requirements) if extra_requirements is not None else [] self._builder = builder or PEXBuilder(tempfile.mkdtemp(), interpreter=self._interpreter) # Note: unrelated to the general pants artifact cache. self._egg_cache_root = os.path.join( PythonSetup(self._config).scratch_dir('artifact_cache', default_name='artifacts'), str(self._interpreter.identity)) self._key_generator = CacheKeyGenerator() self._build_invalidator = BuildInvalidator(self._egg_cache_root)
def __init__(self, context, workdir): """Subclass __init__ methods, if defined, *must* follow this idiom: class MyTask(Task): def __init__(self, *args, **kwargs): super(MyTask, self).__init__(*args, **kwargs) ... This allows us to change Task.__init__()'s arguments without changing every subclass. """ self.context = context self._workdir = workdir # TODO: It would be nice to use self.get_options().cache_key_gen_version here, because then # we could have a separate value for each scope if we really wanted to. However we can't # access per-task options in Task.__init__ because GroupTask.__init__ calls it with the # group task's scope, which isn't currently in the known scopes we generate options for. self._cache_key_generator = CacheKeyGenerator( self.context.options.for_global_scope().cache_key_gen_version) self._read_artifact_cache_spec = None self._write_artifact_cache_spec = None self._artifact_cache = None self._artifact_cache_setup_lock = threading.Lock() self._cache_key_errors = set() self._build_invalidator_dir = os.path.join( self.context.options.for_global_scope().pants_workdir, 'build_invalidator', self.__class__.__name__)
def __init__(self, python_setup, python_repos, ivy_bootstrapper, thrift_binary_factory, interpreter, builder, targets, platforms, extra_requirements=None, log=None): self._python_setup = python_setup self._python_repos = python_repos self._ivy_bootstrapper = ivy_bootstrapper self._thrift_binary_factory = thrift_binary_factory self._interpreter = interpreter self._builder = builder self._targets = targets self._platforms = platforms self._extra_requirements = list(extra_requirements) if extra_requirements else [] self._logger = log or logger # Note: unrelated to the general pants artifact cache. self._artifact_cache_root = os.path.join( self._python_setup.artifact_cache_dir, str(self._interpreter.identity)) self._key_generator = CacheKeyGenerator() self._build_invalidator = BuildInvalidator(self._artifact_cache_root)
def __init__(self, context, workdir): """Subclass __init__ methods, if defined, *must* follow this idiom: class MyTask(Task): def __init__(self, *args, **kwargs): super(MyTask, self).__init__(*args, **kwargs) ... This allows us to change Task.__init__()'s arguments without changing every subclass. """ self.context = context self._workdir = workdir self._cache_key_generator = CacheKeyGenerator( context.config.getdefault('cache_key_gen_version', default='200')) self._read_artifact_cache_spec = None self._write_artifact_cache_spec = None self._artifact_cache = None self._artifact_cache_setup_lock = threading.Lock() self._cache_key_errors = set() default_invalidator_root = os.path.join( self.context.options.for_global_scope().pants_workdir, 'build_invalidator') suffix_type = self.__class__.__name__ self._build_invalidator_dir = os.path.join( context.config.get('tasks', 'build_invalidator', default=default_invalidator_root), suffix_type)
def __init__(self, target, root_dir, extra_targets=None, extra_requirements=None, builder=None, platforms=None, interpreter=None, conn_timeout=None): self._config = Config.load() self._target = target self._root = root_dir self._platforms = platforms self._interpreter = interpreter or PythonInterpreter.get() self._extra_targets = list(extra_targets) if extra_targets is not None else [] self._extra_requirements = list(extra_requirements) if extra_requirements is not None else [] self._builder = builder or PEXBuilder(tempfile.mkdtemp(), interpreter=self._interpreter) # Note: unrelated to the general pants artifact cache. self._egg_cache_root = os.path.join( PythonSetup(self._config).scratch_dir('artifact_cache', default_name='artifacts'), str(self._interpreter.identity)) self._key_generator = CacheKeyGenerator() self._build_invalidator = BuildInvalidator( self._egg_cache_root)
def __init__(self, context, workdir): """Subclass __init__ methods, if defined, *must* follow this idiom: class MyTask(Task): def __init__(self, *args, **kwargs): super(MyTask, self).__init__(*args, **kwargs) ... This allows us to change Task.__init__()'s arguments without changing every subclass. If the subclass does not need its own initialization, this method can (and should) be omitted entirely. """ super(TaskBase, self).__init__() self.context = context self._workdir = workdir # TODO: It would be nice to use self.get_options().cache_key_gen_version here, because then # we could have a separate value for each scope if we really wanted to. However we can't # access per-task options in Task.__init__ because GroupTask.__init__ calls it with the # group task's scope, which isn't currently in the known scopes we generate options for. self._cache_key_generator = CacheKeyGenerator( self.context.options.for_global_scope().cache_key_gen_version) self._cache_key_errors = set() self._build_invalidator_dir = os.path.join( self.context.options.for_global_scope().pants_workdir, 'build_invalidator', self.stable_name()) self._cache_factory = CacheSetup.create_cache_factory_for_task(self) self._options_fingerprinter = OptionsFingerprinter(self.context.build_graph) self._fingerprint = None
def __init__(self, context, targets, extra_requirements=None, builder=None, platforms=None, interpreter=None): self.context = context # TODO: These should come from the caller, and we should not know about config. self._python_setup = PythonSetup(self.context.config) self._python_repos = PythonRepos(self.context.config) self._targets = targets self._extra_requirements = list(extra_requirements) if extra_requirements else [] self._platforms = platforms self._interpreter = interpreter or PythonInterpreter.get() self._builder = builder or PEXBuilder(os.path.realpath(tempfile.mkdtemp()), interpreter=self._interpreter) # Note: unrelated to the general pants artifact cache. self._egg_cache_root = os.path.join( self._python_setup.scratch_dir, 'artifacts', str(self._interpreter.identity)) self._key_generator = CacheKeyGenerator() self._build_invalidator = BuildInvalidator( self._egg_cache_root)
def __init__(self, context, python_setup, python_repos, targets, extra_requirements=None, builder=None, platforms=None, interpreter=None): self.context = context self._python_setup = python_setup self._python_repos = python_repos self._targets = targets self._extra_requirements = list(extra_requirements) if extra_requirements else [] self._platforms = platforms self._interpreter = interpreter or PythonInterpreter.get() self._builder = builder or PEXBuilder(os.path.realpath(tempfile.mkdtemp()), interpreter=self._interpreter) # Note: unrelated to the general pants artifact cache. self._egg_cache_root = os.path.join( self._python_setup.scratch_dir, 'artifacts', str(self._interpreter.identity)) self._key_generator = CacheKeyGenerator() self._build_invalidator = BuildInvalidator( self._egg_cache_root)
def __init__(self, cache_manager, versioned_targets): self._cache_manager = cache_manager self.versioned_targets = versioned_targets self.targets = [vt.target for vt in versioned_targets] # The following line is a no-op if cache_key was set in the VersionedTarget __init__ method. self.cache_key = CacheKeyGenerator.combine_cache_keys([vt.cache_key for vt in versioned_targets]) self.num_chunking_units = self.cache_key.num_chunking_units self.valid = not cache_manager.needs_update(self.cache_key)
def __init__(self, cache_manager, versioned_targets): self._cache_manager = cache_manager self.versioned_targets = versioned_targets self.targets = [vt.target for vt in versioned_targets] # The following line is a no-op if cache_key was set in the VersionedTarget __init__ method. self.cache_key = CacheKeyGenerator.combine_cache_keys( [vt.cache_key for vt in versioned_targets]) self.num_chunking_units = self.cache_key.num_chunking_units self.valid = not cache_manager.needs_update(self.cache_key)
def __init__(self, context, workdir): self.context = context self._workdir = workdir self._cache_key_generator = CacheKeyGenerator( context.config.getdefault('cache_key_gen_version', default=None)) self._read_artifact_cache_spec = None self._write_artifact_cache_spec = None self._artifact_cache = None self._artifact_cache_setup_lock = threading.Lock() default_invalidator_root = os.path.join(self.context.config.getdefault('pants_workdir'), 'build_invalidator') self._build_invalidator_dir = os.path.join( context.config.get('tasks', 'build_invalidator', default=default_invalidator_root), self.product_type())
def __init__(self, context, python_setup, python_repos, interpreter, builder, targets, platforms, extra_requirements=None): self.context = context self._python_setup = python_setup self._python_repos = python_repos self._interpreter = interpreter self._builder = builder self._targets = targets self._platforms = platforms self._extra_requirements = list(extra_requirements) if extra_requirements else [] # Note: unrelated to the general pants artifact cache. self._artifact_cache_root = os.path.join( self._python_setup.artifact_cache_dir, str(self._interpreter.identity)) self._key_generator = CacheKeyGenerator() self._build_invalidator = BuildInvalidator(self._artifact_cache_root)
class PythonChroot(object): _VALID_DEPENDENCIES = { PrepCommand: 'prep', PythonLibrary: 'libraries', PythonRequirementLibrary: 'reqs', PythonBinary: 'binaries', PythonThriftLibrary: 'thrifts', PythonAntlrLibrary: 'antlrs', PythonTests: 'tests' } MEMOIZED_THRIFTS = {} class InvalidDependencyException(Exception): def __init__(self, target): Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target) def __init__(self, context, targets, extra_requirements=None, builder=None, platforms=None, interpreter=None): self.context = context self._targets = targets self._extra_requirements = list(extra_requirements) if extra_requirements else [] self._platforms = platforms self._interpreter = interpreter or PythonInterpreter.get() self._builder = builder or PEXBuilder(os.path.realpath(tempfile.mkdtemp()), interpreter=self._interpreter) # Note: unrelated to the general pants artifact cache. self._egg_cache_root = os.path.join( PythonSetup(self.context.config).scratch_dir('artifact_cache', default_name='artifacts'), str(self._interpreter.identity)) self._key_generator = CacheKeyGenerator() self._build_invalidator = BuildInvalidator( self._egg_cache_root) def delete(self): """Deletes this chroot from disk if it has been dumped.""" safe_rmtree(self.path()) def __del__(self): if os.getenv('PANTS_LEAVE_CHROOT') is None: self.delete() else: self.debug('Left chroot at %s' % self.path()) @property def builder(self): return self._builder def debug(self, msg, indent=0): if os.getenv('PANTS_VERBOSE') is not None: print('%s%s' % (' ' * indent, msg)) def path(self): return os.path.realpath(self._builder.path()) def _dump_library(self, library): def copy_to_chroot(base, path, add_function): src = os.path.join(get_buildroot(), base, path) add_function(src, path) self.debug(' Dumping library: %s' % library) for relpath in library.sources_relative_to_source_root(): try: copy_to_chroot(library.target_base, relpath, self._builder.add_source) except OSError as e: logger.error("Failed to copy {path} for library {library}" .format(path=os.path.join(library.target_base, relpath), library=library)) raise for resources_tgt in library.resources: for resource_file_from_source_root in resources_tgt.sources_relative_to_source_root(): try: copy_to_chroot(resources_tgt.target_base, resource_file_from_source_root, self._builder.add_resource) except OSError as e: logger.error("Failed to copy {path} for resource {resource}" .format(path=os.path.join(resources_tgt.target_base, resource_file_from_source_root), resource=resources_tgt.address.spec)) raise def _dump_requirement(self, req): self.debug(' Dumping requirement: %s' % req) self._builder.add_requirement(req) def _dump_distribution(self, dist): self.debug(' Dumping distribution: .../%s' % os.path.basename(dist.location)) self._builder.add_distribution(dist) def _generate_requirement(self, library, builder_cls): library_key = self._key_generator.key_for_target(library) builder = builder_cls(library, get_buildroot(), self.context.options, '-' + library_key.hash[:8]) cache_dir = os.path.join(self._egg_cache_root, library_key.id) if self._build_invalidator.needs_update(library_key): sdist = builder.build(interpreter=self._interpreter) safe_mkdir(cache_dir) shutil.copy(sdist, os.path.join(cache_dir, os.path.basename(sdist))) self._build_invalidator.update(library_key) return PythonRequirement(builder.requirement_string(), repository=cache_dir, use_2to3=True) def _generate_thrift_requirement(self, library): return self._generate_requirement(library, PythonThriftBuilder) def _generate_antlr_requirement(self, library): return self._generate_requirement(library, PythonAntlrBuilder) def resolve(self, targets): children = defaultdict(OrderedSet) def add_dep(trg): for target_type, target_key in self._VALID_DEPENDENCIES.items(): if isinstance(trg, target_type): children[target_key].add(trg) return elif isinstance(trg, Dependencies): return raise self.InvalidDependencyException(trg) for target in targets: target.walk(add_dep) return children def dump(self): self.debug('Building chroot for %s:' % self._targets) targets = self.resolve(self._targets) for lib in targets['libraries'] | targets['binaries']: self._dump_library(lib) generated_reqs = OrderedSet() if targets['thrifts']: for thr in set(targets['thrifts']): if thr not in self.MEMOIZED_THRIFTS: self.MEMOIZED_THRIFTS[thr] = self._generate_thrift_requirement(thr) generated_reqs.add(self.MEMOIZED_THRIFTS[thr]) generated_reqs.add(PythonRequirement('thrift', use_2to3=True)) for antlr in targets['antlrs']: generated_reqs.add(self._generate_antlr_requirement(antlr)) reqs_from_libraries = OrderedSet() for req_lib in targets['reqs']: for req in req_lib.payload.requirements: reqs_from_libraries.add(req) reqs_to_build = OrderedSet() find_links = [] for req in reqs_from_libraries | generated_reqs | self._extra_requirements: if not req.should_build(self._interpreter.python, Platform.current()): self.debug('Skipping %s based upon version filter' % req) continue reqs_to_build.add(req) self._dump_requirement(req.requirement) if req.repository: find_links.append(req.repository) distributions = resolve_multi( self.context.config, reqs_to_build, interpreter=self._interpreter, platforms=self._platforms, ttl=self.context.options.for_global_scope().python_chroot_requirements_ttl, find_links=find_links) locations = set() for platform, dist_set in distributions.items(): for dist in dist_set: if dist.location not in locations: self._dump_distribution(dist) locations.add(dist.location) if len(targets['binaries']) > 1: print('WARNING: Target has multiple python_binary targets!', file=sys.stderr) return self._builder
class PythonChroot(object): _VALID_DEPENDENCIES = { PythonLibrary: 'libraries', PythonRequirement: 'reqs', PythonBinary: 'binaries', PythonThriftLibrary: 'thrifts', PythonAntlrLibrary: 'antlrs', PythonTests: 'tests' } MEMOIZED_THRIFTS = {} class InvalidDependencyException(Exception): def __init__(self, target): Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target) def __init__(self, target, root_dir, extra_targets=None, builder=None, platforms=None, interpreter=None, conn_timeout=None): self._config = Config.load() self._target = target self._root = root_dir self._platforms = platforms self._interpreter = interpreter or PythonInterpreter.get() self._extra_targets = list(extra_targets) if extra_targets is not None else [] self._builder = builder or PEXBuilder(tempfile.mkdtemp(), interpreter=self._interpreter) # Note: unrelated to the general pants artifact cache. self._egg_cache_root = os.path.join( PythonSetup(self._config).scratch_dir('artifact_cache', default_name='artifacts'), str(self._interpreter.identity)) self._key_generator = CacheKeyGenerator() self._build_invalidator = BuildInvalidator( self._egg_cache_root) def __del__(self): if os.getenv('PANTS_LEAVE_CHROOT') is None: safe_rmtree(self.path()) else: self.debug('Left chroot at %s' % self.path()) @property def builder(self): return self._builder def debug(self, msg, indent=0): if os.getenv('PANTS_VERBOSE') is not None: print('%s%s' % (' ' * indent, msg)) def path(self): return self._builder.path() def _dump_library(self, library): def copy_to_chroot(base, path, add_function): src = os.path.join(self._root, base, path) add_function(src, path) self.debug(' Dumping library: %s' % library) for filename in library.sources: copy_to_chroot(library.target_base, filename, self._builder.add_source) for filename in library.resources: copy_to_chroot(library.target_base, filename, self._builder.add_resource) def _dump_requirement(self, req, dynamic, repo): self.debug(' Dumping requirement: %s%s%s' % (str(req), ' (dynamic)' if dynamic else '', ' (repo: %s)' % repo if repo else '')) self._builder.add_requirement(req, dynamic, repo) def _dump_distribution(self, dist): self.debug(' Dumping distribution: .../%s' % os.path.basename(dist.location)) self._builder.add_distribution(dist) def _generate_requirement(self, library, builder_cls): library_key = self._key_generator.key_for_target(library) builder = builder_cls(library, self._root, self._config, '-' + library_key.hash[:8]) cache_dir = os.path.join(self._egg_cache_root, library_key.id) if self._build_invalidator.needs_update(library_key): sdist = builder.build(interpreter=self._interpreter) safe_mkdir(cache_dir) shutil.copy(sdist, os.path.join(cache_dir, os.path.basename(sdist))) self._build_invalidator.update(library_key) with ParseContext.temp(): return PythonRequirement(builder.requirement_string(), repository=cache_dir, use_2to3=True) def _generate_thrift_requirement(self, library): return self._generate_requirement(library, PythonThriftBuilder) def _generate_antlr_requirement(self, library): return self._generate_requirement(library, PythonAntlrBuilder) def resolve(self, targets): children = defaultdict(OrderedSet) def add_dep(trg): for target_type, target_key in self._VALID_DEPENDENCIES.items(): if isinstance(trg, target_type): children[target_key].add(trg) return raise self.InvalidDependencyException(trg) for target in targets: target.walk(add_dep) return children def dump(self): self.debug('Building PythonBinary %s:' % self._target) targets = self.resolve([self._target] + self._extra_targets) for lib in targets['libraries'] | targets['binaries']: self._dump_library(lib) generated_reqs = OrderedSet() if targets['thrifts']: for thr in set(targets['thrifts']): if thr not in self.MEMOIZED_THRIFTS: self.MEMOIZED_THRIFTS[thr] = self._generate_thrift_requirement(thr) generated_reqs.add(self.MEMOIZED_THRIFTS[thr]) with ParseContext.temp(): # trick pants into letting us add this python requirement, otherwise we get # TargetDefinitionException: Error in target BUILD.temp:thrift: duplicate to # PythonRequirement(thrift) # # TODO(wickman) Instead of just blindly adding a PythonRequirement for thrift, we # should first detect if any explicit thrift requirements have been added and use # those. Only if they have not been supplied should we auto-inject it. generated_reqs.add(PythonRequirement('thrift', use_2to3=True, name='thrift-' + ''.join(random.sample('0123456789abcdef' * 8, 8)))) for antlr in targets['antlrs']: generated_reqs.add(self._generate_antlr_requirement(antlr)) targets['reqs'] |= generated_reqs reqs_to_build = OrderedSet() for req in targets['reqs']: if not req.should_build(self._interpreter.python, Platform.current()): self.debug('Skipping %s based upon version filter' % req) continue reqs_to_build.add(req) self._dump_requirement(req._requirement, False, req._repository) platforms = self._platforms if isinstance(self._target, PythonBinary): platforms = self._target.platforms distributions = resolve_multi( self._config, reqs_to_build, interpreter=self._interpreter, platforms=platforms) locations = set() for platform, dist_set in distributions.items(): for dist in dist_set: if dist.location not in locations: self._dump_distribution(dist) locations.add(dist.location) if len(targets['binaries']) > 1: print('WARNING: Target has multiple python_binary targets!', file=sys.stderr) return self._builder
class PythonChroot(object): _VALID_DEPENDENCIES = { PythonLibrary: 'libraries', PythonRequirementLibrary: 'reqs', PythonBinary: 'binaries', PythonThriftLibrary: 'thrifts', PythonAntlrLibrary: 'antlrs', PythonTests: 'tests' } MEMOIZED_THRIFTS = {} class InvalidDependencyException(Exception): def __init__(self, target): Exception.__init__( self, "Not a valid Python dependency! Found: %s" % target) def __init__(self, target, root_dir, extra_targets=None, extra_requirements=None, builder=None, platforms=None, interpreter=None, conn_timeout=None): self._config = Config.load() self._target = target self._root = root_dir self._platforms = platforms self._interpreter = interpreter or PythonInterpreter.get() self._extra_targets = list( extra_targets) if extra_targets is not None else [] self._extra_requirements = list( extra_requirements) if extra_requirements is not None else [] self._builder = builder or PEXBuilder(tempfile.mkdtemp(), interpreter=self._interpreter) # Note: unrelated to the general pants artifact cache. self._egg_cache_root = os.path.join( PythonSetup(self._config).scratch_dir('artifact_cache', default_name='artifacts'), str(self._interpreter.identity)) self._key_generator = CacheKeyGenerator() self._build_invalidator = BuildInvalidator(self._egg_cache_root) def __del__(self): if os.getenv('PANTS_LEAVE_CHROOT') is None: safe_rmtree(self.path()) else: self.debug('Left chroot at %s' % self.path()) @property def builder(self): return self._builder def debug(self, msg, indent=0): if os.getenv('PANTS_VERBOSE') is not None: print('%s%s' % (' ' * indent, msg)) def path(self): return self._builder.path() def _dump_library(self, library): def copy_to_chroot(base, path, add_function): src = os.path.join(self._root, base, path) add_function(src, path) self.debug(' Dumping library: %s' % library) for relpath in library.sources_relative_to_source_root(): copy_to_chroot(library.target_base, relpath, self._builder.add_source) for resources_tgt in library.resources: for resource_file_from_source_root in resources_tgt.sources_relative_to_source_root( ): copy_to_chroot(resources_tgt.target_base, resource_file_from_source_root, self._builder.add_resource) def _dump_requirement(self, req, dynamic, repo): self.debug(' Dumping requirement: %s%s%s' % (str(req), ' (dynamic)' if dynamic else '', ' (repo: %s)' % repo if repo else '')) self._builder.add_requirement(req, dynamic, repo) def _dump_distribution(self, dist): self.debug(' Dumping distribution: .../%s' % os.path.basename(dist.location)) self._builder.add_distribution(dist) def _generate_requirement(self, library, builder_cls): library_key = self._key_generator.key_for_target(library) builder = builder_cls(library, self._root, self._config, '-' + library_key.hash[:8]) cache_dir = os.path.join(self._egg_cache_root, library_key.id) if self._build_invalidator.needs_update(library_key): sdist = builder.build(interpreter=self._interpreter) safe_mkdir(cache_dir) shutil.copy(sdist, os.path.join(cache_dir, os.path.basename(sdist))) self._build_invalidator.update(library_key) return PythonRequirement(builder.requirement_string(), repository=cache_dir, use_2to3=True) def _generate_thrift_requirement(self, library): return self._generate_requirement(library, PythonThriftBuilder) def _generate_antlr_requirement(self, library): return self._generate_requirement(library, PythonAntlrBuilder) def resolve(self, targets): children = defaultdict(OrderedSet) def add_dep(trg): for target_type, target_key in self._VALID_DEPENDENCIES.items(): if isinstance(trg, target_type): children[target_key].add(trg) return elif isinstance(trg, Dependencies): return raise self.InvalidDependencyException(trg) for target in targets: target.walk(add_dep) return children def dump(self): self.debug('Building PythonBinary %s:' % self._target) targets = self.resolve([self._target] + self._extra_targets) for lib in targets['libraries'] | targets['binaries']: self._dump_library(lib) generated_reqs = OrderedSet() if targets['thrifts']: for thr in set(targets['thrifts']): if thr not in self.MEMOIZED_THRIFTS: self.MEMOIZED_THRIFTS[ thr] = self._generate_thrift_requirement(thr) generated_reqs.add(self.MEMOIZED_THRIFTS[thr]) generated_reqs.add(PythonRequirement('thrift', use_2to3=True)) for antlr in targets['antlrs']: generated_reqs.add(self._generate_antlr_requirement(antlr)) reqs_from_libraries = OrderedSet() for req_lib in targets['reqs']: for req in req_lib.payload.requirements: reqs_from_libraries.add(req) reqs_to_build = OrderedSet() for req in reqs_from_libraries | generated_reqs | self._extra_requirements: if not req.should_build(self._interpreter.python, Platform.current()): self.debug('Skipping %s based upon version filter' % req) continue reqs_to_build.add(req) self._dump_requirement(req._requirement, False, req._repository) platforms = self._platforms if isinstance(self._target, PythonBinary): platforms = self._target.platforms distributions = resolve_multi(self._config, reqs_to_build, interpreter=self._interpreter, platforms=platforms) locations = set() for platform, dist_set in distributions.items(): for dist in dist_set: if dist.location not in locations: self._dump_distribution(dist) locations.add(dist.location) if len(targets['binaries']) > 1: print('WARNING: Target has multiple python_binary targets!', file=sys.stderr) return self._builder
def test_env(content=TEST_CONTENT): with temporary_dir() as d: with tempfile.NamedTemporaryFile() as f: f.write(content) f.flush() yield f, CacheKeyGenerator(), BuildInvalidator(d)
class PythonChroot(object): _VALID_DEPENDENCIES = { PrepCommand: 'prep', PythonLibrary: 'libraries', PythonRequirementLibrary: 'reqs', PythonBinary: 'binaries', PythonThriftLibrary: 'thrifts', PythonAntlrLibrary: 'antlrs', PythonTests: 'tests' } MEMOIZED_THRIFTS = {} class InvalidDependencyException(Exception): def __init__(self, target): Exception.__init__( self, "Not a valid Python dependency! Found: {}".format(target)) @staticmethod def get_platforms(platform_list): return tuple({ Platform.current() if p == 'current' else p for p in platform_list }) def __init__(self, python_setup, python_repos, ivy_bootstrapper, thrift_binary_factory, interpreter, builder, targets, platforms, extra_requirements=None): self._python_setup = python_setup self._python_repos = python_repos self._ivy_bootstrapper = ivy_bootstrapper self._thrift_binary_factory = thrift_binary_factory self._interpreter = interpreter self._builder = builder self._targets = targets self._platforms = platforms self._extra_requirements = list( extra_requirements) if extra_requirements else [] # Note: unrelated to the general pants artifact cache. self._artifact_cache_root = os.path.join( self._python_setup.artifact_cache_dir, str(self._interpreter.identity)) self._key_generator = CacheKeyGenerator() self._build_invalidator = BuildInvalidator(self._artifact_cache_root) def delete(self): """Deletes this chroot from disk if it has been dumped.""" safe_rmtree(self.path()) def debug(self, msg, indent=0): if os.getenv('PANTS_VERBOSE') is not None: print('{}{}'.format(' ' * indent, msg)) def path(self): return os.path.realpath(self._builder.path()) def pex(self): return PEX(self.path(), interpreter=self._interpreter) def package_pex(self, filename): """Package into a PEX zipfile. :param filename: The filename where the PEX should be stored. """ self._builder.build(filename) def _dump_library(self, library): def copy_to_chroot(base, path, add_function): src = os.path.join(get_buildroot(), base, path) add_function(src, path) self.debug(' Dumping library: {}'.format(library)) for relpath in library.sources_relative_to_source_root(): try: copy_to_chroot(library.target_base, relpath, self._builder.add_source) except OSError: logger.error( "Failed to copy {path} for library {library}".format( path=os.path.join(library.target_base, relpath), library=library)) raise for resources_tgt in library.resources: for resource_file_from_source_root in resources_tgt.sources_relative_to_source_root( ): try: copy_to_chroot(resources_tgt.target_base, resource_file_from_source_root, self._builder.add_resource) except OSError: logger.error( "Failed to copy {path} for resource {resource}".format( path=os.path.join(resources_tgt.target_base, resource_file_from_source_root), resource=resources_tgt.address.spec)) raise def _dump_requirement(self, req): self.debug(' Dumping requirement: {}'.format(req)) self._builder.add_requirement(req) def _dump_distribution(self, dist): self.debug(' Dumping distribution: .../{}'.format( os.path.basename(dist.location))) self._builder.add_distribution(dist) def _generate_requirement(self, library, builder_cls): library_key = self._key_generator.key_for_target(library) builder = builder_cls(target=library, root_dir=get_buildroot(), target_suffix='-' + library_key.hash[:8]) cache_dir = os.path.join(self._artifact_cache_root, library_key.id) if self._build_invalidator.needs_update(library_key): sdist = builder.build(interpreter=self._interpreter) safe_mkdir(cache_dir) shutil.copy(sdist, os.path.join(cache_dir, os.path.basename(sdist))) self._build_invalidator.update(library_key) return PythonRequirement(builder.requirement_string(), repository=cache_dir, use_2to3=True) def _generate_thrift_requirement(self, library): thrift_builder = functools.partial( PythonThriftBuilder, thrift_binary_factory=self._thrift_binary_factory, workdir=safe_mkdtemp(dir=self.path(), prefix='thrift.')) return self._generate_requirement(library, thrift_builder) def _generate_antlr_requirement(self, library): antlr_builder = functools.partial( PythonAntlrBuilder, ivy_bootstrapper=self._ivy_bootstrapper, workdir=safe_mkdtemp(dir=self.path(), prefix='antlr.')) return self._generate_requirement(library, antlr_builder) def resolve(self, targets): children = defaultdict(OrderedSet) def add_dep(trg): # Currently we handle all of our code generation, so we don't want to operate over any # synthetic targets injected upstream. # TODO(John Sirois): Revisit this when building a proper python product pipeline. if trg.is_synthetic: return for target_type, target_key in self._VALID_DEPENDENCIES.items(): if isinstance(trg, target_type): children[target_key].add(trg) return elif isinstance(trg, Dependencies): return raise self.InvalidDependencyException(trg) for target in targets: target.walk(add_dep) return children def dump(self): self.debug('Building chroot for {}:'.format(self._targets)) targets = self.resolve(self._targets) for lib in targets['libraries'] | targets['binaries']: self._dump_library(lib) generated_reqs = OrderedSet() if targets['thrifts']: for thr in set(targets['thrifts']): if thr not in self.MEMOIZED_THRIFTS: self.MEMOIZED_THRIFTS[ thr] = self._generate_thrift_requirement(thr) generated_reqs.add(self.MEMOIZED_THRIFTS[thr]) generated_reqs.add(PythonRequirement('thrift', use_2to3=True)) for antlr in targets['antlrs']: generated_reqs.add(self._generate_antlr_requirement(antlr)) reqs_from_libraries = OrderedSet() for req_lib in targets['reqs']: for req in req_lib.payload.requirements: reqs_from_libraries.add(req) reqs_to_build = OrderedSet() find_links = OrderedSet() for req in reqs_from_libraries | generated_reqs | self._extra_requirements: if not req.should_build(self._interpreter.python, Platform.current()): self.debug('Skipping {} based upon version filter'.format(req)) continue reqs_to_build.add(req) self._dump_requirement(req.requirement) if req.repository: find_links.add(req.repository) distributions = self._resolve_multi(reqs_to_build, find_links) locations = set() for platform, dist_set in distributions.items(): for dist in dist_set: if dist.location not in locations: self._dump_distribution(dist) locations.add(dist.location) if len(targets['binaries']) > 1: print('WARNING: Target has multiple python_binary targets!', file=sys.stderr) return self._builder def _resolve_multi(self, requirements, find_links): """Multi-platform dependency resolution for PEX files. Given a pants configuration and a set of requirements, return a list of distributions that must be included in order to satisfy them. That may involve distributions for multiple platforms. :param requirements: A list of :class:`PythonRequirement` objects to resolve. :param find_links: Additional paths to search for source packages during resolution. """ distributions = dict() platforms = self.get_platforms(self._platforms or self._python_setup.platforms) fetchers = self._python_repos.get_fetchers() fetchers.extend(Fetcher([path]) for path in find_links) context = self._python_repos.get_network_context() for platform in platforms: distributions[platform] = resolve( requirements=[req.requirement for req in requirements], interpreter=self._interpreter, fetchers=fetchers, platform=platform, context=context, cache=self._python_setup.resolver_cache_dir, cache_ttl=self._python_setup.resolver_cache_ttl) return distributions
class PythonChroot(object): _VALID_DEPENDENCIES = { PrepCommand: 'prep', PythonLibrary: 'libraries', PythonRequirementLibrary: 'reqs', PythonBinary: 'binaries', PythonThriftLibrary: 'thrifts', PythonAntlrLibrary: 'antlrs', PythonTests: 'tests' } class InvalidDependencyException(Exception): def __init__(self, target): Exception.__init__(self, "Not a valid Python dependency! Found: {}".format(target)) @staticmethod def get_platforms(platform_list): return tuple({Platform.current() if p == 'current' else p for p in platform_list}) def __init__(self, python_setup, python_repos, ivy_bootstrapper, thrift_binary_factory, interpreter, builder, targets, platforms, extra_requirements=None, log=None): self._python_setup = python_setup self._python_repos = python_repos self._ivy_bootstrapper = ivy_bootstrapper self._thrift_binary_factory = thrift_binary_factory self._interpreter = interpreter self._builder = builder self._targets = targets self._platforms = platforms self._extra_requirements = list(extra_requirements) if extra_requirements else [] self._logger = log or logger # Note: unrelated to the general pants artifact cache. self._artifact_cache_root = os.path.join( self._python_setup.artifact_cache_dir, str(self._interpreter.identity)) self._key_generator = CacheKeyGenerator() self._build_invalidator = BuildInvalidator(self._artifact_cache_root) def delete(self): """Deletes this chroot from disk if it has been dumped.""" safe_rmtree(self.path()) def debug(self, msg): self._logger.debug(msg) def path(self): return os.path.realpath(self._builder.path()) def pex(self): return PEX(self.path(), interpreter=self._interpreter) def package_pex(self, filename): """Package into a PEX zipfile. :param filename: The filename where the PEX should be stored. """ self._builder.build(filename) def _dump_library(self, library): def copy_to_chroot(base, path, add_function): src = os.path.join(get_buildroot(), base, path) add_function(src, path) self.debug(' Dumping library: {}'.format(library)) for relpath in library.sources_relative_to_source_root(): try: copy_to_chroot(library.target_base, relpath, self._builder.add_source) except OSError: logger.error("Failed to copy {path} for library {library}" .format(path=os.path.join(library.target_base, relpath), library=library)) raise for resources_tgt in library.resources: for resource_file_from_source_root in resources_tgt.sources_relative_to_source_root(): try: copy_to_chroot(resources_tgt.target_base, resource_file_from_source_root, self._builder.add_resource) except OSError: logger.error("Failed to copy {path} for resource {resource}" .format(path=os.path.join(resources_tgt.target_base, resource_file_from_source_root), resource=resources_tgt.address.spec)) raise def _dump_requirement(self, req): self.debug(' Dumping requirement: {}'.format(req)) self._builder.add_requirement(req) def _dump_distribution(self, dist): self.debug(' Dumping distribution: .../{}'.format(os.path.basename(dist.location))) self._builder.add_distribution(dist) def _generate_requirement(self, library, builder_cls): library_key = self._key_generator.key_for_target(library) builder = builder_cls(target=library, root_dir=get_buildroot(), target_suffix='-' + library_key.hash[:8]) cache_dir = os.path.join(self._artifact_cache_root, library_key.id) if self._build_invalidator.needs_update(library_key): sdist = builder.build(interpreter=self._interpreter) safe_mkdir(cache_dir) shutil.copy(sdist, os.path.join(cache_dir, os.path.basename(sdist))) self._build_invalidator.update(library_key) return PythonRequirement(builder.requirement_string(), repository=cache_dir, use_2to3=True) def _generate_thrift_requirement(self, library): thrift_builder = functools.partial(PythonThriftBuilder, thrift_binary_factory=self._thrift_binary_factory, workdir=safe_mkdtemp(dir=self.path(), prefix='thrift.')) return self._generate_requirement(library, thrift_builder) def _generate_antlr_requirement(self, library): antlr_builder = functools.partial(PythonAntlrBuilder, ivy_bootstrapper=self._ivy_bootstrapper, workdir=safe_mkdtemp(dir=self.path(), prefix='antlr.')) return self._generate_requirement(library, antlr_builder) def resolve(self, targets): children = defaultdict(OrderedSet) def add_dep(trg): # Currently we handle all of our code generation, so we don't want to operate over any # synthetic targets injected upstream. # TODO(John Sirois): Revisit this when building a proper python product pipeline. if trg.is_synthetic: return for target_type, target_key in self._VALID_DEPENDENCIES.items(): if isinstance(trg, target_type): children[target_key].add(trg) return elif isinstance(trg, Dependencies): return raise self.InvalidDependencyException(trg) for target in targets: target.walk(add_dep) return children def dump(self): self.debug('Building chroot for {}:'.format(self._targets)) targets = self.resolve(self._targets) for lib in targets['libraries'] | targets['binaries']: self._dump_library(lib) generated_reqs = OrderedSet() if targets['thrifts']: for thr in targets['thrifts']: generated_reqs.add(self._generate_thrift_requirement(thr)) generated_reqs.add(PythonRequirement('thrift', use_2to3=True)) for antlr in targets['antlrs']: generated_reqs.add(self._generate_antlr_requirement(antlr)) reqs_from_libraries = OrderedSet() for req_lib in targets['reqs']: for req in req_lib.payload.requirements: reqs_from_libraries.add(req) reqs_to_build = OrderedSet() find_links = OrderedSet() for req in reqs_from_libraries | generated_reqs | self._extra_requirements: if not req.should_build(self._interpreter.python, Platform.current()): self.debug('Skipping {} based upon version filter'.format(req)) continue reqs_to_build.add(req) self._dump_requirement(req.requirement) if req.repository: find_links.add(req.repository) distributions = self._resolve_multi(reqs_to_build, find_links) locations = set() for platform, dist_set in distributions.items(): for dist in dist_set: if dist.location not in locations: self._dump_distribution(dist) locations.add(dist.location) if len(targets['binaries']) > 1: print('WARNING: Target has multiple python_binary targets!', file=sys.stderr) return self._builder def _resolve_multi(self, requirements, find_links): """Multi-platform dependency resolution for PEX files. Given a pants configuration and a set of requirements, return a list of distributions that must be included in order to satisfy them. That may involve distributions for multiple platforms. :param requirements: A list of :class:`PythonRequirement` objects to resolve. :param find_links: Additional paths to search for source packages during resolution. """ distributions = dict() platforms = self.get_platforms(self._platforms or self._python_setup.platforms) fetchers = self._python_repos.get_fetchers() fetchers.extend(Fetcher([path]) for path in find_links) context = self._python_repos.get_network_context() for platform in platforms: requirements_cache_dir = os.path.join(self._python_setup.resolver_cache_dir, str(self._interpreter.identity)) distributions[platform] = resolve( requirements=[req.requirement for req in requirements], interpreter=self._interpreter, fetchers=fetchers, platform=platform, context=context, cache=requirements_cache_dir, cache_ttl=self._python_setup.resolver_cache_ttl) return distributions