def _validate_interpreter_constraints(self, constraint_tgts): """Validate that the transitive constraints of the given PythonBinary target are compatible. If no (local) interpreter can satisfy all of the given targets, raises PythonInterpreterCache.UnsatisfiableInterpreterConstraintsError. TODO: This currently does so by finding a concrete local interpreter that matches all of the constraints, but it is possible to do this in memory instead. see https://github.com/pantsbuild/pants/issues/7775 """ PythonInterpreterCache.global_instance( ).select_interpreter_for_targets(constraint_tgts)
def _gather_sources(self, target_roots): with temporary_dir() as cache_dir: interpreter = PythonInterpreter.get() context = self.context( target_roots=target_roots, for_subsystems=[PythonInterpreterCache], options={ PythonSetup.options_scope: { 'interpreter_cache_dir': cache_dir, 'interpreter_search_paths': [os.path.dirname(interpreter.binary)], } }) # We must get an interpreter via the cache, instead of using the value of # PythonInterpreter.get() directly, to ensure that the interpreter has setuptools and # wheel support. interpreter_cache = PythonInterpreterCache.global_instance() interpreters = interpreter_cache.setup( filters=[str(interpreter.identity.requirement)]) context.products.get_data(PythonInterpreter, lambda: interpreters[0]) task = self.create_task(context) task.execute() return context.products.get_data(GatherSources.PYTHON_SOURCES)
def execute(self): if not self.will_be_invoked(): return tool_subsystem = self.tool_subsystem_cls.scoped_instance(self) interpreter_cache = PythonInterpreterCache.global_instance() interpreters = interpreter_cache.setup( filters=tool_subsystem.get_interpreter_constraints()) if not interpreters: raise TaskError( "Found no Python interpreter capable of running the {} tool with " "constraints {}".format( tool_subsystem.options_scope, tool_subsystem.get_interpreter_constraints())) interpreter = min(interpreters) pex_path = self._generate_fingerprinted_pex_path( tool_subsystem, interpreter) if not os.path.exists(pex_path): with self.context.new_workunit( name=f"create-{tool_subsystem.options_scope}-pex", labels=[WorkUnitLabel.PREP]): self._build_tool_pex(tool_subsystem=tool_subsystem, interpreter=interpreter, pex_path=pex_path) tool_instance = self.tool_instance_cls(pex_path, interpreter) self.context.products.register_data(self.tool_instance_cls, tool_instance)
def _create_interpreter_cache(self, setup_options=None): Subsystem.reset(reset_options=True) self.context(for_subsystems=[PythonInterpreterCache], options={ 'python-setup': setup_options, }) return PythonInterpreterCache.global_instance()
def execute(self): """"Run Checkstyle on all found non-synthetic source files.""" python_tgts = self.context.targets( lambda tgt: isinstance(tgt, (PythonTarget)) ) if not python_tgts: return 0 interpreter_cache = PythonInterpreterCache.global_instance() with self.invalidated(self.get_targets(self._is_checked)) as invalidation_check: failure_count = 0 tgts_by_compatibility, _ = interpreter_cache.partition_targets_by_compatibility( [vt.target for vt in invalidation_check.invalid_vts] ) for filters, targets in tgts_by_compatibility.items(): sources = self.calculate_sources([tgt for tgt in targets]) if sources: allowed_interpreters = set(interpreter_cache.setup(filters=filters)) if not allowed_interpreters: raise TaskError('No valid interpreters found for targets: {}\n(filters: {})' .format(targets, filters)) interpreter = min(allowed_interpreters) failure_count += self.checkstyle(interpreter, sources) if failure_count > 0 and self.get_options().fail: raise TaskError('{} Python Style issues found. You may try `./pants fmt <targets>`' .format(failure_count)) return failure_count
def _resolve_requirements(self, target_roots, options=None): with temporary_dir() as cache_dir: options = options or {} python_setup_opts = options.setdefault(PythonSetup.options_scope, {}) python_setup_opts['interpreter_cache_dir'] = cache_dir interpreter = PythonInterpreter.get() python_setup_opts['interpreter_search_paths'] = [ os.path.dirname(interpreter.binary) ] context = self.context(target_roots=target_roots, options=options, for_subsystems=[PythonInterpreterCache]) # We must get an interpreter via the cache, instead of using the value of # PythonInterpreter.get() directly, to ensure that the interpreter has setuptools and # wheel support. interpreter_cache = PythonInterpreterCache.global_instance() interpreters = interpreter_cache.setup( filters=[str(interpreter.identity.requirement)]) context.products.get_data(PythonInterpreter, lambda: interpreters[0]) task = self.create_task(context) task.execute() return context.products.get_data( ResolveRequirements.REQUIREMENTS_PEX)
def _create_interpreter_path_file(self, interpreter_path_file, targets): interpreter_cache = PythonInterpreterCache.global_instance() interpreter = interpreter_cache.select_interpreter_for_targets(targets) safe_mkdir_for(interpreter_path_file) with open(interpreter_path_file, 'w') as outfile: outfile.write('{}\n'.format(interpreter.binary)) for dist, location in interpreter.extras.items(): dist_name, dist_version = dist outfile.write('{}\t{}\t{}\n'.format(dist_name, dist_version, location))
def test_namespace_effective(self): self.create_file('src/thrift/com/foo/one.thrift', contents=dedent(""" namespace py foo.bar struct One {} """)) one = self.make_target(spec='src/thrift/com/foo:one', target_type=PythonThriftLibrary, sources=['one.thrift']) apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one) self.create_file('src/thrift2/com/foo/two.thrift', contents=dedent(""" namespace py foo.baz struct Two {} """)) two = self.make_target(spec='src/thrift2/com/foo:two', target_type=PythonThriftLibrary, sources=['two.thrift']) _, synthetic_target_two = self.generate_single_thrift_target(two) # Confirm separate PYTHONPATH entries, which we need to test namespace packages. self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base) targets = (synthetic_target_one, synthetic_target_two) self.context(for_subsystems=[PythonInterpreterCache, PythonRepos]) interpreter_cache = PythonInterpreterCache.global_instance() python_repos = PythonRepos.global_instance() interpreter = interpreter_cache.select_interpreter_for_targets(targets) # We need setuptools to import namespace packages under python 2 (via pkg_resources), so we # prime the PYTHONPATH with a known good version of setuptools. # TODO(John Sirois): We really should be emitting setuptools in a # `synthetic_target_extra_dependencies` override in `ApacheThriftPyGen`: # https://github.com/pantsbuild/pants/issues/5975 pythonpath = [os.path.join(get_buildroot(), t.target_base) for t in targets] for resolved_dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen)), 'setuptools==40.6.3'], interpreter=interpreter, context=python_repos.get_network_context(), fetchers=python_repos.get_fetchers()): pythonpath.append(resolved_dist.distribution.location) process = subprocess.Popen([interpreter.binary, '-c', 'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'], env={'PYTHONPATH': os.pathsep.join(pythonpath)}, stderr=subprocess.PIPE) _, stderr = process.communicate() self.assertEqual(0, process.returncode, stderr)
def test_namespace_effective(self): self.create_file('src/thrift/com/foo/one.thrift', contents=dedent(""" namespace py foo.bar struct One {} """)) one = self.make_target(spec='src/thrift/com/foo:one', target_type=PythonThriftLibrary, sources=['one.thrift']) apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one) self.create_file('src/thrift2/com/foo/two.thrift', contents=dedent(""" namespace py foo.baz struct Two {} """)) two = self.make_target(spec='src/thrift2/com/foo:two', target_type=PythonThriftLibrary, sources=['two.thrift']) _, synthetic_target_two = self.generate_single_thrift_target(two) # Confirm separate PYTHONPATH entries, which we need to test namespace packages. self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base) targets = (synthetic_target_one, synthetic_target_two) self.context(for_subsystems=[PythonInterpreterCache, PythonRepos]) interpreter_cache = PythonInterpreterCache.global_instance() python_repos = PythonRepos.global_instance() interpreter = interpreter_cache.select_interpreter_for_targets(targets) # We need setuptools to import namespace packages (via pkg_resources), so we prime the # PYTHONPATH with interpreter extras, which Pants always populates with setuptools and wheel. # TODO(John Sirois): We really should be emitting setuptools in a # `synthetic_target_extra_dependencies` override in `ApacheThriftPyGen`: # https://github.com/pantsbuild/pants/issues/5975 pythonpath = list(interpreter.extras.values()) pythonpath.extend(os.path.join(get_buildroot(), t.target_base) for t in targets) for resolved_dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))], interpreter=interpreter, context=python_repos.get_network_context(), fetchers=python_repos.get_fetchers()): pythonpath.append(resolved_dist.distribution.location) process = subprocess.Popen([interpreter.binary, '-c', 'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'], env={'PYTHONPATH': os.pathsep.join(pythonpath)}, stderr=subprocess.PIPE) _, stderr = process.communicate() self.assertEqual(0, process.returncode, stderr)
def execute(self): tool_subsystem = self.tool_subsystem_cls.scoped_instance(self) pex_name = tool_subsystem.options_scope pex_path = os.path.join(self.workdir, self.fingerprint, '{}.pex'.format(pex_name)) interpreter_cache = PythonInterpreterCache.global_instance() interpreter = interpreter_cache.select_interpreter_for_targets([]) if not os.path.exists(pex_path): with self.context.new_workunit(name='create-{}-pex'.format(pex_name), labels=[WorkUnitLabel.PREP]): self._build_tool_pex(tool_subsystem=tool_subsystem, interpreter=interpreter, pex_path=pex_path) tool_instance = self.tool_instance_cls(pex_path, interpreter) self.context.products.register_data(self.tool_instance_cls, tool_instance)
def execute(self): """"Run Checkstyle on all found non-synthetic source files.""" python_tgts = self.context.targets( lambda tgt: isinstance(tgt, (PythonTarget))) if not python_tgts: return 0 interpreter_cache = PythonInterpreterCache.global_instance() with self.invalidated(self.get_targets( self._is_checked)) as invalidation_check: failure_count = 0 tgts_by_compatibility, _ = interpreter_cache.partition_targets_by_compatibility( [vt.target for vt in invalidation_check.invalid_vts]) for filters, targets in tgts_by_compatibility.items(): if self.get_options( ).interpreter_constraints_whitelist is None and not self._constraints_are_whitelisted( filters): deprecated_conditional( lambda: self.get_options( ).interpreter_constraints_whitelist is None, '1.14.0.dev2', "Python linting is currently restricted to targets that match the global " "interpreter constraints: {}. Pants detected unacceptable filters: {}. " "Use the `--interpreter-constraints-whitelist` lint option to whitelist " "compatibiltiy constraints.".format( PythonSetup.global_instance(). interpreter_constraints, filters)) else: sources = self.calculate_sources([tgt for tgt in targets]) if sources: allowed_interpreters = set( interpreter_cache.setup(filters=filters)) if not allowed_interpreters: raise TaskError( 'No valid interpreters found for targets: {}\n(filters: {})' .format(targets, filters)) interpreter = min(allowed_interpreters) failure_count += self.checkstyle(interpreter, sources) if failure_count > 0 and self.get_options().fail: raise TaskError( '{} Python Style issues found. You may try `./pants fmt <targets>`' .format(failure_count)) return failure_count
def execute(self): tool_subsystem = self.tool_subsystem_cls.scoped_instance(self) interpreter_cache = PythonInterpreterCache.global_instance() interpreter = min( interpreter_cache.setup( filters=tool_subsystem.get_interpreter_constraints())) pex_path = self._generate_fingerprinted_pex_path( tool_subsystem, interpreter) if not os.path.exists(pex_path): with self.context.new_workunit(name='create-{}-pex'.format( tool_subsystem.options_scope), labels=[WorkUnitLabel.PREP]): self._build_tool_pex(tool_subsystem=tool_subsystem, interpreter=interpreter, pex_path=pex_path) tool_instance = self.tool_instance_cls(pex_path, interpreter) self.context.products.register_data(self.tool_instance_cls, tool_instance)
def _resolve_requirements(self, target_roots, options=None): with temporary_dir() as cache_dir: options = options or {} python_setup_opts = options.setdefault(PythonSetup.options_scope, {}) python_setup_opts['interpreter_cache_dir'] = cache_dir interpreter = PythonInterpreter.get() python_setup_opts['interpreter_search_paths'] = [os.path.dirname(interpreter.binary)] context = self.context(target_roots=target_roots, options=options, for_subsystems=[PythonInterpreterCache]) # We must get an interpreter via the cache, instead of using the value of # PythonInterpreter.get() directly, to ensure that the interpreter has setuptools and # wheel support. interpreter_cache = PythonInterpreterCache.global_instance() interpreters = interpreter_cache.setup(filters=[str(interpreter.identity.requirement)]) context.products.get_data(PythonInterpreter, lambda: interpreters[0]) task = self.create_task(context) task.execute() return context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX)
def _gather_sources(self, target_roots): with temporary_dir() as cache_dir: interpreter = PythonInterpreter.get() context = self.context(target_roots=target_roots, for_subsystems=[PythonInterpreterCache], options={ PythonSetup.options_scope: { 'interpreter_cache_dir': cache_dir, 'interpreter_search_paths': [os.path.dirname(interpreter.binary)], }}) # We must get an interpreter via the cache, instead of using the value of # PythonInterpreter.get() directly, to ensure that the interpreter has setuptools and # wheel support. interpreter_cache = PythonInterpreterCache.global_instance() interpreters = interpreter_cache.setup(filters=[str(interpreter.identity.requirement)]) context.products.get_data(PythonInterpreter, lambda: interpreters[0]) task = self.create_task(context) task.execute() return context.products.get_data(GatherSources.PYTHON_SOURCES)
def execute(self): if not self.will_be_invoked(): return tool_subsystem = self.tool_subsystem_cls.scoped_instance(self) interpreter_cache = PythonInterpreterCache.global_instance() interpreters = interpreter_cache.setup(filters=tool_subsystem.get_interpreter_constraints()) if not interpreters: raise TaskError('Found no Python interpreter capable of running the {} tool with ' 'constraints {}'.format(tool_subsystem.options_scope, tool_subsystem.get_interpreter_constraints())) interpreter = min(interpreters) pex_path = self._generate_fingerprinted_pex_path(tool_subsystem, interpreter) if not os.path.exists(pex_path): with self.context.new_workunit(name='create-{}-pex'.format(tool_subsystem.options_scope), labels=[WorkUnitLabel.PREP]): self._build_tool_pex(tool_subsystem=tool_subsystem, interpreter=interpreter, pex_path=pex_path) tool_instance = self.tool_instance_cls(pex_path, interpreter) self.context.products.register_data(self.tool_instance_cls, tool_instance)
def _compatible_interpreter(self, unpacked_whls): constraints = PythonSetup.global_instance().compatibility_or_constraints(unpacked_whls.compatibility) allowable_interpreters = PythonInterpreterCache.global_instance().setup(filters=constraints) return min(allowable_interpreters)
def _create_interpreter_cache(self, setup_options=None): Subsystem.reset(reset_options=True) self.context(for_subsystems=[PythonInterpreterCache], options={ 'python-setup': setup_options, }) return PythonInterpreterCache.global_instance()
def test_namespace_effective(self): self.create_file( "src/thrift/com/foo/one.thrift", contents=dedent( """ namespace py foo.bar struct One {} """ ), ) one = self.make_target( spec="src/thrift/com/foo:one", target_type=PythonThriftLibrary, sources=["one.thrift"] ) apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one) self.create_file( "src/thrift2/com/foo/two.thrift", contents=dedent( """ namespace py foo.baz struct Two {} """ ), ) two = self.make_target( spec="src/thrift2/com/foo:two", target_type=PythonThriftLibrary, sources=["two.thrift"] ) _, synthetic_target_two = self.generate_single_thrift_target(two) # Confirm separate PYTHONPATH entries, which we need to test namespace packages. self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base) targets = (synthetic_target_one, synthetic_target_two) self.context(for_subsystems=[PythonInterpreterCache, PythonRepos]) interpreter_cache = PythonInterpreterCache.global_instance() python_repos = PythonRepos.global_instance() interpreter = interpreter_cache.select_interpreter_for_targets(targets) # We need setuptools to import namespace packages under python 2 (via pkg_resources), so we # prime the PYTHONPATH with a known good version of setuptools. # TODO(John Sirois): We really should be emitting setuptools in a # `synthetic_target_extra_dependencies` override in `ApacheThriftPyGen`: # https://github.com/pantsbuild/pants/issues/5975 pythonpath = [os.path.join(get_buildroot(), t.target_base) for t in targets] for resolved_dist in resolve( [f"thrift=={self.get_thrift_version(apache_thrift_gen)}", "setuptools==40.6.3"], interpreter=interpreter, indexes=python_repos.indexes, find_links=python_repos.repos, ): pythonpath.append(resolved_dist.distribution.location) process = subprocess.Popen( [ interpreter.binary, "-c", "from foo.bar.ttypes import One; from foo.baz.ttypes import Two", ], env={"PYTHONPATH": os.pathsep.join(pythonpath)}, stderr=subprocess.PIPE, ) _, stderr = process.communicate() self.assertEqual(0, process.returncode, stderr)
def _interpreter_cache(self): return PythonInterpreterCache.global_instance()
def _interpreter_cache(self): return PythonInterpreterCache.global_instance()
def _compatible_interpreter(self, unpacked_whls): constraints = PythonSetup.global_instance( ).compatibility_or_constraints(unpacked_whls.compatibility) allowable_interpreters = PythonInterpreterCache.global_instance( ).setup(filters=constraints) return min(allowable_interpreters)