class SetupPythonEnvironment(Task): """ Establishes the python intepreter(s) for downstream Python tasks e.g. Resolve, Run, PytestRun. Populates the product namespace (for typename = 'python'): 'intepreters': ordered list of PythonInterpreter objects """ @classmethod def setup_parser(cls, option_group, args, mkflag): option_group.add_option(mkflag("force"), dest="python_setup_force", action="store_true", default=False, help="Force clean and install.") option_group.add_option(mkflag("path"), dest="python_setup_paths", action="append", default=[], help="Add a path to search for interpreters, by default PATH.") option_group.add_option(mkflag("interpreter"), dest="python_interpreter", default=[], action='append', help="Constrain what Python interpreters to use. Uses Requirement " "format from pkg_resources, e.g. 'CPython>=2.6,<3' or 'PyPy'. " "By default, no constraints are used. Multiple constraints may " "be added. They will be ORed together.") option_group.add_option(mkflag("multi"), dest="python_multi", default=False, action='store_true', help="Allow multiple interpreters to be bound to an upstream chroot.") def __init__(self, context): context.products.require('python') self._cache = PythonInterpreterCache(context.config, logger=context.log.debug) super(SetupPythonEnvironment, self).__init__(context) def execute(self, _): ifilters = self.context.options.python_interpreter self._cache.setup(force=self.context.options.python_setup_force, paths=self.context.options.python_setup_paths, filters=ifilters or ['']) all_interpreters = set(self._cache.interpreters) for target in self.context.targets(is_python_root): self.context.log.info('Setting up interpreters for %s' % target) closure = target.closure() self.context.log.debug(' - Target closure: %d targets' % len(closure)) target_compatibilities = [ set(self._cache.matches(getattr(closure_target, 'compatibility', ['']))) for closure_target in closure] target_compatibilities = reduce(set.intersection, target_compatibilities, all_interpreters) self.context.log.debug(' - Target minimum compatibility: %s' % ( ' '.join(interp.version_string for interp in target_compatibilities))) interpreters = self._cache.select_interpreter(target_compatibilities, allow_multiple=self.context.options.python_multi) self.context.log.debug(' - Selected: %s' % interpreters) if not interpreters: raise TaskError('No compatible interpreters for %s' % target) target.interpreters = interpreters
class Py(Command): """Python chroot manipulation.""" __command__ = 'py' def setup_parser(self, parser, args): parser.set_usage('\n' ' %prog py (options) [spec] args\n') parser.disable_interspersed_args() parser.add_option( '-t', '--timeout', dest='conn_timeout', type='int', default=Config.load().getdefault('connection_timeout'), help='Number of seconds to wait for http connections.') parser.add_option( '--pex', dest='pex', default=False, action='store_true', help= 'Dump a .pex of this chroot instead of attempting to execute it.') parser.add_option( '--ipython', dest='ipython', default=False, action='store_true', help='Run the target environment in an IPython interpreter.') parser.add_option( '-r', '--req', dest='extra_requirements', default=[], action='append', help='Additional Python requirements to add to this chroot.') parser.add_option('-i', '--interpreter', dest='interpreter', default=None, help='The interpreter requirement for this chroot.') parser.add_option('-e', '--entry_point', dest='entry_point', default=None, help='The entry point for the generated PEX.') parser.add_option('-v', '--verbose', dest='verbose', default=False, action='store_true', help='Show verbose output.') parser.epilog = """Interact with the chroot of the specified target.""" def __init__(self, run_tracker, root_dir, parser, argv): Command.__init__(self, run_tracker, root_dir, parser, argv) self.target = None self.extra_targets = [] self.config = Config.load() self.interpreter_cache = PythonInterpreterCache(self.config, logger=self.debug) self.interpreter_cache.setup() interpreters = self.interpreter_cache.select_interpreter( list( self.interpreter_cache.matches( [self.options. interpreter] if self.options.interpreter else ['']))) if len(interpreters) != 1: self.error('Unable to detect suitable interpreter.') self.interpreter = interpreters[0] for req in self.options.extra_requirements: with ParseContext.temp(): self.extra_targets.append(PythonRequirement(req, use_2to3=True)) # We parse each arg in the context of the cli usage: # ./pants command (options) [spec] (build args) # ./pants command (options) [spec]... -- (build args) # Our command token and our options are parsed out so we see args of the form: # [spec] (build args) # [spec]... -- (build args) binaries = [] for k in range(len(self.args)): arg = self.args.pop(0) if arg == '--': break def not_a_target(debug_msg): self.debug('Not a target, assuming option: %s.' % e) # We failed to parse the arg as a target or else it was in valid address format but did not # correspond to a real target. Assume this is the 1st of the build args and terminate # processing args for target addresses. self.args.insert(0, arg) target = None try: address = Address.parse(root_dir, arg) target = Target.get(address) if target is None: not_a_target(debug_msg='Unrecognized target') break except Exception as e: not_a_target(debug_msg=e) break for resolved in filter(lambda t: t.is_concrete, target.resolve()): if isinstance(resolved, PythonBinary): binaries.append(resolved) else: self.extra_targets.append(resolved) if len(binaries) == 0: # treat as a chroot pass elif len(binaries) == 1: # We found a binary and are done, the rest of the args get passed to it self.target = binaries[0] else: self.error( 'Can only process 1 binary target, %s contains %d:\n\t%s' % (arg, len(binaries), '\n\t'.join( str(binary.address) for binary in binaries))) if self.target is None: if not self.extra_targets: self.error('No valid target specified!') self.target = self.extra_targets.pop(0) def debug(self, message): if self.options.verbose: print(message, file=sys.stderr) def execute(self): if self.options.pex and self.options.ipython: self.error('Cannot specify both --pex and --ipython!') if self.options.entry_point and self.options.ipython: self.error('Cannot specify both --entry_point and --ipython!') if self.options.verbose: print('Build operating on target: %s %s' % (self.target, 'Extra targets: %s' % ' '.join(map(str, self.extra_targets)) if self.extra_targets else '')) builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter, pex_info=self.target.pexinfo if isinstance( self.target, PythonBinary) else None) if self.options.entry_point: builder.set_entry_point(self.options.entry_point) if self.options.ipython: if not self.config.has_section('python-ipython'): self.error( 'No python-ipython sections defined in your pants.ini!') builder.info.entry_point = self.config.get('python-ipython', 'entry_point') if builder.info.entry_point is None: self.error( 'Must specify entry_point for IPython in the python-ipython section ' 'of your pants.ini!') requirements = self.config.getlist('python-ipython', 'requirements', default=[]) with ParseContext.temp(): for requirement in requirements: self.extra_targets.append(PythonRequirement(requirement)) executor = PythonChroot(self.target, self.root_dir, builder=builder, interpreter=self.interpreter, extra_targets=self.extra_targets, conn_timeout=self.options.conn_timeout) executor.dump() if self.options.pex: pex_name = os.path.join(self.root_dir, 'dist', '%s.pex' % self.target.name) builder.build(pex_name) print('Wrote %s' % pex_name) return 0 else: builder.freeze() pex = PEX(builder.path(), interpreter=self.interpreter) po = pex.run(args=list(self.args), blocking=False) try: return po.wait() except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
class Py(Command): """Python chroot manipulation.""" __command__ = 'py' def setup_parser(self, parser, args): parser.set_usage('\n' ' %prog py (options) [spec] args\n') parser.disable_interspersed_args() parser.add_option('-t', '--timeout', dest='conn_timeout', type='int', default=Config.load().getdefault('connection_timeout'), help='Number of seconds to wait for http connections.') parser.add_option('--pex', dest='pex', default=False, action='store_true', help='Dump a .pex of this chroot instead of attempting to execute it.') parser.add_option('--ipython', dest='ipython', default=False, action='store_true', help='Run the target environment in an IPython interpreter.') parser.add_option('-r', '--req', dest='extra_requirements', default=[], action='append', help='Additional Python requirements to add to this chroot.') parser.add_option('-i', '--interpreter', dest='interpreter', default=None, help='The interpreter requirement for this chroot.') parser.add_option('-e', '--entry_point', dest='entry_point', default=None, help='The entry point for the generated PEX.') parser.add_option('-v', '--verbose', dest='verbose', default=False, action='store_true', help='Show verbose output.') parser.epilog = """Interact with the chroot of the specified target.""" def __init__(self, run_tracker, root_dir, parser, argv): Command.__init__(self, run_tracker, root_dir, parser, argv) self.target = None self.extra_targets = [] self.config = Config.load() self.interpreter_cache = PythonInterpreterCache(self.config, logger=self.debug) self.interpreter_cache.setup() interpreters = self.interpreter_cache.select_interpreter( list(self.interpreter_cache.matches([self.options.interpreter] if self.options.interpreter else ['']))) if len(interpreters) != 1: self.error('Unable to detect suitable interpreter.') self.interpreter = interpreters[0] for req in self.options.extra_requirements: with ParseContext.temp(): self.extra_targets.append(PythonRequirement(req, use_2to3=True)) # We parse each arg in the context of the cli usage: # ./pants command (options) [spec] (build args) # ./pants command (options) [spec]... -- (build args) # Our command token and our options are parsed out so we see args of the form: # [spec] (build args) # [spec]... -- (build args) binaries = [] for k in range(len(self.args)): arg = self.args.pop(0) if arg == '--': break def not_a_target(debug_msg): self.debug('Not a target, assuming option: %s.' % e) # We failed to parse the arg as a target or else it was in valid address format but did not # correspond to a real target. Assume this is the 1st of the build args and terminate # processing args for target addresses. self.args.insert(0, arg) target = None try: address = Address.parse(root_dir, arg) target = Target.get(address) if target is None: not_a_target(debug_msg='Unrecognized target') break except Exception as e: not_a_target(debug_msg=e) break for resolved in filter(lambda t: t.is_concrete, target.resolve()): if isinstance(resolved, PythonBinary): binaries.append(resolved) else: self.extra_targets.append(resolved) if len(binaries) == 0: # treat as a chroot pass elif len(binaries) == 1: # We found a binary and are done, the rest of the args get passed to it self.target = binaries[0] else: self.error('Can only process 1 binary target, %s contains %d:\n\t%s' % ( arg, len(binaries), '\n\t'.join(str(binary.address) for binary in binaries) )) if self.target is None: if not self.extra_targets: self.error('No valid target specified!') self.target = self.extra_targets.pop(0) def debug(self, message): if self.options.verbose: print(message, file=sys.stderr) def execute(self): if self.options.pex and self.options.ipython: self.error('Cannot specify both --pex and --ipython!') if self.options.entry_point and self.options.ipython: self.error('Cannot specify both --entry_point and --ipython!') if self.options.verbose: print('Build operating on target: %s %s' % (self.target, 'Extra targets: %s' % ' '.join(map(str, self.extra_targets)) if self.extra_targets else '')) builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter, pex_info=self.target.pexinfo if isinstance(self.target, PythonBinary) else None) if self.options.entry_point: builder.set_entry_point(self.options.entry_point) if self.options.ipython: if not self.config.has_section('python-ipython'): self.error('No python-ipython sections defined in your pants.ini!') builder.info.entry_point = self.config.get('python-ipython', 'entry_point') if builder.info.entry_point is None: self.error('Must specify entry_point for IPython in the python-ipython section ' 'of your pants.ini!') requirements = self.config.getlist('python-ipython', 'requirements', default=[]) with ParseContext.temp(): for requirement in requirements: self.extra_targets.append(PythonRequirement(requirement)) executor = PythonChroot( self.target, self.root_dir, builder=builder, interpreter=self.interpreter, extra_targets=self.extra_targets, conn_timeout=self.options.conn_timeout) executor.dump() if self.options.pex: pex_name = os.path.join(self.root_dir, 'dist', '%s.pex' % self.target.name) builder.build(pex_name) print('Wrote %s' % pex_name) return 0 else: builder.freeze() pex = PEX(builder.path(), interpreter=self.interpreter) po = pex.run(args=list(self.args), blocking=False) try: return po.wait() except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
class SetupPythonEnvironment(Task): """ Establishes the python intepreter(s) for downstream Python tasks e.g. Resolve, Run, PytestRun. Populates the product namespace (for typename = 'python'): 'intepreters': ordered list of PythonInterpreter objects """ @classmethod def setup_parser(cls, option_group, args, mkflag): option_group.add_option(mkflag("force"), dest="python_setup_force", action="store_true", default=False, help="Force clean and install.") option_group.add_option( mkflag("path"), dest="python_setup_paths", action="append", default=[], help="Add a path to search for interpreters, by default PATH.") option_group.add_option( mkflag("interpreter"), dest="python_interpreter", default=[], action='append', help="Constrain what Python interpreters to use. Uses Requirement " "format from pkg_resources, e.g. 'CPython>=2.6,<3' or 'PyPy'. " "By default, no constraints are used. Multiple constraints may " "be added. They will be ORed together.") option_group.add_option( mkflag("multi"), dest="python_multi", default=False, action='store_true', help= "Allow multiple interpreters to be bound to an upstream chroot.") def __init__(self, context): context.products.require('python') self._cache = PythonInterpreterCache(context.config, logger=context.log.debug) super(SetupPythonEnvironment, self).__init__(context) def execute(self, _): ifilters = self.context.options.python_interpreter self._cache.setup(force=self.context.options.python_setup_force, paths=self.context.options.python_setup_paths, filters=ifilters or ['']) all_interpreters = set(self._cache.interpreters) for target in self.context.targets(is_python_root): self.context.log.info('Setting up interpreters for %s' % target) closure = target.closure() self.context.log.debug(' - Target closure: %d targets' % len(closure)) target_compatibilities = [ set( self._cache.matches( getattr(closure_target, 'compatibility', ['']))) for closure_target in closure ] target_compatibilities = reduce(set.intersection, target_compatibilities, all_interpreters) self.context.log.debug( ' - Target minimum compatibility: %s' % (' '.join(interp.version_string for interp in target_compatibilities))) interpreters = self._cache.select_interpreter( target_compatibilities, allow_multiple=self.context.options.python_multi) self.context.log.debug(' - Selected: %s' % interpreters) if not interpreters: raise TaskError('No compatible interpreters for %s' % target) target.interpreters = interpreters
class Build(Command): """Builds a specified target.""" __command__ = 'build' def setup_parser(self, parser, args): parser.set_usage("\n" " %prog build (options) [spec] (build args)\n" " %prog build (options) [spec]... -- (build args)") parser.add_option("-t", "--timeout", dest="conn_timeout", type="int", default=Config.load().getdefault('connection_timeout'), help="Number of seconds to wait for http connections.") parser.add_option('-i', '--interpreter', dest='interpreter', default=None, help='The interpreter requirement for this chroot.') parser.add_option('-v', '--verbose', dest='verbose', default=False, action='store_true', help='Show verbose output.') parser.disable_interspersed_args() parser.epilog = ('Builds the specified Python target(s). Use ./pants goal for JVM and other ' 'targets.') def __init__(self, run_tracker, root_dir, parser, argv): Command.__init__(self, run_tracker, root_dir, parser, argv) if not self.args: self.error("A spec argument is required") self.config = Config.load() self.interpreter_cache = PythonInterpreterCache(self.config, logger=self.debug) self.interpreter_cache.setup() interpreters = self.interpreter_cache.select_interpreter( list(self.interpreter_cache.matches([self.options.interpreter] if self.options.interpreter else ['']))) if len(interpreters) != 1: self.error('Unable to detect suitable interpreter.') else: self.debug('Selected %s' % interpreters[0]) self.interpreter = interpreters[0] try: specs_end = self.args.index('--') if len(self.args) > specs_end: self.build_args = self.args[specs_end+1:len(self.args)+1] else: self.build_args = [] except ValueError: specs_end = 1 self.build_args = self.args[1:] if len(self.args) > 1 else [] self.targets = OrderedSet() for spec in self.args[0:specs_end]: try: address = Address.parse(root_dir, spec) except: self.error("Problem parsing spec %s: %s" % (spec, traceback.format_exc())) try: target = Target.get(address) except: self.error("Problem parsing BUILD target %s: %s" % (address, traceback.format_exc())) if not target: self.error("Target %s does not exist" % address) self.targets.update(tgt for tgt in target.resolve() if tgt.is_concrete) def debug(self, message): if self.options.verbose: print(message, file=sys.stderr) def execute(self): print("Build operating on targets: %s" % self.targets) python_targets = OrderedSet() for target in self.targets: if target.is_python: python_targets.add(target) else: self.error("Cannot build target %s" % target) if python_targets: status = self._python_build(python_targets) else: status = -1 return status def _python_build(self, targets): try: executor = PythonBuilder(self.run_tracker, self.root_dir) return executor.build( targets, self.build_args, interpreter=self.interpreter, conn_timeout=self.options.conn_timeout) except: self.error("Problem executing PythonBuilder for targets %s: %s" % (targets, traceback.format_exc()))