Exemplo n.º 1
0
class SetupPythonEnvironment(Task):
  """
    Establishes the python intepreter(s) for downstream Python tasks e.g. Resolve, Run, PytestRun.

    Populates the product namespace (for typename = 'python'):
      'intepreters': ordered list of PythonInterpreter objects
  """
  @classmethod
  def setup_parser(cls, option_group, args, mkflag):
    option_group.add_option(mkflag("force"), dest="python_setup_force",
                            action="store_true", default=False,
                            help="Force clean and install.")
    option_group.add_option(mkflag("path"), dest="python_setup_paths",
                            action="append", default=[],
                            help="Add a path to search for interpreters, by default PATH.")
    option_group.add_option(mkflag("interpreter"), dest="python_interpreter",
                            default=[], action='append',
                            help="Constrain what Python interpreters to use.  Uses Requirement "
                                 "format from pkg_resources, e.g. 'CPython>=2.6,<3' or 'PyPy'. "
                                 "By default, no constraints are used.  Multiple constraints may "
                                 "be added.  They will be ORed together.")
    option_group.add_option(mkflag("multi"), dest="python_multi",
                            default=False, action='store_true',
                            help="Allow multiple interpreters to be bound to an upstream chroot.")

  def __init__(self, context):
    context.products.require('python')
    self._cache = PythonInterpreterCache(context.config, logger=context.log.debug)
    super(SetupPythonEnvironment, self).__init__(context)

  def execute(self, _):
    ifilters = self.context.options.python_interpreter
    self._cache.setup(force=self.context.options.python_setup_force,
        paths=self.context.options.python_setup_paths,
        filters=ifilters or [''])
    all_interpreters = set(self._cache.interpreters)
    for target in self.context.targets(is_python_root):
      self.context.log.info('Setting up interpreters for %s' % target)
      closure = target.closure()
      self.context.log.debug('  - Target closure: %d targets' % len(closure))
      target_compatibilities = [
          set(self._cache.matches(getattr(closure_target, 'compatibility', [''])))
          for closure_target in closure]
      target_compatibilities = reduce(set.intersection, target_compatibilities, all_interpreters)
      self.context.log.debug('  - Target minimum compatibility: %s' % (
        ' '.join(interp.version_string for interp in target_compatibilities)))
      interpreters = self._cache.select_interpreter(target_compatibilities,
          allow_multiple=self.context.options.python_multi)
      self.context.log.debug('  - Selected: %s' % interpreters)
      if not interpreters:
        raise TaskError('No compatible interpreters for %s' % target)
      target.interpreters = interpreters
Exemplo n.º 2
0
class Py(Command):
    """Python chroot manipulation."""

    __command__ = 'py'

    def setup_parser(self, parser, args):
        parser.set_usage('\n' '  %prog py (options) [spec] args\n')
        parser.disable_interspersed_args()
        parser.add_option(
            '-t',
            '--timeout',
            dest='conn_timeout',
            type='int',
            default=Config.load().getdefault('connection_timeout'),
            help='Number of seconds to wait for http connections.')
        parser.add_option(
            '--pex',
            dest='pex',
            default=False,
            action='store_true',
            help=
            'Dump a .pex of this chroot instead of attempting to execute it.')
        parser.add_option(
            '--ipython',
            dest='ipython',
            default=False,
            action='store_true',
            help='Run the target environment in an IPython interpreter.')
        parser.add_option(
            '-r',
            '--req',
            dest='extra_requirements',
            default=[],
            action='append',
            help='Additional Python requirements to add to this chroot.')
        parser.add_option('-i',
                          '--interpreter',
                          dest='interpreter',
                          default=None,
                          help='The interpreter requirement for this chroot.')
        parser.add_option('-e',
                          '--entry_point',
                          dest='entry_point',
                          default=None,
                          help='The entry point for the generated PEX.')
        parser.add_option('-v',
                          '--verbose',
                          dest='verbose',
                          default=False,
                          action='store_true',
                          help='Show verbose output.')
        parser.epilog = """Interact with the chroot of the specified target."""

    def __init__(self, run_tracker, root_dir, parser, argv):
        Command.__init__(self, run_tracker, root_dir, parser, argv)

        self.target = None
        self.extra_targets = []
        self.config = Config.load()
        self.interpreter_cache = PythonInterpreterCache(self.config,
                                                        logger=self.debug)
        self.interpreter_cache.setup()
        interpreters = self.interpreter_cache.select_interpreter(
            list(
                self.interpreter_cache.matches(
                    [self.options.
                     interpreter] if self.options.interpreter else [''])))
        if len(interpreters) != 1:
            self.error('Unable to detect suitable interpreter.')
        self.interpreter = interpreters[0]

        for req in self.options.extra_requirements:
            with ParseContext.temp():
                self.extra_targets.append(PythonRequirement(req,
                                                            use_2to3=True))

        # We parse each arg in the context of the cli usage:
        #   ./pants command (options) [spec] (build args)
        #   ./pants command (options) [spec]... -- (build args)
        # Our command token and our options are parsed out so we see args of the form:
        #   [spec] (build args)
        #   [spec]... -- (build args)
        binaries = []
        for k in range(len(self.args)):
            arg = self.args.pop(0)
            if arg == '--':
                break

            def not_a_target(debug_msg):
                self.debug('Not a target, assuming option: %s.' % e)
                # We failed to parse the arg as a target or else it was in valid address format but did not
                # correspond to a real target.  Assume this is the 1st of the build args and terminate
                # processing args for target addresses.
                self.args.insert(0, arg)

            target = None
            try:
                address = Address.parse(root_dir, arg)
                target = Target.get(address)
                if target is None:
                    not_a_target(debug_msg='Unrecognized target')
                    break
            except Exception as e:
                not_a_target(debug_msg=e)
                break

            for resolved in filter(lambda t: t.is_concrete, target.resolve()):
                if isinstance(resolved, PythonBinary):
                    binaries.append(resolved)
                else:
                    self.extra_targets.append(resolved)

        if len(binaries) == 0:
            # treat as a chroot
            pass
        elif len(binaries) == 1:
            # We found a binary and are done, the rest of the args get passed to it
            self.target = binaries[0]
        else:
            self.error(
                'Can only process 1 binary target, %s contains %d:\n\t%s' %
                (arg, len(binaries), '\n\t'.join(
                    str(binary.address) for binary in binaries)))

        if self.target is None:
            if not self.extra_targets:
                self.error('No valid target specified!')
            self.target = self.extra_targets.pop(0)

    def debug(self, message):
        if self.options.verbose:
            print(message, file=sys.stderr)

    def execute(self):
        if self.options.pex and self.options.ipython:
            self.error('Cannot specify both --pex and --ipython!')

        if self.options.entry_point and self.options.ipython:
            self.error('Cannot specify both --entry_point and --ipython!')

        if self.options.verbose:
            print('Build operating on target: %s %s' %
                  (self.target,
                   'Extra targets: %s' % ' '.join(map(str, self.extra_targets))
                   if self.extra_targets else ''))

        builder = PEXBuilder(tempfile.mkdtemp(),
                             interpreter=self.interpreter,
                             pex_info=self.target.pexinfo if isinstance(
                                 self.target, PythonBinary) else None)

        if self.options.entry_point:
            builder.set_entry_point(self.options.entry_point)

        if self.options.ipython:
            if not self.config.has_section('python-ipython'):
                self.error(
                    'No python-ipython sections defined in your pants.ini!')

            builder.info.entry_point = self.config.get('python-ipython',
                                                       'entry_point')
            if builder.info.entry_point is None:
                self.error(
                    'Must specify entry_point for IPython in the python-ipython section '
                    'of your pants.ini!')

            requirements = self.config.getlist('python-ipython',
                                               'requirements',
                                               default=[])

            with ParseContext.temp():
                for requirement in requirements:
                    self.extra_targets.append(PythonRequirement(requirement))

        executor = PythonChroot(self.target,
                                self.root_dir,
                                builder=builder,
                                interpreter=self.interpreter,
                                extra_targets=self.extra_targets,
                                conn_timeout=self.options.conn_timeout)

        executor.dump()

        if self.options.pex:
            pex_name = os.path.join(self.root_dir, 'dist',
                                    '%s.pex' % self.target.name)
            builder.build(pex_name)
            print('Wrote %s' % pex_name)
            return 0
        else:
            builder.freeze()
            pex = PEX(builder.path(), interpreter=self.interpreter)
            po = pex.run(args=list(self.args), blocking=False)
            try:
                return po.wait()
            except KeyboardInterrupt:
                po.send_signal(signal.SIGINT)
                raise
Exemplo n.º 3
0
Arquivo: py.py Projeto: alfss/commons
class Py(Command):
  """Python chroot manipulation."""

  __command__ = 'py'

  def setup_parser(self, parser, args):
    parser.set_usage('\n'
                     '  %prog py (options) [spec] args\n')
    parser.disable_interspersed_args()
    parser.add_option('-t', '--timeout', dest='conn_timeout', type='int',
                      default=Config.load().getdefault('connection_timeout'),
                      help='Number of seconds to wait for http connections.')
    parser.add_option('--pex', dest='pex', default=False, action='store_true',
                      help='Dump a .pex of this chroot instead of attempting to execute it.')
    parser.add_option('--ipython', dest='ipython', default=False, action='store_true',
                      help='Run the target environment in an IPython interpreter.')
    parser.add_option('-r', '--req', dest='extra_requirements', default=[], action='append',
                      help='Additional Python requirements to add to this chroot.')
    parser.add_option('-i', '--interpreter', dest='interpreter', default=None,
                      help='The interpreter requirement for this chroot.')
    parser.add_option('-e', '--entry_point', dest='entry_point', default=None,
                      help='The entry point for the generated PEX.')
    parser.add_option('-v', '--verbose', dest='verbose', default=False, action='store_true',
                      help='Show verbose output.')
    parser.epilog = """Interact with the chroot of the specified target."""

  def __init__(self, run_tracker, root_dir, parser, argv):
    Command.__init__(self, run_tracker, root_dir, parser, argv)

    self.target = None
    self.extra_targets = []
    self.config = Config.load()
    self.interpreter_cache = PythonInterpreterCache(self.config, logger=self.debug)
    self.interpreter_cache.setup()
    interpreters = self.interpreter_cache.select_interpreter(
        list(self.interpreter_cache.matches([self.options.interpreter]
            if self.options.interpreter else [''])))
    if len(interpreters) != 1:
      self.error('Unable to detect suitable interpreter.')
    self.interpreter = interpreters[0]

    for req in self.options.extra_requirements:
      with ParseContext.temp():
        self.extra_targets.append(PythonRequirement(req, use_2to3=True))

    # We parse each arg in the context of the cli usage:
    #   ./pants command (options) [spec] (build args)
    #   ./pants command (options) [spec]... -- (build args)
    # Our command token and our options are parsed out so we see args of the form:
    #   [spec] (build args)
    #   [spec]... -- (build args)
    binaries = []
    for k in range(len(self.args)):
      arg = self.args.pop(0)
      if arg == '--':
        break

      def not_a_target(debug_msg):
        self.debug('Not a target, assuming option: %s.' % e)
        # We failed to parse the arg as a target or else it was in valid address format but did not
        # correspond to a real target.  Assume this is the 1st of the build args and terminate
        # processing args for target addresses.
        self.args.insert(0, arg)

      target = None
      try:
        address = Address.parse(root_dir, arg)
        target = Target.get(address)
        if target is None:
          not_a_target(debug_msg='Unrecognized target')
          break
      except Exception as e:
        not_a_target(debug_msg=e)
        break

      for resolved in filter(lambda t: t.is_concrete, target.resolve()):
        if isinstance(resolved, PythonBinary):
          binaries.append(resolved)
        else:
          self.extra_targets.append(resolved)

    if len(binaries) == 0:
      # treat as a chroot
      pass
    elif len(binaries) == 1:
      # We found a binary and are done, the rest of the args get passed to it
      self.target = binaries[0]
    else:
      self.error('Can only process 1 binary target, %s contains %d:\n\t%s' % (
        arg, len(binaries), '\n\t'.join(str(binary.address) for binary in binaries)
      ))

    if self.target is None:
      if not self.extra_targets:
        self.error('No valid target specified!')
      self.target = self.extra_targets.pop(0)

  def debug(self, message):
    if self.options.verbose:
      print(message, file=sys.stderr)

  def execute(self):
    if self.options.pex and self.options.ipython:
      self.error('Cannot specify both --pex and --ipython!')

    if self.options.entry_point and self.options.ipython:
      self.error('Cannot specify both --entry_point and --ipython!')

    if self.options.verbose:
      print('Build operating on target: %s %s' % (self.target,
        'Extra targets: %s' % ' '.join(map(str, self.extra_targets)) if self.extra_targets else ''))

    builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter,
        pex_info=self.target.pexinfo if isinstance(self.target, PythonBinary) else None)

    if self.options.entry_point:
      builder.set_entry_point(self.options.entry_point)

    if self.options.ipython:
      if not self.config.has_section('python-ipython'):
        self.error('No python-ipython sections defined in your pants.ini!')

      builder.info.entry_point = self.config.get('python-ipython', 'entry_point')
      if builder.info.entry_point is None:
        self.error('Must specify entry_point for IPython in the python-ipython section '
                   'of your pants.ini!')

      requirements = self.config.getlist('python-ipython', 'requirements', default=[])

      with ParseContext.temp():
        for requirement in requirements:
          self.extra_targets.append(PythonRequirement(requirement))

    executor = PythonChroot(
        self.target,
        self.root_dir,
        builder=builder,
        interpreter=self.interpreter,
        extra_targets=self.extra_targets,
        conn_timeout=self.options.conn_timeout)

    executor.dump()

    if self.options.pex:
      pex_name = os.path.join(self.root_dir, 'dist', '%s.pex' % self.target.name)
      builder.build(pex_name)
      print('Wrote %s' % pex_name)
      return 0
    else:
      builder.freeze()
      pex = PEX(builder.path(), interpreter=self.interpreter)
      po = pex.run(args=list(self.args), blocking=False)
      try:
        return po.wait()
      except KeyboardInterrupt:
        po.send_signal(signal.SIGINT)
        raise
Exemplo n.º 4
0
class SetupPythonEnvironment(Task):
    """
    Establishes the python intepreter(s) for downstream Python tasks e.g. Resolve, Run, PytestRun.

    Populates the product namespace (for typename = 'python'):
      'intepreters': ordered list of PythonInterpreter objects
  """
    @classmethod
    def setup_parser(cls, option_group, args, mkflag):
        option_group.add_option(mkflag("force"),
                                dest="python_setup_force",
                                action="store_true",
                                default=False,
                                help="Force clean and install.")
        option_group.add_option(
            mkflag("path"),
            dest="python_setup_paths",
            action="append",
            default=[],
            help="Add a path to search for interpreters, by default PATH.")
        option_group.add_option(
            mkflag("interpreter"),
            dest="python_interpreter",
            default=[],
            action='append',
            help="Constrain what Python interpreters to use.  Uses Requirement "
            "format from pkg_resources, e.g. 'CPython>=2.6,<3' or 'PyPy'. "
            "By default, no constraints are used.  Multiple constraints may "
            "be added.  They will be ORed together.")
        option_group.add_option(
            mkflag("multi"),
            dest="python_multi",
            default=False,
            action='store_true',
            help=
            "Allow multiple interpreters to be bound to an upstream chroot.")

    def __init__(self, context):
        context.products.require('python')
        self._cache = PythonInterpreterCache(context.config,
                                             logger=context.log.debug)
        super(SetupPythonEnvironment, self).__init__(context)

    def execute(self, _):
        ifilters = self.context.options.python_interpreter
        self._cache.setup(force=self.context.options.python_setup_force,
                          paths=self.context.options.python_setup_paths,
                          filters=ifilters or [''])
        all_interpreters = set(self._cache.interpreters)
        for target in self.context.targets(is_python_root):
            self.context.log.info('Setting up interpreters for %s' % target)
            closure = target.closure()
            self.context.log.debug('  - Target closure: %d targets' %
                                   len(closure))
            target_compatibilities = [
                set(
                    self._cache.matches(
                        getattr(closure_target, 'compatibility', [''])))
                for closure_target in closure
            ]
            target_compatibilities = reduce(set.intersection,
                                            target_compatibilities,
                                            all_interpreters)
            self.context.log.debug(
                '  - Target minimum compatibility: %s' %
                (' '.join(interp.version_string
                          for interp in target_compatibilities)))
            interpreters = self._cache.select_interpreter(
                target_compatibilities,
                allow_multiple=self.context.options.python_multi)
            self.context.log.debug('  - Selected: %s' % interpreters)
            if not interpreters:
                raise TaskError('No compatible interpreters for %s' % target)
            target.interpreters = interpreters
Exemplo n.º 5
0
class Build(Command):
  """Builds a specified target."""

  __command__ = 'build'

  def setup_parser(self, parser, args):
    parser.set_usage("\n"
                     "  %prog build (options) [spec] (build args)\n"
                     "  %prog build (options) [spec]... -- (build args)")
    parser.add_option("-t", "--timeout", dest="conn_timeout", type="int",
                      default=Config.load().getdefault('connection_timeout'),
                      help="Number of seconds to wait for http connections.")
    parser.add_option('-i', '--interpreter', dest='interpreter', default=None,
                      help='The interpreter requirement for this chroot.')
    parser.add_option('-v', '--verbose', dest='verbose', default=False, action='store_true',
                      help='Show verbose output.')
    parser.disable_interspersed_args()
    parser.epilog = ('Builds the specified Python target(s). Use ./pants goal for JVM and other '
                     'targets.')

  def __init__(self, run_tracker, root_dir, parser, argv):
    Command.__init__(self, run_tracker, root_dir, parser, argv)

    if not self.args:
      self.error("A spec argument is required")

    self.config = Config.load()
    self.interpreter_cache = PythonInterpreterCache(self.config, logger=self.debug)
    self.interpreter_cache.setup()
    interpreters = self.interpreter_cache.select_interpreter(
        list(self.interpreter_cache.matches([self.options.interpreter]
            if self.options.interpreter else [''])))
    if len(interpreters) != 1:
      self.error('Unable to detect suitable interpreter.')
    else:
      self.debug('Selected %s' % interpreters[0])
    self.interpreter = interpreters[0]

    try:
      specs_end = self.args.index('--')
      if len(self.args) > specs_end:
        self.build_args = self.args[specs_end+1:len(self.args)+1]
      else:
        self.build_args = []
    except ValueError:
      specs_end = 1
      self.build_args = self.args[1:] if len(self.args) > 1 else []

    self.targets = OrderedSet()
    for spec in self.args[0:specs_end]:
      try:
        address = Address.parse(root_dir, spec)
      except:
        self.error("Problem parsing spec %s: %s" % (spec, traceback.format_exc()))

      try:
        target = Target.get(address)
      except:
        self.error("Problem parsing BUILD target %s: %s" % (address, traceback.format_exc()))

      if not target:
        self.error("Target %s does not exist" % address)
      self.targets.update(tgt for tgt in target.resolve() if tgt.is_concrete)

  def debug(self, message):
    if self.options.verbose:
      print(message, file=sys.stderr)

  def execute(self):
    print("Build operating on targets: %s" % self.targets)

    python_targets = OrderedSet()
    for target in self.targets:
      if target.is_python:
        python_targets.add(target)
      else:
        self.error("Cannot build target %s" % target)

    if python_targets:
      status = self._python_build(python_targets)
    else:
      status = -1

    return status

  def _python_build(self, targets):
    try:
      executor = PythonBuilder(self.run_tracker, self.root_dir)
      return executor.build(
        targets,
        self.build_args,
        interpreter=self.interpreter,
        conn_timeout=self.options.conn_timeout)
    except:
      self.error("Problem executing PythonBuilder for targets %s: %s" % (targets,
                                                                         traceback.format_exc()))
Exemplo n.º 6
0
class Build(Command):
  """Builds a specified target."""

  __command__ = 'build'

  def setup_parser(self, parser, args):
    parser.set_usage("\n"
                     "  %prog build (options) [spec] (build args)\n"
                     "  %prog build (options) [spec]... -- (build args)")
    parser.add_option("-t", "--timeout", dest="conn_timeout", type="int",
                      default=Config.load().getdefault('connection_timeout'),
                      help="Number of seconds to wait for http connections.")
    parser.add_option('-i', '--interpreter', dest='interpreter', default=None,
                      help='The interpreter requirement for this chroot.')
    parser.add_option('-v', '--verbose', dest='verbose', default=False, action='store_true',
                      help='Show verbose output.')
    parser.disable_interspersed_args()
    parser.epilog = ('Builds the specified Python target(s). Use ./pants goal for JVM and other '
                     'targets.')

  def __init__(self, run_tracker, root_dir, parser, argv):
    Command.__init__(self, run_tracker, root_dir, parser, argv)

    if not self.args:
      self.error("A spec argument is required")

    self.config = Config.load()
    self.interpreter_cache = PythonInterpreterCache(self.config, logger=self.debug)
    self.interpreter_cache.setup()
    interpreters = self.interpreter_cache.select_interpreter(
        list(self.interpreter_cache.matches([self.options.interpreter]
            if self.options.interpreter else [''])))
    if len(interpreters) != 1:
      self.error('Unable to detect suitable interpreter.')
    else:
      self.debug('Selected %s' % interpreters[0])
    self.interpreter = interpreters[0]

    try:
      specs_end = self.args.index('--')
      if len(self.args) > specs_end:
        self.build_args = self.args[specs_end+1:len(self.args)+1]
      else:
        self.build_args = []
    except ValueError:
      specs_end = 1
      self.build_args = self.args[1:] if len(self.args) > 1 else []

    self.targets = OrderedSet()
    for spec in self.args[0:specs_end]:
      try:
        address = Address.parse(root_dir, spec)
      except:
        self.error("Problem parsing spec %s: %s" % (spec, traceback.format_exc()))

      try:
        target = Target.get(address)
      except:
        self.error("Problem parsing BUILD target %s: %s" % (address, traceback.format_exc()))

      if not target:
        self.error("Target %s does not exist" % address)
      self.targets.update(tgt for tgt in target.resolve() if tgt.is_concrete)

  def debug(self, message):
    if self.options.verbose:
      print(message, file=sys.stderr)

  def execute(self):
    print("Build operating on targets: %s" % self.targets)

    python_targets = OrderedSet()
    for target in self.targets:
      if target.is_python:
        python_targets.add(target)
      else:
        self.error("Cannot build target %s" % target)

    if python_targets:
      status = self._python_build(python_targets)
    else:
      status = -1

    return status

  def _python_build(self, targets):
    try:
      executor = PythonBuilder(self.run_tracker, self.root_dir)
      return executor.build(
        targets,
        self.build_args,
        interpreter=self.interpreter,
        conn_timeout=self.options.conn_timeout)
    except:
      self.error("Problem executing PythonBuilder for targets %s: %s" % (targets,
                                                                         traceback.format_exc()))