Exemplo n.º 1
0
    def dumped_chroot(self, targets):
        python_repos = create_subsystem(PythonRepos)

        with subsystem_instance(IvySubsystem) as ivy_subsystem:
            ivy_bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem)

            with subsystem_instance(
                    ThriftBinary.Factory) as thrift_binary_factory:
                interpreter_cache = PythonInterpreterCache(
                    self.python_setup, python_repos)
                interpreter_cache.setup()
                interpreters = list(
                    interpreter_cache.matches(
                        [self.python_setup.interpreter_requirement]))
                self.assertGreater(len(interpreters), 0)
                interpreter = interpreters[0]

                with temporary_dir() as chroot:
                    pex_builder = PEXBuilder(path=chroot,
                                             interpreter=interpreter)

                    python_chroot = PythonChroot(
                        python_setup=self.python_setup,
                        python_repos=python_repos,
                        ivy_bootstrapper=ivy_bootstrapper,
                        thrift_binary_factory=thrift_binary_factory.create,
                        interpreter=interpreter,
                        builder=pex_builder,
                        targets=targets,
                        platforms=['current'])
                    try:
                        python_chroot.dump()
                        yield pex_builder, python_chroot
                    finally:
                        python_chroot.delete()
Exemplo n.º 2
0
    def _resolve_requirements(self, target_roots, options=None):
        with temporary_dir() as cache_dir:
            options = options or {}
            options.setdefault(PythonSetup.options_scope,
                               {})['interpreter_cache_dir'] = cache_dir
            context = self.context(target_roots=target_roots,
                                   options=options,
                                   for_subsystems=[PythonSetup, PythonRepos])

            # We must get an interpreter via the cache, instead of using PythonInterpreter.get() directly,
            # to ensure that the interpreter has setuptools and wheel support.
            interpreter = PythonInterpreter.get()
            interpreter_cache = PythonInterpreterCache(
                PythonSetup.global_instance(),
                PythonRepos.global_instance(),
                logger=context.log.debug)
            interpreters = interpreter_cache.setup(
                paths=[os.path.dirname(interpreter.binary)],
                filters=[str(interpreter.identity.requirement)])
            context.products.get_data(PythonInterpreter,
                                      lambda: interpreters[0])

            task = self.create_task(context)
            task.execute()

            return context.products.get_data(
                ResolveRequirements.REQUIREMENTS_PEX)
Exemplo n.º 3
0
  def dumped_chroot(self, targets):
    # TODO(benjy): We shouldn't need to mention DistributionLocator here, as IvySubsystem
    # declares it as a dependency. However if we don't then test_antlr() below fails on
    # uninitialized options for that subsystem.  Hopefully my pending (as of 9/2016) change
    # to clean up how we initialize and create instances of subsystems in tests will make
    # this problem go away.
    self.context(for_subsystems=[PythonRepos, PythonSetup, IvySubsystem,
                                 DistributionLocator, ThriftBinary.Factory, BinaryUtil.Factory])
    python_repos = PythonRepos.global_instance()
    ivy_bootstrapper = Bootstrapper(ivy_subsystem=IvySubsystem.global_instance())
    thrift_binary_factory = ThriftBinary.Factory.global_instance().create

    interpreter_cache = PythonInterpreterCache(self.python_setup, python_repos)
    interpreter = interpreter_cache.select_interpreter_for_targets(targets)
    self.assertIsNotNone(interpreter)

    with temporary_dir() as chroot:
      pex_builder = PEXBuilder(path=chroot, interpreter=interpreter)

      python_chroot = PythonChroot(python_setup=self.python_setup,
                                   python_repos=python_repos,
                                   ivy_bootstrapper=ivy_bootstrapper,
                                   thrift_binary_factory=thrift_binary_factory,
                                   interpreter=interpreter,
                                   builder=pex_builder,
                                   targets=targets,
                                   platforms=['current'])
      try:
        python_chroot.dump()
        yield pex_builder, python_chroot
      finally:
        python_chroot.delete()
Exemplo n.º 4
0
  def dumped_chroot(self, targets):
    # TODO(benjy): We shouldn't need to mention DistributionLocator here, as IvySubsystem
    # declares it as a dependency. However if we don't then test_antlr() below fails on
    # uninitialized options for that subsystem.  Hopefully my pending (as of 9/2016) change
    # to clean up how we initialize and create instances of subsystems in tests will make
    # this problem go away.
    self.context(for_subsystems=[PythonRepos, PythonSetup, IvySubsystem,
                                 DistributionLocator, ThriftBinary.Factory, BinaryUtil.Factory])
    python_repos = PythonRepos.global_instance()
    ivy_bootstrapper = Bootstrapper(ivy_subsystem=IvySubsystem.global_instance())
    thrift_binary_factory = ThriftBinary.Factory.global_instance().create

    interpreter_cache = PythonInterpreterCache(self.python_setup, python_repos)
    interpreter_cache.setup()
    interpreters = list(interpreter_cache.matched_interpreters(
      self.python_setup.interpreter_constraints))
    self.assertGreater(len(interpreters), 0)
    interpreter = interpreters[0]

    with temporary_dir() as chroot:
      pex_builder = PEXBuilder(path=chroot, interpreter=interpreter)

      python_chroot = PythonChroot(python_setup=self.python_setup,
                                   python_repos=python_repos,
                                   ivy_bootstrapper=ivy_bootstrapper,
                                   thrift_binary_factory=thrift_binary_factory,
                                   interpreter=interpreter,
                                   builder=pex_builder,
                                   targets=targets,
                                   platforms=['current'])
      try:
        python_chroot.dump()
        yield pex_builder, python_chroot
      finally:
        python_chroot.delete()
Exemplo n.º 5
0
    def __init__(self, *args, **kwargs):
        super(Build, self).__init__(*args, **kwargs)

        if not self.args:
            self.error("A spec argument is required")

        self.config = Config.load()

        interpreters = self.options.interpreters or [b'']
        self.interpreter_cache = PythonInterpreterCache(self.config,
                                                        logger=self.debug)
        self.interpreter_cache.setup(filters=interpreters)
        interpreters = self.interpreter_cache.select_interpreter(
            list(self.interpreter_cache.matches(interpreters)))
        if len(interpreters) != 1:
            self.error('Unable to detect suitable interpreter.')
        else:
            self.debug('Selected %s' % interpreters[0])
        self.interpreter = interpreters[0]

        try:
            specs_end = self.args.index('--')
            if len(self.args) > specs_end:
                self.build_args = self.args[specs_end + 1:len(self.args) + 1]
            else:
                self.build_args = []
        except ValueError:
            specs_end = 1
            self.build_args = self.args[1:] if len(self.args) > 1 else []

        self.targets = OrderedSet()
        spec_parser = SpecParser(self.root_dir, self.build_file_parser)
        self.top_level_addresses = set()

        for spec in self.args[0:specs_end]:
            try:
                addresses = spec_parser.parse_addresses(spec)
            except:
                self.error("Problem parsing spec %s: %s" %
                           (spec, traceback.format_exc()))

            for address in addresses:
                self.top_level_addresses.add(address)
                try:
                    self.build_file_parser.inject_address_closure_into_build_graph(
                        address, self.build_graph)
                    target = self.build_graph.get_target(address)
                except:
                    self.error("Problem parsing BUILD target %s: %s" %
                               (address, traceback.format_exc()))

                if not target:
                    self.error("Target %s does not exist" % address)

                transitive_targets = self.build_graph.transitive_subgraph_of_addresses(
                    [target.address])
                for transitive_target in transitive_targets:
                    self.targets.add(transitive_target)

        self.targets = [target for target in self.targets if target.is_python]
Exemplo n.º 6
0
  def dumped_chroot(self, targets):
    python_repos = create_subsystem(PythonRepos)

    with subsystem_instance(IvySubsystem) as ivy_subsystem:
      ivy_bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem)

      with subsystem_instance(ThriftBinary.Factory) as thrift_binary_factory:
        interpreter_cache = PythonInterpreterCache(self.python_setup, python_repos)
        interpreter_cache.setup()
        interpreters = list(interpreter_cache.matched_interpreters([
          self.python_setup.interpreter_requirement]))
        self.assertGreater(len(interpreters), 0)
        interpreter = interpreters[0]

        with temporary_dir() as chroot:
          pex_builder = PEXBuilder(path=chroot, interpreter=interpreter)

          python_chroot = PythonChroot(python_setup=self.python_setup,
                                       python_repos=python_repos,
                                       ivy_bootstrapper=ivy_bootstrapper,
                                       thrift_binary_factory=thrift_binary_factory.create,
                                       interpreter=interpreter,
                                       builder=pex_builder,
                                       targets=targets,
                                       platforms=['current'])
          try:
            python_chroot.dump()
            yield pex_builder, python_chroot
          finally:
            python_chroot.delete()
Exemplo n.º 7
0
class PythonTask(Task):
  @classmethod
  def setup_parser(cls, option_group, args, mkflag):
    option_group.add_option(mkflag('timeout'), dest='python_conn_timeout', type='int',
                            default=0, help='Number of seconds to wait for http connections.')

  def __init__(self, context, workdir):
    super(PythonTask, self).__init__(context, workdir)
    self.conn_timeout = (self.context.options.python_conn_timeout or
                         self.context.config.getdefault('connection_timeout'))
    compatibilities = self.context.options.interpreter or [b'']

    self.interpreter_cache = PythonInterpreterCache(self.context.config,
                                                    logger=self.context.log.debug)
    # We pass in filters=compatibilities because setting up some python versions
    # (e.g., 3<=python<3.3) crashes, and this gives us an escape hatch.
    self.interpreter_cache.setup(filters=compatibilities)

    # Select a default interpreter to use.
    self._interpreter = self.select_interpreter(compatibilities)

  @property
  def interpreter(self):
    """Subclasses can use this if they're fine with the default interpreter (the usual case)."""
    return self._interpreter

  def select_interpreter(self, compatibilities):
    """Subclasses can use this to be more specific about interpreter selection."""
    interpreters = self.interpreter_cache.select_interpreter(
      list(self.interpreter_cache.matches(compatibilities)))
    if len(interpreters) != 1:
      raise TaskError('Unable to detect suitable interpreter.')
    interpreter = interpreters[0]
    self.context.log.debug('Selected %s' % interpreter)
    return interpreter
Exemplo n.º 8
0
    def execute(self):
        interpreter = None
        python_tgts = self.context.targets(
            lambda tgt: isinstance(tgt, PythonTarget))
        fs = PythonInterpreterFingerprintStrategy(task=self)
        with self.invalidated(python_tgts,
                              fingerprint_strategy=fs) as invalidation_check:
            # If there are no relevant targets, we still go through the motions of selecting
            # an interpreter, to prevent downstream tasks from having to check for this special case.
            if invalidation_check.all_vts:
                target_set_id = VersionedTargetSet.from_versioned_targets(
                    invalidation_check.all_vts).cache_key.hash
            else:
                target_set_id = 'no_targets'
            interpreter_path_file = os.path.join(self.workdir, target_set_id,
                                                 'interpreter.path')
            if not os.path.exists(interpreter_path_file):
                interpreter_cache = PythonInterpreterCache(
                    PythonSetup.global_instance(),
                    PythonRepos.global_instance(),
                    logger=self.context.log.debug)

                # We filter the interpreter cache itself (and not just the interpreters we pull from it)
                # because setting up some python versions (e.g., 3<=python<3.3) crashes, and this gives us
                # an escape hatch.
                filters = self.get_options().constraints or [b'']

                # Cache setup's requirement fetching can hang if run concurrently by another pants proc.
                self.context.acquire_lock()
                try:
                    interpreter_cache.setup(filters=filters)
                finally:
                    self.context.release_lock()

                interpreter = interpreter_cache.select_interpreter_for_targets(
                    python_tgts)
                safe_mkdir_for(interpreter_path_file)
                with open(interpreter_path_file, 'w') as outfile:
                    outfile.write(b'{}\t{}\n'.format(
                        interpreter.binary, str(interpreter.identity)))
                    for dist, location in interpreter.extras.items():
                        dist_name, dist_version = dist
                        outfile.write(b'{}\t{}\t{}\n'.format(
                            dist_name, dist_version, location))

        if not interpreter:
            with open(interpreter_path_file, 'r') as infile:
                lines = infile.readlines()
                binary, identity = lines[0].strip().split('\t')
                extras = {}
                for line in lines[1:]:
                    dist_name, dist_version, location = line.strip().split(
                        '\t')
                    extras[(dist_name, dist_version)] = location

            interpreter = PythonInterpreter(binary,
                                            PythonIdentity.from_path(identity),
                                            extras)

        self.context.products.get_data(PythonInterpreter, lambda: interpreter)
Exemplo n.º 9
0
    def test_setup_using_eggs(self):
        def link_egg(repo_root, requirement):
            existing_dist_location = self._interpreter.get_location(
                requirement)
            if existing_dist_location is not None:
                existing_dist = Package.from_href(existing_dist_location)
                requirement = '{}=={}'.format(existing_dist.name,
                                              existing_dist.raw_version)

            distributions = resolve([requirement],
                                    interpreter=self._interpreter,
                                    precedence=(EggPackage, SourcePackage))
            self.assertEqual(1, len(distributions))
            dist_location = distributions[0].location

            self.assertRegexpMatches(dist_location, r'\.egg$')
            os.symlink(
                dist_location,
                os.path.join(repo_root, os.path.basename(dist_location)))

            return Package.from_href(dist_location).raw_version

        with temporary_dir() as root:
            egg_dir = os.path.join(root, 'eggs')
            os.makedirs(egg_dir)
            setuptools_version = link_egg(egg_dir, 'setuptools')
            wheel_version = link_egg(egg_dir, 'wheel')

            interpreter_requirement = self._interpreter.identity.requirement

            python_setup, python_repos = self.create_python_subsystems(
                setup_options={
                    'interpreter_cache_dir': None,
                    'pants_workdir': os.path.join(root, 'workdir'),
                    'constraints': [interpreter_requirement],
                    'setuptools_version': setuptools_version,
                    'wheel_version': wheel_version,
                },
                repos_options={
                    'indexes': [],
                    'repos': [egg_dir],
                })
            cache = PythonInterpreterCache(python_setup=python_setup,
                                           python_repos=python_repos)

            interpereters = cache.setup(
                paths=[os.path.dirname(self._interpreter.binary)],
                filters=[str(interpreter_requirement)])
            self.assertGreater(len(interpereters), 0)

            def assert_egg_extra(interpreter, name, version):
                location = interpreter.get_location('{}=={}'.format(
                    name, version))
                self.assertIsNotNone(location)
                self.assertIsInstance(Package.from_href(location), EggPackage)

            for interpreter in interpereters:
                assert_egg_extra(interpreter, 'setuptools', setuptools_version)
                assert_egg_extra(interpreter, 'wheel', wheel_version)
Exemplo n.º 10
0
    def test_namespace_effective(self):
        self.create_file('src/thrift/com/foo/one.thrift',
                         contents=dedent("""
    namespace py foo.bar

    struct One {}
    """))
        one = self.make_target(spec='src/thrift/com/foo:one',
                               target_type=PythonThriftLibrary,
                               sources=['one.thrift'])
        apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(
            one)

        self.create_file('src/thrift2/com/foo/two.thrift',
                         contents=dedent("""
    namespace py foo.baz

    struct Two {}
    """))
        two = self.make_target(spec='src/thrift2/com/foo:two',
                               target_type=PythonThriftLibrary,
                               sources=['two.thrift'])
        _, synthetic_target_two = self.generate_single_thrift_target(two)

        # Confirm separate PYTHONPATH entries, which we need to test namespace packages.
        self.assertNotEqual(synthetic_target_one.target_base,
                            synthetic_target_two.target_base)

        targets = (synthetic_target_one, synthetic_target_two)

        python_repos = global_subsystem_instance(PythonRepos)
        python_setup = global_subsystem_instance(PythonSetup)
        interpreter_cache = PythonInterpreterCache(python_setup, python_repos)
        interpreter = interpreter_cache.select_interpreter_for_targets(targets)

        pythonpath = [
            os.path.join(get_buildroot(), t.target_base) for t in targets
        ]
        for dist in resolve(
            ['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))],
                interpreter=interpreter,
                context=python_repos.get_network_context(),
                fetchers=python_repos.get_fetchers()):
            pythonpath.append(dist.location)

        process = subprocess.Popen([
            interpreter.binary, '-c',
            'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'
        ],
                                   env={
                                       'PYTHONPATH':
                                       os.pathsep.join(pythonpath)
                                   },
                                   stderr=subprocess.PIPE)
        _, stderr = process.communicate()
        self.assertEqual(0, process.returncode, stderr)
Exemplo n.º 11
0
 def _interpreter_cache(self):
   interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
                                              PythonRepos.global_instance(),
                                              logger=self.context.log.debug)
   # Cache setup's requirement fetching can hang if run concurrently by another pants proc.
   self.context.acquire_lock()
   try:
     interpreter_cache.setup()
   finally:
     self.context.release_lock()
   return interpreter_cache
Exemplo n.º 12
0
 def _create_interpreter_path_file(self, interpreter_path_file, targets):
   interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
                                              PythonRepos.global_instance(),
                                              logger=self.context.log.debug)
   interpreter = interpreter_cache.select_interpreter_for_targets(targets)
   safe_mkdir_for(interpreter_path_file)
   with open(interpreter_path_file, 'w') as outfile:
     outfile.write(b'{}\n'.format(interpreter.binary))
     for dist, location in interpreter.extras.items():
       dist_name, dist_version = dist
       outfile.write(b'{}\t{}\t{}\n'.format(dist_name, dist_version, location))
Exemplo n.º 13
0
    def _cache_current_interpreter(self):
        cache = PythonInterpreterCache(self.config())

        # We only need to cache the current interpreter, avoid caching for every interpreter on the
        # PATH.
        current_interpreter = PythonInterpreter.get()
        for cached_interpreter in cache.setup(
                filters=[current_interpreter.identity.requirement]):
            if cached_interpreter == current_interpreter:
                return cached_interpreter
        raise RuntimeError('Could not find suitable interpreter to run tests.')
Exemplo n.º 14
0
  def test_setup_using_eggs(self):
    def link_egg(repo_root, requirement):
      existing_dist_location = self._interpreter.get_location(requirement)
      if existing_dist_location is not None:
        existing_dist = Package.from_href(existing_dist_location)
        requirement = '{}=={}'.format(existing_dist.name, existing_dist.raw_version)

      distributions = resolve([requirement],
                              interpreter=self._interpreter,
                              precedence=(EggPackage, SourcePackage))
      self.assertEqual(1, len(distributions))
      dist_location = distributions[0].location

      self.assertRegexpMatches(dist_location, r'\.egg$')
      os.symlink(dist_location, os.path.join(repo_root, os.path.basename(dist_location)))

      return Package.from_href(dist_location).raw_version

    with temporary_dir() as root:
      egg_dir = os.path.join(root, 'eggs')
      os.makedirs(egg_dir)
      setuptools_version = link_egg(egg_dir, 'setuptools')
      wheel_version = link_egg(egg_dir, 'wheel')

      interpreter_requirement = self._interpreter.identity.requirement

      self.context(for_subsystems=[PythonSetup, PythonRepos], options={
        PythonSetup.options_scope: {
          'interpreter_cache_dir': None,
          'pants_workdir': os.path.join(root, 'workdir'),
          'constraints': [interpreter_requirement],
          'setuptools_version': setuptools_version,
          'wheel_version': wheel_version,
        },
        PythonRepos.options_scope: {
          'indexes': [],
          'repos': [egg_dir],
        }
      })
      cache = PythonInterpreterCache(PythonSetup.global_instance(), PythonRepos.global_instance())

      interpereters = cache.setup(paths=[os.path.dirname(self._interpreter.binary)],
                                  filters=[str(interpreter_requirement)])
      self.assertGreater(len(interpereters), 0)

      def assert_egg_extra(interpreter, name, version):
        location = interpreter.get_location('{}=={}'.format(name, version))
        self.assertIsNotNone(location)
        self.assertIsInstance(Package.from_href(location), EggPackage)

      for interpreter in interpereters:
        assert_egg_extra(interpreter, 'setuptools', setuptools_version)
        assert_egg_extra(interpreter, 'wheel', wheel_version)
Exemplo n.º 15
0
    def _validate_interpreter_constraints(self, constraint_tgts):
        """Validate that the transitive constraints of the given PythonBinary target are compatible.

    If no (local) interpreter can satisfy all of the given targets, raises
    PythonInterpreterCache.UnsatisfiableInterpreterConstraintsError.

    TODO: This currently does so by finding a concrete local interpreter that matches all of the
    constraints, but it is possible to do this in memory instead.
      see https://github.com/pantsbuild/pants/issues/7775
    """
        PythonInterpreterCache.global_instance(
        ).select_interpreter_for_targets(constraint_tgts)
Exemplo n.º 16
0
 def _interpreter_cache(self):
     interpreter_cache = PythonInterpreterCache(
         PythonSetup.global_instance(),
         PythonRepos.global_instance(),
         logger=self.context.log.debug)
     # Cache setup's requirement fetching can hang if run concurrently by another pants proc.
     self.context.acquire_lock()
     try:
         interpreter_cache.setup()
     finally:
         self.context.release_lock()
     return interpreter_cache
Exemplo n.º 17
0
class SetupPythonEnvironment(Task):
  """
    Establishes the python intepreter(s) for downstream Python tasks e.g. Resolve, Run, PytestRun.

    Populates the product namespace (for typename = 'python'):
      'intepreters': ordered list of PythonInterpreter objects
  """
  @classmethod
  def setup_parser(cls, option_group, args, mkflag):
    option_group.add_option(mkflag("force"), dest="python_setup_force",
                            action="store_true", default=False,
                            help="Force clean and install.")
    option_group.add_option(mkflag("path"), dest="python_setup_paths",
                            action="append", default=[],
                            help="Add a path to search for interpreters, by default PATH.")
    option_group.add_option(mkflag("interpreter"), dest="python_interpreter",
                            default=[], action='append',
                            help="Constrain what Python interpreters to use.  Uses Requirement "
                                 "format from pkg_resources, e.g. 'CPython>=2.6,<3' or 'PyPy'. "
                                 "By default, no constraints are used.  Multiple constraints may "
                                 "be added.  They will be ORed together.")
    option_group.add_option(mkflag("multi"), dest="python_multi",
                            default=False, action='store_true',
                            help="Allow multiple interpreters to be bound to an upstream chroot.")

  def __init__(self, context, workdir):
    context.products.require('python')
    self._cache = PythonInterpreterCache(context.config, logger=context.log.debug)
    super(SetupPythonEnvironment, self).__init__(context, workdir)

  def execute(self):
    ifilters = self.context.options.python_interpreter
    self._cache.setup(force=self.context.options.python_setup_force,
        paths=self.context.options.python_setup_paths,
        filters=ifilters or [b''])
    all_interpreters = set(self._cache.interpreters)
    for target in self.context.targets(is_python_root):
      self.context.log.info('Setting up interpreters for %s' % target)
      closure = target.closure()
      self.context.log.debug('  - Target closure: %d targets' % len(closure))
      target_compatibilities = [
          set(self._cache.matches(getattr(closure_target, 'compatibility', [''])))
          for closure_target in closure]
      target_compatibilities = reduce(set.intersection, target_compatibilities, all_interpreters)
      self.context.log.debug('  - Target minimum compatibility: %s' % (
        ' '.join(interp.version_string for interp in target_compatibilities)))
      interpreters = self._cache.select_interpreter(target_compatibilities,
          allow_multiple=self.context.options.python_multi)
      self.context.log.debug('  - Selected: %s' % interpreters)
      if not interpreters:
        raise TaskError('No compatible interpreters for %s' % target)
      target.interpreters = interpreters
Exemplo n.º 18
0
 def _create_interpreter_path_file(self, interpreter_path_file, targets):
     interpreter_cache = PythonInterpreterCache(
         PythonSetup.global_instance(),
         PythonRepos.global_instance(),
         logger=self.context.log.debug)
     interpreter = interpreter_cache.select_interpreter_for_targets(targets)
     safe_mkdir_for(interpreter_path_file)
     with open(interpreter_path_file, 'w') as outfile:
         outfile.write(b'{}\n'.format(interpreter.binary))
         for dist, location in interpreter.extras.items():
             dist_name, dist_version = dist
             outfile.write(b'{}\t{}\t{}\n'.format(dist_name, dist_version,
                                                  location))
Exemplo n.º 19
0
  def _cache_current_interpreter(self):
    cache = PythonInterpreterCache(self.config())

    # We only need to cache the current interpreter, avoid caching for every interpreter on the
    # PATH.
    current_interpreter = PythonInterpreter.get()
    current_id = (current_interpreter.binary, current_interpreter.identity)
    for cached_interpreter in cache.setup(filters=[current_interpreter.identity.requirement]):
      # TODO(John Sirois): Revert to directly comparing interpreters when
      # https://github.com/pantsbuild/pex/pull/31 is in, released and consumed by pants.
      if (cached_interpreter.binary, cached_interpreter.identity) == current_id:
        return cached_interpreter
    raise RuntimeError('Could not find suitable interpreter to run tests.')
Exemplo n.º 20
0
  def test_namespace_effective(self):
    self.create_file('src/thrift/com/foo/one.thrift', contents=dedent("""
    namespace py foo.bar

    struct One {}
    """))
    one = self.make_target(spec='src/thrift/com/foo:one',
                           target_type=PythonThriftLibrary,
                           sources=['one.thrift'])
    apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one)

    self.create_file('src/thrift2/com/foo/two.thrift', contents=dedent("""
    namespace py foo.baz

    struct Two {}
    """))
    two = self.make_target(spec='src/thrift2/com/foo:two',
                           target_type=PythonThriftLibrary,
                           sources=['two.thrift'])
    _, synthetic_target_two = self.generate_single_thrift_target(two)

    # Confirm separate PYTHONPATH entries, which we need to test namespace packages.
    self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base)

    targets = (synthetic_target_one, synthetic_target_two)

    python_repos = global_subsystem_instance(PythonRepos)
    python_setup = global_subsystem_instance(PythonSetup)
    interpreter_cache = PythonInterpreterCache(python_setup, python_repos)
    interpreter = interpreter_cache.select_interpreter_for_targets(targets)

    # We need setuptools to import namespace packages (via pkg_resources), so we prime the
    # PYTHONPATH with interpreter extras, which Pants always populates with setuptools and wheel.
    # TODO(John Sirois): We really should be emitting setuptools in a
    # `synthetic_target_extra_dependencies` override in `ApacheThriftPyGen`:
    #   https://github.com/pantsbuild/pants/issues/5975
    pythonpath = interpreter.extras.values()
    pythonpath.extend(os.path.join(get_buildroot(), t.target_base) for t in targets)
    for dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))],
                        interpreter=interpreter,
                        context=python_repos.get_network_context(),
                        fetchers=python_repos.get_fetchers()):
      pythonpath.append(dist.location)

    process = subprocess.Popen([interpreter.binary,
                                '-c',
                                'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'],
                               env={'PYTHONPATH': os.pathsep.join(pythonpath)},
                               stderr=subprocess.PIPE)
    _, stderr = process.communicate()
    self.assertEqual(0, process.returncode, stderr)
Exemplo n.º 21
0
  def test_namespace_effective(self):
    self.create_file('src/thrift/com/foo/one.thrift', contents=dedent("""
    namespace py foo.bar

    struct One {}
    """))
    one = self.make_target(spec='src/thrift/com/foo:one',
                           target_type=PythonThriftLibrary,
                           sources=['one.thrift'])
    apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one)

    self.create_file('src/thrift2/com/foo/two.thrift', contents=dedent("""
    namespace py foo.baz

    struct Two {}
    """))
    two = self.make_target(spec='src/thrift2/com/foo:two',
                           target_type=PythonThriftLibrary,
                           sources=['two.thrift'])
    _, synthetic_target_two = self.generate_single_thrift_target(two)

    # Confirm separate PYTHONPATH entries, which we need to test namespace packages.
    self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base)

    targets = (synthetic_target_one, synthetic_target_two)

    python_repos = global_subsystem_instance(PythonRepos)
    python_setup = global_subsystem_instance(PythonSetup)
    interpreter_cache = PythonInterpreterCache(python_setup, python_repos)
    interpreter = interpreter_cache.select_interpreter_for_targets(targets)

    # We need setuptools to import namespace packages (via pkg_resources), so we prime the
    # PYTHONPATH with interpreter extras, which Pants always populates with setuptools and wheel.
    # TODO(John Sirois): We really should be emitting setuptools in a
    # `synthetic_target_extra_dependencies` override in `ApacheThriftPyGen`:
    #   https://github.com/pantsbuild/pants/issues/5975
    pythonpath = interpreter.extras.values()
    pythonpath.extend(os.path.join(get_buildroot(), t.target_base) for t in targets)
    for dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))],
                        interpreter=interpreter,
                        context=python_repos.get_network_context(),
                        fetchers=python_repos.get_fetchers()):
      pythonpath.append(dist.location)

    process = subprocess.Popen([interpreter.binary,
                                '-c',
                                'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'],
                               env={'PYTHONPATH': os.pathsep.join(pythonpath)},
                               stderr=subprocess.PIPE)
    _, stderr = process.communicate()
    self.assertEqual(0, process.returncode, stderr)
Exemplo n.º 22
0
    def interpreter_cache(self):
        if self._interpreter_cache is None:
            self._interpreter_cache = PythonInterpreterCache(
                self.context.config, logger=self.context.log.debug)

            # Cache setup's requirement fetching can hang if run concurrently by another pants proc.
            self.context.acquire_lock()
            try:
                # We pass in filters=compatibilities because setting up some python versions
                # (e.g., 3<=python<3.3) crashes, and this gives us an escape hatch.
                self._interpreter_cache.setup(filters=self._compatibilities)
            finally:
                self.context.release_lock()
        return self._interpreter_cache
Exemplo n.º 23
0
  def execute(self):
    interpreter = None
    python_tgts = self.context.targets(lambda tgt: isinstance(tgt, PythonTarget))
    fs = PythonInterpreterFingerprintStrategy(task=self)
    with self.invalidated(python_tgts, fingerprint_strategy=fs) as invalidation_check:
      # If there are no relevant targets, we still go through the motions of selecting
      # an interpreter, to prevent downstream tasks from having to check for this special case.
      if invalidation_check.all_vts:
        target_set_id = VersionedTargetSet.from_versioned_targets(
            invalidation_check.all_vts).cache_key.hash
      else:
        target_set_id = 'no_targets'
      interpreter_path_file = os.path.join(self.workdir, target_set_id, 'interpreter.path')
      if not os.path.exists(interpreter_path_file):
        interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
                                                   PythonRepos.global_instance(),
                                                   logger=self.context.log.debug)

        # We filter the interpreter cache itself (and not just the interpreters we pull from it)
        # because setting up some python versions (e.g., 3<=python<3.3) crashes, and this gives us
        # an escape hatch.
        filters = self.get_options().constraints or [b'']

        # Cache setup's requirement fetching can hang if run concurrently by another pants proc.
        self.context.acquire_lock()
        try:
          interpreter_cache.setup(filters=filters)
        finally:
          self.context.release_lock()

        interpreter = interpreter_cache.select_interpreter_for_targets(python_tgts)
        safe_mkdir_for(interpreter_path_file)
        with open(interpreter_path_file, 'w') as outfile:
          outfile.write(b'{}\t{}\n'.format(interpreter.binary, str(interpreter.identity)))
          for dist, location in interpreter.extras.items():
            dist_name, dist_version = dist
            outfile.write(b'{}\t{}\t{}\n'.format(dist_name, dist_version, location))

    if not interpreter:
      with open(interpreter_path_file, 'r') as infile:
        lines = infile.readlines()
        binary, identity = lines[0].strip().split('\t')
        extras = {}
        for line in lines[1:]:
          dist_name, dist_version, location = line.strip().split('\t')
          extras[(dist_name, dist_version)] = location

      interpreter = PythonInterpreter(binary, PythonIdentity.from_path(identity), extras)

    self.context.products.get_data(PythonInterpreter, lambda: interpreter)
Exemplo n.º 24
0
  def _setup_test(self, constraints=None):
    mock_setup = mock.MagicMock().return_value
    type(mock_setup).interpreter_constraints = mock.PropertyMock(return_value=constraints)

    with temporary_dir() as path:
      mock_setup.interpreter_cache_dir = path
      cache = PythonInterpreterCache(mock_setup, mock.MagicMock())

      def set_interpreters(_):
        cache._interpreters.add(self._interpreter)

      cache._setup_cached = mock.Mock(side_effect=set_interpreters)
      cache._setup_paths = mock.Mock()
      yield cache, path
Exemplo n.º 25
0
  def __init__(self, *args, **kwargs):
    super(PythonTask, self).__init__(*args, **kwargs)
    self.conn_timeout = (self.context.options.python_conn_timeout or
                         self.context.config.getdefault('connection_timeout'))
    compatibilities = self.context.options.interpreter or [b'']

    self.interpreter_cache = PythonInterpreterCache(self.context.config,
                                                    logger=self.context.log.debug)
    # We pass in filters=compatibilities because setting up some python versions
    # (e.g., 3<=python<3.3) crashes, and this gives us an escape hatch.
    self.interpreter_cache.setup(filters=compatibilities)

    # Select a default interpreter to use.
    self._interpreter = self.select_interpreter(compatibilities)
Exemplo n.º 26
0
  def _setup_test(self, constraints=None, mock_setup_paths_interpreters=None):
    mock_setup = mock.MagicMock().return_value
    type(mock_setup).interpreter_constraints = mock.PropertyMock(return_value=constraints)

    with temporary_dir() as path:
      mock_setup.interpreter_cache_dir = path
      cache = PythonInterpreterCache(mock_setup, mock.MagicMock())
      cache._setup_cached = mock.Mock(return_value=[self._interpreter])
      if mock_setup_paths_interpreters:
        cache._setup_paths = mock.Mock(return_value=[PythonInterpreter.from_binary(mock_setup_paths_interpreters[0]),
                                                     PythonInterpreter.from_binary(mock_setup_paths_interpreters[1])])
      else:
        cache._setup_paths = mock.Mock(return_value=[])
      yield cache, path
Exemplo n.º 27
0
    def _setup_test(self, constraints=None):
        mock_setup = mock.MagicMock().return_value
        type(mock_setup).interpreter_constraints = mock.PropertyMock(
            return_value=constraints)

        with temporary_dir() as path:
            mock_setup.interpreter_cache_dir = path
            cache = PythonInterpreterCache(mock_setup, mock.MagicMock())

            def set_interpreters(_):
                cache._interpreters.add(self._interpreter)

            cache._setup_cached = mock.Mock(side_effect=set_interpreters)
            cache._setup_paths = mock.Mock()
            yield cache, path
Exemplo n.º 28
0
 def execute(self):
   """"Run Checkstyle on all found non-synthetic source files."""
   python_tgts = self.context.targets(
     lambda tgt: isinstance(tgt, (PythonTarget))
   )
   if not python_tgts:
     return 0
   interpreter_cache = PythonInterpreterCache.global_instance()
   with self.invalidated(self.get_targets(self._is_checked)) as invalidation_check:
     failure_count = 0
     tgts_by_compatibility, _ = interpreter_cache.partition_targets_by_compatibility(
       [vt.target for vt in invalidation_check.invalid_vts]
     )
     for filters, targets in tgts_by_compatibility.items():
       sources = self.calculate_sources([tgt for tgt in targets])
       if sources:
         allowed_interpreters = set(interpreter_cache.setup(filters=filters))
         if not allowed_interpreters:
           raise TaskError('No valid interpreters found for targets: {}\n(filters: {})'
                           .format(targets, filters))
         interpreter = min(allowed_interpreters)
         failure_count += self.checkstyle(interpreter, sources)
     if failure_count > 0 and self.get_options().fail:
       raise TaskError('{} Python Style issues found. You may try `./pants fmt <targets>`'
                       .format(failure_count))
     return failure_count
Exemplo n.º 29
0
  def execute(self):
    python_tgts_and_reqs = self.context.targets(
      lambda tgt: isinstance(tgt, (PythonTarget, PythonRequirementLibrary))
    )
    if not python_tgts_and_reqs:
      return
    python_tgts = [tgt for tgt in python_tgts_and_reqs if isinstance(tgt, PythonTarget)]
    fs = PythonInterpreterFingerprintStrategy()
    with self.invalidated(python_tgts, fingerprint_strategy=fs) as invalidation_check:
      if (PythonSetup.global_instance().interpreter_search_paths
          and PythonInterpreterCache.pex_python_paths()):
        self.context.log.warn("Detected both PEX_PYTHON_PATH and "
                              "--python-setup-interpreter-search-paths. Ignoring "
                              "--python-setup-interpreter-search-paths.")
      # If there are no relevant targets, we still go through the motions of selecting
      # an interpreter, to prevent downstream tasks from having to check for this special case.
      if invalidation_check.all_vts:
        target_set_id = VersionedTargetSet.from_versioned_targets(
            invalidation_check.all_vts).cache_key.hash
      else:
        target_set_id = 'no_targets'
      interpreter_path_file = self._interpreter_path_file(target_set_id)
      if not os.path.exists(interpreter_path_file):
        self._create_interpreter_path_file(interpreter_path_file, python_tgts)

    interpreter = self._get_interpreter(interpreter_path_file)
    self.context.products.register_data(PythonInterpreter, interpreter)
Exemplo n.º 30
0
    def _gather_sources(self, target_roots):
        with temporary_dir() as cache_dir:
            interpreter = PythonInterpreter.get()
            context = self.context(
                target_roots=target_roots,
                for_subsystems=[PythonInterpreterCache],
                options={
                    PythonSetup.options_scope: {
                        'interpreter_cache_dir':
                        cache_dir,
                        'interpreter_search_paths':
                        [os.path.dirname(interpreter.binary)],
                    }
                })

            # We must get an interpreter via the cache, instead of using the value of
            # PythonInterpreter.get() directly, to ensure that the interpreter has setuptools and
            # wheel support.
            interpreter_cache = PythonInterpreterCache.global_instance()
            interpreters = interpreter_cache.setup(
                filters=[str(interpreter.identity.requirement)])
            context.products.get_data(PythonInterpreter,
                                      lambda: interpreters[0])

            task = self.create_task(context)
            task.execute()

            return context.products.get_data(GatherSources.PYTHON_SOURCES)
Exemplo n.º 31
0
 def _create_interpreter_cache(self, setup_options=None):
     Subsystem.reset(reset_options=True)
     self.context(for_subsystems=[PythonInterpreterCache],
                  options={
                      'python-setup': setup_options,
                  })
     return PythonInterpreterCache.global_instance()
Exemplo n.º 32
0
    def execute(self):
        python_tgts = self.context.targets(
            lambda tgt: isinstance(tgt, PythonTarget))
        fs = PythonInterpreterFingerprintStrategy()
        with self.invalidated(python_tgts,
                              fingerprint_strategy=fs) as invalidation_check:
            if (PythonSetup.global_instance().interpreter_search_paths
                    and PythonInterpreterCache.pex_python_paths()):
                self.context.log.warn(
                    "Detected both PEX_PYTHON_PATH and "
                    "--python-setup-interpreter-search-paths. Ignoring "
                    "--python-setup-interpreter-search-paths.")
            # If there are no relevant targets, we still go through the motions of selecting
            # an interpreter, to prevent downstream tasks from having to check for this special case.
            if invalidation_check.all_vts:
                target_set_id = VersionedTargetSet.from_versioned_targets(
                    invalidation_check.all_vts).cache_key.hash
            else:
                target_set_id = 'no_targets'
            interpreter_path_file = self._interpreter_path_file(target_set_id)
            if not os.path.exists(interpreter_path_file):
                self._create_interpreter_path_file(interpreter_path_file,
                                                   python_tgts)

        interpreter = self._get_interpreter(interpreter_path_file)
        self.context.products.register_data(PythonInterpreter, interpreter)
Exemplo n.º 33
0
    def execute(self):
        if not self.will_be_invoked():
            return

        tool_subsystem = self.tool_subsystem_cls.scoped_instance(self)

        interpreter_cache = PythonInterpreterCache.global_instance()
        interpreters = interpreter_cache.setup(
            filters=tool_subsystem.get_interpreter_constraints())
        if not interpreters:
            raise TaskError(
                "Found no Python interpreter capable of running the {} tool with "
                "constraints {}".format(
                    tool_subsystem.options_scope,
                    tool_subsystem.get_interpreter_constraints()))
        interpreter = min(interpreters)

        pex_path = self._generate_fingerprinted_pex_path(
            tool_subsystem, interpreter)
        if not os.path.exists(pex_path):
            with self.context.new_workunit(
                    name=f"create-{tool_subsystem.options_scope}-pex",
                    labels=[WorkUnitLabel.PREP]):
                self._build_tool_pex(tool_subsystem=tool_subsystem,
                                     interpreter=interpreter,
                                     pex_path=pex_path)

        tool_instance = self.tool_instance_cls(pex_path, interpreter)
        self.context.products.register_data(self.tool_instance_cls,
                                            tool_instance)
Exemplo n.º 34
0
    def _resolve_requirements(self, target_roots, options=None):
        with temporary_dir() as cache_dir:
            options = options or {}
            python_setup_opts = options.setdefault(PythonSetup.options_scope,
                                                   {})
            python_setup_opts["interpreter_cache_dir"] = cache_dir
            interpreter = PythonInterpreter.get()
            python_setup_opts["interpreter_search_paths"] = [
                os.path.dirname(interpreter.binary)
            ]
            context = self.context(target_roots=target_roots,
                                   options=options,
                                   for_subsystems=[PythonInterpreterCache])

            # We must get an interpreter via the cache, instead of using the value of
            # PythonInterpreter.get() directly, to ensure that the interpreter has setuptools and
            # wheel support.
            interpreter_cache = PythonInterpreterCache.global_instance()
            interpreters = interpreter_cache.setup(
                filters=[str(interpreter.identity.requirement)])
            context.products.get_data(PythonInterpreter,
                                      lambda: interpreters[0])

            task = self.create_task(context)
            task.execute()

            return context.products.get_data(
                ResolveRequirements.REQUIREMENTS_PEX)
Exemplo n.º 35
0
  def _setup_test(self, interpreter_requirement=None):
    mock_setup = mock.MagicMock().return_value

    # Explicitly set a repo-wide requirement that excludes our one interpreter.
    type(mock_setup).interpreter_requirement = mock.PropertyMock(
      return_value=interpreter_requirement)

    with temporary_dir() as path:
      mock_setup.interpreter_cache_dir = path
      cache = PythonInterpreterCache(mock_setup, mock.MagicMock())

      def set_interpreters(_):
        cache._interpreters.add(self._interpreter)

      cache._setup_cached = mock.Mock(side_effect=set_interpreters)
      cache._setup_paths = mock.Mock()
      yield cache, path
Exemplo n.º 36
0
  def _setup_test(self, interpreter_requirement=None):
    mock_setup = mock.MagicMock().return_value

    # Explicitly set a repo-wide requirement that excludes our one interpreter.
    type(mock_setup).interpreter_requirement = mock.PropertyMock(
      return_value=interpreter_requirement)

    with temporary_dir() as path:
      mock_setup.interpreter_cache_dir = path
      cache = PythonInterpreterCache(mock_setup, mock.MagicMock())

      def set_interpreters(_):
        cache._interpreters.add(self._interpreter)

      cache._setup_cached = mock.Mock(side_effect=set_interpreters)
      cache._setup_paths = mock.Mock()
      yield cache, path
Exemplo n.º 37
0
  def test_cache_setup_with_no_filters_uses_repo_default_excluded(self, MockSetup):
    interpreter = PythonInterpreter.get()

    mock_setup = MockSetup.return_value
    type(mock_setup).interpreter_requirement = mock.PropertyMock(return_value=None)

    with temporary_dir() as path:
      mock_setup.scratch_dir.return_value = path

      cache = PythonInterpreterCache(mock.MagicMock())

      def set_interpreters(_):
        cache._interpreters.add(interpreter)

      cache._setup_cached = mock.Mock(side_effect=set_interpreters)

      self.assertEqual(cache.setup(), [interpreter])
Exemplo n.º 38
0
  def _gather_sources(self, target_roots):
    context = self.context(target_roots=target_roots, for_subsystems=[PythonSetup, PythonRepos])

    # We must get an interpreter via the cache, instead of using PythonInterpreter.get() directly,
    # to ensure that the interpreter has setuptools and wheel support.
    interpreter = PythonInterpreter.get()
    interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
                                               PythonRepos.global_instance(),
                                               logger=context.log.debug)
    interpreters = interpreter_cache.setup(paths=[os.path.dirname(interpreter.binary)],
                                           filters=[str(interpreter.identity.requirement)])
    context.products.get_data(PythonInterpreter, lambda: interpreters[0])

    task = self.create_task(context)
    task.execute()

    return context.products.get_data(GatherSources.PYTHON_SOURCES)
Exemplo n.º 39
0
 def _setup_cache_at(self, path, constraints=None):
     setup_options = {'interpreter_cache_dir': path}
     if constraints:
         setup_options.update(interpreter_constraints=constraints)
     python_setup, python_repos = self.create_python_subsystems(
         setup_options=setup_options)
     return PythonInterpreterCache(python_setup=python_setup,
                                   python_repos=python_repos)
Exemplo n.º 40
0
  def _gather_sources(self, target_roots):
    context = self.context(target_roots=target_roots, for_subsystems=[PythonSetup, PythonRepos])

    # We must get an interpreter via the cache, instead of using PythonInterpreter.get() directly,
    # to ensure that the interpreter has setuptools and wheel support.
    interpreter = PythonInterpreter.get()
    interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
                                               PythonRepos.global_instance(),
                                               logger=context.log.debug)
    interpreters = interpreter_cache.setup(paths=[os.path.dirname(interpreter.binary)],
                                           filters=[str(interpreter.identity.requirement)])
    context.products.get_data(PythonInterpreter, lambda: interpreters[0])

    task = self.create_task(context)
    task.execute()

    return context.products.get_data(GatherSources.PYTHON_SOURCES)
Exemplo n.º 41
0
 def execute(self):
     """"Run Checkstyle on all found non-synthetic source files."""
     python_tgts = self.context.targets(
         lambda tgt: isinstance(tgt, (PythonTarget)))
     if not python_tgts:
         return 0
     interpreter_cache = PythonInterpreterCache(
         PythonSetup.global_instance(),
         PythonRepos.global_instance(),
         logger=self.context.log.debug)
     with self.invalidated(self.get_targets(
             self._is_checked)) as invalidation_check:
         failure_count = 0
         tgts_by_compatibility, _ = interpreter_cache.partition_targets_by_compatibility(
             [vt.target for vt in invalidation_check.invalid_vts])
         for filters, targets in tgts_by_compatibility.items():
             if self.get_options(
             ).interpreter_constraints_whitelist is None and not self._constraints_are_whitelisted(
                     filters):
                 deprecated_conditional(
                     lambda: self.get_options(
                     ).interpreter_constraints_whitelist is None,
                     '1.14.0.dev2',
                     "Python linting is currently restricted to targets that match the global "
                     "interpreter constraints: {}. Pants detected unacceptable filters: {}. "
                     "Use the `--interpreter-constraints-whitelist` lint option to whitelist "
                     "compatibiltiy constraints.".format(
                         PythonSetup.global_instance().
                         interpreter_constraints, filters))
             else:
                 sources = self.calculate_sources([tgt for tgt in targets])
                 if sources:
                     allowed_interpreters = set(
                         interpreter_cache.setup(filters=filters))
                     if not allowed_interpreters:
                         raise TaskError(
                             'No valid interpreters found for targets: {}\n(filters: {})'
                             .format(targets, filters))
                     interpreter = min(allowed_interpreters)
                     failure_count += self.checkstyle(interpreter, sources)
         if failure_count > 0 and self.get_options().fail:
             raise TaskError(
                 '{} Python Style issues found. You may try `./pants fmt <targets>`'
                 .format(failure_count))
         return failure_count
  def test_namespace_effective(self):
    self.create_file('src/thrift/com/foo/one.thrift', contents=dedent("""
    namespace py foo.bar

    struct One {}
    """))
    one = self.make_target(spec='src/thrift/com/foo:one',
                           target_type=PythonThriftLibrary,
                           sources=['one.thrift'])
    apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one)

    self.create_file('src/thrift2/com/foo/two.thrift', contents=dedent("""
    namespace py foo.baz

    struct Two {}
    """))
    two = self.make_target(spec='src/thrift2/com/foo:two',
                           target_type=PythonThriftLibrary,
                           sources=['two.thrift'])
    _, synthetic_target_two = self.generate_single_thrift_target(two)

    # Confirm separate PYTHONPATH entries, which we need to test namespace packages.
    self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base)

    targets = (synthetic_target_one, synthetic_target_two)

    python_repos = global_subsystem_instance(PythonRepos)
    python_setup = global_subsystem_instance(PythonSetup)
    interpreter_cache = PythonInterpreterCache(python_setup, python_repos)
    interpreter = interpreter_cache.select_interpreter_for_targets(targets)

    pythonpath = [os.path.join(get_buildroot(), t.target_base) for t in targets]
    for dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))],
                        interpreter=interpreter,
                        context=python_repos.get_network_context(),
                        fetchers=python_repos.get_fetchers()):
      pythonpath.append(dist.location)

    process = subprocess.Popen([interpreter.binary,
                                '-c',
                                'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'],
                               env={'PYTHONPATH': os.pathsep.join(pythonpath)},
                               stderr=subprocess.PIPE)
    _, stderr = process.communicate()
    self.assertEqual(0, process.returncode, stderr)
Exemplo n.º 43
0
    def _resolve_requirements(self, target_roots, options=None):
        context = self.context(target_roots=target_roots, options=options)

        # We must get an interpreter via the cache, instead of using PythonInterpreter.get() directly,
        # to ensure that the interpreter has setuptools and wheel support.
        interpreter = PythonInterpreter.get()
        interpreter_cache = PythonInterpreterCache(
            PythonSetup.global_instance(), PythonRepos.global_instance(), logger=context.log.debug
        )
        interpreters = interpreter_cache.setup(
            paths=[os.path.dirname(interpreter.binary)], filters=[str(interpreter.identity.requirement)]
        )
        context.products.get_data(PythonInterpreter, lambda: interpreters[0])

        task = self.create_task(context)
        task.execute()

        return context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX)
Exemplo n.º 44
0
  def _do_test(self, interpreter_requirement, filters, expected):
    mock_setup = mock.MagicMock().return_value

    # Explicitly set a repo-wide requirement that excludes our one interpreter.
    type(mock_setup).interpreter_requirement = mock.PropertyMock(
      return_value=interpreter_requirement)

    with temporary_dir() as path:
      mock_setup.scratch_dir = path
      cache = PythonInterpreterCache(mock_setup, mock.MagicMock())

      def set_interpreters(_):
        cache._interpreters.add(self._interpreter)

      cache._setup_cached = mock.Mock(side_effect=set_interpreters)
      cache._setup_paths = mock.Mock()

      self.assertEqual(cache.setup(filters=filters), expected)
Exemplo n.º 45
0
  def test_cache_setup_with_filter_overrides_repo_default(self, MockSetup):
    interpreter = PythonInterpreter.get()

    mock_setup = MockSetup.return_value
    # Explicitly set a repo-wide requirement that excludes our one interpreter
    type(mock_setup).interpreter_requirement = mock.PropertyMock(
        return_value=self._make_bad_requirement(interpreter.identity.requirement))

    with temporary_dir() as path:
      mock_setup.scratch_dir.return_value = path

      cache = PythonInterpreterCache(mock.MagicMock())

      def set_interpreters(_):
        cache._interpreters.add(interpreter)

      cache._setup_cached = mock.Mock(side_effect=set_interpreters)

      self.assertEqual(cache.setup(filters=(str(interpreter.identity.requirement),)), [interpreter])
Exemplo n.º 46
0
    def test_cache_setup_with_no_filters_uses_repo_default_excluded(
            self, MockSetup):
        interpreter = PythonInterpreter.get()

        mock_setup = MockSetup.return_value
        type(mock_setup).interpreter_requirement = mock.PropertyMock(
            return_value=None)

        with temporary_dir() as path:
            mock_setup.scratch_dir.return_value = path

            cache = PythonInterpreterCache(mock.MagicMock())

            def set_interpreters(_):
                cache._interpreters.add(interpreter)

            cache._setup_cached = mock.Mock(side_effect=set_interpreters)

            self.assertEqual(cache.setup(), [interpreter])
Exemplo n.º 47
0
  def __init__(self, *args, **kwargs):
    super(Build, self).__init__(*args, **kwargs)

    if not self.args:
      self.error("A spec argument is required")

    self.config = Config.load()

    interpreters = self.options.interpreters or [b'']
    self.interpreter_cache = PythonInterpreterCache(self.config, logger=self.debug)
    self.interpreter_cache.setup(filters=interpreters)
    interpreters = self.interpreter_cache.select_interpreter(
        list(self.interpreter_cache.matches(interpreters)))
    if len(interpreters) != 1:
      self.error('Unable to detect suitable interpreter.')
    else:
      self.debug('Selected %s' % interpreters[0])
    self.interpreter = interpreters[0]

    try:
      specs_end = self.args.index('--')
      if len(self.args) > specs_end:
        self.build_args = self.args[specs_end+1:len(self.args)+1]
      else:
        self.build_args = []
    except ValueError:
      specs_end = 1
      self.build_args = self.args[1:] if len(self.args) > 1 else []

    self.targets = OrderedSet()
    spec_parser = SpecParser(self.root_dir, self.build_file_parser)
    self.top_level_addresses = set()

    for spec in self.args[0:specs_end]:
      try:
        addresses = spec_parser.parse_addresses(spec)
      except:
        self.error("Problem parsing spec %s: %s" % (spec, traceback.format_exc()))

      for address in addresses:
        self.top_level_addresses.add(address)
        try:
          self.build_file_parser.inject_address_closure_into_build_graph(address, self.build_graph)
          target = self.build_graph.get_target(address)
        except:
          self.error("Problem parsing BUILD target %s: %s" % (address, traceback.format_exc()))

        if not target:
          self.error("Target %s does not exist" % address)

        transitive_targets = self.build_graph.transitive_subgraph_of_addresses([target.address])
        for transitive_target in transitive_targets:
          self.targets.add(transitive_target)

    self.targets = [target for target in self.targets if target.is_python]
Exemplo n.º 48
0
    def _setup_test(self,
                    constraints=None,
                    mock_setup_paths_interpreters=None):
        mock_setup = mock.MagicMock().return_value
        type(mock_setup).interpreter_constraints = mock.PropertyMock(
            return_value=constraints)

        with temporary_dir() as path:
            mock_setup.interpreter_cache_dir = path
            cache = PythonInterpreterCache(mock_setup, mock.MagicMock())
            cache._setup_cached = mock.Mock(return_value=[self._interpreter])
            if mock_setup_paths_interpreters:
                cache._setup_paths = mock.Mock(return_value=[
                    PythonInterpreter.from_binary(
                        mock_setup_paths_interpreters[0]),
                    PythonInterpreter.from_binary(
                        mock_setup_paths_interpreters[1])
                ])
            else:
                cache._setup_paths = mock.Mock(return_value=[])
            yield cache, path
Exemplo n.º 49
0
  def test_cache_setup_with_no_filters_uses_repo_default_excluded(self, MockSetup):
    # This is the interpreter we'll inject into the cache
    interpreter = PythonInterpreter.get()

    mock_setup = MockSetup.return_value
    # Explicitly set a repo-wide requirement that excludes our one interpreter
    type(mock_setup).interpreter_requirement = mock.PropertyMock(
        return_value=self._make_bad_requirement(interpreter.identity.requirement))

    with temporary_dir() as path:
      mock_setup.scratch_dir.return_value = path

      cache = PythonInterpreterCache(mock.MagicMock())

      def set_interpreters(_):
        cache._interpreters.add(interpreter)

      cache._setup_cached = mock.Mock(side_effect=set_interpreters)
      cache._setup_paths = mock.Mock()

      self.assertEqual(len(cache.setup()), 0)
Exemplo n.º 50
0
class PythonTask(Task):
    @classmethod
    def setup_parser(cls, option_group, args, mkflag):
        option_group.add_option(
            mkflag('timeout'),
            dest='python_conn_timeout',
            type='int',
            default=0,
            help='Number of seconds to wait for http connections.')

    def __init__(self, context, workdir):
        super(PythonTask, self).__init__(context, workdir)
        self.conn_timeout = (
            self.context.options.python_conn_timeout
            or self.context.config.getdefault('connection_timeout'))
        compatibilities = self.context.options.interpreter or [b'']

        self.interpreter_cache = PythonInterpreterCache(
            self.context.config, logger=self.context.log.debug)
        # We pass in filters=compatibilities because setting up some python versions
        # (e.g., 3<=python<3.3) crashes, and this gives us an escape hatch.
        self.interpreter_cache.setup(filters=compatibilities)

        # Select a default interpreter to use.
        self._interpreter = self.select_interpreter(compatibilities)

    @property
    def interpreter(self):
        """Subclasses can use this if they're fine with the default interpreter (the usual case)."""
        return self._interpreter

    def select_interpreter(self, compatibilities):
        """Subclasses can use this to be more specific about interpreter selection."""
        interpreters = self.interpreter_cache.select_interpreter(
            list(self.interpreter_cache.matches(compatibilities)))
        if len(interpreters) != 1:
            raise TaskError('Unable to detect suitable interpreter.')
        interpreter = interpreters[0]
        self.context.log.debug('Selected %s' % interpreter)
        return interpreter
Exemplo n.º 51
0
    def test_cache_setup_with_filter_overrides_repo_default(self, MockSetup):
        interpreter = PythonInterpreter.get()

        mock_setup = MockSetup.return_value
        # Explicitly set a repo-wide requirement that excludes our one interpreter
        type(mock_setup).interpreter_requirement = mock.PropertyMock(
            return_value=self._make_bad_requirement(
                interpreter.identity.requirement))

        with temporary_dir() as path:
            mock_setup.scratch_dir.return_value = path

            cache = PythonInterpreterCache(mock.MagicMock())

            def set_interpreters(_):
                cache._interpreters.add(interpreter)

            cache._setup_cached = mock.Mock(side_effect=set_interpreters)

            self.assertEqual(
                cache.setup(filters=(str(interpreter.identity.requirement), )),
                [interpreter])
Exemplo n.º 52
0
  def __init__(self, context, workdir):
    super(PythonTask, self).__init__(context, workdir)
    self.conn_timeout = (self.context.options.python_conn_timeout or
                         self.context.config.getdefault('connection_timeout'))
    compatibilities = self.context.options.interpreter or [b'']

    self.interpreter_cache = PythonInterpreterCache(self.context.config,
                                                    logger=self.context.log.debug)
    # We pass in filters=compatibilities because setting up some python versions
    # (e.g., 3<=python<3.3) crashes, and this gives us an escape hatch.
    self.interpreter_cache.setup(filters=compatibilities)

    # Select a default interpreter to use.
    self._interpreter = self.select_interpreter(compatibilities)
Exemplo n.º 53
0
  def interpreter_cache(self):
    if self._interpreter_cache is None:
      self._interpreter_cache = PythonInterpreterCache(self.context.config,
                                                       logger=self.context.log.debug)

      # Cache setup's requirement fetching can hang if run concurrently by another pants proc.
      self.context.acquire_lock()
      try:
        # We pass in filters=compatibilities because setting up some python versions
        # (e.g., 3<=python<3.3) crashes, and this gives us an escape hatch.
        self._interpreter_cache.setup(filters=self._compatibilities)
      finally:
        self.context.release_lock()
    return self._interpreter_cache
  def _resolve_requirements(self, target_roots, options=None):
    with temporary_dir() as cache_dir:
      options = options or {}
      python_setup_opts = options.setdefault(PythonSetup.options_scope, {})
      python_setup_opts['interpreter_cache_dir'] = cache_dir
      interpreter = PythonInterpreter.get()
      python_setup_opts['interpreter_search_paths'] = [os.path.dirname(interpreter.binary)]
      context = self.context(target_roots=target_roots, options=options,
                             for_subsystems=[PythonInterpreterCache])

      # We must get an interpreter via the cache, instead of using the value of
      # PythonInterpreter.get() directly, to ensure that the interpreter has setuptools and
      # wheel support.
      interpreter_cache = PythonInterpreterCache.global_instance()
      interpreters = interpreter_cache.setup(filters=[str(interpreter.identity.requirement)])
      context.products.get_data(PythonInterpreter, lambda: interpreters[0])

      task = self.create_task(context)
      task.execute()

      return context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX)
Exemplo n.º 55
0
  def _gather_sources(self, target_roots):
    with temporary_dir() as cache_dir:
      interpreter = PythonInterpreter.get()
      context = self.context(target_roots=target_roots,
                             for_subsystems=[PythonInterpreterCache],
                             options={
                               PythonSetup.options_scope: {
                                 'interpreter_cache_dir': cache_dir,
                                 'interpreter_search_paths': [os.path.dirname(interpreter.binary)],
                               }})

      # We must get an interpreter via the cache, instead of using the value of
      # PythonInterpreter.get() directly, to ensure that the interpreter has setuptools and
      # wheel support.
      interpreter_cache = PythonInterpreterCache.global_instance()
      interpreters = interpreter_cache.setup(filters=[str(interpreter.identity.requirement)])
      context.products.get_data(PythonInterpreter, lambda: interpreters[0])

      task = self.create_task(context)
      task.execute()

      return context.products.get_data(GatherSources.PYTHON_SOURCES)
Exemplo n.º 56
0
  def execute(self):
    if not self.will_be_invoked():
      return

    tool_subsystem = self.tool_subsystem_cls.scoped_instance(self)

    interpreter_cache = PythonInterpreterCache.global_instance()
    interpreters = interpreter_cache.setup(filters=tool_subsystem.get_interpreter_constraints())
    if not interpreters:
      raise TaskError('Found no Python interpreter capable of running the {} tool with '
                      'constraints {}'.format(tool_subsystem.options_scope,
                                              tool_subsystem.get_interpreter_constraints()))
    interpreter = min(interpreters)

    pex_path = self._generate_fingerprinted_pex_path(tool_subsystem, interpreter)
    if not os.path.exists(pex_path):
      with self.context.new_workunit(name='create-{}-pex'.format(tool_subsystem.options_scope),
                                     labels=[WorkUnitLabel.PREP]):
        self._build_tool_pex(tool_subsystem=tool_subsystem,
                             interpreter=interpreter,
                             pex_path=pex_path)

    tool_instance = self.tool_instance_cls(pex_path, interpreter)
    self.context.products.register_data(self.tool_instance_cls, tool_instance)
Exemplo n.º 57
0
class Build(Command):
  """Builds a specified target."""

  __command__ = 'build'

  def setup_parser(self, parser, args):
    parser.set_usage("\n"
                     "  %prog build (options) [spec] (build args)\n"
                     "  %prog build (options) [spec]... -- (build args)")
    parser.add_option("-t", "--timeout", dest="conn_timeout", type="int",
                      default=Config.load().getdefault('connection_timeout'),
                      help="Number of seconds to wait for http connections.")
    parser.add_option('-i', '--interpreter', dest='interpreters', default=[], action='append',
                      help="Constrain what Python interpreters to use.  Uses Requirement "
                           "format from pkg_resources, e.g. 'CPython>=2.6,<3' or 'PyPy'. "
                           "By default, no constraints are used.  Multiple constraints may "
                           "be added.  They will be ORed together.")
    parser.add_option('-v', '--verbose', dest='verbose', default=False, action='store_true',
                      help='Show verbose output.')
    parser.add_option('-f', '--fast', dest='fast', default=False, action='store_true',
                      help='Run tests in a single chroot.')
    parser.disable_interspersed_args()
    parser.epilog = ('Builds the specified Python target(s). Use ./pants goal for JVM and other '
                     'targets.')

  def __init__(self, *args, **kwargs):
    super(Build, self).__init__(*args, **kwargs)

    if not self.args:
      self.error("A spec argument is required")

    self.config = Config.load()

    interpreters = self.options.interpreters or [b'']
    self.interpreter_cache = PythonInterpreterCache(self.config, logger=self.debug)
    self.interpreter_cache.setup(filters=interpreters)
    interpreters = self.interpreter_cache.select_interpreter(
        list(self.interpreter_cache.matches(interpreters)))
    if len(interpreters) != 1:
      self.error('Unable to detect suitable interpreter.')
    else:
      self.debug('Selected %s' % interpreters[0])
    self.interpreter = interpreters[0]

    try:
      specs_end = self.args.index('--')
      if len(self.args) > specs_end:
        self.build_args = self.args[specs_end+1:len(self.args)+1]
      else:
        self.build_args = []
    except ValueError:
      specs_end = 1
      self.build_args = self.args[1:] if len(self.args) > 1 else []

    self.targets = OrderedSet()
    spec_parser = CmdLineSpecParser(self.root_dir, self.build_file_parser)
    self.top_level_addresses = set()

    specs = self.args[0:specs_end]
    addresses = spec_parser.parse_addresses(specs)

    for address in addresses:
      self.top_level_addresses.add(address)
      try:
        self.build_file_parser.inject_address_closure_into_build_graph(address, self.build_graph)
        target = self.build_graph.get_target(address)
      except:
        self.error("Problem parsing BUILD target %s: %s" % (address, traceback.format_exc()))

      if not target:
        self.error("Target %s does not exist" % address)

      transitive_targets = self.build_graph.transitive_subgraph_of_addresses([target.address])
      for transitive_target in transitive_targets:
        self.targets.add(transitive_target)

    self.targets = [target for target in self.targets if target.is_python]

  def debug(self, message):
    if self.options.verbose:
      print(message, file=sys.stderr)

  def execute(self):
    print("Build operating on top level addresses: %s" % self.top_level_addresses)

    python_targets = OrderedSet()
    for target in self.targets:
      if target.is_python:
        python_targets.add(target)
      else:
        self.error("Cannot build target %s" % target)

    if python_targets:
      status = self._python_build(python_targets)
    else:
      status = -1

    return status

  def _python_build(self, targets):
    try:
      executor = PythonBuilder(self.run_tracker)
      return executor.build(
        targets,
        self.build_args,
        interpreter=self.interpreter,
        conn_timeout=self.options.conn_timeout,
        fast_tests=self.options.fast)
    except:
      self.error("Problem executing PythonBuilder for targets %s: %s" % (targets,
                                                                         traceback.format_exc()))
Exemplo n.º 58
0
 def _interpreter_cache(self):
   return PythonInterpreterCache.global_instance()