Example #1
0
  def test_locate(self):
    with pytest.raises(ParseContext.ContextError):
      ParseContext.locate()

    with temporary_dir() as root_dir:
      a_context = ParseContext(create_buildfile(root_dir, 'a'))
      b_context = ParseContext(create_buildfile(root_dir, 'b'))

      def test_in_a():
        self.assertEquals(a_context, ParseContext.locate())
        return b_context.do_in_context(lambda: ParseContext.locate())

      self.assertEquals(b_context, a_context.do_in_context(test_in_a))
Example #2
0
def PythonEgg(glob, name=None):
  """Refers to pre-built Python eggs in the file system. (To instead fetch
  eggs in a ``pip``/``easy_install`` way, use ``python_requirement``)

  E.g., ``egg(name='foo', glob='foo-0.1-py2.6.egg')`` would pick up the
  file ``foo-0.1-py2.6.egg`` from the ``BUILD`` file's directory; targets
  could depend on it by name ``foo``.

  :param string glob: File glob pattern.
  :param string name: Target name; by default uses the egg's project name.
  """
  # TODO(John Sirois): Rationalize with globs handling in ParseContext
  eggs = fsglob(ParseContext.path(glob))

  requirements = set()
  for egg in eggs:
    if os.path.isdir(egg):
      metadata = PathMetadata(egg, os.path.join(egg, 'EGG-INFO'))
    else:
      metadata = EggMetadata(zipimporter(egg))
    dist = Distribution.from_filename(egg, metadata=metadata)
    requirements.add(dist.as_requirement())

  if len(requirements) > 1:
    raise ValueError('Got multiple egg versions! => %s' % requirements)

  return PythonRequirement(str(requirements.pop()), name=name)
Example #3
0
    def test_binary_target_injected_into_minified_dependencies_with_provider(
            self):
        with ParseContext.temp():
            bar = python_library(name='bar',
                                 provides=setup_py(
                                     name='bar',
                                     version='0.0.0',
                                 ).with_binaries(bar_binary=pants(':bar_bin')))

            bar_bin = python_binary(name='bar_bin',
                                    entry_point='bar.bin.bar',
                                    dependencies=[pants(':bar_bin_dep')])

            bar_bin_dep = python_library(name='bar_bin_dep',
                                         provides=setup_py(
                                             name='bar_bin_dep',
                                             version='0.0.0',
                                         ))

        assert SetupPy.minified_dependencies(bar) == OrderedSet(
            [bar_bin, bar_bin_dep])
        entry_points = dict(SetupPy.iter_entry_points(bar))
        assert entry_points == {'bar_binary': 'bar.bin.bar'}

        with self.run_execute(bar, recursive=False) as setup_py_command:
            setup_py_command.run_one.assert_called_with(bar)

        with self.run_execute(bar, recursive=True) as setup_py_command:
            setup_py_command.run_one.assert_has_calls(
                [call(bar), call(bar_bin_dep)], any_order=True)
 def test_python_binary_with_source_no_entry_point(self):
     with ParseContext.temp('src'):
         assert PythonBinary(name='binary1',
                             source='blork.py').entry_point == 'blork'
         assert PythonBinary(
             name='binary2',
             source='bin/blork.py').entry_point == 'bin.blork'
Example #5
0
 def test_validation(self):
     with ParseContext.temp():
         repo = Repository(name="myRepo", url="myUrl", push_db="myPushDb")
         Artifact(org="testOrg",
                  name="testName",
                  repo=repo,
                  description="Test")
         self.assertRaises(ValueError,
                           Artifact,
                           org=1,
                           name="testName",
                           repo=repo,
                           description="Test")
         self.assertRaises(ValueError,
                           Artifact,
                           org="testOrg",
                           name=1,
                           repo=repo,
                           description="Test")
         self.assertRaises(ValueError,
                           Artifact,
                           org="testOrg",
                           name="testName",
                           repo=1,
                           description="Test")
         self.assertRaises(ValueError,
                           Artifact,
                           org="testOrg",
                           name="testName",
                           repo=repo,
                           description=1)
    def test_validation(self):
        with ParseContext.temp('PythonTargetTest/test_validation'):

            # Adding a JVM Artifact as a provides on a PythonTarget doesn't make a lot of sense. This test
            # sets up that very scenario, and verifies that pants throws a TargetDefinitionException.
            self.assertRaises(TargetDefinitionException,
                              PythonTarget,
                              name="one",
                              sources=[],
                              provides=Artifact(org='com.twitter',
                                                name='one-jar',
                                                repo=Repository(
                                                    name='internal',
                                                    url=None,
                                                    push_db=None,
                                                    exclusives=None)))

            name = "test-with-PythonArtifact"
            pa = PythonArtifact(name='foo', version='1.0', description='foo')

            # This test verifies that adding a 'setup_py' provides to a PythonTarget is okay.
            self.assertEquals(
                PythonTarget(name=name, provides=pa, sources=[]).name, name)
            name = "test-with-none"

            # This test verifies that having no provides is okay.
            self.assertEquals(
                PythonTarget(name=name, provides=None, sources=[]).name, name)
Example #7
0
def extract_target(java_targets, is_classpath):
  primary_target = InternalTarget.sort_targets(java_targets)[0]

  with ParseContext.temp(primary_target.target_base):
    internal_deps, jar_deps = _extract_target(java_targets, is_classpath)

    # TODO(John Sirois): make an empty source set work in ant/compile.xml
    sources = [ '__no_source__' ]

    all_deps = OrderedSet()
    all_deps.update(internal_deps)
    all_deps.update(jar_deps)

    if is_java(primary_target):
      return JavaLibrary('ide',
                         sources,
                         dependencies = all_deps,
                         is_meta = True)
    elif is_scala(primary_target):
      return ScalaLibrary('ide',
                          sources,
                          dependencies = all_deps,
                          is_meta = True)
    else:
      raise TypeError("Cannot generate IDE configuration for targets: %s" % java_targets)
Example #8
0
  def test_binary_target_injected_into_minified_dependencies(self):
    with ParseContext.temp():
      foo = python_library(
        name = 'foo',
        provides = setup_py(
          name = 'foo',
          version = '0.0.0',
        ).with_binaries(
          foo_binary = pants(':foo_bin')
        )
      )

      foo_bin = python_binary(
        name = 'foo_bin',
        entry_point = 'foo.bin.foo',
        dependencies = [ pants(':foo_bin_dep') ]
      )

      foo_bin_dep = python_library(
        name = 'foo_bin_dep'
      )

    assert SetupPy.minified_dependencies(foo) == OrderedSet([foo_bin, foo_bin_dep])
    entry_points = dict(SetupPy.iter_entry_points(foo))
    assert entry_points == {'foo_binary': 'foo.bin.foo'}

    with self.run_execute(foo, recursive=False) as setup_py_command:
      setup_py_command.run_one.assert_called_with(foo)

    with self.run_execute(foo, recursive=True) as setup_py_command:
      setup_py_command.run_one.assert_called_with(foo)
Example #9
0
    def __init__(self, spec, exclusives=None):
        """
    :param string spec: target address. E.g.,
      src/main/java/com/twitter/common/util/BUILD:util
    """
        # it's critical the spec is parsed 1st, the results are needed elsewhere in constructor flow
        parse_context = ParseContext.locate()

        def parse_address():
            if spec.startswith(':'):
                # the :[target] could be in a sibling BUILD - so parse using the canonical address
                pathish = "%s:%s" % (parse_context.buildfile.canonical_relpath,
                                     spec[1:])
                return Address.parse(parse_context.buildfile.root_dir, pathish,
                                     False)
            else:
                return Address.parse(parse_context.buildfile.root_dir, spec,
                                     False)

        try:
            self.address = parse_address()
        except IOError as e:
            self.address = parse_context.buildfile.relpath
            raise TargetDefinitionException(
                self, '%s%s' % (self._DEFINITION_ERROR_MSG, e))

        # We must disable the re-init check, because our funky __getattr__ breaks it.
        # We're not involved in any multiple inheritance, so it's OK to disable it here.
        super(Pants, self).__init__(self.address.target_name,
                                    reinit_check=False,
                                    exclusives=exclusives)
Example #10
0
    def test_sibling_references(self):
        with temporary_dir() as root_dir:
            buildfile = create_buildfile(root_dir,
                                         'a',
                                         name='BUILD',
                                         content=dedent("""
          dependencies(name='util',
            dependencies=[
              jar(org='com.twitter', name='util', rev='0.0.1')
            ]
          )
        """).strip())
            sibling = create_buildfile(root_dir,
                                       'a',
                                       name='BUILD.sibling',
                                       content=dedent("""
          dependencies(name='util-ex',
            dependencies=[
              pants(':util'),
              jar(org='com.twitter', name='util-ex', rev='0.0.1')
            ]
          )
        """).strip())
            ParseContext(buildfile).parse()

            utilex = Target.get(
                Address.parse(root_dir, 'a:util-ex', is_relative=False))
            utilex_deps = set(utilex.resolve())

            util = Target.get(
                Address.parse(root_dir, 'a:util', is_relative=False))
            util_deps = set(util.resolve())

            self.assertEquals(util_deps, util_deps.intersection(utilex_deps))
    def __init__(self, spec, exclusives=None):
        """
    :param string spec: target address. E.g., `src/java/com/twitter/common/util/BUILD\:util`
    """
        # it's critical the spec is parsed 1st, the results are needed elsewhere in constructor flow
        parse_context = ParseContext.locate()

        def parse_address():
            if spec.startswith(':'):
                # the :[target] could be in a sibling BUILD - so parse using the canonical address
                pathish = "%s:%s" % (parse_context.buildfile.canonical_relpath,
                                     spec[1:])
                return Address.parse(parse_context.buildfile.root_dir, pathish,
                                     False)
            else:
                return Address.parse(parse_context.buildfile.root_dir, spec,
                                     False)

        try:
            self.address = parse_address()
        except IOError as e:
            self.address = parse_context.buildfile.relpath
            raise TargetDefinitionException(
                self, '%s%s' % (self._DEFINITION_ERROR_MSG, e))

        # We must disable the re-init check, because our funky __getattr__ breaks it.
        # We're not involved in any multiple inheritance, so it's OK to disable it here.
        super(Pants, self).__init__(
            self.address.target_name,
            reinit_check=False,
            exclusives=exclusives)
Example #12
0
 def test_python_binary_with_entry_point_and_source(self):
   with ParseContext.temp('src'):
     assert 'blork' == PythonBinary(
         name = 'binary1', entry_point = 'blork', source='blork.py').entry_point
     assert 'blork:main' == PythonBinary(
         name = 'binary2', entry_point = 'blork:main', source='blork.py').entry_point
     assert 'bin.blork:main' == PythonBinary(
         name = 'binary3', entry_point = 'bin.blork:main', source='bin/blork.py').entry_point
Example #13
0
 def test_validation(self):
     with ParseContext.temp('JarLibraryTest/test_validation'):
         target = Target(name='mybird')
         JarLibrary(name="test", dependencies=target)
         self.assertRaises(TargetDefinitionException,
                           JarLibrary,
                           name="test1",
                           dependencies=None)
Example #14
0
def create_dependencies(depmap):
    target_map = {}
    with ParseContext.temp():
        for name, deps in depmap.items():
            target_map[name] = python_library(
                name=name,
                provides=setup_py(name=name, version='0.0.0'),
                dependencies=[pants(':%s' % dep) for dep in deps])
    return target_map
Example #15
0
 def __init__(self, target, msg):
   address = getattr(target, 'address', None)
   if address is None:
     try:
       location = ParseContext.locate().current_buildfile
     except ParseContext.ContextError:
       location = 'unknown location'
     address = 'unknown target of type %s in %s' % (target.__class__.__name__, location)
   super(Exception, self).__init__('Error with %s: %s' % (address, msg))
Example #16
0
  def test_validation(self):
    with ParseContext.temp('InternalTargetTest/test_validation'):
      InternalTarget(name="valid", dependencies=None)
      self.assertRaises(TargetDefinitionException, InternalTarget,
                        name=1, dependencies=None)

      InternalTarget(name="valid2", dependencies=Target(name='mybird'))
      self.assertRaises(TargetDefinitionException, InternalTarget,
                        name='valid3', dependencies=1)
    def dump(self):
        self.debug("Building PythonBinary %s:" % self._target)

        targets = self.resolve([self._target] + self._extra_targets)

        for lib in targets["libraries"] | targets["binaries"]:
            self._dump_library(lib)

        generated_reqs = OrderedSet()
        if targets["thrifts"]:
            for thr in set(targets["thrifts"]):
                if thr not in self.MEMOIZED_THRIFTS:
                    self.MEMOIZED_THRIFTS[thr] = self._generate_thrift_requirement(thr)
                generated_reqs.add(self.MEMOIZED_THRIFTS[thr])
            with ParseContext.temp():
                # trick pants into letting us add this python requirement, otherwise we get
                # TargetDefinitionException: Error in target BUILD.temp:thrift: duplicate to
                # PythonRequirement(thrift)
                #
                # TODO(wickman) Instead of just blindly adding a PythonRequirement for thrift, we
                # should first detect if any explicit thrift requirements have been added and use
                # those.  Only if they have not been supplied should we auto-inject it.
                generated_reqs.add(
                    PythonRequirement(
                        "thrift", use_2to3=True, name="thrift-" + "".join(random.sample("0123456789abcdef" * 8, 8))
                    )
                )

        for antlr in targets["antlrs"]:
            generated_reqs.add(self._generate_antlr_requirement(antlr))

        targets["reqs"] |= generated_reqs
        reqs_to_build = OrderedSet()
        for req in targets["reqs"]:
            if not req.should_build(self._interpreter.python, Platform.current()):
                self.debug("Skipping %s based upon version filter" % req)
                continue
            reqs_to_build.add(req)
            self._dump_requirement(req._requirement, False, req._repository)

        platforms = self._platforms
        if isinstance(self._target, PythonBinary):
            platforms = self._target.platforms
        distributions = resolve_multi(self._config, reqs_to_build, interpreter=self._interpreter, platforms=platforms)

        locations = set()
        for platform, dist_set in distributions.items():
            for dist in dist_set:
                if dist.location not in locations:
                    self._dump_distribution(dist)
                locations.add(dist.location)

        if len(targets["binaries"]) > 1:
            print("WARNING: Target has multiple python_binary targets!", file=sys.stderr)

        return self._builder
Example #18
0
  def execute(self):
    if self.options.pex and self.options.ipython:
      self.error('Cannot specify both --pex and --ipython!')

    if self.options.entry_point and self.options.ipython:
      self.error('Cannot specify both --entry_point and --ipython!')

    if self.options.verbose:
      print('Build operating on target: %s %s' % (self.target,
        'Extra targets: %s' % ' '.join(map(str, self.extra_targets)) if self.extra_targets else ''))

    builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter,
        pex_info=self.target.pexinfo if isinstance(self.target, PythonBinary) else None)

    if self.options.entry_point:
      builder.set_entry_point(self.options.entry_point)

    if self.options.ipython:
      if not self.config.has_section('python-ipython'):
        self.error('No python-ipython sections defined in your pants.ini!')

      builder.info.entry_point = self.config.get('python-ipython', 'entry_point')
      if builder.info.entry_point is None:
        self.error('Must specify entry_point for IPython in the python-ipython section '
                   'of your pants.ini!')

      requirements = self.config.getlist('python-ipython', 'requirements', default=[])

      with ParseContext.temp():
        for requirement in requirements:
          self.extra_targets.append(PythonRequirement(requirement))

    executor = PythonChroot(
        self.target,
        self.root_dir,
        builder=builder,
        interpreter=self.interpreter,
        extra_targets=self.extra_targets,
        conn_timeout=self.options.conn_timeout)

    executor.dump()

    if self.options.pex:
      pex_name = os.path.join(self.root_dir, 'dist', '%s.pex' % self.target.name)
      builder.build(pex_name)
      print('Wrote %s' % pex_name)
      return 0
    else:
      builder.freeze()
      pex = PEX(builder.path(), interpreter=self.interpreter)
      po = pex.run(args=list(self.args), blocking=False)
      try:
        return po.wait()
      except KeyboardInterrupt:
        po.send_signal(signal.SIGINT)
        raise
Example #19
0
  def execute(self):
    if self.options.pex and self.options.ipython:
      self.error('Cannot specify both --pex and --ipython!')

    if self.options.entry_point and self.options.ipython:
      self.error('Cannot specify both --entry_point and --ipython!')

    if self.options.verbose:
      print('Build operating on target: %s %s' % (self.target,
        'Extra targets: %s' % ' '.join(map(str, self.extra_targets)) if self.extra_targets else ''))

    builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter,
        pex_info=self.target.pexinfo if isinstance(self.target, PythonBinary) else None)

    if self.options.entry_point:
      builder.set_entry_point(self.options.entry_point)

    if self.options.ipython:
      if not self.config.has_section('python-ipython'):
        self.error('No python-ipython sections defined in your pants.ini!')

      builder.info.entry_point = self.config.get('python-ipython', 'entry_point')
      if builder.info.entry_point is None:
        self.error('Must specify entry_point for IPython in the python-ipython section '
                   'of your pants.ini!')

      requirements = self.config.getlist('python-ipython', 'requirements', default=[])

      with ParseContext.temp():
        for requirement in requirements:
          self.extra_targets.append(PythonRequirement(requirement))

    executor = PythonChroot(
        self.target,
        self.root_dir,
        builder=builder,
        interpreter=self.interpreter,
        extra_targets=self.extra_targets,
        conn_timeout=self.options.conn_timeout)

    executor.dump()

    if self.options.pex:
      pex_name = os.path.join(self.root_dir, 'dist', '%s.pex' % self.target.name)
      builder.build(pex_name)
      print('Wrote %s' % pex_name)
      return 0
    else:
      builder.freeze()
      pex = PEX(builder.path(), interpreter=self.interpreter)
      po = pex.run(args=list(self.args), blocking=False)
      try:
        return po.wait()
      except KeyboardInterrupt:
        po.send_signal(signal.SIGINT)
        raise
Example #20
0
def create_dependencies(depmap):
  target_map = {}
  with ParseContext.temp():
    for name, deps in depmap.items():
      target_map[name] = python_library(
        name=name,
        provides=setup_py(name=name, version='0.0.0'),
        dependencies=[pants(':%s' % dep) for dep in deps]
      )
  return target_map
Example #21
0
 def __init__(self, target, msg):
     address = getattr(target, 'address', None)
     if address is None:
         try:
             location = ParseContext.locate().current_buildfile
         except ParseContext.ContextError:
             location = 'unknown location'
         address = 'unknown target of type %s in %s' % (
             target.__class__.__name__, location)
     super(Exception, self).__init__('Error with %s: %s' % (address, msg))
Example #22
0
 def __init__(self, name, dependencies=None, num_sources=0, exclusives=None):
   with ParseContext.temp():
     InternalTarget.__init__(self, name, dependencies, exclusives=exclusives)
     TargetWithSources.__init__(self, name, exclusives=exclusives)
   self.num_sources = num_sources
   self.declared_exclusives = defaultdict(set)
   if exclusives is not None:
     for k in exclusives:
       self.declared_exclusives[k] = set([exclusives[k]])
   self.exclusives = None
Example #23
0
 def generate_test_targets(cls):
   if cls.TESTING_TARGETS is None:
     with ParseContext.temp():
       cls.TESTING_TARGETS = [
         PythonRequirement('pytest'),
         PythonRequirement('pytest-cov'),
         PythonRequirement('unittest2', version_filter=lambda py, pl: py.startswith('2')),
         PythonRequirement('unittest2py3k', version_filter=lambda py, pl: py.startswith('3'))
       ]
   return cls.TESTING_TARGETS
Example #24
0
def get_syms():
    r = {}
    vc = ParseContext.default_globals()
    for s in vc:
        if s in PREDEFS: continue
        if s[0].isupper():
            continue  # REMIND see both jvm_binary and JvmBinary??
        o = vc[s]
        r[s] = o
    return r
Example #25
0
 def test_python_binary_with_entry_point_and_source_mismatch(self):
   with ParseContext.temp('src'):
     with pytest.raises(TargetDefinitionException):
       PythonBinary(name = 'binary1', entry_point = 'blork', source='hork.py')
     with pytest.raises(TargetDefinitionException):
       PythonBinary(name = 'binary2', entry_point = 'blork:main', source='hork.py')
     with pytest.raises(TargetDefinitionException):
       PythonBinary(name = 'binary3', entry_point = 'bin.blork', source='blork.py')
     with pytest.raises(TargetDefinitionException):
       PythonBinary(name = 'binary4', entry_point = 'bin.blork', source='bin.py')
def get_syms():
    r = {}
    vc = ParseContext.default_globals()
    for s in vc:
        if s in PREDEFS:
            continue
        if s[0].isupper():
            continue  # REMIND see both jvm_binary and JvmBinary??
        o = vc[s]
        r[s] = o
    return r
Example #27
0
  def __init__(self, base=None, mapper=None, relative_to=None):
    """
    :param mapper: Function that takes a path string and returns a path string. Takes a path in
      the source tree, returns a path to use in the resulting bundle. By default, an identity
      mapper.
    :param string relative_to: Set up a simple mapping from source path to bundle path.
      E.g., ``relative_to='common'`` removes that prefix from all files in the application bundle.
    """
    if mapper and relative_to:
      raise ValueError("Must specify exactly one of 'mapper' or 'relative_to'")

    if relative_to:
      base = base or ParseContext.path(relative_to)
      if not os.path.isdir(base):
        raise ValueError('Could not find a directory to bundle relative to at %s' % base)
      self.mapper = RelativeToMapper(base)
    else:
      self.mapper = mapper or RelativeToMapper(base or ParseContext.path())

    self.filemap = {}
Example #28
0
 def test_python_binary_with_entry_point_and_source(self):
     with ParseContext.temp('src'):
         assert 'blork' == PythonBinary(name='binary1',
                                        entry_point='blork',
                                        source='blork.py').entry_point
         assert 'blork:main' == PythonBinary(name='binary2',
                                             entry_point='blork:main',
                                             source='blork.py').entry_point
         assert 'bin.blork:main' == PythonBinary(
             name='binary3',
             entry_point='bin.blork:main',
             source='bin/blork.py').entry_point
Example #29
0
    def get(address):
        """Returns the specified module target if already parsed; otherwise, parses the buildfile in the
    context of its parent directory and returns the parsed target."""
        def lookup():
            return Target._targets_by_address.get(address, None)

        target = lookup()
        if target:
            return target
        else:
            ParseContext(address.buildfile).parse()
            return lookup()
Example #30
0
 def test_validation(self):
   with ParseContext.temp():
     repo = Repository(name="myRepo", url="myUrl", push_db="myPushDb")
     Artifact(org="testOrg", name="testName", repo=repo, description="Test")
     self.assertRaises(ValueError, Artifact,
                       org=1, name="testName", repo=repo, description="Test")
     self.assertRaises(ValueError, Artifact,
                       org="testOrg", name=1, repo=repo, description="Test")
     self.assertRaises(ValueError, Artifact,
                       org="testOrg", name="testName", repo=1, description="Test")
     self.assertRaises(ValueError, Artifact,
                       org="testOrg", name="testName", repo=repo, description=1)
Example #31
0
  def _generate_requirement(self, library, builder_cls):
    library_key = self._key_generator.key_for_target(library)
    builder = builder_cls(library, self._root, self._config, '-' + library_key.hash[:8])

    cache_dir = os.path.join(self._egg_cache_root, library_key.id)
    if self._build_invalidator.needs_update(library_key):
      sdist = builder.build(interpreter=self._interpreter)
      safe_mkdir(cache_dir)
      shutil.copy(sdist, os.path.join(cache_dir, os.path.basename(sdist)))
      self._build_invalidator.update(library_key)

    with ParseContext.temp():
      return PythonRequirement(builder.requirement_string(), repository=cache_dir, use_2to3=True)
Example #32
0
 def test_parse(self):
     with temporary_dir() as root_dir:
         buildfile = create_buildfile(root_dir,
                                      'a',
                                      content=dedent("""
       with open('%s/a/b', 'w') as b:
         b.write('jack spratt')
     """ % root_dir).strip())
         b_file = os.path.join(root_dir, 'a', 'b')
         self.assertFalse(os.path.exists(b_file))
         ParseContext(buildfile).parse()
         with open(b_file, 'r') as b:
             self.assertEquals('jack spratt', b.read())
Example #33
0
    def test_binary_cycle(self):
        with ParseContext.temp():
            foo = python_library(name='foo',
                                 provides=setup_py(
                                     name='foo',
                                     version='0.0.0',
                                 ).with_binaries(foo_binary=pants(':foo_bin')))

            foo_bin = python_binary(name='foo_bin',
                                    entry_point='foo.bin.foo',
                                    dependencies=[pants(':foo')])

        with pytest.raises(TargetDefinitionException):
            SetupPy.minified_dependencies(foo)
Example #34
0
    def test_on_context_exit(self):
        with temporary_dir() as root_dir:
            parse_context = ParseContext(create_buildfile(root_dir, 'a'))
            with pytest.raises(parse_context.ContextError):
                parse_context.on_context_exit(lambda: 37)

        with temporary_dir() as root_dir:
            buildfile = create_buildfile(root_dir,
                                         'a',
                                         content=dedent("""
          import os
          from twitter.pants.base.parse_context import ParseContext
          def leave_a_trail(file, contents=''):
            with open(file, 'w') as b:
              b.write(contents)
          b_file = os.path.join(os.path.dirname(__file__), 'b')
          ParseContext.locate().on_context_exit(leave_a_trail, b_file, contents='42')
          assert not os.path.exists(b_file), 'Expected context exit action to be delayed.'
        """).strip())
            b_file = os.path.join(root_dir, 'a', 'b')
            self.assertFalse(os.path.exists(b_file))
            ParseContext(buildfile).parse()
            with open(b_file, 'r') as b:
                self.assertEquals('42', b.read())
Example #35
0
    def get_all_addresses(cls, buildfile):
        """Returns all of the target addresses in the specified buildfile if already parsed; otherwise,
    parses the buildfile to find all the addresses it contains and then returns them."""
        def lookup():
            if buildfile in cls._addresses_by_buildfile:
                return cls._addresses_by_buildfile[buildfile]
            else:
                return OrderedSet()

        addresses = lookup()
        if addresses:
            return addresses
        else:
            ParseContext(buildfile).parse()
            return lookup()
Example #36
0
  def test_on_context_exit(self):
    with temporary_dir() as root_dir:
      parse_context = ParseContext(create_buildfile(root_dir, 'a'))
      with pytest.raises(parse_context.ContextError):
        parse_context.on_context_exit(lambda: 37)

    with temporary_dir() as root_dir:
      buildfile = create_buildfile(root_dir, 'a',
        content=dedent("""
          import os
          from twitter.pants.base import ParseContext
          def leave_a_trail(file, contents=''):
            with open(file, 'w') as b:
              b.write(contents)
          b_file = os.path.join(os.path.dirname(__file__), 'b')
          ParseContext.locate().on_context_exit(leave_a_trail, b_file, contents='42')
          assert not os.path.exists(b_file), 'Expected context exit action to be delayed.'
        """).strip()
      )
      b_file = os.path.join(root_dir, 'a', 'b')
      self.assertFalse(os.path.exists(b_file))
      ParseContext(buildfile).parse()
      with open(b_file, 'r') as b:
        self.assertEquals('42', b.read())
Example #37
0
 def generate_test_targets(cls):
     if cls.TESTING_TARGETS is None:
         with ParseContext.temp():
             cls.TESTING_TARGETS = [
                 PythonRequirement('pytest'),
                 PythonRequirement('pytest-cov'),
                 PythonRequirement('coverage==3.6b1'),
                 PythonRequirement(
                     'unittest2',
                     version_filter=lambda py, pl: py.startswith('2')),
                 PythonRequirement(
                     'unittest2py3k',
                     version_filter=lambda py, pl: py.startswith('3'))
             ]
     return cls.TESTING_TARGETS
Example #38
0
  def test_validation(self):
    with ParseContext.temp('PythonTargetTest/test_validation'):

      # Adding a JVM Artifact as a provides on a PythonTarget doesn't make a lot of sense. This test
      # sets up that very scenario, and verifies that pants throws a TargetDefinitionException.
      self.assertRaises(TargetDefinitionException, PythonTarget, name="one", sources=[],
        provides=Artifact(org='com.twitter', name='one-jar',
        repo=Repository(name='internal', url=None, push_db=None, exclusives=None)))

      name = "test-with-PythonArtifact"
      pa = PythonArtifact(name='foo', version='1.0', description='foo')

      # This test verifies that adding a 'setup_py' provides to a PythonTarget is okay.
      self.assertEquals(PythonTarget(name=name, provides=pa, sources=[]).name, name)
      name = "test-with-none"

      # This test verifies that having no provides is okay.
      self.assertEquals(PythonTarget(name=name, provides=None, sources=[]).name, name)
Example #39
0
 def test_python_binary_with_entry_point_and_source_mismatch(self):
     with ParseContext.temp('src'):
         with pytest.raises(TargetDefinitionException):
             PythonBinary(name='binary1',
                          entry_point='blork',
                          source='hork.py')
         with pytest.raises(TargetDefinitionException):
             PythonBinary(name='binary2',
                          entry_point='blork:main',
                          source='hork.py')
         with pytest.raises(TargetDefinitionException):
             PythonBinary(name='binary3',
                          entry_point='bin.blork',
                          source='blork.py')
         with pytest.raises(TargetDefinitionException):
             PythonBinary(name='binary4',
                          entry_point='bin.blork',
                          source='bin.py')
Example #40
0
    def test_locate(self):
        with pytest.raises(ParseContext.ContextError):
            ParseContext.locate()

        with temporary_dir() as root_dir:
            a_context = ParseContext(create_buildfile(root_dir, 'a'))
            b_context = ParseContext(create_buildfile(root_dir, 'b'))

            def test_in_a():
                self.assertEquals(a_context, ParseContext.locate())
                return b_context.do_in_context(lambda: ParseContext.locate())

            self.assertEquals(b_context, a_context.do_in_context(test_in_a))
Example #41
0
  def dump(self):
    self.debug('Building PythonBinary %s:' % self._target)

    targets = self.resolve([self._target] + self._extra_targets)

    for lib in targets['libraries'] | targets['binaries']:
      self._dump_library(lib)

    generated_reqs = OrderedSet()
    if targets['thrifts']:
      for thr in set(targets['thrifts']):
        if thr not in self.MEMOIZED_THRIFTS:
          self.MEMOIZED_THRIFTS[thr] = self._generate_thrift_requirement(thr)
        generated_reqs.add(self.MEMOIZED_THRIFTS[thr])
      with ParseContext.temp():
        # trick pants into letting us add this python requirement, otherwise we get
        # TargetDefinitionException: Error in target BUILD.temp:thrift: duplicate to
        # PythonRequirement(thrift)
        #
        # TODO(wickman) Instead of just blindly adding a PythonRequirement for thrift, we
        # should first detect if any explicit thrift requirements have been added and use
        # those.  Only if they have not been supplied should we auto-inject it.
        generated_reqs.add(PythonRequirement('thrift', use_2to3=True,
            name='thrift-' + ''.join(random.sample('0123456789abcdef' * 8, 8))))

    for antlr in targets['antlrs']:
      generated_reqs.add(self._generate_antlr_requirement(antlr))

    targets['reqs'] |= generated_reqs
    for req in targets['reqs']:
      if not req.should_build(self._interpreter.python, Platform.current()):
        self.debug('Skipping %s based upon version filter' % req)
        continue
      self._dump_requirement(req._requirement, False, req._repository)

    reqs_to_build = (req for req in targets['reqs']
        if req.should_build(self._interpreter.python, Platform.current()))
    for dist in self._resolver.resolve(reqs_to_build, interpreter=self._interpreter):
      self._dump_distribution(dist)

    if len(targets['binaries']) > 1:
      print('WARNING: Target has multiple python_binary targets!', file=sys.stderr)

    return self._builder
Example #42
0
  def test_binary_cycle(self):
    with ParseContext.temp():
      foo = python_library(
        name = 'foo',
        provides = setup_py(
          name = 'foo',
          version = '0.0.0',
        ).with_binaries(
          foo_binary = pants(':foo_bin')
        )
      )

      foo_bin = python_binary(
        name = 'foo_bin',
        entry_point = 'foo.bin.foo',
        dependencies = [ pants(':foo') ]
      )

    with pytest.raises(TargetDefinitionException):
      SetupPy.minified_dependencies(foo)
Example #43
0
  def find(cls, target):
    """Finds the source root for the given target.

    If none is registered, returns the parent directory of the target's BUILD file.
    """
    target_path = os.path.relpath(target.address.buildfile.parent_path, get_buildroot())

    def _find():
      for root_dir, types in cls._TYPES_BY_ROOT.items():
        if target_path.startswith(root_dir):  # The only candidate root for this target.
          # Validate the target type, if restrictions were specified.
          if types and not isinstance(target, tuple(types)):
            # TODO: Find a way to use the BUILD file aliases in the error message, instead
            # of target.__class__.__name__. E.g., java_tests instead of JavaTests.
            raise TargetDefinitionException(target,
                'Target type %s not allowed under %s' % (target.__class__.__name__, root_dir))
          return root_dir
      return None

    # Try already registered roots
    root = _find()
    if root:
      return root

    # Fall back to searching the ancestor path for a root.
    # TODO(John Sirois): We currently allow for organic growth of maven multi-module layout style
    # projects (for example) and do not require a global up-front registration of all source roots
    # and instead do lazy resolution here.  This allows for parse cycles that lead to surprising
    # runtime errors.  Re-consider allowing lazy source roots at all.
    for buildfile in reversed(target.address.buildfile.ancestors()):
      if buildfile not in cls._SEARCHED:
        ParseContext(buildfile).parse()
        cls._SEARCHED.add(buildfile)
        root = _find()
        if root:
          return root

    # Finally, resolve files relative to the BUILD file parent dir as the target base
    return target_path
Example #44
0
  def test_binary_target_injected_into_minified_dependencies_with_provider(self):
    with ParseContext.temp():
      bar = python_library(
        name = 'bar',
        provides = setup_py(
          name = 'bar',
          version = '0.0.0',
        ).with_binaries(
          bar_binary = pants(':bar_bin')
        )
      )

      bar_bin = python_binary(
        name = 'bar_bin',
        entry_point = 'bar.bin.bar',
        dependencies = [ pants(':bar_bin_dep') ]
      )

      bar_bin_dep = python_library(
        name = 'bar_bin_dep',
        provides = setup_py(
          name = 'bar_bin_dep',
          version = '0.0.0',
        )
      )

    assert SetupPy.minified_dependencies(bar) == OrderedSet([bar_bin, bar_bin_dep])
    entry_points = dict(SetupPy.iter_entry_points(bar))
    assert entry_points == {'bar_binary': 'bar.bin.bar'}

    with self.run_execute(bar, recursive=False) as setup_py_command:
      setup_py_command.run_one.assert_called_with(bar)

    with self.run_execute(bar, recursive=True) as setup_py_command:
      setup_py_command.run_one.assert_has_calls([
          call(bar),
          call(bar_bin_dep)
      ], any_order=True)
Example #45
0
    def __init__(self, name, reinit_check=True, exclusives=None):
        """
    :param string name: The target name.
    """
        # See "get_all_exclusives" below for an explanation of the exclusives parameter.
        # This check prevents double-initialization in multiple-inheritance situations.
        # TODO(John Sirois): fix target inheritance - use super() to linearize or use alternatives to
        # multiple inheritance.
        if not reinit_check or not hasattr(self, '_initialized'):
            if not isinstance(name, Compatibility.string):
                self.address = '%s:%s' % (
                    ParseContext.locate().current_buildfile, str(name))
                raise TargetDefinitionException(
                    self, "Invalid target name: %s" % name)
            self.name = name
            self.description = None

            self.address = self._locate()

            # TODO(John Sirois): Transition all references to self.identifier to eliminate id builtin
            # ambiguity
            self.id = self._create_id()

            self._register()

            self.labels = set()

            self._initialized = True

            self.declared_exclusives = collections.defaultdict(set)
            if exclusives is not None:
                for k in exclusives:
                    self.declared_exclusives[k].add(exclusives[k])
            self.exclusives = None

            # For synthetic codegen targets this will be the original target from which
            # the target was synthesized.
            self._derived_from = self
Example #46
0
    def test_binary_target_injected_into_minified_dependencies(self):
        with ParseContext.temp():
            foo = python_library(name='foo',
                                 provides=setup_py(
                                     name='foo',
                                     version='0.0.0',
                                 ).with_binaries(foo_binary=pants(':foo_bin')))

            foo_bin = python_binary(name='foo_bin',
                                    entry_point='foo.bin.foo',
                                    dependencies=[pants(':foo_bin_dep')])

            foo_bin_dep = python_library(name='foo_bin_dep')

        assert SetupPy.minified_dependencies(foo) == OrderedSet(
            [foo_bin, foo_bin_dep])
        entry_points = dict(SetupPy.iter_entry_points(foo))
        assert entry_points == {'foo_binary': 'foo.bin.foo'}

        with self.run_execute(foo, recursive=False) as setup_py_command:
            setup_py_command.run_one.assert_called_with(foo)

        with self.run_execute(foo, recursive=True) as setup_py_command:
            setup_py_command.run_one.assert_called_with(foo)
Example #47
0
  def __init__(self, name, reinit_check=True, exclusives=None):
    """
    :param string name: The target name.
    """
    # See "get_all_exclusives" below for an explanation of the exclusives parameter.
    # This check prevents double-initialization in multiple-inheritance situations.
    # TODO(John Sirois): fix target inheritance - use super() to linearize or use alternatives to
    # multiple inheritance.
    if not reinit_check or not hasattr(self, '_initialized'):
      if not isinstance(name, Compatibility.string):
        self.address = '%s:%s' % (ParseContext.locate().current_buildfile, str(name))
        raise TargetDefinitionException(self, "Invalid target name: %s" % name)
      self.name = name
      self.description = None

      self.address = self._locate()

      # TODO(John Sirois): Transition all references to self.identifier to eliminate id builtin
      # ambiguity
      self.id = self._create_id()

      self._register()

      self.labels = set()

      self._initialized = True

      self.declared_exclusives = collections.defaultdict(set)
      if exclusives is not None:
        for k in exclusives:
          self.declared_exclusives[k].add(exclusives[k])
      self.exclusives = None

      # For synthetic codegen targets this will be the original target from which
      # the target was synthesized.
      self._derived_from = self
Example #48
0
 def test_in_a():
   self.assertEquals(a_context, ParseContext.locate())
   return b_context.do_in_context(lambda: ParseContext.locate())
Example #49
0
  def __init__(self, run_tracker, root_dir, parser, argv):
    Command.__init__(self, run_tracker, root_dir, parser, argv)

    self.target = None
    self.extra_targets = []
    self.config = Config.load()
    self.interpreter_cache = PythonInterpreterCache(self.config, logger=self.debug)
    self.interpreter_cache.setup()
    interpreters = self.interpreter_cache.select_interpreter(
        list(self.interpreter_cache.matches([self.options.interpreter]
            if self.options.interpreter else [''])))
    if len(interpreters) != 1:
      self.error('Unable to detect suitable interpreter.')
    self.interpreter = interpreters[0]

    for req in self.options.extra_requirements:
      with ParseContext.temp():
        self.extra_targets.append(PythonRequirement(req, use_2to3=True))

    # We parse each arg in the context of the cli usage:
    #   ./pants command (options) [spec] (build args)
    #   ./pants command (options) [spec]... -- (build args)
    # Our command token and our options are parsed out so we see args of the form:
    #   [spec] (build args)
    #   [spec]... -- (build args)
    binaries = []
    for k in range(len(self.args)):
      arg = self.args.pop(0)
      if arg == '--':
        break

      def not_a_target(debug_msg):
        self.debug('Not a target, assuming option: %s.' % e)
        # We failed to parse the arg as a target or else it was in valid address format but did not
        # correspond to a real target.  Assume this is the 1st of the build args and terminate
        # processing args for target addresses.
        self.args.insert(0, arg)

      target = None
      try:
        address = Address.parse(root_dir, arg)
        target = Target.get(address)
        if target is None:
          not_a_target(debug_msg='Unrecognized target')
          break
      except Exception as e:
        not_a_target(debug_msg=e)
        break

      for resolved in filter(lambda t: t.is_concrete, target.resolve()):
        if isinstance(resolved, PythonBinary):
          binaries.append(resolved)
        else:
          self.extra_targets.append(resolved)

    if len(binaries) == 0:
      # treat as a chroot
      pass
    elif len(binaries) == 1:
      # We found a binary and are done, the rest of the args get passed to it
      self.target = binaries[0]
    else:
      self.error('Can only process 1 binary target, %s contains %d:\n\t%s' % (
        arg, len(binaries), '\n\t'.join(str(binary.address) for binary in binaries)
      ))

    if self.target is None:
      if not self.extra_targets:
        self.error('No valid target specified!')
      self.target = self.extra_targets.pop(0)
Example #50
0
 def test_validation(self):
   basedir = 'PantsTargetTest/test_validation'
   with ParseContext.temp(basedir):
     self.assertRaises(TargetDefinitionException, Pants, spec='fake')
     self.assertRaises(TargetDefinitionException, Pants, spec='%s:fake' % basedir)
Example #51
0
 def _locate(self):
   parse_context = ParseContext.locate()
   return Address(parse_context.current_buildfile, self.name)
Example #52
0
 def _post_construct(self, func, *args, **kwargs):
   """Registers a command to invoke after this target's BUILD file is parsed."""
   ParseContext.locate().on_context_exit(func, *args, **kwargs)
Example #53
0
    def __init__(self, run_tracker, root_dir, parser, argv):
        Command.__init__(self, run_tracker, root_dir, parser, argv)

        self.target = None
        self.extra_targets = []
        self.config = Config.load()
        self.interpreter_cache = PythonInterpreterCache(self.config,
                                                        logger=self.debug)
        self.interpreter_cache.setup()
        interpreters = self.interpreter_cache.select_interpreter(
            list(
                self.interpreter_cache.matches(
                    [self.options.
                     interpreter] if self.options.interpreter else [''])))
        if len(interpreters) != 1:
            self.error('Unable to detect suitable interpreter.')
        self.interpreter = interpreters[0]

        for req in self.options.extra_requirements:
            with ParseContext.temp():
                self.extra_targets.append(PythonRequirement(req,
                                                            use_2to3=True))

        # We parse each arg in the context of the cli usage:
        #   ./pants command (options) [spec] (build args)
        #   ./pants command (options) [spec]... -- (build args)
        # Our command token and our options are parsed out so we see args of the form:
        #   [spec] (build args)
        #   [spec]... -- (build args)
        binaries = []
        for k in range(len(self.args)):
            arg = self.args.pop(0)
            if arg == '--':
                break

            def not_a_target(debug_msg):
                self.debug('Not a target, assuming option: %s.' % e)
                # We failed to parse the arg as a target or else it was in valid address format but did not
                # correspond to a real target.  Assume this is the 1st of the build args and terminate
                # processing args for target addresses.
                self.args.insert(0, arg)

            target = None
            try:
                address = Address.parse(root_dir, arg)
                target = Target.get(address)
                if target is None:
                    not_a_target(debug_msg='Unrecognized target')
                    break
            except Exception as e:
                not_a_target(debug_msg=e)
                break

            for resolved in filter(lambda t: t.is_concrete, target.resolve()):
                if isinstance(resolved, PythonBinary):
                    binaries.append(resolved)
                else:
                    self.extra_targets.append(resolved)

        if len(binaries) == 0:
            # treat as a chroot
            pass
        elif len(binaries) == 1:
            # We found a binary and are done, the rest of the args get passed to it
            self.target = binaries[0]
        else:
            self.error(
                'Can only process 1 binary target, %s contains %d:\n\t%s' %
                (arg, len(binaries), '\n\t'.join(
                    str(binary.address) for binary in binaries)))

        if self.target is None:
            if not self.extra_targets:
                self.error('No valid target specified!')
            self.target = self.extra_targets.pop(0)
Example #54
0
 def locate(self):
   parse_context = ParseContext.locate()
   return Address(parse_context.buildfile, self.name, self.is_meta)