Пример #1
0
    def dump(self):
        self.debug('Building PythonBinary %s:' % self._target)

        targets = self.resolve([self._target] + self._extra_targets)

        for lib in targets['libraries'] | targets['binaries']:
            self._dump_library(lib)

        generated_reqs = OrderedSet()
        if targets['thrifts']:
            for thr in set(targets['thrifts']):
                if thr not in self.MEMOIZED_THRIFTS:
                    self.MEMOIZED_THRIFTS[
                        thr] = self._generate_thrift_requirement(thr)
                generated_reqs.add(self.MEMOIZED_THRIFTS[thr])
            with ParseContext.temp():
                # trick pants into letting us add this python requirement, otherwise we get
                # TargetDefinitionException: Error in target BUILD.temp:thrift: duplicate to
                # PythonRequirement(thrift)
                #
                # TODO(wickman) Instead of just blindly adding a PythonRequirement for thrift, we
                # should first detect if any explicit thrift requirements have been added and use
                # those.  Only if they have not been supplied should we auto-inject it.
                generated_reqs.add(
                    PythonRequirement(
                        'thrift',
                        use_2to3=True,
                        name='thrift-' +
                        ''.join(random.sample('0123456789abcdef' * 8, 8))))

        for antlr in targets['antlrs']:
            generated_reqs.add(self._generate_antlr_requirement(antlr))

        targets['reqs'] |= generated_reqs
        for req in targets['reqs']:
            if not req.should_build(self._interpreter.python,
                                    Platform.current()):
                self.debug('Skipping %s based upon version filter' % req)
                continue
            self._dump_requirement(req._requirement, False, req._repository)

        reqs_to_build = (
            req for req in targets['reqs']
            if req.should_build(self._interpreter.python, Platform.current()))
        for dist in self._resolver.resolve(reqs_to_build,
                                           interpreter=self._interpreter):
            self._dump_distribution(dist)

        if len(targets['binaries']) > 1:
            print('WARNING: Target has multiple python_binary targets!',
                  file=sys.stderr)

        return self._builder
Пример #2
0
 def test_get_current_platform(self):
     expected_platforms = [Platform.current(), 'linux-x86_64']
     self.assertEqual(
         expected_platforms,
         list(
             get_platforms(self.config.getlist('python-setup',
                                               'platforms'))))
Пример #3
0
def main():
  parser = configure_clp()
  options, args = parser.parse_args()
  verbosity = 5 if options.verbosity else -1

  with Tracer.env_override(
      PEX_VERBOSE=verbosity,
      TWITTER_COMMON_PYTHON_HTTP=verbosity,
      PYTHON_VERBOSE=verbosity):

    pex_builder = build_pex(args, options)

    if options.pex_name is not None:
      log('Saving PEX file to %s' % options.pex_name, v=options.verbosity)
      tmp_name = options.pex_name + '~'
      safe_delete(tmp_name)
      pex_builder.build(tmp_name)
      os.rename(tmp_name, options.pex_name)
      return 0

    if options.platform != Platform.current():
      log('WARNING: attempting to run PEX with differing platform!')

    pex_builder.freeze()

    log('Running PEX file at %s with args %s' % (pex_builder.path(), args), v=options.verbosity)
    pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
    return pex.run(args=list(args))
Пример #4
0
  def dump(self):
    self.debug('Building PythonBinary %s:' % self._target)

    targets = self.resolve([self._target] + self._extra_targets)

    for lib in targets['libraries'] | targets['binaries']:
      self._dump_library(lib)

    generated_reqs = OrderedSet()
    if targets['thrifts']:
      for thr in set(targets['thrifts']):
        if thr not in self.MEMOIZED_THRIFTS:
          self.MEMOIZED_THRIFTS[thr] = self._generate_thrift_requirement(thr)
        generated_reqs.add(self.MEMOIZED_THRIFTS[thr])
      with ParseContext.temp():
        # trick pants into letting us add this python requirement, otherwise we get
        # TargetDefinitionException: Error in target BUILD.temp:thrift: duplicate to
        # PythonRequirement(thrift)
        #
        # TODO(wickman) Instead of just blindly adding a PythonRequirement for thrift, we
        # should first detect if any explicit thrift requirements have been added and use
        # those.  Only if they have not been supplied should we auto-inject it.
        generated_reqs.add(PythonRequirement('thrift', use_2to3=True,
            name='thrift-' + ''.join(random.sample('0123456789abcdef' * 8, 8))))

    for antlr in targets['antlrs']:
      generated_reqs.add(self._generate_antlr_requirement(antlr))

    targets['reqs'] |= generated_reqs
    reqs_to_build = OrderedSet()
    for req in targets['reqs']:
      if not req.should_build(self._interpreter.python, Platform.current()):
        self.debug('Skipping %s based upon version filter' % req)
        continue
      reqs_to_build.add(req)
      self._dump_requirement(req._requirement, False, req._repository)

    platforms = self._platforms
    if isinstance(self._target, PythonBinary):
      platforms = self._target.platforms
    distributions = resolve_multi(
         self._config,
         reqs_to_build,
         interpreter=self._interpreter,
         platforms=platforms)

    locations = set()
    for platform, dist_set in distributions.items():
      for dist in dist_set:
        if dist.location not in locations:
          self._dump_distribution(dist)
        locations.add(dist.location)

    if len(targets['binaries']) > 1:
      print('WARNING: Target has multiple python_binary targets!', file=sys.stderr)

    return self._builder
Пример #5
0
    def dump(self):
        self.debug('Building PythonBinary %s:' % self._target)
        targets = self.resolve([self._target] + self._extra_targets)

        for lib in targets['libraries'] | targets['binaries']:
            self._dump_library(lib)

        generated_reqs = OrderedSet()
        if targets['thrifts']:
            for thr in set(targets['thrifts']):
                if thr not in self.MEMOIZED_THRIFTS:
                    self.MEMOIZED_THRIFTS[
                        thr] = self._generate_thrift_requirement(thr)
                generated_reqs.add(self.MEMOIZED_THRIFTS[thr])

            generated_reqs.add(PythonRequirement('thrift', use_2to3=True))

        for antlr in targets['antlrs']:
            generated_reqs.add(self._generate_antlr_requirement(antlr))

        reqs_from_libraries = OrderedSet()
        for req_lib in targets['reqs']:
            for req in req_lib.payload.requirements:
                reqs_from_libraries.add(req)

        reqs_to_build = OrderedSet()
        for req in reqs_from_libraries | generated_reqs | self._extra_requirements:
            if not req.should_build(self._interpreter.python,
                                    Platform.current()):
                self.debug('Skipping %s based upon version filter' % req)
                continue
            reqs_to_build.add(req)
            self._dump_requirement(req._requirement, False, req._repository)

        platforms = self._platforms
        if isinstance(self._target, PythonBinary):
            platforms = self._target.platforms
        distributions = resolve_multi(self._config,
                                      reqs_to_build,
                                      interpreter=self._interpreter,
                                      platforms=platforms)

        locations = set()
        for platform, dist_set in distributions.items():
            for dist in dist_set:
                if dist.location not in locations:
                    self._dump_distribution(dist)
                locations.add(dist.location)

        if len(targets['binaries']) > 1:
            print('WARNING: Target has multiple python_binary targets!',
                  file=sys.stderr)

        return self._builder
Пример #6
0
 def test_unknown(self):
   with pytest.raises(Platform.UnknownPlatformError):
     Platform.compatible('macosx-10.0-morfgorf', 'macosx-10.1-morfgorf')
   with pytest.raises(Platform.UnknownPlatformError):
     Platform.compatible('macosx-10.0-x86_64', 'macosx-10.1-morfgorf')
   with pytest.raises(Platform.UnknownPlatformError):
     Platform.compatible('macosx-10.0-morfgorf', 'macosx-10.1-x86_64')
Пример #7
0
  def dump(self):
    self.debug('Building PythonBinary %s:' % self._target)
    targets = self.resolve([self._target] + self._extra_targets)

    for lib in targets['libraries'] | targets['binaries']:
      self._dump_library(lib)

    generated_reqs = OrderedSet()
    if targets['thrifts']:
      for thr in set(targets['thrifts']):
        if thr not in self.MEMOIZED_THRIFTS:
          self.MEMOIZED_THRIFTS[thr] = self._generate_thrift_requirement(thr)
        generated_reqs.add(self.MEMOIZED_THRIFTS[thr])

      generated_reqs.add(PythonRequirement('thrift', use_2to3=True))

    for antlr in targets['antlrs']:
      generated_reqs.add(self._generate_antlr_requirement(antlr))

    reqs_from_libraries = OrderedSet()
    for req_lib in targets['reqs']:
      for req in req_lib.payload.requirements:
        reqs_from_libraries.add(req)

    reqs_to_build = OrderedSet()
    for req in reqs_from_libraries | generated_reqs | self._extra_requirements:
      if not req.should_build(self._interpreter.python, Platform.current()):
        self.debug('Skipping %s based upon version filter' % req)
        continue
      reqs_to_build.add(req)
      self._dump_requirement(req._requirement, False, req._repository)

    platforms = self._platforms
    if isinstance(self._target, PythonBinary):
      platforms = self._target.platforms
    distributions = resolve_multi(
         self._config,
         reqs_to_build,
         interpreter=self._interpreter,
         platforms=platforms)

    locations = set()
    for platform, dist_set in distributions.items():
      for dist in dist_set:
        if dist.location not in locations:
          self._dump_distribution(dist)
        locations.add(dist.location)

    if len(targets['binaries']) > 1:
      print('WARNING: Target has multiple python_binary targets!', file=sys.stderr)

    return self._builder
Пример #8
0
  def test_versioning(self):
    # Major versions incompatible
    assert not Platform.compatible('macosx-9.1-x86_64', 'macosx-10.0-x86_64')
    assert not Platform.compatible('macosx-10.0-x86_64', 'macosx-9.1-x86_64')

    # Platforms equal
    assert Platform.compatible('macosx-10.0-x86_64', 'macosx-10.0-x86_64')

    # Minor versions less than
    assert Platform.compatible('macosx-10.0-x86_64', 'macosx-10.1-x86_64')
    assert not Platform.compatible('macosx-10.1-x86_64', 'macosx-10.0-x86_64')
    assert Platform.compatible('macosx-10.9-x86_64', 'macosx-10.10-x86_64')
    assert not Platform.compatible('macosx-10.10-x86_64', 'macosx-10.9-x86_64')
Пример #9
0
 def __init__(self,
              caches=(),
              install_cache=None,
              fetcher=None,
              fetcher_provider=None,
              platform=Platform.current(),
              python=sys.version[:3]):
   assert (fetcher is not None) + (fetcher_provider is not None) == 1, (
     "At most one of fetcher or fetcher_provider should be supplied")
   self._subcaches = [Resolver.Subcache(cache, self) for cache in caches]
   self._fetcher = fetcher
   self._fetcher_provider = fetcher_provider
   self._install_cache = install_cache
   self._ws = WorkingSet([])
   with self.timed('Calling environment super'):
     super(Resolver, self).__init__(search_path=[], platform=platform, python=python)
Пример #10
0
  def __init__(self, target, root_dir, extra_targets=None, builder=None):
    self._config = Config.load()

    self._target = target
    self._root = root_dir
    self._cache = BuildCache(os.path.join(self._config.get('python-setup', 'artifact_cache'),
      '%s' % PythonIdentity.get()))
    self._extra_targets = list(extra_targets) if extra_targets is not None else []
    self._resolver = PythonResolver([self._target] + self._extra_targets)
    self._builder = builder or PEXBuilder(tempfile.mkdtemp())
    self._platforms = (Platform.current(),)
    self._pythons = (sys.version[:3],)

    # TODO(wickman) Should this be in the binary builder?
    if isinstance(self._target, PythonBinary):
      self._platforms = self._target._platforms
      self._pythons = self._target._interpreters
Пример #11
0
    def __init__(self, target, root_dir, extra_targets=None, builder=None):
        self._config = Config.load()

        self._target = target
        self._root = root_dir
        self._cache = BuildCache(
            os.path.join(self._config.get('python-setup', 'artifact_cache'),
                         '%s' % PythonIdentity.get()))
        self._extra_targets = list(
            extra_targets) if extra_targets is not None else []
        self._resolver = PythonResolver([self._target] + self._extra_targets)
        self._builder = builder or PEXBuilder(tempfile.mkdtemp())
        self._platforms = (Platform.current(), )
        self._pythons = (sys.version[:3], )

        # TODO(wickman) Should this be in the binary builder?
        if isinstance(self._target, PythonBinary):
            self._platforms = self._target._platforms
            self._pythons = self._target._interpreters
Пример #12
0
  def test_platform_subsets(self):
    # Pure platform subset
    assert Platform.compatible('macosx-10.0-i386', 'macosx-10.0-intel')

    # Version and platform subset
    assert Platform.compatible('macosx-10.0-i386', 'macosx-10.1-intel')
    assert Platform.compatible('macosx-10.0-x86_64', 'macosx-10.1-intel')

    # Intersecting sets of platform but not pure subset
    assert Platform.compatible('macosx-10.0-fat', 'macosx-10.1-intel')

    # Non-intersecting sets of platform
    assert not Platform.compatible('macosx-10.0-ppc', 'macosx-10.1-intel')

    # Test our common case
    assert Platform.compatible('macosx-10.4-x86_64', 'macosx-10.7-intel')
Пример #13
0
    def __init__(self,
                 name,
                 source=None,
                 dependencies=None,
                 entry_point=None,
                 inherit_path=False,
                 zip_safe=True,
                 repositories=None,
                 indices=None,
                 ignore_errors=False,
                 allow_pypi=False,
                 platforms=(Platform.current(), ),
                 interpreters=(sys.version[:3], )):
        """
      name: target name

      source: the python source file that becomes this binary's __main__ [optional]
              if none specified, drops into an interpreter by default

      dependencies: a list of other PythonLibrary or Pants targets this binary depends upon

      entry_point: the default entry point for this binary (by default drops
                   into the entry point defined by @source)

      inherit_path: inherit the sys.path of the environment that this binary runs in

      zip_safe: whether or not this binary is safe to run in compacted (zip-file) form

      repositories: a list of repositories to query for dependencies

      indices: a list of indices to use for packages

      allow_pypi: whether or not this binary should be allowed to hit pypi for dependency
                  management

      platforms: the platforms to target when building this binary.  by
                 default the current platform.

      interpreters: the interpreter versions to target when building this binary.  by default the
                    current interpreter version (specify in the form: '2.6', '2.7', '3.2' etc.)
    """
        if source is None and dependencies is None:
            raise TargetDefinitionException(
                'ERROR: no source or dependencies declared for target %s' %
                name)
        if source and entry_point:
            raise TargetDefinitionException(
                'Can only declare an entry_point if no source binary is specified.'
            )
        if not isinstance(platforms, (list, tuple)) or not isinstance(
                interpreters, (list, tuple)):
            raise TargetDefinitionException(
                'platforms and interpreters must be lists or tuples.')

        self._entry_point = entry_point
        self._inherit_path = bool(inherit_path)
        self._zip_safe = bool(zip_safe)
        self._platforms = platforms
        self._interpreters = interpreters
        self._repositories = repositories or []
        self._indices = indices or []
        self._allow_pypi = bool(allow_pypi)
        self._ignore_errors = bool(ignore_errors)

        PythonTarget.__init__(self,
                              name, [] if source is None else [source],
                              dependencies=dependencies)
Пример #14
0
 def test_get_current_platform(self):
     expected_platforms = [Platform.current(), "linux-x86_64"]
     self.assertEqual(expected_platforms, list(get_platforms(self.config.getlist("python-setup", "platforms"))))
Пример #15
0
 def translate(platform):
   return Platform.current() if platform == 'current' else platform
Пример #16
0
        close_fds=True,
        cwd=get_buildroot()
      )
      with _safe_open(self._pidfile, 'w') as pidfile:
        pidfile.write('%d' % process.pid)
      log.debug('Spawned ng server @ %d' % process.pid)
      # Prevents finally blocks being executed, unlike sys.exit(). We don't want to execute finally
      # blocks because we might, e.g., clean up tempfiles that the parent still needs.
      os._exit(0)


# Pick implementations for killall and _find. We don't use psutil, as it uses
# native code and so is not portable, leading to packaging and deployment headaches.
# TODO: Extract this to a class and add a paired test guarded by
# http://pytest.org/latest/skipping.html#skipping.
plat = Platform.current()
if plat.startswith('linux') or plat.startswith('macosx'):
  # TODO: add other platforms as needed, after checking that these cmds work there as expected.

  # Returns the cmd's output, as a list of lines, including the newline characters.
  def _run_cmd(cmd):
    runcmd = cmd + ' && echo "\n${PIPESTATUS[*]}"'
    popen = subprocess.Popen(runcmd, shell=True, executable='/bin/bash', bufsize=-1, close_fds=True,
                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    (stdout_data, _) = popen.communicate()
    stdout_data_lines = [line for line in stdout_data.strip().split('\n') if line]
    if not stdout_data_lines:
      raise NailgunError('No output for command (%s)' % runcmd)
    try:
      # Get the return codes of each piped cmd.
      piped_return_codes = [int(x) for x in stdout_data_lines[-1].split(' ') if x]
Пример #17
0
 def platform_compatible():
   return Platform.compatible(dist.platform, self.platform)
Пример #18
0
def configure_clp():
    usage = (
        "%prog [options]\n\n"
        "%prog builds a PEX (Python Executable) file based on the given specifications: "
        "sources, requirements, their dependencies and other options"
    )

    parser = OptionParser(usage=usage, version="%prog 0.2")

    parser.add_option(
        "--pypi",
        "--no-pypi",
        dest="pypi",
        default=True,
        action="callback",
        callback=parse_bool,
        help="Whether to use pypi to resolve dependencies; Default: use pypi",
    )

    parser.add_option(
        "--python",
        dest="python",
        default=None,
        help="The Python interpreter to use to build the pex.  Either specify an explicit "
        "path to an interpreter, or specify a binary accessible on $PATH. "
        "Default: Use current interpreter.",
    )

    parser.add_option(
        "--platform",
        dest="platform",
        default=Platform.current(),
        help="The platform for which to build the PEX.  Default: %%default",
    )

    parser.add_option(
        "--zip-safe",
        "--not-zip-safe",
        dest="zip_safe",
        default=True,
        action="callback",
        callback=parse_bool,
        help="Whether or not the sources in the pex file are zip safe.  If they are "
        "not zip safe, they will be written to disk prior to execution; "
        "Default: zip safe.",
    )

    parser.add_option(
        "--always-write-cache",
        dest="always_write_cache",
        default=False,
        action="store_true",
        help="Always write the internally cached eggs to disk prior to invoking "
        "the pex source code.  This can use less memory in RAM constrained "
        "environments. [Default: %default]",
    )

    parser.add_option(
        "--ignore-errors",
        dest="ignore_errors",
        default=False,
        action="store_true",
        help="Ignore run-time requirement resolution errors when invoking the pex. " "[Default: %default]",
    )

    parser.add_option(
        "--inherit-path",
        dest="inherit_path",
        default=False,
        action="store_true",
        help="Inherit the contents of sys.path (including site-packages) running the pex. " "[Default: %default]",
    )

    parser.add_option(
        "--cache-dir",
        dest="cache_dir",
        default=os.path.expanduser("~/.pex/build"),
        help="The local cache directory to use for speeding up requirement " "lookups; [Default: %default]",
    )

    parser.add_option(
        "-o",
        "-p",
        "--output-file",
        "--pex-name",
        dest="pex_name",
        default=None,
        help="The name of the generated .pex file: Omiting this will run PEX " "immediately and not save it to a file.",
    )

    parser.add_option(
        "-e",
        "--entry-point",
        dest="entry_point",
        default=None,
        help="The entry point for this pex; Omiting this will enter the python "
        "REPL with sources and requirements available for import.  Can be "
        "either a module or EntryPoint (module:function) format.",
    )

    parser.add_option(
        "-r",
        "--requirement",
        dest="requirements",
        metavar="REQUIREMENT",
        default=[],
        action="append",
        help="requirement to be included; may be specified multiple times.",
    )

    parser.add_option(
        "--repo",
        dest="repos",
        metavar="PATH",
        default=[],
        action="append",
        help="Additional repository path (directory or URL) to look for requirements.",
    )

    parser.add_option(
        "-s",
        "--source-dir",
        dest="source_dirs",
        metavar="DIR",
        default=[],
        action="append",
        help="Source to be packaged; This <DIR> should be a pip-installable project " "with a setup.py.",
    )

    parser.add_option(
        "-v", "--verbosity", dest="verbosity", default=False, action="store_true", help="Turn on logging verbosity."
    )

    return parser
Пример #19
0
  def __init__(self,
               name,
               source=None,
               dependencies=None,
               entry_point=None,
               inherit_path=False,
               zip_safe=True,
               repositories=None,
               indices=None,
               ignore_errors=False,
               allow_pypi=False,
               platforms=(Platform.current(),),
               interpreters=(sys.version[:3],)):
    """
      name: target name

      source: the python source file that becomes this binary's __main__ [optional]
              if none specified, drops into an interpreter by default

      dependencies: a list of other PythonLibrary or Pants targets this binary depends upon

      entry_point: the default entry point for this binary (by default drops
                   into the entry point defined by @source)

      inherit_path: inherit the sys.path of the environment that this binary runs in

      zip_safe: whether or not this binary is safe to run in compacted (zip-file) form

      repositories: a list of repositories to query for dependencies

      indices: a list of indices to use for packages

      allow_pypi: whether or not this binary should be allowed to hit pypi for dependency
                  management

      platforms: the platforms to target when building this binary.  by
                 default the current platform.

      interpreters: the interpreter versions to target when building this binary.  by default the
                    current interpreter version (specify in the form: '2.6', '2.7', '3.2' etc.)
    """
    if source is None and dependencies is None:
      raise TargetDefinitionException(
          'ERROR: no source or dependencies declared for target %s' % name)
    if source and entry_point:
      raise TargetDefinitionException(
          'Can only declare an entry_point if no source binary is specified.')
    if not isinstance(platforms, (list, tuple)) or not isinstance(interpreters, (list, tuple)):
      raise TargetDefinitionException('platforms and interpreters must be lists or tuples.')

    self._entry_point = entry_point
    self._inherit_path = bool(inherit_path)
    self._zip_safe = bool(zip_safe)
    self._platforms = platforms
    self._interpreters = interpreters
    self._repositories = repositories or []
    self._indices = indices or []
    self._allow_pypi = bool(allow_pypi)
    self._ignore_errors = bool(ignore_errors)

    PythonTarget.__init__(self, name, [] if source is None else [source], dependencies=dependencies)
Пример #20
0
 def test_pure_python(self):
   assert Platform.compatible(None, None)
   assert Platform.compatible(None, 'i386')
   assert Platform.compatible(None, 'universal')
Пример #21
0
                                       stderr=err_fd,
                                       close_fds=True,
                                       cwd=get_buildroot())
            with _safe_open(self._pidfile, 'w') as pidfile:
                pidfile.write('%d' % process.pid)
            log.debug('Spawned ng server @ %d' % process.pid)
            # Prevents finally blocks being executed, unlike sys.exit(). We don't want to execute finally
            # blocks because we might, e.g., clean up tempfiles that the parent still needs.
            os._exit(0)


# Pick implementations for killall and _find. We don't use psutil, as it uses
# native code and so is not portable, leading to packaging and deployment headaches.
# TODO: Extract this to a class and add a paired test guarded by
# http://pytest.org/latest/skipping.html#skipping.
plat = Platform.current()
if plat.startswith('linux') or plat.startswith('macosx'):
    # TODO: add other platforms as needed, after checking that these cmds work there as expected.

    # Returns the cmd's output, as a list of lines, including the newline characters.
    def _run_cmd(cmd):
        runcmd = cmd + ' && echo "\n${PIPESTATUS[*]}"'
        popen = subprocess.Popen(runcmd,
                                 shell=True,
                                 executable='/bin/bash',
                                 bufsize=-1,
                                 close_fds=True,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)
        (stdout_data, _) = popen.communicate()
        stdout_data_lines = [
Пример #22
0
 def test_get_current_platform(self):
   expected_platforms = [Platform.current(), 'linux-x86_64']
   self.assertEqual(expected_platforms,
                    list(get_platforms(self.config.getlist('python-setup', 'platforms'))))
Пример #23
0
class NailgunProcessManager(object):
  """A container for some gnarly process id munging logic."""
  # Verify that the gnarly logic works on this platform.
  plat = Platform.current()
  if not (plat.startswith('linux') or plat.startswith('macosx')):
    raise NotImplementedError('Platform %s not supported by pants.' % plat)

  @staticmethod
  def _run_cmd(cmd):
    # Returns the cmd's output, as a list of lines, including the newline characters.
    runcmd = cmd + ' && echo "${PIPESTATUS[*]}"'
    popen = subprocess.Popen(runcmd, shell=True, executable='/bin/bash', bufsize=-1, close_fds=True,
                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    (stdout_data, _) = popen.communicate()
    stdout_data_lines = [line for line in stdout_data.strip().split('\n') if line]
    if not stdout_data_lines:
      return None #raise NailgunError('No output for command (%s)' % runcmd)
    try:
      # Get the return codes of each piped cmd.
      piped_return_codes = [int(x) for x in stdout_data_lines[-1].split(' ') if x]
    except ValueError:
      raise NailgunError('Failed to parse result (%s) for command (%s)' % (stdout_data_lines, cmd))
      # Drop the echoing of PIPESTATUS, which our caller doesn't care about.
    stdout_data_lines = stdout_data_lines[:-1]
    failed = any(piped_return_codes)
    if failed:
      raise NailgunError('Failed to execute cmd: "%s". Exit codes: %s. Output: "%s"' %\
                         (cmd, piped_return_codes, ''.join(stdout_data_lines)))
    return stdout_data_lines

  @staticmethod
  def _find_matching_pids(strs):
    # Grep all processes whose cmd-lines contain all the strs, except for the grep process itself.
    filters = ' | '.join(["grep -F -e '%s'" % s for s in strs])
    data = NailgunProcessManager._run_cmd(
      "ps axwww | %s | (grep -v grep || true) | cut -b 1-5" % filters)
    pids = [int(x.strip()) for x in data if x]
    return pids

  @staticmethod
  def _find_ngs(everywhere=False):
    arg = NailgunTask.PANTS_NG_ARG_PREFIX if everywhere else NailgunTask.PANTS_NG_ARG
    return NailgunProcessManager._find_matching_pids([arg])

  @staticmethod
  def killall(log, everywhere=False):
    for pid in NailgunProcessManager._find_ngs(everywhere=everywhere):
      try:
        if log:
          NailgunTask._log_kill(log, pid)
        os.kill(pid, signal.SIGKILL)
      except OSError:
        pass

  DIGITS_RE = re.compile('^\d+$')
  @staticmethod
  def find(identifier_arg):
    pids = NailgunProcessManager._find_matching_pids([NailgunTask.PANTS_NG_ARG, identifier_arg])
    if len(pids) != 1:
      return None
    pid = pids[0]

    # Expected output of the lsof cmd: pPID\nn[::127.0.0.1]:PORT
    lines = NailgunProcessManager._run_cmd('lsof -a -p %s -i TCP -s TCP:LISTEN -P -Fn' % pid)
    if lines is None or len(lines) != 2 or lines[0] != 'p%s' % pid:
      return None
    port = lines[1][lines[1].rfind(':') + 1:].strip()
    if not NailgunProcessManager.DIGITS_RE.match(port):
      return None
    return pid, int(port)
Пример #24
0
  def test_cross_platform(self):
    assert not Platform.compatible('linux-x86_64', 'macosx-10.0-x86_64')

    # TODO(wickman): Should we do extended platform support beyond OS X?
    assert not Platform.compatible('linux-i386', 'linux-x86_64')
Пример #25
0
 def translate(platform):
     return Platform.current() if platform == 'current' else platform
Пример #26
0
 def can_add(self, dist):
   return Platform.distribution_compatible(dist, python=self.python, platform=self.platform)
Пример #27
0
def configure_clp():
  usage = (
      '%prog [options]\n\n'
      '%prog builds a PEX (Python Executable) file based on the given specifications: '
      'sources, requirements, their dependencies and other options')

  parser = OptionParser(usage=usage, version='%prog 0.2')

  parser.add_option(
      '--pypi', '--no-pypi',
      dest='pypi',
      default=True,
      action='callback',
      callback=parse_bool,
      help='Whether to use pypi to resolve dependencies; Default: use pypi')

  parser.add_option(
      '--wheel', '--no-wheel',
      dest='use_wheel',
      default=True,
      action='callback',
      callback=parse_bool,
      help='Whether to allow wheel distributions; Default: allow wheels')

  parser.add_option(
      '--build', '--no-build',
      dest='allow_builds',
      default=True,
      action='callback',
      callback=parse_bool,
      help='Whether to allow building of distributions from source; Default: allow builds')

  parser.add_option(
      '--python',
      dest='python',
      default=None,
      help='The Python interpreter to use to build the pex.  Either specify an explicit '
           'path to an interpreter, or specify a binary accessible on $PATH. '
           'Default: Use current interpreter.')

  parser.add_option(
      '--platform',
      dest='platform',
      default=Platform.current(),
      help='The platform for which to build the PEX.  Default: %%default')

  parser.add_option(
      '--zip-safe', '--not-zip-safe',
      dest='zip_safe',
      default=True,
      action='callback',
      callback=parse_bool,
      help='Whether or not the sources in the pex file are zip safe.  If they are '
           'not zip safe, they will be written to disk prior to execution; '
           'Default: zip safe.')

  parser.add_option(
      '--always-write-cache',
      dest='always_write_cache',
      default=False,
      action='store_true',
      help='Always write the internally cached distributions to disk prior to invoking '
           'the pex source code.  This can use less memory in RAM constrained '
           'environments. [Default: %default]')

  parser.add_option(
      '--ignore-errors',
      dest='ignore_errors',
      default=False,
      action='store_true',
      help='Ignore run-time requirement resolution errors when invoking the pex. '
           '[Default: %default]')

  parser.add_option(
      '--inherit-path',
      dest='inherit_path',
      default=False,
      action='store_true',
      help='Inherit the contents of sys.path (including site-packages) running the pex. '
           '[Default: %default]')

  parser.add_option(
      '--cache-dir',
      dest='cache_dir',
      default=os.path.expanduser('~/.pex/build'),
      help='The local cache directory to use for speeding up requirement '
           'lookups; [Default: %default]')

  parser.add_option(
      '-o', '-p', '--output-file', '--pex-name',
      dest='pex_name',
      default=None,
      help='The name of the generated .pex file: Omiting this will run PEX '
           'immediately and not save it to a file.')

  parser.add_option(
      '-e', '--entry-point',
      dest='entry_point',
      default=None,
      help='The entry point for this pex; Omiting this will enter the python '
           'REPL with sources and requirements available for import.  Can be '
           'either a module or EntryPoint (module:function) format.')

  parser.add_option(
      '-r', '--requirement',
      dest='requirements',
      metavar='REQUIREMENT',
      default=[],
      action='append',
      help='requirement to be included; may be specified multiple times.')

  parser.add_option(
      '--repo',
      dest='repos',
      metavar='PATH',
      default=[],
      action='append',
      help='Additional repository path (directory or URL) to look for requirements.')

  parser.add_option(
      '-i', '--index',
      dest='indices',
      metavar='URL',
      default=[],
      action='append',
      help='Additional cheeseshop indices to use to satisfy requirements.')

  parser.add_option(
      '-s', '--source-dir',
      dest='source_dirs',
      metavar='DIR',
      default=[],
      action='append',
      help='Source to be packaged; This <DIR> should be a pip-installable project '
           'with a setup.py.')

  parser.add_option(
      '-v', '--verbosity',
      dest='verbosity',
      default=False,
      action='store_true',
      help='Turn on logging verbosity.')

  return parser
Пример #28
0
 def __init__(self, interpreter, platform=None):
   platform = platform or Platform.current()
   self.__interpreter = interpreter
   super(PantsEnvironment, self).__init__(
       search_path=[], python=interpreter.python, platform=platform)