예제 #1
0
def select_binary(base_path, version, name, config=None):
  """Selects a binary matching the current os and architecture.

  :raises: :class:`pants.binary_util.BinaryUtil.BinaryNotFound` if no binary of the given version
    and name could be found.
  """
  # TODO(John Sirois): finish doc of the path structure expexcted under base_path
  config = config or Config.load()
  bootstrap_dir = config.getdefault('pants_bootstrapdir')

  binary_path = select_binary_base_path(base_path, version, name)
  bootstrapped_binary_path = os.path.join(bootstrap_dir, binary_path)
  if not os.path.exists(bootstrapped_binary_path):
    downloadpath = bootstrapped_binary_path + '~'
    try:
      with select_binary_stream(base_path, version, name, config) as stream:
        with safe_open(downloadpath, 'wb') as bootstrapped_binary:
          bootstrapped_binary.write(stream())
        os.rename(downloadpath, bootstrapped_binary_path)
        chmod_plus_x(bootstrapped_binary_path)
    finally:
      safe_delete(downloadpath)

  log.debug('Selected {binary} binary bootstrapped to: {path}'
            .format(binary=name, path=bootstrapped_binary_path))
  return bootstrapped_binary_path
예제 #2
0
def test_simple_process_filesystem_isolator():
    with temporary_dir() as td:
        taskpath = make_taskpath(td)
        sandbox = setup_sandbox(td, taskpath)

        test_isolator_path = os.path.join(td, "fake-mesos-containerier")
        with open(test_isolator_path, "w") as fd:
            # We use a fake version of the mesos-containerizer binary that just echoes out its args so
            # we can assert on them in the process's output.
            fd.write("\n".join(["#!/bin/sh", 'echo "$@"']))

            fd.close()

            chmod_plus_x(test_isolator_path)

            p = TestProcess(
                "process", "echo hello world", 0, taskpath, sandbox, mesos_containerizer_path=test_isolator_path
            )
            p.start()

        rc = wait_for_rc(taskpath.getpath("process_checkpoint"))
        assert rc == 0
        assert_log_content(
            taskpath,
            "stdout",
            "launch --unshare_namespace_mnt --rootfs=/some/path/taskfs --user=None "
            '--command={"shell":true,"value":"echo hello world"}\n',
        )
예제 #3
0
def test_simple_process_filesystem_isolator():
    with temporary_dir() as td:
        taskpath = make_taskpath(td)
        sandbox = setup_sandbox(td, taskpath)

        test_isolator_path = os.path.join(td, 'fake-mesos-containerier')
        with open(test_isolator_path, 'w') as fd:
            # We use a fake version of the mesos-containerizer binary that just echoes out its args so
            # we can assert on them in the process's output.
            fd.write('\n'.join(['#!/bin/sh', 'echo "$@"']))

            fd.close()

            chmod_plus_x(test_isolator_path)

            p = TestProcess('process',
                            'echo hello world',
                            0,
                            taskpath,
                            sandbox,
                            mesos_containerizer_path=test_isolator_path,
                            container_sandbox=sandbox)
            p.start()

        rc = wait_for_rc(taskpath.getpath('process_checkpoint'))
        assert rc == 0
        assert_log_content(
            taskpath, 'stdout',
            'launch --unshare_namespace_mnt --working_directory=%s --rootfs=/some/path/taskfs '
            '--user=None --command={"shell": false, "arguments": ["/bin/bash", "-c", '
            '"echo hello world"], "value": "/bin/bash"}\n' % (sandbox))
예제 #4
0
 def distribution(self, files=None, executables=None):
     with temporary_dir() as jdk:
         for f in maybe_list(files or ()):
             touch(os.path.join(jdk, f))
         for exe in maybe_list(executables or (), expected_type=self.EXE):
             path = os.path.join(jdk, exe.name)
             with safe_open(path, 'w') as fp:
                 fp.write(exe.contents or '')
             chmod_plus_x(path)
         yield jdk
예제 #5
0
 def write(self, filename):
   chroot = self._env.chroot().dup()
   chroot.zip(filename + '~')
   with open(filename, "wb") as pexfile:
     pexfile.write(Compatibility.to_bytes('%s\n' % self._identity.hashbang()))
     with open(filename + '~', 'rb') as pexfile_zip:
       pexfile.write(pexfile_zip.read())
   chroot.delete()
   os.unlink(filename + '~')
   chmod_plus_x(filename)
예제 #6
0
 def write(self, filename):
   chroot = self._env.chroot().dup()
   chroot.zip(filename + '~')
   with open(filename, "w") as pexfile:
     pexfile.write('#!/usr/bin/env python2.6\n')
     with open(filename + '~') as pexfile_zip:
       pexfile.write(pexfile_zip.read())
   chroot.delete()
   os.unlink(filename + '~')
   chmod_plus_x(filename)
예제 #7
0
 def jre(self, env_var):
   with temporary_dir() as jre:
     path = os.path.join(jre, 'java')
     with safe_open(path, 'w') as fp:
       fp.write(textwrap.dedent("""
           #!/bin/sh
           echo ${env_var}
         """.format(env_var=env_var)).strip())
     chmod_plus_x(path)
     yield jre
예제 #8
0
 def distribution(self, files=None, executables=None):
   with temporary_dir() as jdk:
     for f in maybe_list(files or ()):
       touch(os.path.join(jdk, f))
     for exe in maybe_list(executables or (), expected_type=self.EXE):
       path = os.path.join(jdk, exe.name)
       with safe_open(path, 'w') as fp:
         fp.write(exe.contents or '')
       chmod_plus_x(path)
     yield jdk
예제 #9
0
def select_binary(base_path, version, name, config=None):
    """Selects a binary matching the current os and architecture.

  Raises TaskError if no binary of the given version and name could be found.
  """
    # TODO(John Sirois): finish doc of the path structure expexcted under base_path
    config = config or Config.load()
    bootstrap_dir = config.getdefault('pants_bootstrapdir',
                                      default=os.path.expanduser('~/.pants.d'))
    baseurl = config.getdefault('pants_support_baseurl')
    timeout_secs = config.getdefault('pants_support_fetch_timeout_secs',
                                     type=int,
                                     default=30)

    sysname, _, release, _, machine = os.uname()
    os_id = _ID_BY_OS[sysname.lower()]
    if os_id:
        middle_path = _PATH_BY_ID[os_id(release, machine)]
        if middle_path:
            binary_path = os.path.join(base_path,
                                       *(middle_path + [version, name]))
            bootstrapped_binary_path = os.path.join(bootstrap_dir, binary_path)
            if not os.path.exists(bootstrapped_binary_path):
                url = posixpath.join(baseurl, binary_path)
                log.info('Fetching %s binary from: %s' % (name, url))
                downloadpath = bootstrapped_binary_path + '~'
                try:
                    with closing(
                            urllib_request.urlopen(
                                url, timeout=timeout_secs)) as binary:
                        with safe_open(downloadpath,
                                       'wb') as bootstrapped_binary:
                            bootstrapped_binary.write(binary.read())

                    os.rename(downloadpath, bootstrapped_binary_path)
                    chmod_plus_x(bootstrapped_binary_path)
                except (IOError, urllib_error.HTTPError,
                        urllib_error.URLError) as e:
                    raise TaskError('Failed to fetch binary from %s: %s' %
                                    (url, e))
                finally:
                    safe_delete(downloadpath)
            log.debug('Selected %s binary bootstrapped to: %s' %
                      (name, bootstrapped_binary_path))
            return bootstrapped_binary_path
    raise TaskError('No %s binary found for: %s' %
                    (name, (sysname, release, machine)))
예제 #10
0
 def build(self, filename):
   self.freeze()
   try:
     os.unlink(filename + '~')
     self._logger.warn('Previous binary unexpectedly exists, cleaning: %s' % (filename + '~'))
   except OSError:
     # The expectation is that the file does not exist, so continue
     pass
   with open(filename + '~', 'ab') as pexfile:
     assert os.path.getsize(pexfile.name) == 0
     # TODO(wickman) Make this tunable
     pexfile.write(Compatibility.to_bytes('%s\n' % PythonIdentity.get().hashbang()))
   self._chroot.zip(filename + '~', mode='a')
   if os.path.exists(filename):
     os.unlink(filename)
   os.rename(filename + '~', filename)
   chmod_plus_x(filename)
예제 #11
0
 def build(self, filename):
   self.freeze()
   try:
     os.unlink(filename + '~')
     print('WARNING: Previous binary unexpectedly exists, cleaning: %s' % (filename + '~'))
   except OSError:
     # The expectation is that the file does not exist, so continue
     pass
   with open(filename + '~', 'ab') as pexfile:
     assert os.path.getsize(pexfile.name) == 0
     # TODO(wickman) Make this tunable
     pexfile.write(Compatibility.to_bytes('%s\n' % PythonIdentity.get().hashbang()))
   self._chroot.zip(filename + '~', mode='a')
   if os.path.exists(filename):
     os.unlink(filename)
   os.rename(filename + '~', filename)
   chmod_plus_x(filename)
예제 #12
0
def test_simple_process_filesystem_isolator_launch_info():
  with temporary_dir() as td:
    taskpath = make_taskpath(td)
    sandbox = setup_sandbox(td, taskpath)

    test_isolator_path = os.path.join(td, 'fake-mesos-containerier')
    with open(test_isolator_path, 'w') as fd:
      # We use a fake version of the mesos-containerizer binary that just echoes out its args so
      # we can assert on them in the process's output. Also imitates a failure when there are not
      # enough arguments, this is used to find the version of the binary (by checking the failure
      # message)
      fd.write('\n'.join([
        '#!/bin/sh',
        'if [ "$#" -eq 1 ]; then',
        '  { echo "launch_info" >&2; };',
        'else',
        '  echo "$@";',
        'fi'
      ]))

      fd.close()

      chmod_plus_x(test_isolator_path)

      p = TestProcess(
          'process',
          'echo hello world',
          0,
          taskpath,
          sandbox,
          mesos_containerizer_path=test_isolator_path,
          container_sandbox=sandbox)
      p.start()

    rc = wait_for_rc(taskpath.getpath('process_checkpoint'))
    assert rc == 0
    assert_log_content(
        taskpath,
        'stdout',
        'launch --unshare_namespace_mnt --launch_info={"command": {"shell": false, "arguments": '
        '["/bin/bash", "-c", "echo hello world"], "value": "/bin/bash"}, '
        '"working_directory": "%s", "user": "******", "rootfs": "/some/path/taskfs"}\n' % (sandbox))
예제 #13
0
    def start(self, timeout=MAX_WAIT):
        """Fork the task runner and return once the underlying task is running, up to timeout."""
        self.forking.set()

        try:
            chmod_plus_x(self._runner_pex)
        except OSError as e:
            if e.errno != errno.EPERM:
                raise TaskError('Failed to chmod +x runner: %s' % e)

        self._monitor = TaskMonitor(TaskPath(root=self._checkpoint_root),
                                    self._task_id)

        cmdline_args = self._cmdline()
        log.info('Forking off runner with cmdline: %s' %
                 ' '.join(cmdline_args))

        try:
            self._popen = subprocess.Popen(cmdline_args)
        except OSError as e:
            raise TaskError(e)

        self.forked.set()

        log.debug('Waiting for task to start.')

        def is_started():
            return self._monitor and (self._monitor.active
                                      or self._monitor.finished)

        waited = Amount(0, Time.SECONDS)
        while not is_started() and waited < timeout:
            log.debug('  - sleeping...')
            self._clock.sleep(self.POLL_INTERVAL.as_(Time.SECONDS))
            waited += self.POLL_INTERVAL

        if not is_started():
            log.error('Task did not start with in deadline, forcing loss.')
            self.lose()
            raise TaskError('Task did not start within deadline.')
  def start(self, timeout=MAX_WAIT):
    """Fork the task runner and return once the underlying task is running, up to timeout."""
    self.forking.set()

    try:
      chmod_plus_x(self._runner_pex)
    except OSError as e:
      if e.errno != errno.EPERM:
        raise TaskError('Failed to chmod +x runner: %s' % e)

    self._monitor = TaskMonitor(TaskPath(root=self._checkpoint_root), self._task_id)

    cmdline_args = self._cmdline()
    log.info('Forking off runner with cmdline: %s' % ' '.join(cmdline_args))

    try:
      self._popen = subprocess.Popen(cmdline_args)
    except OSError as e:
      raise TaskError(e)

    self.forked.set()

    self.wait_start(timeout=timeout)
  def start(self, timeout=MAX_WAIT):
    """Fork the task runner and return once the underlying task is running, up to timeout."""
    self.forking.set()

    try:
      chmod_plus_x(self._runner_pex)
    except OSError as e:
      if e.errno != errno.EPERM:
        raise TaskError('Failed to chmod +x runner: %s' % e)

    self._monitor = TaskMonitor(TaskPath(root=self._checkpoint_root), self._task_id)

    cmdline_args = self._cmdline()
    log.info('Forking off runner with cmdline: %s' % ' '.join(cmdline_args))

    try:
      self._popen = subprocess.Popen(cmdline_args)
    except OSError as e:
      raise TaskError(e)

    self.forked.set()

    log.debug('Waiting for task to start.')

    def is_started():
      return self._monitor and (self._monitor.active or self._monitor.finished)

    waited = Amount(0, Time.SECONDS)
    while not is_started() and waited < timeout:
      log.debug('  - sleeping...')
      self._clock.sleep(self.POLL_INTERVAL.as_(Time.SECONDS))
      waited += self.POLL_INTERVAL

    if not is_started():
      log.error('Task did not start with in deadline, forcing loss.')
      self.lose()
      raise TaskError('Task did not start within deadline.')
예제 #16
0
def test_simple_process_filesystem_isolator_launch_info():
    with temporary_dir() as td:
        taskpath = make_taskpath(td)
        sandbox = setup_sandbox(td, taskpath)

        test_isolator_path = os.path.join(td, 'fake-mesos-containerier')
        with open(test_isolator_path, 'w') as fd:
            # We use a fake version of the mesos-containerizer binary that just echoes out its args so
            # we can assert on them in the process's output. Also imitates a failure when there are not
            # enough arguments, this is used to find the version of the binary (by checking the failure
            # message)
            fd.write('\n'.join([
                '#!/bin/sh', 'if [ "$#" -eq 1 ]; then',
                '  { echo "launch_info" >&2; };', 'else', '  echo "$@";', 'fi'
            ]))

            fd.close()

            chmod_plus_x(test_isolator_path)

            p = FakeProcess('process',
                            'echo hello world',
                            0,
                            taskpath,
                            sandbox,
                            mesos_containerizer_path=test_isolator_path,
                            container_sandbox=sandbox)
            p.start()

        rc = wait_for_rc(taskpath.getpath('process_checkpoint'))
        assert rc == 0
        assert_log_content(
            taskpath, 'stdout',
            'launch --unshare_namespace_mnt --launch_info={"command": {"shell": false, "arguments": '
            '["/bin/bash", "-c", "echo hello world"], "value": "/bin/bash"}, '
            '"working_directory": "%s", "user": "******", "rootfs": "/some/path/taskfs"}\n'
            % (sandbox))
예제 #17
0
def select_binary(base_path, version, name, config=None):
  """Selects a binary matching the current os and architecture.

  Raises TaskError if no binary of the given version and name could be found.
  """
  # TODO(John Sirois): finish doc of the path structure expexcted under base_path
  config = config or Config.load()
  cachedir = config.getdefault('pants_cachedir', default=os.path.expanduser('~/.pants.d'))
  baseurl = config.getdefault('pants_support_baseurl')
  timeout_secs = config.getdefault('pants_support_fetch_timeout_secs', type=int, default=30)

  sysname, _, release, _, machine = os.uname()
  os_id = _ID_BY_OS[sysname.lower()]
  if os_id:
    middle_path = _PATH_BY_ID[os_id(release, machine)]
    if middle_path:
      binary_path = os.path.join(base_path, *(middle_path + [version, name]))
      cached_binary_path = os.path.join(cachedir, binary_path)
      if not os.path.exists(cached_binary_path):
        url = posixpath.join(baseurl, binary_path)
        log.info('Fetching %s binary from: %s' % (name, url))
        downloadpath = cached_binary_path + '~'
        try:
          with closing(urllib_request.urlopen(url, timeout=timeout_secs)) as binary:
            with safe_open(downloadpath, 'wb') as cached_binary:
              cached_binary.write(binary.read())

          os.rename(downloadpath, cached_binary_path)
          chmod_plus_x(cached_binary_path)
        except (IOError, urllib_error.HTTPError, urllib_error.URLError) as e:
          raise TaskError('Failed to fetch binary from %s: %s' % (url, e))
        finally:
          safe_delete(downloadpath)
      log.debug('Selected %s binary cached at: %s' % (name, cached_binary_path))
      return cached_binary_path
  raise TaskError('No %s binary found for: %s' % (name, (sysname, release, machine)))
예제 #18
0
def test_simple_process_filesystem_isolator():
  with temporary_dir() as td:
    taskpath = make_taskpath(td)
    sandbox = setup_sandbox(td, taskpath)

    test_isolator_path = os.path.join(td, 'fake-mesos-containerier')
    with open(test_isolator_path, 'w') as fd:
      # We use a fake version of the mesos-containerizer binary that just echoes out its args so
      # we can assert on them in the process's output.
      fd.write('\n'.join([
        '#!/bin/sh',
        'echo "$@"'
      ]))

      fd.close()

      chmod_plus_x(test_isolator_path)

      p = TestProcess(
          'process',
          'echo hello world',
          0,
          taskpath,
          sandbox,
          mesos_containerizer_path=test_isolator_path,
          container_sandbox=sandbox)
      p.start()

    rc = wait_for_rc(taskpath.getpath('process_checkpoint'))
    assert rc == 0
    assert_log_content(
        taskpath,
        'stdout',
        'launch --unshare_namespace_mnt --working_directory=%s --rootfs=/some/path/taskfs '
        '--user=None --command={"shell":true,"value":"/bin/bash -c \'echo hello world\'"}\n' % (
            sandbox))