Exemple #1
0
def make_executor(proxy_driver,
                  checkpoint_root,
                  task,
                  ports={},
                  fast_status=False,
                  runner_class=ThermosTaskRunner,
                  status_providers=()):

    status_manager_class = FastStatusManager if fast_status else StatusManager
    runner_provider = make_provider(checkpoint_root, runner_class)
    te = FastThermosExecutor(
        runner_provider=runner_provider,
        status_manager_class=status_manager_class,
        sandbox_provider=DefaultTestSandboxProvider,
        status_providers=status_providers,
    )

    ExecutorTimeout(te.launched,
                    proxy_driver,
                    timeout=Amount(100, Time.MILLISECONDS)).start()
    task_description = make_task(task, assigned_ports=ports, instanceId=0)
    te.launchTask(proxy_driver, task_description)

    te.status_manager_started.wait()
    sampled_metrics = te.metrics.sample()
    assert 'kill_manager.enabled' in sampled_metrics
    for checker in te._chained_checker._status_checkers:  # hacky
        assert ('%s.enabled' % checker.name()) in sampled_metrics

    while len(proxy_driver.method_calls['sendStatusUpdate']) < 2:
        time.sleep(0.1)

    # make sure startup was kosher
    updates = proxy_driver.method_calls['sendStatusUpdate']
    assert len(updates) == 2
    status_updates = [arg_tuple[0][0] for arg_tuple in updates]
    assert status_updates[0].state == mesos_pb2.TASK_STARTING
    assert status_updates[1].state == mesos_pb2.TASK_RUNNING

    # wait for the runner to bind to a task
    while True:
        runner = TaskRunner.get(task_description.task_id.value,
                                checkpoint_root)
        if runner:
            break
        time.sleep(0.1)

    assert te.launched.is_set()
    return runner, te
Exemple #2
0
def make_executor(
    proxy_driver,
    checkpoint_root,
    task,
    ports={},
    fast_status=False,
    runner_class=ThermosTaskRunner,
    status_providers=[HealthCheckerProvider()],
    assert_task_is_running=True,
    stop_timeout_in_secs=120):

  status_manager_class = FastStatusManager if fast_status else StatusManager
  runner_provider = make_provider(checkpoint_root, runner_class)
  te = FastThermosExecutor(
      runner_provider=runner_provider,
      status_manager_class=status_manager_class,
      sandbox_provider=DefaultTestSandboxProvider(),
      status_providers=status_providers,
      stop_timeout_in_secs=stop_timeout_in_secs
  )

  ExecutorTimeout(te.launched, proxy_driver, timeout=Amount(100, Time.MILLISECONDS)).start()
  task_description = make_task(task, assigned_ports=ports, instanceId=0)
  te.launchTask(proxy_driver, task_description)

  te.status_manager_started.wait()

  while len(proxy_driver.method_calls['sendStatusUpdate']) < 2:
    time.sleep(0.1)

  # make sure startup was kosher
  updates = proxy_driver.method_calls['sendStatusUpdate']
  assert len(updates) == 2
  status_updates = [arg_tuple[0][0] for arg_tuple in updates]
  assert status_updates[0].state == mesos_pb2.TASK_STARTING

  runner = None
  if assert_task_is_running:
    assert status_updates[1].state == mesos_pb2.TASK_RUNNING
    # wait for the runner to bind to a task
    while True:
      runner = TaskRunner.get(task_description.task_id.value, checkpoint_root)
      if runner:
        break
      time.sleep(0.1)

  assert te.launched.is_set()
  return runner, te
def make_executor(
    proxy_driver,
    checkpoint_root,
    task,
    ports={},
    fast_status=False,
    runner_class=ThermosTaskRunner,
    status_providers=[HealthCheckerProvider()],
    assert_task_is_running=True,
    stop_timeout_in_secs=120):

  status_manager_class = FastStatusManager if fast_status else StatusManager
  runner_provider = make_provider(checkpoint_root, runner_class)
  te = FastThermosExecutor(
      runner_provider=runner_provider,
      status_manager_class=status_manager_class,
      sandbox_provider=DefaultTestSandboxProvider(),
      status_providers=status_providers,
      stop_timeout_in_secs=stop_timeout_in_secs
  )

  ExecutorTimeout(te.launched, proxy_driver, timeout=Amount(100, Time.MILLISECONDS)).start()
  task_description = make_task(task, assigned_ports=ports, instanceId=0)
  te.launchTask(proxy_driver, task_description)

  te.status_manager_started.wait()

  while len(proxy_driver.method_calls['sendStatusUpdate']) < 2:
    time.sleep(0.1)

  # make sure startup was kosher
  updates = proxy_driver.method_calls['sendStatusUpdate']
  assert len(updates) == 2
  status_updates = [arg_tuple[0][0] for arg_tuple in updates]
  assert status_updates[0].state == mesos_pb2.TASK_STARTING

  runner = None
  if assert_task_is_running:
    assert status_updates[1].state == mesos_pb2.TASK_RUNNING
    # wait for the runner to bind to a task
    while True:
      runner = TaskRunner.get(task_description.task_id.value, checkpoint_root)
      if runner:
        break
      time.sleep(0.1)

  assert te.launched.is_set()
  return runner, te
def make_executor(
    proxy_driver,
    checkpoint_root,
    task,
    ports={},
    fast_status=False,
    runner_class=ThermosTaskRunner,
    status_providers=()):

  status_manager_class = FastStatusManager if fast_status else StatusManager
  runner_provider = make_provider(checkpoint_root, runner_class)
  te = FastThermosExecutor(
      runner_provider=runner_provider,
      status_manager_class=status_manager_class,
      sandbox_provider=DefaultTestSandboxProvider,
      status_providers=status_providers,
  )

  ExecutorTimeout(te.launched, proxy_driver, timeout=Amount(100, Time.MILLISECONDS)).start()
  task_description = make_task(task, assigned_ports=ports, instanceId=0)
  te.launchTask(proxy_driver, task_description)

  te.status_manager_started.wait()
  sampled_metrics = te.metrics.sample()
  assert 'kill_manager.enabled' in sampled_metrics
  for checker in te._chained_checker._status_checkers:  # hacky
    assert ('%s.enabled' % checker.name()) in sampled_metrics

  while len(proxy_driver.method_calls['sendStatusUpdate']) < 2:
    time.sleep(0.1)

  # make sure startup was kosher
  updates = proxy_driver.method_calls['sendStatusUpdate']
  assert len(updates) == 2
  status_updates = [arg_tuple[0][0] for arg_tuple in updates]
  assert status_updates[0].state == mesos_pb2.TASK_STARTING
  assert status_updates[1].state == mesos_pb2.TASK_RUNNING

  # wait for the runner to bind to a task
  while True:
    runner = TaskRunner.get(task_description.task_id.value, checkpoint_root)
    if runner:
      break
    time.sleep(0.1)

  assert te.launched.is_set()
  return runner, te
  def test_coordinator_dead_kill(self):
    runner = self.start_runner()
    tm = TaskMonitor(runner.pathspec, runner.task_id)
    self.wait_until_running(tm)
    process_state, run_number = tm.get_active_processes()[0]
    assert process_state.process == 'ignorant_process'
    assert run_number == 0

    os.kill(runner.po.pid, signal.SIGKILL)
    os.kill(process_state.coordinator_pid, signal.SIGKILL)
    os.kill(process_state.pid, signal.SIGKILL)

    killer = TaskRunner.get(runner.task_id, runner.root)
    assert killer is not None
    killer.kill(force=True)

    state = tm.get_state()
    assert len(state.processes['ignorant_process']) == 1
    assert state.processes['ignorant_process'][0].state == ProcessState.LOST
    def test_coordinator_dead_kill(self):
        runner = self.start_runner()
        tm = TaskMonitor(runner.tempdir, runner.task_id)
        self.wait_until_running(tm)
        process_state, run_number = tm.get_active_processes()[0]
        assert process_state.process == 'ignorant_process'
        assert run_number == 0

        os.kill(runner.po.pid, signal.SIGKILL)
        os.kill(process_state.coordinator_pid, signal.SIGKILL)
        os.kill(process_state.pid, signal.SIGKILL)

        killer = TaskRunner.get(runner.task_id, runner.root)
        assert killer is not None
        killer.kill(force=True)

        state = tm.get_state()
        assert len(state.processes['ignorant_process']) == 1
        assert state.processes['ignorant_process'][
            0].state == ProcessState.LOST
  def test_preemption_wait(self):
    runner = self.start_runner()
    tm = TaskMonitor(runner.pathspec, runner.task_id)
    self.wait_until_running(tm)
    process_state, run_number = tm.get_active_processes()[0]
    assert process_state.process == 'ignorant_process'
    assert run_number == 0

    preempter = TaskRunner.get(runner.task_id, runner.root)
    assert preempter is not None
    now = time.time()
    preempter.kill(force=True, preemption_wait=Amount(1, Time.SECONDS))
    duration = time.time() - now

    # This is arbitrary, but make sure we finish within half a second of
    # requested preemption wait.
    assert abs(duration - 1.0) < 0.5

    assert preempter.state.statuses[-1].state == TaskState.KILLED
    assert preempter.state.processes['ignorant_process'][-1].state == ProcessState.KILLED
    def test_preemption_wait(self):
        runner = self.start_runner()
        tm = TaskMonitor(runner.tempdir, runner.task_id)
        self.wait_until_running(tm)
        process_state, run_number = tm.get_active_processes()[0]
        assert process_state.process == 'ignorant_process'
        assert run_number == 0

        preempter = TaskRunner.get(runner.task_id, runner.root)
        assert preempter is not None
        now = time.time()
        preempter.kill(force=True, preemption_wait=Amount(1, Time.SECONDS))
        duration = time.time() - now

        # This is arbitrary, but make sure we finish within half a second of
        # requested preemption wait.
        assert abs(duration - 1.0) < 0.5

        assert preempter.state.statuses[-1].state == TaskState.KILLED
        assert preempter.state.processes['ignorant_process'][
            -1].state == ProcessState.KILLED