def restart_job(self, cluster, role, environment, jobname, jobspec=None, instances=[]):
        """Method to restart aurora job"""

        job_key = AuroraJobKey(cluster, role, environment, jobname)
        logger.info("request to restart => %s", job_key.to_path())

        instances = self.pack_instance_list(instances)
        try:
            config = self.make_job_config(job_key, jobspec)
        except Exception as e:
            return(job_key.to_path(), ["Failed to restart Aurora job",
                                       "Can not create job configuration object because", str(e)])

        # these are the default values from apache.aurora.client.commands.core.restart()
        updater_config = UpdaterConfig(
            1,          # options.batch_size
            60,         # options.restart_threshold
            30,         # options.watch_secs
            0,          # options.max_per_shard_failures
            0           # options.max_total_failures
        )

        api = make_client(job_key.cluster)
        # instances = all shards, health check = 3 sec
        resp = api.restart(job_key, instances, updater_config, 3, config=config)
        if resp.responseCode != ResponseCode.OK:
            logger.warning("aurora -- restart job failed")
            responseStr = self.response_string(resp)
            logger.warning(responseStr)
            return(job_key.to_path(), ["Error reported by aurora client:", responseStr])

        logger.info("aurora -- restart job successful")
        return(job_key.to_path(), None)
Example #2
0
 def get_status_query_response(cls):
     query_response = Response()
     query_response.responseCode = ResponseCode.OK
     query_response.result = Result()
     summaries = GetJobUpdateSummariesResult()
     query_response.result.getJobUpdateSummariesResult = summaries
     summaries.updateSummaries = [
         JobUpdateSummary(
             updateId="hello",
             jobKey=AuroraJobKey('west', 'mcc', 'test', 'hello'),
             user="******",
             state=JobUpdateState(status=JobUpdateStatus.ROLLING_FORWARD,
                                  createdTimestampMs=1411404927,
                                  lastModifiedTimestampMs=14114056030)),
         JobUpdateSummary(
             updateId="goodbye",
             jobKey=AuroraJobKey('west', 'mch', 'prod', 'goodbye'),
             user="******",
             state=JobUpdateState(status=JobUpdateStatus.ROLLING_BACK,
                                  createdTimestampMs=1411300632,
                                  lastModifiedTimestampMs=14114092632)),
         JobUpdateSummary(updateId="gasp",
                          jobKey=AuroraJobKey('west', 'mcq', 'devel',
                                              'gasp'),
                          user="******",
                          state=JobUpdateState(
                              status=JobUpdateStatus.ROLL_FORWARD_PAUSED,
                              createdTimestampMs=1411600891,
                              lastModifiedTimestampMs=1411800891))
     ]
     return query_response
Example #3
0
 def execute(self, context):
   cluster = context.options.cluster
   api = context.get_api(cluster)
   response = api.query_job_updates(
       role=context.options.role,
       job_key=context.options.jobspec,
       user=context.options.user,
       update_statuses=context.options.status)
   context.log_response_and_raise(response)
   if context.options.write_json:
     result = []
     for summary in response.result.getJobUpdateSummariesResult.updateSummaries:
       job_entry = {
           "jobkey": AuroraJobKey.from_thrift(cluster, summary.jobKey).to_path(),
           "id": summary.updateId,
           "user": summary.user,
           "started": summary.state.createdTimestampMs,
           "lastModified": summary.state.lastModifiedTimestampMs,
           "status": JobUpdateStatus._VALUES_TO_NAMES[summary.state.status]
       }
       result.append(job_entry)
     context.print_out(json.dumps(result, indent=2, separators=[',', ': '], sort_keys=False))
   else:
     for summary in response.result.getJobUpdateSummariesResult.updateSummaries:
       created = summary.state.createdTimestampMs
       lastMod = summary.state.lastModifiedTimestampMs
       context.print_out("Job: %s, Id: %s, User: %s, Status: %s" % (
           AuroraJobKey.from_thrift(cluster, summary.jobKey).to_path(),
           summary.updateId,
           summary.user,
           JobUpdateStatus._VALUES_TO_NAMES[summary.state.status]))
       context.print_out("Created: %s, Last Modified %s" % (created, lastMod), indent=2)
   return EXIT_OK
Example #4
0
  def execute(self, context):
    update_filter = context.options.filter
    cluster = update_filter.cluster
    if (update_filter.role is not None
        and update_filter.env is not None
        and update_filter.job is not None):

      job_key = AuroraJobKey(
          cluster=cluster,
          role=update_filter.role,
          env=update_filter.env,
          name=update_filter.job)
    else:
      job_key = None

    api = context.get_api(cluster)

    filter_statuses = set()
    for status in context.options.status:
      filter_statuses = filter_statuses.union(set(self.STATUS_GROUPS[status]))

    response = api.query_job_updates(
        role=update_filter.role if job_key is None else None,
        job_key=job_key,
        update_statuses=filter_statuses if filter_statuses else None,
        user=context.options.user)
    context.log_response_and_raise(response)

    # The API does not offer a way to query by environment, so if that filter is requested, we
    # perform a more broad role-based query and filter here.
    summaries = response.result.getJobUpdateSummariesResult.updateSummaries
    if job_key is None and update_filter.env is not None:
      summaries = [s for s in summaries if s.key.job.environment == update_filter.env]

    if context.options.write_json:
      result = []
      for summary in summaries:
        job_entry = {
            "job": AuroraJobKey.from_thrift(cluster, summary.key.job).to_path(),
            "id": summary.key.id,
            "user": summary.user,
            "started": format_timestamp(summary.state.createdTimestampMs),
            "last_modified": format_timestamp(summary.state.lastModifiedTimestampMs),
            "status": JobUpdateStatus._VALUES_TO_NAMES[summary.state.status]
        }
        result.append(job_entry)
      context.print_out(json.dumps(result, indent=2, separators=[',', ': '], sort_keys=False))
    else:
      if summaries:
        context.print_out(self.HEADER)
      for summary in summaries:
        context.print_out(self.FORMAT_STR.format(
            AuroraJobKey.from_thrift(cluster, summary.key.job).to_path(),
            summary.key.id,
            JobUpdateStatus._VALUES_TO_NAMES[summary.state.status],
            summary.user,
            format_timestamp(summary.state.createdTimestampMs),
            format_timestamp(summary.state.lastModifiedTimestampMs))
        )
    return EXIT_OK
Example #5
0
 def test_query_job_updates(self):
     """Test querying job updates."""
     api, mock_proxy = self.mock_api()
     job_key = AuroraJobKey("foo", "role", "env", "name")
     query = JobUpdateQuery(jobKey=job_key.to_thrift(), updateStatuses=set([JobUpdateStatus.ROLLING_FORWARD]))
     api.query_job_updates(job_key=job_key, update_statuses=query.updateStatuses)
     mock_proxy.getJobUpdateSummaries.assert_called_once_with(query)
Example #6
0
 def setUp(self):
   self._scheduler = Mock()
   self._sla = Sla(self._scheduler)
   self._cluster = Cluster(name='cl')
   self._role = 'mesos'
   self._name = 'job'
   self._env = 'test'
   self._job_key = AuroraJobKey(self._cluster.name, self._role, self._env, self._name)
   self._min_count = 1
Example #7
0
 def test_query_job_updates(self):
   """Test querying job updates."""
   api, mock_proxy = self.mock_api()
   job_key = AuroraJobKey("foo", "role", "env", "name")
   query = JobUpdateQuery(
       jobKey=job_key.to_thrift(),
       updateStatuses={JobUpdateStatus.ROLLING_FORWARD})
   api.query_job_updates(job_key=job_key, update_statuses=query.updateStatuses)
   mock_proxy.getJobUpdateSummaries.assert_called_once_with(query)
Example #8
0
  def test_add_instances(self):
    """Test adding instances."""
    api, mock_proxy = self.mock_api()
    job_key = AuroraJobKey("foo", "role", "env", "name")
    mock_proxy.addInstances.return_value = self.create_simple_success_response()
    api.add_instances(job_key, 1, 10)

    mock_proxy.addInstances.assert_called_once_with(
        InstanceKey(jobKey=job_key.to_thrift(), instanceId=1),
        10)
Example #9
0
    def test_add_instances(self):
        """Test adding instances."""
        api, mock_proxy = self.mock_api()
        job_key = AuroraJobKey("foo", "role", "env", "name")
        mock_proxy.addInstances.return_value = self.create_simple_success_response(
        )
        api.add_instances(job_key, 1, 10)

        mock_proxy.addInstances.assert_called_once_with(
            InstanceKey(jobKey=job_key.to_thrift(), instanceId=1), 10)
Example #10
0
 def assert_kill_calls(cls, api, instance_range=None, instances=None):
     if instances:
         kill_calls = [
             call(AuroraJobKey.from_path(cls.TEST_JOBSPEC), instances)
         ]
     else:
         kill_calls = [
             call(AuroraJobKey.from_path(cls.TEST_JOBSPEC), [i])
             for i in instance_range
         ]
     assert api.kill_job.mock_calls == kill_calls
Example #11
0
    def test_inequality(self):
        base = AuroraJobKey('cluster', 'role', 'env', 'name')
        keys = [
            AuroraJobKey('XXXXXXX', 'role', 'env', 'name'),
            AuroraJobKey('cluster', 'XXXX', 'env', 'name'),
            AuroraJobKey('cluster', 'role', 'XXX', 'name'),
            AuroraJobKey('cluster', 'role', 'env', 'XXXX')
        ]

        for key in keys:
            assert base != key
            assert not (base == key)
Example #12
0
def instance_specifier(spec_str):
  if spec_str is None or spec_str == '':
    raise ValueError('Instance specifier must be non-empty')
  parts = spec_str.split('/')
  if len(parts) == 4:
    jobkey = AuroraJobKey(*parts)
    return TaskInstanceKey(jobkey, ALL_INSTANCES)
  elif len(parts) != 5:
    raise ArgumentTypeError('Instance specifier must be a CLUSTER/ROLE/ENV/JOB/INSTANCES tuple')
  (cluster, role, env, name, instance_str) = parts
  jobkey = AuroraJobKey(cluster, role, env, name)
  instances = parse_instances(instance_str)
  return TaskInstanceKey(jobkey, instances)
Example #13
0
    def test_task_query(self):
        jobs = set([
            AuroraJobKey(self._cluster.name, self._role, self._env, 'j1'),
            AuroraJobKey(self._cluster.name, self._role, self._env, 'j2'),
            AuroraJobKey(self._cluster.name, self._role, self._env, 'j3'),
            AuroraJobKey(self._cluster.name, self._role, self._env, 'j4')
        ])

        query = task_query(job_keys=jobs)
        assert len(jobs) == len(
            query.jobKeys), 'Expected length:%s, Actual:%s' % (
                len(jobs), len(query.jobKeys))
        assert LIVE_STATES == query.statuses, 'Expected:%s, Actual:%s' % (
            LIVE_STATES, query.statuses)
Example #14
0
 def test_disambiguate_args_or_die_unambiguous_with_no_config(self):
     expected = (self._api,
                 AuroraJobKey(self.CLUSTER.name, self.ROLE, self.ENV,
                              self.NAME), None)
     result = LiveJobDisambiguator.disambiguate_args_or_die(
         [self.JOB_PATH], None, client_factory=lambda *_: self._api)
     assert result == expected
Example #15
0
  def test_killall_job(self):
    """Test kill client-side API logic."""
    mock_context = FakeAuroraCommandContext()
    mock_scheduler_proxy = Mock()
    with contextlib.nested(
        patch('time.sleep'),
        patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context),
        patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):

      api = mock_context.get_api('west')
      mock_scheduler_proxy.getTasksStatus.return_value = self.create_status_call_result()
      api.kill_job.return_value = self.get_kill_job_response()
      mock_scheduler_proxy.killTasks.return_value = self.get_kill_job_response()
      mock_context.add_expected_status_query_result(self.create_status_call_result(
          self.create_mock_task(ScheduleStatus.KILLED)))
      with temporary_file() as fp:
        fp.write(self.get_valid_config())
        fp.flush()
        cmd = AuroraCommandLine()
        cmd.execute(['job', 'killall', '--no-batching', '--config=%s' % fp.name,
            'west/bozo/test/hello'])

      # Now check that the right API calls got made.
      assert api.kill_job.call_count == 1
      api.kill_job.assert_called_with(AuroraJobKey.from_path('west/bozo/test/hello'), None)
      self.assert_scheduler_called(api, self.get_expected_task_query(), 2)
  def test_start_update_command_line_succeeds(self):
    mock_context = FakeAuroraCommandContext()
    resp = self.create_simple_success_response()
    resp.result = Result(startJobUpdateResult=StartJobUpdateResult(updateId="id"))
    with contextlib.nested(
        patch('apache.aurora.client.cli.update.Update.create_context', return_value=mock_context),
        patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):
      mock_api = mock_context.get_api(self.TEST_CLUSTER)
      mock_api.start_job_update.return_value = resp
      with temporary_file() as fp:
        fp.write(self.get_valid_config())
        fp.flush()
        cmd = AuroraCommandLine()
        result = cmd.execute(['beta-update', 'start', self.TEST_JOBSPEC, fp.name])
        assert result == EXIT_OK

      update_url_msg = StartUpdate.UPDATE_MSG_TEMPLATE % (
          mock_context.get_update_page(mock_api, AuroraJobKey.from_path(self.TEST_JOBSPEC), "id"))

      assert mock_api.start_job_update.call_count == 1
      args, kwargs = mock_api.start_job_update.call_args
      assert isinstance(args[0], AuroraConfig)
      assert args[1] is None
      assert mock_context.get_out() == [update_url_msg]
      assert mock_context.get_err() == []
  def test_start_update_command_line_succeeds(self):
    mock_context = FakeAuroraCommandContext()
    resp = self.create_simple_success_response()
    resp.result = Result(startJobUpdateResult=StartJobUpdateResult(updateId="id"))
    with contextlib.nested(
        patch('apache.aurora.client.cli.update.Update.create_context', return_value=mock_context),
        patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):
      mock_api = mock_context.get_api('west')
      mock_api.start_job_update.return_value = resp
      with temporary_file() as fp:
        fp.write(self.get_valid_config())
        fp.flush()
        cmd = AuroraCommandLine()
        result = cmd.execute(['beta-update', 'start', self.TEST_JOBSPEC, fp.name])
        assert result == EXIT_OK

      update_url_msg = StartUpdate.UPDATE_MSG_TEMPLATE % (
          mock_context.get_update_page(mock_api, AuroraJobKey.from_path(self.TEST_JOBSPEC), "id"))

      assert mock_api.start_job_update.call_count == 1
      args, kwargs = mock_api.start_job_update.call_args
      assert isinstance(args[0], AuroraConfig)
      assert args[1] is None
      assert mock_context.get_out() == [update_url_msg]
      assert mock_context.get_err() == []
Example #18
0
    def test_kill_lock_error_batches(self):
        """Verify that the batch kill path short circuits and includes the lock error message."""
        command = KillCommand()

        jobkey = AuroraJobKey("cluster", "role", "env", "job")

        mock_options = mock_verb_options(command)
        mock_options.instance_spec = TaskInstanceKey(jobkey, [1])
        mock_options.no_batching = False

        fake_context = FakeAuroraCommandContext()
        fake_context.set_options(mock_options)

        fake_context.add_expected_query_result(
            AuroraClientCommandTest.create_query_call_result(
                AuroraClientCommandTest.create_scheduled_task(
                    1, ScheduleStatus.RUNNING)))

        mock_api = fake_context.get_api('test')
        mock_api.kill_job.return_value = AuroraClientCommandTest.create_blank_response(
            ResponseCode.LOCK_ERROR, "Error.")

        with pytest.raises(Context.CommandError):
            command.execute(fake_context)

        mock_api.kill_job.assert_called_once_with(
            jobkey, mock_options.instance_spec.instance)
        self.assert_lock_message(fake_context)
Example #19
0
  def test_simple_successful_killall_job(self):
    """Run a test of the "kill" command against a mocked-out API:
    Verifies that the kill command sends the right API RPCs, and performs the correct
    tests on the result."""

    mock_options = self.setup_mock_options()
    mock_config = Mock()
    (mock_api, mock_scheduler_proxy) = self.create_mock_api()
    mock_api.kill_job.return_value = self.get_kill_job_response()
    mock_scheduler_proxy.killTasks.return_value = self.get_kill_job_response()
    mock_query_results = [
        self.create_mock_status_query_result(ScheduleStatus.RUNNING),
        self.create_mock_status_query_result(ScheduleStatus.KILLING),
        self.create_mock_status_query_result(ScheduleStatus.KILLED),
    ]
    mock_scheduler_proxy.getTasksWithoutConfigs.side_effect = mock_query_results
    with contextlib.nested(
        patch('time.sleep'),
        patch('apache.aurora.client.commands.core.make_client',
            return_value=mock_api),
        patch('twitter.common.app.get_options', return_value=mock_options),
        patch('apache.aurora.client.commands.core.get_job_config', return_value=mock_config)):

      with temporary_file() as fp:
        fp.write(self.get_valid_config())
        fp.flush()
        killall(['west/mchucarroll/test/hello', fp.name], mock_options)

      # Now check that the right API calls got made.
      self.assert_kill_job_called(mock_api)
      mock_api.kill_job.assert_called_with(
        AuroraJobKey(cluster=self.TEST_CLUSTER, role=self.TEST_ROLE, env=self.TEST_ENV,
            name=self.TEST_JOB), None, config=mock_config)
      self.assert_scheduler_called(mock_api, self.get_expected_task_query(), 3)
Example #20
0
    def test_killall_job(self):
        """Test kill client-side API logic."""
        mock_context = FakeAuroraCommandContext()
        mock_scheduler_proxy = Mock()
        with contextlib.nested(
                patch('threading._Event.wait'),
                patch('apache.aurora.client.cli.jobs.Job.create_context',
                      return_value=mock_context),
                patch('apache.aurora.client.factory.CLUSTERS',
                      new=self.TEST_CLUSTERS)):

            api = mock_context.get_api('west')
            mock_scheduler_proxy.getTasksWithoutConfigs.return_value = self.create_status_call_result(
            )
            api.kill_job.return_value = self.get_kill_job_response()
            mock_scheduler_proxy.killTasks.return_value = self.get_kill_job_response(
            )
            mock_context.add_expected_status_query_result(
                self.create_status_call_result(
                    self.create_mock_task(ScheduleStatus.KILLED)))
            with temporary_file() as fp:
                fp.write(self.get_valid_config())
                fp.flush()
                cmd = AuroraCommandLine()
                cmd.execute([
                    'job', 'killall', '--no-batching',
                    '--config=%s' % fp.name, 'west/bozo/test/hello'
                ])

            # Now check that the right API calls got made.
            assert api.kill_job.call_count == 1
            api.kill_job.assert_called_with(
                AuroraJobKey.from_path('west/bozo/test/hello'), None)
            self.assert_scheduler_called(api, self.get_expected_task_query(),
                                         2)
Example #21
0
  def test_successful_batched_killall_job(self):
    """Run a test of the "kill" command against a mocked-out API:
    Verifies that the kill command sends the right API RPCs, and performs the correct
    tests on the result."""

    mock_options = self.setup_mock_options()
    mock_options.batch_size = 5
    mock_config = Mock()
    (mock_api, mock_scheduler_proxy) = self.create_mock_api()
    mock_api.kill_job.return_value = self.get_kill_job_response()
    mock_api.check_status.return_value = self.create_status_call_result()
    with contextlib.nested(
        patch('apache.aurora.client.commands.core.make_client',
            return_value=mock_api),
        patch('twitter.common.app.get_options', return_value=mock_options),
        patch('apache.aurora.client.commands.core.get_job_config', return_value=mock_config),
        patch('apache.aurora.client.commands.core.JobMonitor')):

      with temporary_file() as fp:
        fp.write(self.get_valid_config())
        fp.flush()
        killall(['west/mchucarroll/test/hello', fp.name], mock_options)

      # Now check that the right API calls got made.
      assert mock_api.kill_job.call_count == 4
      mock_api.kill_job.assert_called_with(
        AuroraJobKey(cluster=self.TEST_CLUSTER, role=self.TEST_ROLE, env=self.TEST_ENV,
            name=self.TEST_JOB), [15, 16, 17, 18, 19])
Example #22
0
    def test_kill_job_with_instances_batched_large(self):
        """Test kill client-side API logic."""
        mock_context = FakeAuroraCommandContext()
        with contextlib.nested(
                patch('threading._Event.wait'),
                patch('apache.aurora.client.cli.jobs.Job.create_context',
                      return_value=mock_context),
                patch('apache.aurora.client.factory.CLUSTERS',
                      new=self.TEST_CLUSTERS)):
            api = mock_context.get_api('west')
            status_result = self.create_status_call_result()
            mock_context.add_expected_status_query_result(status_result)
            api.kill_job.return_value = self.get_kill_job_response()
            mock_context.add_expected_status_query_result(
                self.create_status_call_result(
                    self.create_mock_task(ScheduleStatus.KILLED)))

            with temporary_file() as fp:
                fp.write(self.get_valid_config())
                fp.flush()
                cmd = AuroraCommandLine()
                cmd.execute([
                    'job', 'kill',
                    '--config=%s' % fp.name, 'west/bozo/test/hello/0,2,4-13'
                ])

            # Now check that the right API calls got made.
            assert api.kill_job.call_count == 3
            api.kill_job.assert_called_with(
                AuroraJobKey.from_path('west/bozo/test/hello'), [12, 13])
            # Expect total 5 calls (3 from JobMonitor).
            self.assert_scheduler_called(
                api, self.get_expected_task_query([12, 13]), 5)
Example #23
0
    def test_create_with_lock(self):
        command = CreateJobCommand()

        jobkey = AuroraJobKey("cluster", "role", "env", "job")
        mock_options = mock_verb_options(command)
        mock_options.jobspec = jobkey
        mock_options.config_file = "/tmp/whatever"

        fake_context = FakeAuroraCommandContext()
        fake_context.set_options(mock_options)

        mock_config = create_autospec(spec=AuroraConfig,
                                      spec_set=True,
                                      instance=True)
        mock_job_config = Mock()
        mock_job_config.has_cron_schedule.return_value = False
        mock_config.raw.return_value = mock_job_config
        fake_context.get_job_config = Mock(return_value=mock_config)
        mock_api = fake_context.get_api("test")

        mock_api.create_job.return_value = AuroraClientCommandTest.create_blank_response(
            ResponseCode.JOB_UPDATING_ERROR, "Error.")

        with pytest.raises(Context.CommandError):
            command.execute(fake_context)

        mock_api.create_job.assert_called_once_with(mock_config)
        self.assert_lock_message(fake_context)
Example #24
0
    def test_update_status(self):
        mock_context = FakeAuroraCommandContext()
        api = mock_context.get_api('west')
        api.query_job_updates.return_value = self.get_status_query_response()
        api.get_job_update_details.return_value = self.get_update_details_response(
        )

        with contextlib.nested(
                patch('apache.aurora.client.cli.update.Update.create_context',
                      return_value=mock_context),
                patch('apache.aurora.client.factory.CLUSTERS',
                      new=self.TEST_CLUSTERS)):
            cmd = AuroraCommandLine()
            result = cmd.execute(
                ["beta-update", "status", "west/mcc/test/hello"])
            assert result == EXIT_OK
            assert mock_context.get_out() == [
                "Job: west/mcc/test/hello, UpdateID: fake-update-identifier",
                "Started YYYY-MM-DD HH:MM:SS, last updated: YYYY-MM-DD HH:MM:SS",
                "Current status: ROLLING_FORWARD", "Update events:",
                "  Status: ROLLING_FORWARD at YYYY-MM-DD HH:MM:SS",
                "  Status: ROLL_FORWARD_PAUSED at YYYY-MM-DD HH:MM:SS",
                "  Status: ROLLING_FORWARD at YYYY-MM-DD HH:MM:SS",
                "Instance events:",
                "  Instance 1 at YYYY-MM-DD HH:MM:SS: INSTANCE_UPDATING",
                "  Instance 2 at YYYY-MM-DD HH:MM:SS: INSTANCE_UPDATING",
                "  Instance 1 at YYYY-MM-DD HH:MM:SS: INSTANCE_UPDATED",
                "  Instance 2 at YYYY-MM-DD HH:MM:SS: INSTANCE_UPDATED"
            ]
            mock_context.get_api("west").query_job_updates.assert_called_with(
                job_key=AuroraJobKey('west', 'mcc', 'test', 'hello'))
 def create_probe_hosts(cls, num_hosts, predicted, safe, safe_in):
     hosts = defaultdict(list)
     for i in range(num_hosts):
         host_name = "h%s" % i
         job = AuroraJobKey.from_path("west/role/env/job%s" % i)
         hosts[host_name].append(DomainUpTimeSlaVector.JobUpTimeDetails(job, predicted, safe, safe_in))
     return hosts
Example #26
0
  def test_killall_job_something_else(self):
    """Test kill client-side API logic."""
    mock_context = FakeAuroraCommandContext()
    mock_scheduler_proxy = create_autospec(spec=SchedulerThriftApiSpec, instance=True)
    with contextlib.nested(
        patch('threading._Event.wait'),
        patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context),
        patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):

      api = mock_context.get_api('west')
      api.kill_job.return_value = self.get_kill_job_response()
      mock_context.add_expected_status_query_result(self.create_status_call_result())
      mock_scheduler_proxy.killTasks.return_value = self.get_kill_job_response()
      mock_context.add_expected_status_query_result(self.create_status_call_result(
          self.create_mock_task(ScheduleStatus.KILLED)))
      with temporary_file() as fp:
        fp.write(self.get_valid_config())
        fp.flush()
        cmd = AuroraCommandLine()
        cmd.execute(['job', 'killall', '--config=%s' % fp.name, 'west/bozo/test/hello'])

      # Now check that the right API calls got made.
      assert api.kill_job.call_count == 4
      instances = [15, 16, 17, 18, 19]
      api.kill_job.assert_called_with(AuroraJobKey.from_path('west/bozo/test/hello'), instances)
      self.assert_scheduler_called(api, self.get_expected_task_query(instances), 6)
Example #27
0
  def disambiguate_args_or_die(cls, args, options, client_factory=AuroraClientAPI):
    """
    Returns a (AuroraClientAPI, AuroraJobKey, AuroraConfigFile:str) tuple
    if one can be found given the args, potentially querying the scheduler with the returned client.
    Calls die() with an appropriate error message otherwise.

    Arguments:
      args: args from app command invocation.
      options: options from app command invocation. must have env and cluster attributes.
      client_factory: a callable (cluster) -> AuroraClientAPI.
    """
    if not len(args) > 0:
      die('job path is required')
    try:
      job_key = AuroraJobKey.from_path(args[0])
      client = client_factory(job_key.cluster)
      config_file = args[1] if len(args) > 1 else None  # the config for hooks
      return client, job_key, config_file
    except AuroraJobKey.Error:
      log.warning("Failed to parse job path, falling back to compatibility mode")
      role = args[0] if len(args) > 0 else None
      name = args[1] if len(args) > 1 else None
      env = None
      config_file = None  # deprecated form does not support hooks functionality
      cluster = options.cluster
      if not cluster:
        die('cluster is required')
      client = client_factory(cluster)
      return client, cls._disambiguate_or_die(client, role, env, name), config_file
Example #28
0
def run(args, options):
    """usage: run cluster/role/env/job cmd

  Runs a shell command on all machines currently hosting shards of a single job.

  This feature supports the same command line wildcards that are used to
  populate a job's commands.

  This means anything in the {{mesos.*}} and {{thermos.*}} namespaces.
  """
    # TODO(William Farner): Add support for invoking on individual shards.
    # TODO(Kevin Sweeney): Restore the ability to run across jobs with globs (See MESOS-3010).
    if not args:
        die("job path is required")
    job_path = args.pop(0)
    new_cmd = ["task", "run"]
    instances_spec = job_path
    if options.num_threads != 1:
        new_cmd.append("--threads=%s" % options.num_threads)
    if options.ssh_user is not None:
        new_cmd.append("--ssh-user=%s" % options.ssh_user)
    if options.executor_sandbox:
        new_cmd.append("--executor-sandbox")
    new_cmd.append('"%s"' % " ".join(args))
    v1_deprecation_warning("ssh", new_cmd)

    try:
        cluster_name, role, env, name = AuroraJobKey.from_path(job_path)
    except AuroraJobKey.Error as e:
        die('Invalid job path "%s": %s' % (job_path, e))

    command = " ".join(args)
    cluster = CLUSTERS[cluster_name]
    dcr = DistributedCommandRunner(cluster, role, env, [name], options.ssh_user)
    dcr.run(command, parallelism=options.num_threads, executor_sandbox=options.executor_sandbox)
Example #29
0
  def test_safe_domain_override_jobs(self):
    """Test successful execution of the sla_list_safe_domain command with override_jobs option."""
    mock_vector = self.create_mock_vector(self.create_hosts(3, 80, 100))
    with temporary_file() as fp:
      fp.write('west/role/env/job1 30 200s')
      fp.flush()
      mock_options = self.setup_mock_options(override=fp.name)
      with contextlib.nested(
          patch('apache.aurora.client.commands.admin.AuroraClientAPI',
              new=create_autospec(spec=AuroraClientAPI)),
          patch('apache.aurora.client.commands.admin.print_results'),
          patch('apache.aurora.client.commands.admin.CLUSTERS', new=self.TEST_CLUSTERS),
          patch('twitter.common.app.get_options', return_value=mock_options)
      ) as (
          mock_api,
          mock_print_results,
          test_clusters,
          mock_options):

        mock_api.return_value.sla_get_safe_domain_vector.return_value = mock_vector

        sla_list_safe_domain(['west', '50', '100s'])

        job_key = AuroraJobKey.from_path('west/role/env/job1')
        override = {job_key: JobUpTimeLimit(job_key, 30, 200)}
        mock_vector.get_safe_hosts.assert_called_once_with(50.0, 100.0, override, DEFAULT_GROUPING)
        mock_print_results.assert_called_once_with(['h0', 'h1', 'h2'])
Example #30
0
def get_job_config(job_spec, config_file, options):
  try:
    job_key = AuroraJobKey.from_path(job_spec)
    select_cluster = job_key.cluster
    select_env = job_key.env
    select_role = job_key.role
    jobname = job_key.name
  except AuroraJobKey.Error:
    deprecation_warning('Please refer to your job in CLUSTER/ROLE/ENV/NAME format.')
    select_cluster = options.cluster if options.cluster else None
    select_env = options.env
    select_role = None
    jobname = job_spec
  try:
    json_option = options.json
  except AttributeError:
    json_option = False
  try:
    bindings = options.bindings
  except AttributeError:
    bindings = ()
  return get_config(
      jobname,
      config_file,
      json_option,
      bindings,
      select_cluster=select_cluster,
      select_role=select_role,
      select_env=select_env)
Example #31
0
 def create_hosts(cls, num_hosts, percentage, duration):
   hosts = defaultdict(list)
   for i in range(num_hosts):
     host_name = 'h%s' % i
     job = AuroraJobKey.from_path('west/role/env/job%s' % i)
     hosts[host_name].append(JobUpTimeLimit(job, percentage, duration))
   return [hosts]
Example #32
0
    def setUp(self):
        self.RETURN_VALUE = "foo"
        test_obj = self

        class FakeAuroraClientAPI(object):
            def kill_job(self, job_key, instances=None, lock=None):
                test_obj.API_CALL = functools.partial(self.kill_job, job_key, instances, lock)
                return test_obj.RETURN_VALUE

            def restart(self, job_key, shards, restart_settings):
                test_obj.API_CALL = functools.partial(self.restart, job_key, shards, restart_settings)
                return test_obj.RETURN_VALUE

            def start_cronjob(self, job_key):
                test_obj.API_CALL = functools.partial(self.start_cronjob, job_key)
                return test_obj.RETURN_VALUE

        self._patch_bases(NonHookedAuroraClientAPI, (FakeAuroraClientAPI,))
        self.api = NonHookedAuroraClientAPI()

        # Test args passed in to check that these are proxied un-modified
        self.test_job_key = AuroraJobKey.from_path("a/b/c/d")
        self.test_config = "bar"
        self.test_shards = "baz"
        self.test_lock = "lock"
        self.health_check_interval_seconds = "baa"
Example #33
0
  def test_kill_job_with_instances_batched_large(self):
    """Test kill client-side API logic."""
    mock_context = FakeAuroraCommandContext()
    with contextlib.nested(
        patch('threading._Event.wait'),
        patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context),
        patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):
      api = mock_context.get_api('west')
      status_result = self.create_status_call_result()
      mock_context.add_expected_status_query_result(status_result)
      api.kill_job.return_value = self.get_kill_job_response()
      mock_context.add_expected_status_query_result(self.create_status_call_result(
          self.create_mock_task(ScheduleStatus.KILLED)))

      with temporary_file() as fp:
        fp.write(self.get_valid_config())
        fp.flush()
        cmd = AuroraCommandLine()
        cmd.execute(['job', 'kill', '--config=%s' % fp.name, 'west/bozo/test/hello/0,2,4-13'])

      # Now check that the right API calls got made.
      assert api.kill_job.call_count == 3
      api.kill_job.assert_called_with(AuroraJobKey.from_path('west/bozo/test/hello'),
          [12, 13])
      # Expect total 5 calls (3 from JobMonitor).
      self.assert_scheduler_called(api, self.get_expected_task_query([12, 13]), 5)
Example #34
0
    def test_safe_domain_override_jobs(self):
        """Test successful execution of the sla_list_safe_domain command with override_jobs option."""
        mock_vector = self.create_mock_vector(self.create_hosts(3, 80, 100))
        with temporary_file() as fp:
            fp.write('west/role/env/job1 30 200s')
            fp.flush()
            mock_options = self.setup_mock_options(override=fp.name)
            with contextlib.nested(
                    patch(
                        'apache.aurora.client.commands.admin.AuroraClientAPI',
                        new=Mock(spec=AuroraClientAPI)),
                    patch('apache.aurora.client.commands.admin.print_results'),
                    patch('apache.aurora.client.commands.admin.CLUSTERS',
                          new=self.TEST_CLUSTERS),
                    patch('twitter.common.app.get_options',
                          return_value=mock_options)) as (mock_api,
                                                          mock_print_results,
                                                          test_clusters,
                                                          mock_options):

                mock_api.return_value.sla_get_safe_domain_vector.return_value = mock_vector

                sla_list_safe_domain(['west', '50', '100s'])

                job_key = AuroraJobKey.from_path('west/role/env/job1')
                override = {job_key: JobUpTimeLimit(job_key, 30, 200)}
                mock_vector.get_safe_hosts.assert_called_once_with(
                    50.0, 100.0, override, DEFAULT_GROUPING)
                mock_print_results.assert_called_once_with(['h0', 'h1', 'h2'])
Example #35
0
 def create_hosts(cls, num_hosts, percentage, duration):
     hosts = defaultdict(list)
     for i in range(num_hosts):
         host_name = 'h%s' % i
         job = AuroraJobKey.from_path('west/role/env/job%s' % i)
         hosts[host_name].append(JobUpTimeLimit(job, percentage, duration))
     return [hosts]
Example #36
0
 def create_probe_hosts(cls, num_hosts, predicted, safe, safe_in):
   hosts = defaultdict(list)
   for i in range(num_hosts):
     host_name = 'h%s' % i
     job = AuroraJobKey.from_path('west/role/env/job%s' % i)
     hosts[host_name].append(JobUpTimeDetails(job, predicted, safe, safe_in))
   return [hosts]
Example #37
0
  def test_killall_job_wait_until_timeout(self):
    """Test kill client-side API logic."""
    mock_context = FakeAuroraCommandContext()
    mock_scheduler_proxy = create_autospec(spec=SchedulerThriftApiSpec, instance=True)
    with contextlib.nested(
        patch('threading._Event.wait'),
        patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context),
        patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):

      api = mock_context.get_api('west')
      mock_scheduler_proxy.getTasksWithoutConfigs.return_value = self.create_status_call_result()
      api.kill_job.return_value = self.get_kill_job_response()
      mock_scheduler_proxy.killTasks.return_value = self.get_kill_job_response()
      for _ in range(8):
        mock_context.add_expected_status_query_result(self.create_status_call_result(
            self.create_mock_task(ScheduleStatus.RUNNING)))

      with temporary_file() as fp:
        fp.write(self.get_valid_config())
        fp.flush()
        cmd = AuroraCommandLine()
        assert EXIT_TIMEOUT == cmd.execute(
            ['job', 'killall', '--no-batching', '--config=%s' % fp.name, 'west/bozo/test/hello'])

      # Now check that the right API calls got made.
      assert api.kill_job.call_count == 1
      api.kill_job.assert_called_with(AuroraJobKey.from_path('west/bozo/test/hello'), None)
      self.assert_scheduler_called(api, self.get_expected_task_query(), 8)
Example #38
0
def get_job_config(job_spec, config_file, options):
    try:
        job_key = AuroraJobKey.from_path(job_spec)
        select_cluster = job_key.cluster
        select_env = job_key.env
        select_role = job_key.role
        jobname = job_key.name
    except AuroraJobKey.Error:
        deprecation_warning(
            'Please refer to your job in CLUSTER/ROLE/ENV/NAME format.')
        select_cluster = options.cluster if options.cluster else None
        select_env = options.env
        select_role = None
        jobname = job_spec
    try:
        json_option = options.json
    except AttributeError:
        json_option = False
    try:
        bindings = options.bindings
    except AttributeError:
        bindings = ()
    return get_config(jobname,
                      config_file,
                      json_option,
                      bindings,
                      select_cluster=select_cluster,
                      select_role=select_role,
                      select_env=select_env)
  def setUp(self):
    self.RETURN_VALUE = 'foo'
    test_obj = self

    class FakeAuroraClientAPI(object):

      def cancel_update(self, job_key):
        test_obj.API_CALL = functools.partial(self.cancel_update, job_key)
        return test_obj.RETURN_VALUE

      def kill_job(self, job_key, instances=None, lock=None):
        test_obj.API_CALL = functools.partial(self.kill_job, job_key, instances, lock)
        return test_obj.RETURN_VALUE

      def restart(self, job_key, shards, updater_config, health_check_interval_seconds):
        test_obj.API_CALL = functools.partial(self.restart, job_key, shards,
            updater_config, health_check_interval_seconds)
        return test_obj.RETURN_VALUE

      def start_cronjob(self, job_key):
        test_obj.API_CALL = functools.partial(self.start_cronjob, job_key)
        return test_obj.RETURN_VALUE

    self._patch_bases(NonHookedAuroraClientAPI, (FakeAuroraClientAPI, ))
    self.api = NonHookedAuroraClientAPI()

    # Test args passed in to check that these are proxied un-modified
    self.test_job_key = AuroraJobKey.from_path('a/b/c/d')
    self.test_config = 'bar'
    self.test_shards = 'baz'
    self.test_lock = 'lock'
    self.test_updater_config = 'blah'
    self.health_check_interval_seconds = 'baa'
Example #40
0
    def test_restart_with_lock(self):
        command = RestartCommand()

        jobkey = AuroraJobKey("cluster", "role", "env", "job")
        mock_options = mock_verb_options(command)
        mock_options.instance_spec = TaskInstanceKey(jobkey, [])

        fake_context = FakeAuroraCommandContext()
        fake_context.set_options(mock_options)

        mock_api = fake_context.get_api("test")
        mock_api.restart.return_value = AuroraClientCommandTest.create_blank_response(
            ResponseCode.LOCK_ERROR, "Error.")

        with pytest.raises(Context.CommandError):
            command.execute(fake_context)

        updater_config = UpdaterConfig(mock_options.batch_size,
                                       mock_options.restart_threshold,
                                       mock_options.watch_secs,
                                       mock_options.max_per_instance_failures,
                                       mock_options.max_total_failures)

        mock_api.restart.assert_called_once_with(
            jobkey,
            mock_options.instance_spec.instance,
            updater_config,
            mock_options.healthcheck_interval_seconds,
            config=None)
        self.assert_lock_message(fake_context)
Example #41
0
  def setUp(self):
    self.RETURN_VALUE = 'foo'
    test_obj = self

    class FakeAuroraClientAPI(object):

      def kill_job(self, job_key, instances=None, lock=None):
        test_obj.API_CALL = functools.partial(self.kill_job, job_key, instances, lock)
        return test_obj.RETURN_VALUE

      def restart(self, job_key, shards, restart_settings):
        test_obj.API_CALL = functools.partial(self.restart, job_key, shards, restart_settings)
        return test_obj.RETURN_VALUE

      def start_cronjob(self, job_key):
        test_obj.API_CALL = functools.partial(self.start_cronjob, job_key)
        return test_obj.RETURN_VALUE

    self._patch_bases(NonHookedAuroraClientAPI, (FakeAuroraClientAPI, ))
    self.api = NonHookedAuroraClientAPI()

    # Test args passed in to check that these are proxied un-modified
    self.test_job_key = AuroraJobKey.from_path('a/b/c/d')
    self.test_config = 'bar'
    self.test_shards = 'baz'
    self.test_lock = 'lock'
    self.health_check_interval_seconds = 'baa'
Example #42
0
def really_killall(args, options):
    """Helper for testing purposes: make it easier to mock out the actual kill process,
  while testing hooks in the command dispatch process.
  """
    maybe_disable_hooks(options)
    job_key = AuroraJobKey.from_path(args[0])
    config_file = args[1] if len(args) > 1 else None  # the config for hooks
    new_cmd = ["job", "killall", args[0]]
    if config_file is not None:
        new_cmd.append("--config=%s" % config_file)
    if options.open_browser:
        new_cmd.append("--open-browser")
    if options.batch_size is not None:
        new_cmd.append("--batch-size=%s" % options.batch_size)
    if options.max_total_failures is not None:
        new_cmd.append("--max-total-failures=%s" % options.max_total_failures)
    v1_deprecation_warning("killall", new_cmd)

    config = get_job_config(job_key.to_path(), config_file,
                            options) if config_file else None
    api = make_client(job_key.cluster)
    if options.batch_size is not None:
        kill_in_batches(api, job_key, None, options.batch_size,
                        options.max_failures_option)
    else:
        resp = api.kill_job(job_key, None, config=config)
        check_and_log_response(resp)
    handle_open(api.scheduler_proxy.scheduler_client().url, job_key.role,
                job_key.env, job_key.name)
    wait_kill_tasks(api.scheduler_proxy, job_key)
Example #43
0
def really_killall(args, options):
  """Helper for testing purposes: make it easier to mock out the actual kill process,
  while testing hooks in the command dispatch process.
  """
  maybe_disable_hooks(options)
  job_key = AuroraJobKey.from_path(args[0])
  config_file = args[1] if len(args) > 1 else None  # the config for hooks
  new_cmd = ["job", "killall", args[0]]
  if config_file is not None:
    new_cmd.append("--config=%s" % config_file)
  if options.open_browser:
    new_cmd.append("--open-browser")
  if options.batch_size is not None:
    new_cmd.append("--batch-size=%s" % options.batch_size)
  if options.max_total_failures is not None:
    new_cmd.append("--max-total-failures=%s" % options.max_total_failures)
  v1_deprecation_warning("killall", new_cmd)

  config = get_job_config(job_key.to_path(), config_file, options) if config_file else None
  api = make_client(job_key.cluster)
  if options.batch_size is not None:
    kill_in_batches(api, job_key, None, options.batch_size, options.max_failures_option)
  else:
    resp = api.kill_job(job_key, None, config=config)
    check_and_log_response(resp)
  handle_open(api.scheduler_proxy.scheduler_client().url, job_key.role, job_key.env, job_key.name)
  wait_kill_tasks(api.scheduler_proxy, job_key)
Example #44
0
  def execute(self, context):
    job = context.options.instance_spec.jobkey
    instances = (None if context.options.instance_spec.instance == ALL_INSTANCES else
        context.options.instance_spec.instance)
    config = context.get_job_config(job, context.options.config_file)
    if config.raw().has_cron_schedule():
      raise context.CommandError(
          EXIT_COMMAND_FAILURE,
          "Cron jobs may only be updated with \"aurora cron schedule\" command")

    api = context.get_api(config.cluster())
    try:
      resp = api.start_job_update(config, context.options.message, instances)
    except AuroraClientAPI.UpdateConfigError as e:
      raise context.CommandError(EXIT_INVALID_CONFIGURATION, e.message)

    context.log_response_and_raise(resp, err_code=EXIT_API_ERROR,
        err_msg="Failed to start update due to error:")

    if resp.result:
      update_key = resp.result.startJobUpdateResult.key
      url = get_update_page(
        api,
        AuroraJobKey.from_thrift(config.cluster(), update_key.job),
        resp.result.startJobUpdateResult.key.id)
      context.print_out(self.UPDATE_MSG_TEMPLATE % url)

      if context.options.wait:
        return wait_for_update(context, self._clock, api, update_key)
    else:
      context.print_out(combine_messages(resp))

    return EXIT_OK
Example #45
0
 def create_probe_hosts(cls, num_hosts, predicted, safe, safe_in):
   hosts = defaultdict(list)
   for i in range(num_hosts):
     host_name = 'h%s' % i
     job = AuroraJobKey.from_path('west/role/env/job%s' % i)
     hosts[host_name].append(JobUpTimeDetails(job, predicted, safe, safe_in))
   return [hosts]
Example #46
0
 def assert_cancel_update_called(cls, mock_api):
     # Running cancel update should result in calling the API cancel_update
     # method once, with an AuroraJobKey parameter.
     assert mock_api.cancel_update.call_count == 1
     assert mock_api.cancel_update.called_with(AuroraJobKey(
         cls.TEST_CLUSTER, cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB),
                                               config=None)
Example #47
0
    def get_jobs_matching_key(self, key):
        """Finds all jobs matching a key containing wildcard segments.
    This is potentially slow!
    TODO(mchucarroll): insert a warning to users about slowness if the key contains wildcards!
    """
        def is_fully_bound(key):
            """Helper that checks if a key contains wildcards."""
            return not any(
                '*' in component
                for component in [key.cluster, key.role, key.env, key.name])

        def filter_job_list(jobs, role, env, name):
            """Filter a list of jobs to get just the jobs that match the pattern from a key"""
            return [
                job for job in jobs if fnmatch(job.role, role)
                and fnmatch(job.env, env) and fnmatch(job.name, name)
            ]

        # For cluster, we can expand the list of things we're looking for directly.
        # For other key elements, we need to just get a list of the jobs on the clusters, and filter
        # it for things that match.
        if key.cluster == '*':
            clusters_to_search = CLUSTERS
        else:
            clusters_to_search = [key.cluster]
        if is_fully_bound(key):
            return [AuroraJobKey(key.cluster, key.role, key.env, key.name)]
        else:
            jobs = filter_job_list(
                self.get_job_list(clusters_to_search, key.role), key.role,
                key.env, key.name)
            return jobs
 def create_hosts(cls, num_hosts, percentage, duration):
     hosts = defaultdict(list)
     for i in range(num_hosts):
         host_name = "h%s" % i
         job = AuroraJobKey.from_path("west/role/env/job%s" % i)
         hosts[host_name].append(DomainUpTimeSlaVector.JobUpTimeLimit(job, percentage, duration))
     return hosts
Example #49
0
 def execute(self, context):
     cluster = context.options.cluster
     api = context.get_api(cluster)
     response = api.query_job_updates(
         role=context.options.role,
         job_key=context.options.jobspec,
         user=context.options.user,
         update_statuses=context.options.status)
     context.log_response_and_raise(response)
     if context.options.write_json:
         result = []
         for summary in response.result.getJobUpdateSummariesResult.updateSummaries:
             job_entry = {
                 "jobkey":
                 AuroraJobKey.from_thrift(cluster,
                                          summary.jobKey).to_path(),
                 "id":
                 summary.updateId,
                 "user":
                 summary.user,
                 "started":
                 summary.state.createdTimestampMs,
                 "lastModified":
                 summary.state.lastModifiedTimestampMs,
                 "status":
                 JobUpdateStatus._VALUES_TO_NAMES[summary.state.status]
             }
             result.append(job_entry)
         context.print_out(
             json.dumps(result,
                        indent=2,
                        separators=[',', ': '],
                        sort_keys=False))
     else:
         for summary in response.result.getJobUpdateSummariesResult.updateSummaries:
             created = summary.state.createdTimestampMs
             lastMod = summary.state.lastModifiedTimestampMs
             context.print_out(
                 "Job: %s, Id: %s, User: %s, Status: %s" %
                 (AuroraJobKey.from_thrift(
                     cluster, summary.jobKey).to_path(), summary.updateId,
                  summary.user,
                  JobUpdateStatus._VALUES_TO_NAMES[summary.state.status]))
             context.print_out("Created: %s, Last Modified %s" %
                               (created, lastMod),
                               indent=2)
     return EXIT_OK
Example #50
0
 def setUp(self):
   self._command = StartUpdate()
   self._job_key = AuroraJobKey.from_thrift("cluster", UPDATE_KEY.job)
   self._mock_options = mock_verb_options(self._command)
   self._mock_options.instance_spec = TaskInstanceKey(self._job_key, None)
   self._fake_context = FakeAuroraCommandContext()
   self._fake_context.set_options(self._mock_options)
   self._mock_api = self._fake_context.get_api('UNUSED')
Example #51
0
 def setUp(self):
     self._command = ListUpdates()
     self._job_key = AuroraJobKey.from_thrift("cluster", UPDATE_KEY.job)
     self._mock_options = mock_verb_options(self._command)
     self._mock_options.filter = UpdateFilter(cluster=self.TEST_CLUSTER, role=None, env=None, job=None)
     self._fake_context = FakeAuroraCommandContext()
     self._fake_context.set_options(self._mock_options)
     self._mock_api = self._fake_context.get_api("UNUSED")
Example #52
0
def ssh(args, options):
  """usage: ssh cluster/role/env/job shard [args...]

  Initiate an SSH session on the machine that a shard is running on.
  """
  if not args:
    die('Job path is required')
  job_path = args.pop(0)
  try:
    cluster_name, role, env, name = AuroraJobKey.from_path(job_path)
  except AuroraJobKey.Error as e:
    die('Invalid job path "%s": %s' % (job_path, e))
  if not args:
    die('Shard is required')
  try:
    shard = int(args.pop(0))
  except ValueError:
    die('Shard must be an integer')

  newcmd = ["task", "ssh", "%s/%s" % (job_path, shard)]
  if len(options.tunnels) > 0:
    newcmd.append("--tunnels=%s" % options.tunnels)
  if options.ssh_user is not None:
    newcmd.append("--ssh-user=%s" % options.ssh_user)
  if options.executor_sandbox:
    newcmd.append("--executor-sandbox")
  if len(args) > 0:
    newcmd.append("--command=\"%s\"" % " ".join(args))
  v1_deprecation_warning("ssh", newcmd)

  api = make_client(cluster_name)
  resp = api.query(api.build_query(role, name, set([int(shard)]), env=env))
  check_and_log_response(resp)

  first_task = resp.result.scheduleStatusResult.tasks[0]
  remote_cmd = 'bash' if not args else ' '.join(args)
  command = DistributedCommandRunner.substitute(remote_cmd, first_task,
      api.cluster, executor_sandbox=options.executor_sandbox)

  ssh_command = ['ssh', '-t']


  role = first_task.assignedTask.task.owner.role
  slave_host = first_task.assignedTask.slaveHost

  for tunnel in options.tunnels:
    try:
      port, name = tunnel.split(':')
      port = int(port)
    except ValueError:
      die('Could not parse tunnel: %s.  Must be of form PORT:NAME' % tunnel)
    if name not in first_task.assignedTask.assignedPorts:
      die('Task %s has no port named %s' % (first_task.assignedTask.taskId, name))
    ssh_command += [
        '-L', '%d:%s:%d' % (port, slave_host, first_task.assignedTask.assignedPorts[name])]

  ssh_command += ['%s@%s' % (options.ssh_user or role, slave_host), command]
  return subprocess.call(ssh_command)
Example #53
0
 def setUp(self):
     self._scheduler = Mock()
     self._sla = Sla(self._scheduler)
     self._cluster = Cluster(name="cl")
     self._role = "mesos"
     self._name = "job"
     self._env = "test"
     self._job_key = AuroraJobKey(self._cluster.name, self._role, self._env, self._name)
     self._min_count = 1
Example #54
0
  def test_successful_schedule(self):
    mock_context = FakeAuroraCommandContext()
    key = AuroraJobKey("west", "bozo", "test", "hello")
    with contextlib.nested(
        patch('apache.aurora.client.cli.cron.CronNoun.create_context', return_value=mock_context)):

      api = mock_context.get_api('west')
      api.schedule_cron.return_value = self.create_simple_success_response()
      with temporary_file() as fp:
        fp.write(self.get_valid_cron_config())
        fp.flush()
        cmd = AuroraCommandLine()
        cmd.execute(['cron', 'schedule', key.to_path(), fp.name])

      # Now check that the right API calls got made.
      # Check that create_job was called exactly once, with an AuroraConfig parameter.
      assert api.schedule_cron.call_count == 1
      assert isinstance(api.schedule_cron.call_args[0][0], AuroraConfig)

      # The last text printed out to the user should contain a url to the job
      assert mock_context.get_job_page(api, key) in mock_context.out[-1]
Example #55
0
def killall(args, options):
  """usage: killall cluster/role/env/job
  Kills all tasks in a running job, blocking until all specified tasks have been terminated.
  """

  job_key = AuroraJobKey.from_path(args[0])
  config_file = args[1] if len(args) > 1 else None  # the config for hooks
  config = get_job_config(job_key.to_path(), config_file, options) if config_file else None
  api = make_client(job_key.cluster)
  resp = api.kill_job(job_key, None, config=config)
  check_and_log_response(resp)
  handle_open(api.scheduler_proxy.scheduler_client().url, job_key.role, job_key.env, job_key.name)
    def cancel_update_job(self, cluster, role, environment, jobname, jobspec=None):
        """Method to cancel an update of aurora job"""

        job_key = AuroraJobKey(cluster, role, environment, jobname)
        logger.info("request to cancel update of => %s", job_key.to_path())

        cmd_output = ""
        try:
            cmd_args = [job_key.to_path(),]

            # aurora client requires jobspec be passed as file, no reading from STDIN
            jobspec_file = self.make_jobspec_file(jobspec)
            if jobspec_file is not None:
                cmd_args.append(jobspec_file.name)

            cmd_output = subprocess.check_output(
                [self.aurora_cmd, "cancel_update"] + cmd_args, stderr=subprocess.STDOUT)

        except subprocess.CalledProcessError as e:
            logger.warning("aurora client exit status: %d, details follow" % e.returncode)
            for s in e.output.splitlines():
                logger.warning("> %s" % s)
            logger.warning("----------------------------------------")

            return(job_key.to_path(), ["Error reported by aurora client:"] + e.output.splitlines())

        finally:
            if jobspec_file: jobspec_file.close()

        if self.is_aurora_command_successful(cmd_output):
            logger.info("aurora -- cancel update successful")
            return(job_key.to_path(), None)
        else:
            logger.warning("aurora -- cancel update job")
            return(job_key.to_path(), ["Error reported by aurora client:"] + cmd_output.splitlines())