def test_load_json():
with temporary_file() as fp:
fp.write(MESOS_CONFIG)
fp.flush()
env = AuroraConfigLoader.load(fp.name)
job = env["jobs"][0]
with temporary_file() as fp:
fp.write(json.dumps(job.get()))
fp.flush()
new_job = AuroraConfigLoader.load_json(fp.name)
assert new_job == job
def test_override_single_variable():
with temporary_file() as output:
# test that the override takes place
with environment_as(HORK = 'BORK'):
subprocess.Popen([sys.executable, '-c', 'import os; print os.environ["HORK"]'],
stdout=output).wait()
output.seek(0)
assert output.read() == 'BORK\n'
# test that the variable is cleared
with temporary_file() as new_output:
subprocess.Popen([sys.executable, '-c', 'import os; print os.environ.has_key("HORK")'],
stdout=new_output).wait()
new_output.seek(0)
assert new_output.read() == 'False\n'
def test_updater_simple_with_instances(self):
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_health_check = self.setup_health_checks(mock_api)
mock_quota_check = self.setup_quota_check()
mock_job_monitor = self.setup_job_monitor()
fake_mux = self.FakeSchedulerMux()
self.setup_mock_scheduler_for_simple_update(mock_api)
with contextlib.nested(
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.api.instance_watcher.StatusHealthCheck',
return_value=mock_health_check),
patch('apache.aurora.client.api.updater.JobMonitor', return_value=mock_job_monitor),
patch('apache.aurora.client.api.updater.QuotaCheck', return_value=mock_quota_check),
patch('apache.aurora.client.api.updater.SchedulerMux', return_value=fake_mux),
patch('time.time', side_effect=functools.partial(self.fake_time, self)),
patch('time.sleep', return_value=None),
patch('threading._Event.wait')):
with temporary_file() as fp:
fp.write(self.get_service_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(['job', 'update', 'west/bozo/test/hello/1', fp.name])
mock_scheduler_proxy = mock_api.scheduler_proxy
assert mock_scheduler_proxy.acquireLock.call_count == 1
assert mock_scheduler_proxy.addInstances.call_count == 1
assert mock_scheduler_proxy.killTasks.call_count == 1
self.assert_correct_status_calls(mock_scheduler_proxy)
assert mock_scheduler_proxy.releaseLock.call_count == 1
def use_cached_files(self, cache_key):
# This implementation fetches the appropriate tarball and extracts it.
remote_path = self._remote_path_for_key(cache_key)
try:
# Send an HTTP request for the tarball.
response = self._request('GET', remote_path)
if response is None:
return None
done = False
with temporary_file() as outfile:
total_bytes = 0
# Read the data in a loop.
while not done:
data = response.read(self.READ_SIZE)
outfile.write(data)
if len(data) < self.READ_SIZE:
done = True
total_bytes += len(data)
outfile.close()
self.log.debug('Read %d bytes from artifact cache at %s' %
(total_bytes,self._url_string(remote_path)))
# Extract the tarfile.
artifact = TarballArtifact(self.artifact_root, outfile.name, self.compress)
artifact.extract()
return artifact
except Exception as e:
self.log.warn('Error while reading from remote artifact cache: %s' % e)
return None
def test_safe_domain_exclude_hosts(self):
"""Test successful execution of the sla_list_safe_domain command with exclude hosts option."""
mock_vector = self.create_mock_vector(self.create_hosts(3, 80, 100))
with temporary_file() as fp:
fp.write('h1')
fp.flush()
mock_options = self.setup_mock_options(exclude=fp.name)
with contextlib.nested(
patch('apache.aurora.client.commands.admin.AuroraClientAPI',
new=create_autospec(spec=AuroraClientAPI)),
patch('apache.aurora.client.commands.admin.print_results'),
patch('apache.aurora.client.commands.admin.CLUSTERS', new=self.TEST_CLUSTERS),
patch('twitter.common.app.get_options', return_value=mock_options)
) as (
mock_api,
mock_print_results,
test_clusters,
mock_options):
mock_api.return_value.sla_get_safe_domain_vector.return_value = mock_vector
sla_list_safe_domain(['west', '50', '100s'])
mock_vector.get_safe_hosts.assert_called_once_with(50.0, 100.0, {}, DEFAULT_GROUPING)
mock_print_results.assert_called_once_with(['h0', 'h2'])
def test_simple_successful_create_job_output(self):
"""Run a test of the "create" command against a mocked-out API:
Verifies that the creation command generates the correct output.
"""
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context)):
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.PENDING))
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.RUNNING))
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.RUNNING))
api = mock_context.get_api('west')
api.create_job.return_value = self.get_createjob_response()
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
result = cmd.execute(['job', 'create', '--wait-until=RUNNING', 'west/bozo/test/hello',
fp.name])
assert result == EXIT_OK
assert mock_context.get_out() == [
"Job create succeeded: job url=http://something_or_other/scheduler/bozo/test/hello"]
assert mock_context.get_err() == []
def test_updater_simple_small_doesnt_warn(self):
mock_out = IOMock()
mock_err = IOMock()
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_health_check = self.setup_health_checks(mock_api)
mock_quota_check = self.setup_quota_check()
mock_job_monitor = self.setup_job_monitor()
fake_mux = self.FakeSchedulerMux()
self.setup_mock_scheduler_for_simple_update(mock_api)
# This doesn't work, because:
# - The mock_context stubs out the API.
# - the test relies on using live code in the API.
with contextlib.nested(
patch('apache.aurora.client.cli.jobs.AuroraCommandContext.print_out',
side_effect=mock_out.put),
patch('apache.aurora.client.cli.jobs.AuroraCommandContext.print_err',
side_effect=mock_err.put),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.api.instance_watcher.StatusHealthCheck',
return_value=mock_health_check),
patch('apache.aurora.client.api.updater.JobMonitor', return_value=mock_job_monitor),
patch('apache.aurora.client.api.updater.QuotaCheck', return_value=mock_quota_check),
patch('apache.aurora.client.api.updater.SchedulerMux', return_value=fake_mux),
patch('time.time', side_effect=functools.partial(self.fake_time, self)),
patch('time.sleep', return_value=None),
patch('threading._Event.wait')):
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(['job', 'update', 'west/bozo/test/hello', fp.name])
assert mock_out.get() == ['Update completed successfully']
assert mock_err.get() == []
def test_simple_successful_kill_job(self):
"""Run a test of the "kill" command against a mocked-out API:
Verifies that the kill command sends the right API RPCs, and performs the correct
tests on the result."""
mock_options = self.setup_mock_options()
mock_config = Mock()
mock_api_factory = self.setup_mock_api_factory()
with contextlib.nested(
patch('apache.aurora.client.commands.core.make_client_factory',
return_value=mock_api_factory),
patch('twitter.common.app.get_options', return_value=mock_options),
patch('apache.aurora.client.commands.core.get_job_config', return_value=mock_config)) as (
mock_make_client_factory,
options, mock_get_job_config):
mock_api = mock_api_factory.return_value
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
kill(['west/mchucarroll/test/hello', fp.name], mock_options)
# Now check that the right API calls got made.
self.assert_kill_job_called(mock_api)
mock_api.kill_job.assert_called_with(
AuroraJobKey(cluster=self.TEST_CLUSTER, role=self.TEST_ROLE, env=self.TEST_ENV,
name=self.TEST_JOB), None, config=mock_config)
self.assert_scheduler_called(mock_api)
assert mock_make_client_factory.call_count == 1
def test_diff_server_error(self):
"""Test the diff command if the user passes a config with an error in it."""
mock_options = self.setup_mock_options()
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_scheduler_proxy.getTasksStatus.return_value = self.create_failed_status_response()
self.setup_populate_job_config(mock_scheduler_proxy)
with contextlib.nested(
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('twitter.common.app.get_options', return_value=mock_options),
patch('subprocess.call', return_value=0),
patch('json.loads', return_value=Mock())) as (
mock_scheduler_proxy_class,
mock_clusters,
options,
subprocess_patch,
json_patch):
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
result = cmd.execute(['job', 'diff', 'west/bozo/test/hello', fp.name])
assert result == EXIT_INVALID_PARAMETER
# In this error case, we should have called the server getTasksStatus;
# but since it fails, we shouldn't call populateJobConfig or subprocess.
mock_scheduler_proxy.getTasksStatus.assert_called_with(
TaskQuery(jobName='hello', environment='test', owner=Identity(role='bozo'),
statuses=ACTIVE_STATES))
assert mock_scheduler_proxy.populateJobConfig.call_count == 0
assert subprocess_patch.call_count == 0
def test_kill_job_with_instances_batched_large(self):
"""Test kill client-side API logic."""
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context',
return_value=mock_context),
patch('apache.aurora.client.factory.CLUSTERS',
new=self.TEST_CLUSTERS)):
api = mock_context.get_api('west')
status_result = self.create_status_call_result()
mock_context.add_expected_status_query_result(status_result)
api.kill_job.return_value = self.get_kill_job_response()
mock_context.add_expected_status_query_result(
self.create_status_call_result(
self.create_mock_task(ScheduleStatus.KILLED)))
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute([
'job', 'kill',
'--config=%s' % fp.name, 'west/bozo/test/hello/0,2,4-13'
])
# Now check that the right API calls got made.
assert api.kill_job.call_count == 3
api.kill_job.assert_called_with(
AuroraJobKey.from_path('west/bozo/test/hello'), [12, 13])
# Expect total 5 calls (3 from JobMonitor).
self.assert_scheduler_called(
api, self.get_expected_task_query([12, 13]), 5)
def test_kill_job_with_invalid_instances_strict(self):
"""Test kill client-side API logic."""
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context',
return_value=mock_context),
patch('apache.aurora.client.factory.CLUSTERS',
new=self.TEST_CLUSTERS)):
api = mock_context.get_api('west')
self.setup_get_tasks_status_calls(api.scheduler_proxy)
api.kill_job.return_value = self.get_kill_job_response()
mock_context.add_expected_status_query_result(
self.create_status_call_result(
self.create_mock_task(ScheduleStatus.KILLED)))
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute([
'job', 'kill',
'--config=%s' % fp.name, '--no-batching', '--strict',
'west/bozo/test/hello/0,2,4-6,11-20'
])
# Now check that the right API calls got made.
assert api.kill_job.call_count == 0
def test_perform_maintenance_hosts_failed_default_sla(self):
with temporary_file() as fp:
mock_options = self.make_mock_options()
mock_options.post_drain_script = None
mock_options.grouping = 'by_host'
mock_options.unsafe_hosts_filename = fp.name
mock_api, mock_scheduler_proxy = self.create_mock_api()
mock_scheduler_proxy.startMaintenance.return_value = self.create_start_maintenance_result(
)
mock_scheduler_proxy.drainHosts.return_value = self.create_start_maintenance_result(
)
mock_vector = self.create_mock_probe_hosts_vector([
self.create_probe_hosts(self.HOSTNAMES[0], 95, False, None),
self.create_probe_hosts(self.HOSTNAMES[1], 95, False, None),
self.create_probe_hosts(self.HOSTNAMES[2], 95, False, None)
])
with contextlib.nested(
patch('apache.aurora.client.api.SchedulerProxy',
return_value=mock_scheduler_proxy),
patch(
'apache.aurora.client.api.sla.Sla.get_domain_uptime_vector',
return_value=mock_vector),
patch('apache.aurora.admin.maintenance.CLUSTERS',
new=self.TEST_CLUSTERS),
patch('threading._Event.wait'),
patch('twitter.common.app.get_options',
return_value=mock_options)):
host_drain([self.TEST_CLUSTER])
mock_scheduler_proxy.startMaintenance.assert_called_with(
Hosts(set(self.HOSTNAMES)))
def test_record_stats_written(self):
timings = {"compile": {'checkstyle': [0.00057005882263183594]}}
with temporary_file() as temp_fd:
temp_filename = temp_fd.name
bs = BuildTimeStats(temp_filename)
self.set_up_mocks()
bs.record_stats(timings, 100)
self.mox.VerifyAll()
self.mox.UnsetStubs()
self.set_up_mocks()
#Test append
timings = {"compile": {'checkstyle': [0.00057005882263183594]}}
bs.record_stats(timings, 100)
with open(temp_filename, 'r') as stats_file:
lines = stats_file.readlines()
self.assertEquals(len(lines), 2)
self.mox.VerifyAll()
exp_calls = [{
"/tmp/test/current_run.DEBUG":
"/tmp/dummy/buildtime_uploader.global.DEBUG"
}, {
"/tmp/test/current_run.INFO":
"/tmp/dummy/buildtime_uploader.global.INFO"
}, {
"/tmp/test/current_run.FATAL":
"/tmp/dummy/buildtime_uploader.global.FATAL"
}, {
"/tmp/test/current_run.ERROR":
"/tmp/dummy/buildtime_uploader.global.ERROR"
}, {
"/tmp/test/current_run.WARNING":
"/tmp/dummy/buildtime_uploader.global.WARNING"
}]
self.assertEqual(bs.get_log_append_calls(), exp_calls)
def test_start_update_command_line_succeeds(self):
mock_context = FakeAuroraCommandContext()
resp = self.create_simple_success_response()
resp.result = Result(startJobUpdateResult=StartJobUpdateResult(updateId="id"))
with contextlib.nested(
patch('apache.aurora.client.cli.update.Update.create_context', return_value=mock_context),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):
mock_api = mock_context.get_api(self.TEST_CLUSTER)
mock_api.start_job_update.return_value = resp
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
result = cmd.execute(['beta-update', 'start', self.TEST_JOBSPEC, fp.name])
assert result == EXIT_OK
update_url_msg = StartUpdate.UPDATE_MSG_TEMPLATE % (
mock_context.get_update_page(mock_api, AuroraJobKey.from_path(self.TEST_JOBSPEC), "id"))
assert mock_api.start_job_update.call_count == 1
args, kwargs = mock_api.start_job_update.call_args
assert isinstance(args[0], AuroraConfig)
assert args[1] is None
assert mock_context.get_out() == [update_url_msg]
assert mock_context.get_err() == []
def test_updater_simple_small_doesnt_warn(self):
mock_out = IOMock()
mock_err = IOMock()
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_health_check = self.setup_health_checks(mock_api)
mock_quota_check = self.setup_quota_check()
mock_job_monitor = self.setup_job_monitor()
fake_mux = self.FakeSchedulerMux()
self.setup_mock_scheduler_for_simple_update(mock_api)
with contextlib.nested(
patch('apache.aurora.client.cli.jobs.AuroraCommandContext.print_out',
side_effect=mock_out.put),
patch('apache.aurora.client.cli.jobs.AuroraCommandContext.print_err',
side_effect=mock_err.put),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.api.instance_watcher.StatusHealthCheck',
return_value=mock_health_check),
patch('apache.aurora.client.api.updater.JobMonitor', return_value=mock_job_monitor),
patch('apache.aurora.client.api.updater.QuotaCheck', return_value=mock_quota_check),
patch('apache.aurora.client.api.updater.SchedulerMux', return_value=fake_mux),
patch('time.time', side_effect=functools.partial(self.fake_time, self)),
patch('time.sleep', return_value=None),
patch('threading._Event.wait')):
with temporary_file() as fp:
fp.write(self.get_service_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(['job', 'update', 'west/bozo/test/hello', fp.name])
assert mock_out.get() == ['Update completed successfully']
assert mock_err.get() == [CLIENT_UPDATER_DEPRECATION]
def do_test_artifact_cache(self, artifact_cache):
key = CacheKey("muppet_key", "fake_hash", 42, [])
with temporary_file(artifact_cache.artifact_root) as f:
# Write the file.
f.write(TEST_CONTENT1)
path = f.name
f.close()
# Cache it.
self.assertFalse(artifact_cache.has(key))
self.assertFalse(bool(artifact_cache.use_cached_files(key)))
artifact_cache.insert(key, [path])
self.assertTrue(artifact_cache.has(key))
# Stomp it.
with open(path, "w") as outfile:
outfile.write(TEST_CONTENT2)
# Recover it from the cache.
self.assertTrue(bool(artifact_cache.use_cached_files(key)))
# Check that it was recovered correctly.
with open(path, "r") as infile:
content = infile.read()
self.assertEquals(content, TEST_CONTENT1)
# Delete it.
artifact_cache.delete(key)
self.assertFalse(artifact_cache.has(key))
def test_successful_diff(self):
"""Test the diff command."""
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
with contextlib.nested(
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('subprocess.call', return_value=0),
patch('json.loads', return_value=Mock())) as (_, _, subprocess_patch, _):
mock_scheduler_proxy.getTasksStatus.return_value = self.create_status_response()
self.setup_populate_job_config(mock_scheduler_proxy)
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(['job', 'diff', 'west/bozo/test/hello', fp.name])
# Diff should get the task status, populate a config, and run diff.
mock_scheduler_proxy.getTasksStatus.assert_called_with(
TaskQuery(jobName='hello', environment='test', owner=Identity(role='bozo'),
statuses=ACTIVE_STATES))
assert mock_scheduler_proxy.populateJobConfig.call_count == 1
assert isinstance(mock_scheduler_proxy.populateJobConfig.call_args[0][0], JobConfiguration)
assert (mock_scheduler_proxy.populateJobConfig.call_args[0][0].key ==
JobKey(environment=u'test', role=u'bozo', name=u'hello'))
# Subprocess should have been used to invoke diff with two parameters.
assert subprocess_patch.call_count == 1
assert len(subprocess_patch.call_args[0][0]) == 3
assert subprocess_patch.call_args[0][0][0] == os.environ.get('DIFF_VIEWER', 'diff')
def test_kill_job_with_instances_batched_maxerrors(self):
"""Test kill client-side API logic."""
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('apache.aurora.client.cli.jobs.Job.create_context',
return_value=mock_context),
patch('apache.aurora.client.factory.CLUSTERS',
new=self.TEST_CLUSTERS)):
api = mock_context.get_api('west')
status_result = self.create_status_call_result()
mock_context.add_expected_status_query_result(status_result)
api.kill_job.return_value = self.create_error_response()
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute([
'job', 'kill', '--max-total-failures=1',
'--config=%s' % fp.name, 'west/bozo/test/hello/0,2,4-13'
])
# Now check that the right API calls got made. We should have aborted after the second batch.
assert api.kill_job.call_count == 2
assert api.scheduler_proxy.getTasksWithoutConfigs.call_count == 0
def test_plugin_runs_in_create_job(self):
"""Run a test of the "create" command against a mocked-out API:
Verifies that the creation command sends the right API RPCs, and performs the correct
tests on the result."""
# We'll patch out create_context, which will give us a fake context
# object, and everything can be stubbed through that.
mock_context = FakeAuroraCommandContext()
with patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context):
# After making the client, create sets up a job monitor.
# The monitor uses TaskQuery to get the tasks. It's called at least twice:once before
# the job is created, and once after. So we need to set up mocks for the query results.
mock_query = self.create_mock_query()
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.INIT))
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.RUNNING))
api = mock_context.get_api('west')
api.create_job.return_value = self.get_createjob_response()
# This is the real test: invoke create as if it had been called by the command line.
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.register_plugin(BogusPlugin())
cmd.execute(['job', 'create', '--bogosity=maximum', '--wait_until=RUNNING',
'west/bozo/test/hello', fp.name])
# Now check that the right API calls got made.
# Check that create_job was called exactly once, with an AuroraConfig parameter.
self.assert_create_job_called(api)
self.assert_scheduler_called(api, mock_query, 2)
# Check that the plugin did its job.
assert mock_context.bogosity == "maximum"
def test_kill_job_with_empty_instances_batched(self):
"""Test kill client-side API logic."""
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context',
return_value=mock_context),
patch('apache.aurora.client.factory.CLUSTERS',
new=self.TEST_CLUSTERS)):
api = mock_context.get_api('west')
# set up an empty instance list in the getTasksWithoutConfigs response
status_response = self.create_simple_success_response()
schedule_status = Mock(spec=ScheduleStatusResult)
status_response.result.scheduleStatusResult = schedule_status
schedule_status.tasks = []
mock_context.add_expected_status_query_result(status_response)
api.kill_job.return_value = self.get_kill_job_response()
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute([
'job', 'kill',
'--config=%s' % fp.name, 'west/bozo/test/hello/0,2,4-13'
])
# Now check that the right API calls got made.
assert api.kill_job.call_count == 0
def test_perform_maintenance_hosts_failed_default_sla(self):
with temporary_file() as fp:
mock_options = self.make_mock_options()
mock_options.post_drain_script = None
mock_options.grouping = 'by_host'
mock_options.unsafe_hosts_filename = fp.name
mock_api, mock_scheduler_proxy = self.create_mock_api()
mock_scheduler_proxy.startMaintenance.return_value = self.create_start_maintenance_result()
mock_scheduler_proxy.drainHosts.return_value = self.create_start_maintenance_result()
mock_vector = self.create_mock_probe_hosts_vector([
self.create_probe_hosts(self.HOSTNAMES[0], 95, False, None),
self.create_probe_hosts(self.HOSTNAMES[1], 95, False, None),
self.create_probe_hosts(self.HOSTNAMES[2], 95, False, None)
])
with contextlib.nested(
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.api.sla.Sla.get_domain_uptime_vector',
return_value=mock_vector),
patch('apache.aurora.client.commands.maintenance.CLUSTERS', new=self.TEST_CLUSTERS),
patch('twitter.common.app.get_options', return_value=mock_options)):
host_drain([self.TEST_CLUSTER])
mock_scheduler_proxy.startMaintenance.assert_called_with(Hosts(set(self.HOSTNAMES)))
def test_kill_job_with_instances_deep_api(self):
"""Test kill client-side API logic."""
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.api.SchedulerProxy',
return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS',
new=self.TEST_CLUSTERS)):
mock_scheduler_proxy.killTasks.return_value = self.get_kill_job_response(
)
self.setup_get_tasks_status_calls(mock_scheduler_proxy)
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute([
'job', 'kill',
'--config=%s' % fp.name, 'west/bozo/test/hello/0,2,4-6'
])
# Now check that the right API calls got made.
assert mock_scheduler_proxy.killTasks.call_count == 1
mock_scheduler_proxy.killTasks.assert_called_with(
TaskQuery(jobName='hello',
environment='test',
instanceIds=frozenset([0, 2, 4, 5, 6]),
owner=Identity(role='bozo')), None)
def test_perform_maintenance_partial_sla_failure(self, mock_check_sla, mock_start_maintenance,
mock_drain_hosts, mock_operate_on_hosts, mock_complete_maintenance):
mock_callback = mock.Mock()
failed_host = 'us-west-001.example.com'
mock_check_sla.return_value = set([failed_host])
drained_hosts = set(TEST_HOSTNAMES) - set([failed_host])
maintenance = HostMaintenance(DEFAULT_CLUSTER, 'quiet')
with temporary_file() as fp:
with group_by_rack():
maintenance.perform_maintenance(
TEST_HOSTNAMES,
callback=mock_callback,
grouping_function='by_rack',
output_file=fp.name)
with open(fp.name, 'r') as fpr:
content = fpr.read()
assert failed_host in content
mock_start_maintenance.assert_called_once_with(TEST_HOSTNAMES)
assert mock_check_sla.call_count == 1
assert mock_drain_hosts.call_count == 1
assert mock_drain_hosts.call_args_list == [mock.call(Hosts(drained_hosts))]
assert mock_operate_on_hosts.call_count == 1
assert mock_operate_on_hosts.call_args_list == [
mock.call(Hosts(drained_hosts), mock_callback)]
assert mock_complete_maintenance.call_count == 2
assert mock_complete_maintenance.call_args_list == [
mock.call(Hosts(set([failed_host]))), mock.call(Hosts(drained_hosts))]
def test_killall_job_output(self):
"""Test kill output."""
mock_context = FakeAuroraCommandContext()
mock_scheduler_proxy = Mock()
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context',
return_value=mock_context),
patch('apache.aurora.client.factory.CLUSTERS',
new=self.TEST_CLUSTERS)):
api = mock_context.get_api('west')
mock_scheduler_proxy.getTasksWithoutConfigs.return_value = self.create_status_call_result(
)
api.kill_job.return_value = self.get_kill_job_response()
mock_scheduler_proxy.killTasks.return_value = self.get_kill_job_response(
)
mock_context.add_expected_status_query_result(
self.create_status_call_result(
self.create_mock_task(ScheduleStatus.KILLED)))
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute([
'job', 'killall', '--no-batching',
'--config=%s' % fp.name, 'west/bozo/test/hello'
])
assert mock_context.get_out() == ['job killall succeeded']
assert mock_context.get_err() == []
def test_simple_successful_create_job_with_bindings(self):
"""Run a test of the "create" command against a mocked-out API:
Verifies that the creation command sends the right API RPCs, and performs the correct
tests on the result."""
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context)):
mock_query = self.create_query()
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.PENDING))
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.RUNNING))
api = mock_context.get_api('west')
api.create_job.return_value = self.get_createjob_response()
# This is the real test: invoke create as if it had been called by the command line.
with temporary_file() as fp:
fp.write(self.get_unbound_test_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(['job', 'create', '--wait-until=RUNNING', '--bind', 'cluster_binding=west',
'--bind', 'instances_binding=20', '--bind', 'TEST_BATCH=1',
'west/bozo/test/hello',
fp.name])
# Now check that the right API calls got made.
# Check that create_job was called exactly once, with an AuroraConfig parameter.
self.assert_create_job_called(api)
self.assert_scheduler_called(api, mock_query, 2)
def test_kill_job_with_instances_batched_output(self):
"""Test kill client-side API logic."""
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context',
return_value=mock_context),
patch('apache.aurora.client.factory.CLUSTERS',
new=self.TEST_CLUSTERS)):
api = mock_context.get_api('west')
status_result = self.create_status_call_result()
mock_context.add_expected_status_query_result(status_result)
api.kill_job.return_value = self.get_kill_job_response()
mock_context.add_expected_status_query_result(
self.create_status_call_result(
self.create_mock_task(ScheduleStatus.KILLED)))
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute([
'job', 'kill',
'--config=%s' % fp.name, 'west/bozo/test/hello/0,2,4-6'
])
assert mock_context.get_out() == [
'Successfully killed shards [0, 2, 4, 5, 6]', 'job kill succeeded'
]
assert mock_context.get_err() == []
def test_kill_job_with_instances_batched_large(self):
"""Test kill client-side API logic."""
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):
api = mock_context.get_api('west')
status_result = self.create_status_call_result()
mock_context.add_expected_status_query_result(status_result)
api.kill_job.return_value = self.get_kill_job_response()
mock_context.add_expected_status_query_result(self.create_status_call_result(
self.create_mock_task(ScheduleStatus.KILLED)))
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(['job', 'kill', '--config=%s' % fp.name, 'west/bozo/test/hello/0,2,4-13'])
# Now check that the right API calls got made.
assert api.kill_job.call_count == 3
api.kill_job.assert_called_with(AuroraJobKey.from_path('west/bozo/test/hello'),
[12, 13])
# Expect total 5 calls (3 from JobMonitor).
self.assert_scheduler_called(api, self.get_expected_task_query([12, 13]), 5)
def test_kill_job_with_instances_batched_maxerrors_output(self):
"""Test kill client-side API logic."""
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('apache.aurora.client.cli.jobs.Job.create_context',
return_value=mock_context),
patch('apache.aurora.client.factory.CLUSTERS',
new=self.TEST_CLUSTERS)):
api = mock_context.get_api('west')
status_result = self.create_status_call_result()
mock_context.add_expected_status_query_result(status_result)
api.kill_job.return_value = self.create_error_response()
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute([
'job', 'kill', '--max-total-failures=1',
'--config=%s' % fp.name, 'west/bozo/test/hello/0,2,4-13'
])
assert mock_context.get_out() == []
print(mock_context.get_err())
assert mock_context.get_err() == [
'Kill of shards [0, 2, 4, 5, 6] failed with error; see log for details',
'Kill of shards [7, 8, 9, 10, 11] failed with error; see log for details',
'Exceeded maximum number of errors while killing instances'
]
def use_cached_files(self, cache_key):
path = self._path_for_key(cache_key)
response = self._request('GET', path)
if response is None:
return False
expected_size = int(response.getheader('content-length', -1))
if expected_size == -1:
raise Exception, 'No content-length header in HTTP response'
read_size = 4 * 1024 * 1024 # 4 MB
done = False
if self.context:
self.context.log.info('Reading %d bytes' % expected_size)
with temporary_file() as outfile:
total_bytes = 0
while not done:
data = response.read(read_size)
outfile.write(data)
if len(data) < read_size:
done = True
total_bytes += len(data)
if self.context:
self.context.log.debug('Read %d bytes' % total_bytes)
outfile.close()
if total_bytes != expected_size:
raise Exception, 'Read only %d bytes from %d expected' % (total_bytes, expected_size)
mode = 'r:bz2' if self.compress else 'r'
with open_tar(outfile.name, mode) as tarfile:
tarfile.extractall(self.artifact_root)
return True
def test_killall_job(self):
"""Test kill client-side API logic."""
mock_context = FakeAuroraCommandContext()
mock_scheduler_proxy = Mock()
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context',
return_value=mock_context),
patch('apache.aurora.client.factory.CLUSTERS',
new=self.TEST_CLUSTERS)):
api = mock_context.get_api('west')
mock_scheduler_proxy.getTasksWithoutConfigs.return_value = self.create_status_call_result(
)
api.kill_job.return_value = self.get_kill_job_response()
mock_scheduler_proxy.killTasks.return_value = self.get_kill_job_response(
)
mock_context.add_expected_status_query_result(
self.create_status_call_result(
self.create_mock_task(ScheduleStatus.KILLED)))
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute([
'job', 'killall', '--no-batching',
'--config=%s' % fp.name, 'west/bozo/test/hello'
])
# Now check that the right API calls got made.
assert api.kill_job.call_count == 1
api.kill_job.assert_called_with(
AuroraJobKey.from_path('west/bozo/test/hello'), None)
self.assert_scheduler_called(api, self.get_expected_task_query(),
2)
def test_compute_stats(self):
executed_goals = {'resolve-idl':{ 'idl': [0.00072813034057617188],
'extract': [3.0994415283203125e-06] },
"thriftstore-codegen": {'thriftstore-codegen': [0.0001010894775390625] },
"gen": {'tweetypie-fetch': [0.028632879257202148], 'thrift': [0.016566991806030273],
'protoc': [0.0038318634033203125], 'antlr': [0.0020389556884765625],
'thriftstore-dml-gen': [0.0022170543670654297],
'tweetypie-clean': [0.0054290294647216797] },
"resolve": {'ivy': [0.00097703933715820312] },
"compile": {'checkstyle': [0.00057005882263183594]},
"test": {'junit': [9.1075897216796875e-05], 'specs': [0.0015749931335449219]}
}
with temporary_file() as temp_fd:
bs = BuildTimeStats(temp_fd.name)
actual_timings = bs.compute_stats(executed_goals, 100)
expected_timings =[{'phase': 'resolve', 'total': 0.00097703933715820312, 'goal': 'ivy'},
{'phase': 'resolve-idl', 'total': 0.00072813034057617188, 'goal': 'idl'},
{'phase': 'resolve-idl', 'total': 3.0994415283203125e-06, 'goal': 'extract'},
{'phase': 'resolve-idl', 'total': 0.00073122978210449219, 'goal': 'phase_total'},
{'phase': 'compile', 'total': 0.00057005882263183594, 'goal': 'checkstyle'},
{'phase': 'thriftstore-codegen', 'total': 0.0001010894775390625, 'goal': 'thriftstore-codegen'},
{'phase': 'test', 'total': 9.1075897216796875e-05, 'goal': 'junit'},
{'phase': 'test', 'total': 0.0015749931335449219, 'goal': 'specs'},
{'phase': 'test', 'total': 0.0016660690307617188, 'goal': 'phase_total'},
{'phase': 'gen', 'total': 0.0038318634033203125, 'goal': 'protoc'},
{'phase': 'gen', 'total': 0.0020389556884765625, 'goal': 'antlr'},
{'phase': 'gen', 'total': 0.028632879257202148, 'goal': 'tweetypie-fetch'},
{'phase': 'gen', 'total': 0.0054290294647216797, 'goal': 'tweetypie-clean'},
{'phase': 'gen', 'total': 0.0022170543670654297, 'goal': 'thriftstore-dml-gen'},
{'phase': 'gen', 'total': 0.016566991806030273, 'goal': 'thrift'},
{'phase': 'gen', 'total': 0.058716773986816406, 'goal': 'phase_total'},
{'phase': 'cmd_total', 'total': 100, 'goal': 'cmd_total'}]
self.assertEqual(actual_timings, expected_timings )
def test_safe_domain_override_jobs(self):
"""Test successful execution of the sla_list_safe_domain command with override_jobs option."""
mock_vector = self.create_mock_vector(self.create_hosts(3, 80, 100))
with temporary_file() as fp:
fp.write('west/role/env/job1 30 200s')
fp.flush()
mock_options = self.setup_mock_options(override=fp.name)
with contextlib.nested(
patch('apache.aurora.admin.admin.make_admin_client',
return_value=create_autospec(spec=AuroraClientAPI)),
patch('apache.aurora.admin.admin.print_results'),
patch('apache.aurora.admin.admin.CLUSTERS',
new=self.TEST_CLUSTERS),
patch('twitter.common.app.get_options',
return_value=mock_options)) as (mock_api,
mock_print_results,
test_clusters,
mock_options):
mock_api.return_value.sla_get_safe_domain_vector.return_value = mock_vector
sla_list_safe_domain(['west', '50', '100s'])
job_key = AuroraJobKey.from_path('west/role/env/job1')
override = {job_key: JobUpTimeLimit(job_key, 30, 200)}
mock_vector.get_safe_hosts.assert_called_once_with(
50.0, 100.0, override, DEFAULT_GROUPING)
mock_print_results.assert_called_once_with(['h0', 'h1', 'h2'])
def test_parse_script(self, mock_subprocess):
with temporary_file() as fp:
mock_popen = mock.Mock()
mock_popen.wait.return_value = 0
mock_subprocess.Popen.return_value = mock_popen
parse_script(fp.name)('h1')
assert mock_popen.wait.call_count == 1
def test_simple_successful_killall_job(self):
"""Run a test of the "kill" command against a mocked-out API:
Verifies that the kill command sends the right API RPCs, and performs the correct
tests on the result."""
mock_options = self.setup_mock_options()
mock_config = Mock()
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_api.kill_job.return_value = self.get_kill_job_response()
mock_scheduler_proxy.killTasks.return_value = self.get_kill_job_response()
mock_query_results = [
self.create_mock_status_query_result(ScheduleStatus.RUNNING),
self.create_mock_status_query_result(ScheduleStatus.KILLING),
self.create_mock_status_query_result(ScheduleStatus.KILLED),
]
mock_scheduler_proxy.getTasksWithoutConfigs.side_effect = mock_query_results
with contextlib.nested(
patch('time.sleep'),
patch('apache.aurora.client.commands.core.make_client',
return_value=mock_api),
patch('twitter.common.app.get_options', return_value=mock_options),
patch('apache.aurora.client.commands.core.get_job_config', return_value=mock_config)):
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
killall(['west/mchucarroll/test/hello', fp.name], mock_options)
# Now check that the right API calls got made.
self.assert_kill_job_called(mock_api)
mock_api.kill_job.assert_called_with(
AuroraJobKey(cluster=self.TEST_CLUSTER, role=self.TEST_ROLE, env=self.TEST_ENV,
name=self.TEST_JOB), None, config=mock_config)
self.assert_scheduler_called(mock_api, self.get_expected_task_query(), 3)
def test_diff_invalid_config(self):
"""Test the diff command if the user passes a config with an error in it."""
mock_options = self.setup_mock_options()
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_scheduler_proxy.getTasksStatus.return_value = self.create_status_response()
self.setup_populate_job_config(mock_scheduler_proxy)
with contextlib.nested(
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('twitter.common.app.get_options', return_value=mock_options),
patch('subprocess.call', return_value=0),
patch('json.loads', return_value=Mock())) as (
mock_scheduler_proxy_class,
mock_clusters,
options,
subprocess_patch,
json_patch):
with temporary_file() as fp:
fp.write(self.get_invalid_config('stupid="me"',))
fp.flush()
cmd = AuroraCommandLine()
result = cmd.execute(['job', 'diff', 'west/bozo/test/hello', fp.name])
assert result == EXIT_INVALID_CONFIGURATION
assert mock_scheduler_proxy.getTasksStatus.call_count == 0
assert mock_scheduler_proxy.populateJobConfig.call_count == 0
assert subprocess_patch.call_count == 0
def test_successful_batched_killall_job(self):
"""Run a test of the "kill" command against a mocked-out API:
Verifies that the kill command sends the right API RPCs, and performs the correct
tests on the result."""
mock_options = self.setup_mock_options()
mock_options.batch_size = 5
mock_config = Mock()
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_api.kill_job.return_value = self.get_kill_job_response()
mock_api.check_status.return_value = self.create_status_call_result()
with contextlib.nested(
patch('apache.aurora.client.commands.core.make_client',
return_value=mock_api),
patch('twitter.common.app.get_options', return_value=mock_options),
patch('apache.aurora.client.commands.core.get_job_config', return_value=mock_config),
patch('apache.aurora.client.commands.core.JobMonitor')):
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
killall(['west/mchucarroll/test/hello', fp.name], mock_options)
# Now check that the right API calls got made.
assert mock_api.kill_job.call_count == 4
mock_api.kill_job.assert_called_with(
AuroraJobKey(cluster=self.TEST_CLUSTER, role=self.TEST_ROLE, env=self.TEST_ENV,
name=self.TEST_JOB), [15, 16, 17, 18, 19])
def test_failed_create_job_with_incomplete_bindings(self):
"""Run a test of the "create" command against a mocked-out API:
Verifies that the creation command sends the right API RPCs, and performs the correct
tests on the result."""
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context)):
# This is the real test: invoke create as if it had been called by the command line.
with temporary_file() as fp:
fp.write(self.get_unbound_test_config())
fp.flush()
cmd = AuroraCommandLine()
result = cmd.execute(['job', 'create', '--wait-until=RUNNING',
'--bind', 'cluster_binding=west',
'west/bozo/test/hello',
fp.name])
assert result == EXIT_INVALID_CONFIGURATION
assert mock_context.get_out() == []
assert mock_context.get_err() == [
"Error loading configuration: "
"TypeCheck(FAILED): MesosJob[update_config] failed: "
"UpdateConfig[batch_size] failed: u'{{TEST_BATCH}}' not an integer"]
def test_kill_job_api_level(self):
"""Test kill client-side API logic."""
mock_options = self.setup_mock_options()
mock_config = Mock()
mock_config.hooks = []
mock_config.raw.return_value.enable_hooks.return_value.get.return_value = False
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_scheduler_proxy.killTasks.return_value = self.get_kill_job_response()
mock_query_results = [
self.create_mock_status_query_result(ScheduleStatus.RUNNING),
self.create_mock_status_query_result(ScheduleStatus.KILLING),
self.create_mock_status_query_result(ScheduleStatus.KILLED),
]
mock_scheduler_proxy.getTasksWithoutConfigs.side_effect = mock_query_results
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.factory.make_client', return_value=mock_api),
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('twitter.common.app.get_options', return_value=mock_options),
patch('apache.aurora.client.commands.core.get_job_config', return_value=mock_config)):
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
killall(['west/mchucarroll/test/hello', fp.name], mock_options)
# Now check that the right API calls got made.
assert mock_scheduler_proxy.killTasks.call_count == 1
query = self.get_expected_task_query()
mock_scheduler_proxy.killTasks.assert_called_with(query, None)
self.assert_scheduler_called(mock_api, query, 3)
def test_kill_job_api_level_with_shards(self):
"""Test kill client-side API logic."""
mock_options = self.setup_mock_options()
mock_options.shards = [0, 1, 2, 3]
mock_config = Mock()
mock_config.hooks = []
mock_config.raw.return_value.enable_hooks.return_value.get.return_value = False
(mock_api, mock_scheduler) = self.setup_mock_api()
mock_api_factory = Mock(return_value=mock_api)
mock_scheduler.killTasks.return_value = self.get_kill_job_response()
with contextlib.nested(
patch('apache.aurora.client.factory.make_client_factory', return_value=mock_api_factory),
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('twitter.common.app.get_options', return_value=mock_options),
patch('apache.aurora.client.commands.core.get_job_config', return_value=mock_config)) as (
mock_api_factory_patch,
mock_scheduler_proxy_class,
mock_clusters,
options, mock_get_job_config):
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
kill(['west/mchucarroll/test/hello', fp.name], mock_options)
# Now check that the right API calls got made.
self.assert_scheduler_called(mock_api)
assert mock_scheduler.killTasks.call_count == 1
query = self.get_expected_task_query([0, 1, 2, 3])
mock_scheduler.killTasks.assert_called_with(query, None)
def test_kill_job_api_level_with_shards_batched(self):
"""Test kill client-side API logic."""
mock_options = self.setup_mock_options()
mock_options.batch_size = 2
mock_options.shards = [0, 1, 2, 3]
mock_config = Mock()
mock_config.hooks = []
mock_config.raw.return_value.enable_hooks.return_value.get.return_value = False
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_api.check_status.return_value = self.create_status_call_result()
mock_scheduler_proxy.getTasksWithoutConfigs.return_value = self.create_status_call_result()
mock_scheduler_proxy.killTasks.return_value = self.get_kill_job_response()
with contextlib.nested(
patch('apache.aurora.client.factory.make_client', return_value=mock_api),
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('twitter.common.app.get_options', return_value=mock_options),
patch('apache.aurora.client.commands.core.get_job_config', return_value=mock_config),
patch('apache.aurora.client.commands.core.JobMonitor')):
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
kill(['west/mchucarroll/test/hello', fp.name], mock_options)
# Now check that the right API calls got made.
assert mock_scheduler_proxy.killTasks.call_count == 2
query = self.get_expected_task_query([2, 3])
mock_scheduler_proxy.killTasks.assert_called_with(query, None)
def test_kill_job_api_level_with_shards_batched_and_some_errors(self):
"""Test kill client-side API logic."""
mock_options = self.setup_mock_options()
mock_options.batch_size = 2
mock_options.shards = [0, 1, 2, 3]
mock_config = Mock()
mock_config.hooks = []
mock_config.raw.return_value.enable_hooks.return_value.get.return_value = False
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_api.check_status.return_value = self.create_status_call_result()
mock_scheduler_proxy.getTasksWithoutConfigs.return_value = self.create_status_call_result()
mock_api.kill_job.side_effect = [
self.get_kill_job_error_response(), self.get_kill_job_response()]
with contextlib.nested(
patch('apache.aurora.client.factory.make_client', return_value=mock_api),
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('twitter.common.app.get_options', return_value=mock_options),
patch('apache.aurora.client.commands.core.get_job_config', return_value=mock_config)):
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
# We should get an exception in this case, because the one of the two calls fails.
self.assertRaises(SystemExit, kill, ['west/mchucarroll/test/hello', fp.name], mock_options)
# killTasks should still have gotten called twice - the first error shouldn't abort
# the second batch.
assert mock_scheduler_proxy.killTasks.call_count == 2
query = self.get_expected_task_query([2, 3])
mock_scheduler_proxy.killTasks.assert_called_with(query, None)
def test_probe_hosts_with_file(self):
"""Tests successful execution of the sla_probe_hosts command with host filename."""
mock_vector = self.create_mock_probe_hosts_vector(
[self.create_probe_hosts(1, 80, False, None)])
with temporary_file() as fp:
fp.write('h0')
fp.flush()
mock_options = self.setup_mock_options(filename=fp.name)
with contextlib.nested(
patch('apache.aurora.admin.admin.make_client',
new=create_autospec(spec=AuroraClientAPI)),
patch('apache.aurora.admin.admin.print_results'),
patch('apache.aurora.admin.admin.CLUSTERS',
new=self.TEST_CLUSTERS),
patch('twitter.common.app.get_options',
return_value=mock_options)) as (mock_api,
mock_print_results,
test_clusters,
options):
mock_api.return_value.sla_get_safe_domain_vector.return_value = mock_vector
sla_probe_hosts(['west', '90', '200s'])
mock_api.return_value.sla_get_safe_domain_vector.assert_called_once_with(
mock_options.min_instance_count, ['h0'])
mock_vector.probe_hosts.assert_called_once_with(
90.0, 200.0, mock_options.grouping)
mock_print_results.assert_called_once_with(
['h0\twest/role/env/job0\t80.00\tFalse\tn/a'])
def test_perform_maintenance_hosts_failed_custom_sla(self):
with temporary_file() as fp:
mock_options = self.make_mock_options()
mock_options.post_drain_script = None
mock_options.grouping = 'by_host'
mock_options.percentage = 50
mock_options.duration = '10m'
mock_options.reason = 'Test overrides'
mock_options.unsafe_hosts_filename = fp.name
mock_api, mock_scheduler_proxy = self.create_mock_api()
mock_scheduler_proxy.startMaintenance.return_value = self.create_start_maintenance_result()
mock_scheduler_proxy.drainHosts.return_value = self.create_start_maintenance_result()
mock_vector = self.create_mock_probe_hosts_vector([
self.create_probe_hosts(self.HOSTNAMES[0], 95, False, None),
self.create_probe_hosts(self.HOSTNAMES[1], 95, False, None),
self.create_probe_hosts(self.HOSTNAMES[2], 95, False, None)
])
with contextlib.nested(
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.api.sla.Sla.get_domain_uptime_vector',
return_value=mock_vector),
patch('apache.aurora.client.commands.maintenance.CLUSTERS', new=self.TEST_CLUSTERS),
patch('apache.aurora.admin.admin_util.log_admin_message'),
patch('twitter.common.app.get_options', return_value=mock_options)) as (_, _, _, log, _):
host_drain([self.TEST_CLUSTER])
assert 'Test overrides' in log.call_args[0][1]
mock_scheduler_proxy.startMaintenance.assert_called_with(Hosts(set(self.HOSTNAMES)))
def test_safe_domain_exclude_hosts(self):
"""Test successful execution of the sla_list_safe_domain command with exclude hosts option."""
mock_vector = self.create_mock_vector(self.create_hosts(3, 80, 100))
with temporary_file() as fp:
fp.write('h1')
fp.flush()
mock_options = self.setup_mock_options(exclude=fp.name)
with contextlib.nested(
patch('apache.aurora.admin.admin.make_client',
new=create_autospec(spec=AuroraClientAPI)),
patch('apache.aurora.admin.admin.print_results'),
patch('apache.aurora.admin.admin.CLUSTERS',
new=self.TEST_CLUSTERS),
patch('twitter.common.app.get_options',
return_value=mock_options)) as (mock_api,
mock_print_results,
test_clusters,
mock_options):
mock_api.return_value.sla_get_safe_domain_vector.return_value = mock_vector
sla_list_safe_domain(['west', '50', '100s'])
mock_vector.get_safe_hosts.assert_called_once_with(
50.0, 100.0, {}, DEFAULT_GROUPING)
mock_print_results.assert_called_once_with(['h0', 'h2'])
def test_make_zk_auth_config_validation():
invalid_configs = [
# No credential in auth
{'auth': [{'scheme': 's'}],
'acl': [{'scheme': 's', 'credential': 'c', 'permissions': {'read': True}}]},
# Acl is not a list
{'auth': [{'scheme': 's', 'credential': 'c'}],
'acl': {'scheme': 's', 'credential': 'c', 'permissions': {'read': True}}},
# No credential in acl
{'auth': [{'scheme': 's', 'credential': 'c'}],
'acl': [{'scheme': 's', 'permissions': {'read': True}}]},
# permissions is not an object
{'auth': [{'scheme': 's', 'credential': 'c'}],
'acl': [{'scheme': 's', 'credential': 'c', 'permissions': 'non-object'}]},
# permissions object has unrecognized property
{'auth': [{'scheme': 's', 'credential': 'c'}],
'acl': [{'scheme': 's', 'credential': 'c', 'permissions': {'extraprop': True}}]},
# non boolean property in permissions object
{'auth': [{'scheme': 's', 'credential': 'c'}],
'acl': [{'scheme': 's', 'credential': 'c', 'permissions': {'read': 'non-bool'}}]},
]
for invalid_config in invalid_configs:
with temporary_file() as fp:
fp.write(json.dumps(invalid_config))
fp.flush()
with pytest.raises(SystemExit):
make_zk_auth(fp.name)
def do_test_artifact_cache(self, artifact_cache):
key = CacheKey('muppet_key', 'fake_hash', 42, [])
with temporary_file(artifact_cache.artifact_root) as f:
# Write the file.
f.write(TEST_CONTENT1)
path = f.name
f.close()
# Cache it.
self.assertFalse(artifact_cache.has(key))
self.assertFalse(bool(artifact_cache.use_cached_files(key)))
artifact_cache.insert(key, [path])
self.assertTrue(artifact_cache.has(key))
# Stomp it.
with open(path, 'w') as outfile:
outfile.write(TEST_CONTENT2)
# Recover it from the cache.
self.assertTrue(bool(artifact_cache.use_cached_files(key)))
# Check that it was recovered correctly.
with open(path, 'r') as infile:
content = infile.read()
self.assertEquals(content, TEST_CONTENT1)
# Delete it.
artifact_cache.delete(key)
self.assertFalse(artifact_cache.has(key))
def test_simple_successful_create_job_open_page(self):
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
# TODO(maxim): Patching threading.Event with all possible namespace/patch/mock
# combinations did not produce the desired effect. Investigate why (AURORA-510)
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context)):
mock_query = self.create_query()
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.PENDING))
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.RUNNING))
api = mock_context.get_api('west')
api.create_job.return_value = self.get_createjob_response()
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
result = cmd.execute(['job', 'create', '--wait-until=RUNNING', '--open-browser',
'west/bozo/test/hello',
fp.name])
assert result == EXIT_OK
self.assert_create_job_called(api)
self.assert_scheduler_called(api, mock_query, 2)
assert mock_context.showed_urls == ["http://something_or_other/scheduler/bozo/test/hello"]
def test_large_with_instances_doesnt_warn(self):
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_health_check = self.setup_health_checks(mock_api)
mock_quota_check = self.setup_quota_check()
mock_job_monitor = self.setup_job_monitor()
fake_mux = self.FakeSchedulerMux()
self.setup_mock_scheduler_for_simple_update(mock_api, count=20)
config = self.get_valid_config()
config = config.replace("instances = 20", "instances = 200")
with contextlib.nested(
patch('apache.aurora.client.factory.CLUSTERS',
new=self.TEST_CLUSTERS),
patch('apache.aurora.client.api.SchedulerProxy',
return_value=mock_scheduler_proxy),
patch(
'apache.aurora.client.api.instance_watcher.StatusHealthCheck',
return_value=mock_health_check),
patch('apache.aurora.client.api.updater.JobMonitor',
return_value=mock_job_monitor),
patch('apache.aurora.client.api.updater.QuotaCheck',
return_value=mock_quota_check),
patch('apache.aurora.client.api.updater.SchedulerMux',
return_value=fake_mux),
patch('time.time',
side_effect=functools.partial(self.fake_time, self)),
patch('threading._Event.wait')):
with patch('time.sleep') as sleep:
with temporary_file() as fp:
fp.write(config)
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(
['job', 'update', 'west/bozo/test/hello/1,3', fp.name])
assert sleep.call_count == 0
def test_create_job_startup_fails(self):
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context)):
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.PENDING))
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.RUNNING))
# We need to override the side_effect behavior of check_status in the context.
def check_status_side_effect(*args):
return self.create_error_response()
mock_context.get_api("west").check_status.side_effect = check_status_side_effect
api = mock_context.get_api('west')
api.create_job.return_value = self.get_createjob_response()
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
result = cmd.execute(['job', 'create', '--wait-until=RUNNING', 'west/bozo/test/hello',
fp.name])
assert result == EXIT_COMMAND_FAILURE
assert mock_context.get_out() == []
assert mock_context.get_err() == ["Error occurred while creating job west/bozo/test/hello"]
def test_simple_successful_create_job_with_bindings(self):
"""Run a test of the "create" command against a mocked-out API:
Verifies that the creation command sends the right API RPCs, and performs the correct
tests on the result."""
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context)):
mock_query = self.create_mock_query()
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.PENDING))
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.RUNNING))
api = mock_context.get_api('west')
api.create_job.return_value = self.get_createjob_response()
mock_context.enable_reveal_errors()
# This is the real test: invoke create as if it had been called by the command line.
with temporary_file() as fp:
fp.write(self.get_unbound_test_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(['job', 'create', '--wait-until=RUNNING', '--bind', 'cluster_binding=west',
'--bind', 'instances_binding=20', 'west/bozo/test/hello',
fp.name])
# Now check that the right API calls got made.
# Check that create_job was called exactly once, with an AuroraConfig parameter.
self.assert_create_job_called(api)
self.assert_scheduler_called(api, mock_query, 2)
def test_probe_hosts_with_file(self):
"""Tests successful execution of the sla_probe_hosts command with host filename."""
mock_vector = self.create_mock_probe_hosts_vector([self.create_probe_hosts(1, 80, False, None)])
with temporary_file() as fp:
fp.write('h0')
fp.flush()
mock_options = self.setup_mock_options(filename=fp.name)
with contextlib.nested(
patch('apache.aurora.client.commands.admin.AuroraClientAPI',
new=create_autospec(spec=AuroraClientAPI)),
patch('apache.aurora.client.commands.admin.print_results'),
patch('apache.aurora.client.commands.admin.CLUSTERS', new=self.TEST_CLUSTERS),
patch('twitter.common.app.get_options', return_value=mock_options)
) as (
mock_api,
mock_print_results,
test_clusters,
options):
mock_api.return_value.sla_get_safe_domain_vector.return_value = mock_vector
sla_probe_hosts(['west', '90', '200s'])
mock_api.return_value.sla_get_safe_domain_vector.assert_called_once_with(
mock_options.min_instance_count, ['h0'])
mock_vector.probe_hosts.assert_called_once_with(90.0, 200.0, mock_options.grouping)
mock_print_results.assert_called_once_with([
'h0\twest/role/env/job0\t80.00\tFalse\tn/a'
])
def test_simple_successful_create_job_open_page(self):
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
# TODO(maxim): Patching threading.Event with all possible namespace/patch/mock
# combinations did not produce the desired effect. Investigate why (AURORA-510)
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context',
return_value=mock_context)):
mock_query = self.create_query()
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.PENDING))
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.RUNNING))
api = mock_context.get_api('west')
api.create_job.return_value = self.get_createjob_response()
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
result = cmd.execute([
'job', 'create', '--wait-until=RUNNING', '--open-browser',
'west/bozo/test/hello', fp.name
])
assert result == EXIT_OK
self.assert_create_job_called(api)
self.assert_scheduler_called(api, mock_query, 2)
assert self.mock_webbrowser.mock_calls == [
call("http://something_or_other/scheduler/bozo/test/hello")
]
def test_killall_job_something_else(self):
"""Test kill client-side API logic."""
mock_context = FakeAuroraCommandContext()
mock_scheduler_proxy = create_autospec(spec=SchedulerThriftApiSpec, instance=True)
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):
api = mock_context.get_api('west')
api.kill_job.return_value = self.get_kill_job_response()
mock_context.add_expected_status_query_result(self.create_status_call_result())
mock_scheduler_proxy.killTasks.return_value = self.get_kill_job_response()
mock_context.add_expected_status_query_result(self.create_status_call_result(
self.create_mock_task(ScheduleStatus.KILLED)))
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(['job', 'killall', '--config=%s' % fp.name, 'west/bozo/test/hello'])
# Now check that the right API calls got made.
assert api.kill_job.call_count == 4
instances = [15, 16, 17, 18, 19]
api.kill_job.assert_called_with(AuroraJobKey.from_path('west/bozo/test/hello'), instances)
self.assert_scheduler_called(api, self.get_expected_task_query(instances), 6)
def test_create_job_delayed(self):
"""Run a test of the "create" command against a mocked-out API:
this time, make the monitor check status several times before successful completion.
"""
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context',
return_value=mock_context)):
mock_query = self.create_query()
for result in [
ScheduleStatus.PENDING, ScheduleStatus.PENDING,
ScheduleStatus.RUNNING
]:
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(result))
api = mock_context.get_api('west')
api.create_job.return_value = self.get_createjob_response()
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute([
'job', 'create', '--wait-until=RUNNING',
'west/bozo/test/hello', fp.name
])
self.assert_create_job_called(api)
self.assert_scheduler_called(api, mock_query, 3)
def test_killall_job_wait_until_timeout(self):
"""Test kill client-side API logic."""
mock_context = FakeAuroraCommandContext()
mock_scheduler_proxy = create_autospec(spec=SchedulerThriftApiSpec, instance=True)
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):
api = mock_context.get_api('west')
mock_scheduler_proxy.getTasksWithoutConfigs.return_value = self.create_status_call_result()
api.kill_job.return_value = self.get_kill_job_response()
mock_scheduler_proxy.killTasks.return_value = self.get_kill_job_response()
for _ in range(8):
mock_context.add_expected_status_query_result(self.create_status_call_result(
self.create_mock_task(ScheduleStatus.RUNNING)))
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
assert EXIT_TIMEOUT == cmd.execute(
['job', 'killall', '--no-batching', '--config=%s' % fp.name, 'west/bozo/test/hello'])
# Now check that the right API calls got made.
assert api.kill_job.call_count == 1
api.kill_job.assert_called_with(AuroraJobKey.from_path('west/bozo/test/hello'), None)
self.assert_scheduler_called(api, self.get_expected_task_query(), 8)
def test_simple_successful_create_job_output(self):
"""Run a test of the "create" command against a mocked-out API:
Verifies that the creation command generates the correct output.
"""
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context',
return_value=mock_context)):
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.PENDING))
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.RUNNING))
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.RUNNING))
api = mock_context.get_api('west')
api.create_job.return_value = self.get_createjob_response()
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
result = cmd.execute([
'job', 'create', '--wait-until=RUNNING',
'west/bozo/test/hello', fp.name
])
assert result == EXIT_OK
assert mock_context.get_out() == [
"Job create succeeded: job url=http://something_or_other/scheduler/bozo/test/hello"
]
assert mock_context.get_err() == []
def test_download_path(self):
with temporary_file() as fd:
fd.close()
downloaded, path = self.expect_download(path_or_fd=fd.name)
self.assertEqual(path, fd.name)
with open(path) as fp:
self.assertEqual(downloaded, fp.read())
def test_create_job_startup_fails(self):
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('threading._Event.wait'),
patch('apache.aurora.client.cli.jobs.Job.create_context',
return_value=mock_context)):
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.PENDING))
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.RUNNING))
# We need to override the side_effect behavior of check_status in the context.
def check_status_side_effect(*args):
return self.create_error_response()
mock_context.get_api(
"west").check_status.side_effect = check_status_side_effect
api = mock_context.get_api('west')
api.create_job.return_value = self.get_createjob_response()
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
result = cmd.execute([
'job', 'create', '--wait-until=RUNNING',
'west/bozo/test/hello', fp.name
])
assert result == EXIT_COMMAND_FAILURE
assert mock_context.get_out() == []
assert mock_context.get_err() == [
"Error occurred while creating job west/bozo/test/hello"
]
def setUp(self):
with temporary_file() as ini:
ini.write(
'''
[DEFAULT]
answer: 42
scale: 1.2
path: /a/b/%(answer)s
embed: %(path)s::foo
disclaimer:
Let it be known
that.
[a]
fast: True
list: [1, 2, 3, %(answer)s]
[b]
preempt: False
dict: {
'a': 1,
'b': %(answer)s,
'c': ['%(answer)s', %(answer)s]
}
''')
ini.close()
self.config = Config.load(configpath=ini.name)
def test_restart_failed_restart_output(self):
self.reset_mock_io()
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_health_check = self.setup_health_checks(mock_api)
self.setup_mock_scheduler_for_simple_restart(mock_api)
mock_scheduler_proxy.restartShards.return_value = self.create_error_response(
)
with contextlib.nested(
patch('apache.aurora.client.api.SchedulerProxy',
return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS',
new=self.TEST_CLUSTERS),
patch(
'apache.aurora.client.api.instance_watcher.StatusHealthCheck',
return_value=mock_health_check),
patch('time.time',
side_effect=functools.partial(self.fake_time, self)),
patch(
'apache.aurora.client.cli.context.AuroraCommandContext.print_out',
side_effect=self.mock_print_out),
patch(
'apache.aurora.client.cli.context.AuroraCommandContext.print_err',
side_effect=self.mock_print_err),
patch('threading._Event.wait')):
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute([
'job', 'restart', '--batch-size=5', 'west/bozo/test/hello',
'--config', fp.name
])
assert self.MOCK_OUT == []
assert 'Error restarting job west/bozo/test/hello; see log for details' in self.MOCK_ERR
