def test_dedicated_portmap():
base_job = Job(name='hello_world',
role='john_doe',
cluster='test-cluster',
task=Task(name='main',
processes=[],
resources=Resources(cpu=0.1,
ram=64 * MB,
disk=64 * MB)))
config._validate_announce_configuration(AuroraConfig(base_job))
config._validate_announce_configuration(
AuroraConfig(base_job(constraints={'dedicated': 'mesos-team'})))
config._validate_announce_configuration(
AuroraConfig(
base_job(constraints={'dedicated': 'mesos-team'},
announce=Announcer(portmap={'http': 80}))))
with pytest.raises(ValueError):
config._validate_announce_configuration(
AuroraConfig(base_job(announce=Announcer(portmap={'http': 80}))))
with pytest.raises(ValueError):
config._validate_announce_configuration(
AuroraConfig(
base_job(announce=Announcer(portmap={'http': 80}),
constraints={'foo': 'bar'})))
def test_config_with_ports():
hwc = HELLO_WORLD(
task=HELLO_WORLD.task()(
processes=[Process(name="hello_world", cmdline="echo {{thermos.ports[http]}} {{thermos.ports[admin]}}")]
)
)
config = AuroraConfig(hwc)
job = config.job()
assert job.taskConfig.requestedPorts == set(["http", "admin"])
def test_config_with_ports():
hwc = HELLO_WORLD(task=HELLO_WORLD.task()(processes=[
Process(
name='hello_world',
cmdline='echo {{thermos.ports[http]}} {{thermos.ports[admin]}}')
]))
config = AuroraConfig(hwc)
job = config.job()
assert job.taskConfig.requestedPorts == set(['http', 'admin'])
def test_pick():
env = AuroraConfigLoader.load(BytesIO(MESOS_CONFIG))
hello_world = env['jobs'][0]
assert AuroraConfig.pick(env, 'hello_world', None) == hello_world
env['jobs'][0] = env['jobs'][0](name='something_{{else}}')
assert str(AuroraConfig.pick(env, 'something_else', [{'else': 'else'}]).name()) == (
'something_else')
def test_config_with_ports():
hwc = HELLO_WORLD(task=HELLO_WORLD.task()(processes=[
Process(
name='hello_world',
cmdline='echo {{thermos.ports[http]}} {{thermos.ports[admin]}}')
]))
config = AuroraConfig(hwc)
job = config.job()
assert Resource(namedPort='http') in list(job.taskConfig.resources)
assert Resource(namedPort='admin') in list(job.taskConfig.resources)
def test_pick():
env = AuroraConfigLoader.load(BytesIO(MESOS_CONFIG))
hello_world = env['jobs'][0]
assert AuroraConfig.pick(env, 'hello_world', None) == hello_world
env['jobs'][0] = env['jobs'][0](name='something_{{else}}')
assert str(
AuroraConfig.pick(env, 'something_else', [{
'else': 'else'
}]).name()) == ('something_else')
def test_pick():
with temporary_file() as fp:
fp.write(MESOS_CONFIG)
fp.flush()
env = AuroraConfigLoader.load(fp.name)
hello_world = env['jobs'][0]
assert AuroraConfig.pick(env, 'hello_world', None) == hello_world
env['jobs'][0] = env['jobs'][0](name='something_{{else}}')
assert str(AuroraConfig.pick(env, 'something_else', [{'else': 'else'}]).name()) == (
'something_else')
def test_config_with_ports():
hwc = HELLO_WORLD(
task=HELLO_WORLD.task()(
processes=[
Process(name='hello_world',
cmdline='echo {{thermos.ports[http]}} {{thermos.ports[admin]}}')
]
)
)
config = AuroraConfig(hwc)
job = config.job()
assert Resource(namedPort='http') in list(job.taskConfig.resources)
assert Resource(namedPort='admin') in list(job.taskConfig.resources)
def test_inject_default_environment():
base_job = Job(
name='hello_world', role='john_doe', cluster='smf1-test',
task=Task(name='main', processes=[],
resources=Resources(cpu=0.1, ram=64 * MB, disk=64 * MB)))
no_env_config = AuroraConfig(base_job)
config._inject_default_environment(no_env_config)
assert no_env_config.environment() == DEFAULT_ENVIRONMENT
test_env_config = AuroraConfig(base_job(environment='test'))
config._inject_default_environment(test_env_config)
assert test_env_config.environment() == 'test'
def test_simple_config():
with temporary_file() as fp:
fp.write(MESOS_CONFIG)
fp.flush()
proxy_config1 = AuroraConfig.load(fp.name)
proxy_config2 = AuroraConfig.load(fp.name, name="hello_world")
assert proxy_config1.job()
assert proxy_config1._job == proxy_config2._job
assert proxy_config1._job == REIFIED_CONFIG
assert proxy_config1.name() == 'hello_world'
assert proxy_config1.role() == 'john_doe'
assert proxy_config1.cluster() == 'smf1-test'
assert proxy_config1.ports() == set()
def test_simple_config():
with temporary_file() as fp:
fp.write(MESOS_CONFIG)
fp.flush()
proxy_config1 = AuroraConfig.load(fp.name)
proxy_config2 = AuroraConfig.load(fp.name, name="hello_world")
assert proxy_config1.job()
assert proxy_config1._job == proxy_config2._job
assert proxy_config1._job == REIFIED_CONFIG
assert proxy_config1.name() == "hello_world"
assert proxy_config1.role() == "john_doe"
assert proxy_config1.cluster() == "smf1-test"
assert proxy_config1.ports() == set()
def test_config_with_task_links():
tl = Map(String, String)
unresolved_tl = {
"foo": "http://%host%:{{thermos.ports[foo]}}",
"bar": "http://%host%:{{thermos.ports[bar]}}/{{mesos.instance}}",
}
resolved_tl = {"foo": "http://%host%:%port:foo%", "bar": "http://%host%:%port:bar%/%shard_id%"}
aurora_config = AuroraConfig(HELLO_WORLD(task_links=tl(unresolved_tl)))
assert aurora_config.task_links() == tl(resolved_tl)
assert aurora_config.job().taskConfig.taskLinks == frozendict(resolved_tl)
bad_tl = {"foo": "{{thermos.ports.bad}}"}
with pytest.raises(AuroraConfig.InvalidConfig):
AuroraConfig(HELLO_WORLD(task_links=tl(bad_tl))).job()
def test_pick():
with temporary_file() as fp:
fp.write(MESOS_CONFIG)
fp.flush()
env = AuroraConfigLoader.load(fp.name)
hello_world = env['jobs'][0]
assert AuroraConfig.pick(env, 'hello_world', None) == hello_world
env['jobs'][0] = env['jobs'][0](name='something_{{else}}')
assert str(
AuroraConfig.pick(env, 'something_else', [{
'else': 'else'
}]).name()) == ('something_else')
def test_inject_default_environment():
base_job = Job(
name="hello_world",
role="john_doe",
cluster="test-cluster",
task=Task(name="main", processes=[], resources=Resources(cpu=0.1, ram=64 * MB, disk=64 * MB)),
)
no_env_config = AuroraConfig(base_job)
config._inject_default_environment(no_env_config)
assert no_env_config.environment() == DEFAULT_ENVIRONMENT
test_env_config = AuroraConfig(base_job(environment="test"))
config._inject_default_environment(test_env_config)
assert test_env_config.environment() == "test"
def test_config_with_task_links():
tl = Map(String, String)
unresolved_tl = {
'foo': 'http://%host%:{{thermos.ports[foo]}}',
'bar': 'http://%host%:{{thermos.ports[bar]}}/{{mesos.instance}}',
}
resolved_tl = {
'foo': 'http://%host%:%port:foo%',
'bar': 'http://%host%:%port:bar%/%shard_id%'
}
aurora_config = AuroraConfig(HELLO_WORLD(task_links=tl(unresolved_tl)))
assert aurora_config.task_links() == tl(resolved_tl)
assert aurora_config.job().taskConfig.taskLinks == frozendict(resolved_tl)
bad_tl = {'foo': '{{thermos.ports.bad}}'}
with pytest.raises(AuroraConfig.InvalidConfig):
AuroraConfig(HELLO_WORLD(task_links=tl(bad_tl))).job()
def test_environment_names():
base_job = Job(name='hello_world',
role='john_doe',
cluster='test-cluster',
task=Task(name='main',
processes=[],
resources=Resources(cpu=0.1,
ram=64 * MB,
disk=64 * MB)))
with pytest.raises(ValueError):
config._validate_environment_name(AuroraConfig(base_job))
for env_name in GOOD_ENV:
config._validate_environment_name(
AuroraConfig(base_job(environment=env_name)))
for env_name in BAD_ENV:
with pytest.raises(ValueError):
config._validate_environment_name(
AuroraConfig(base_job(environment=env_name)))
def test_update_config_passes_with_min_consecutive_successes_zero():
base_job = Job(
name='hello_world',
role='john_doe',
cluster='test-cluster',
health_check_config=HealthCheckConfig(min_consecutive_successes=0),
task=Task(name='main',
processes=[],
resources=Resources(cpu=0.1, ram=64 * MB, disk=64 * MB)))
config._validate_update_config(AuroraConfig(base_job))
def _start_release(self, api, context):
package, config_content = get_config(
context.options.jobspec, context.options.version or 'latest')
config = AuroraConfig.loads_json(config_content)
resp = api.start_job_update(config, message='Release started by %s.' % getpass.getuser())
if not resp.result:
return package, None
return package, resp.result.startJobUpdateResult.key.id
def test_update_config_passes_with_default_values():
base_job = Job(name='hello_world',
role='john_doe',
cluster='test-cluster',
task=Task(name='main',
processes=[],
resources=Resources(cpu=0.1,
ram=64 * MB,
disk=64 * MB)))
config._validate_update_config(AuroraConfig(base_job))
def test_update_config_fails_with_min_consecutive_successes_negative():
base_job = Job(
name='hello_world',
role='john_doe',
cluster='test-cluster',
health_check_config=HealthCheckConfig(min_consecutive_successes=-1),
task=Task(name='main',
processes=[],
resources=Resources(cpu=0.1, ram=64 * MB, disk=64 * MB)))
with pytest.raises(SystemExit):
config._validate_update_config(AuroraConfig(base_job))
def test_health_check_config_http_ok():
base_job = Job(name='hello_bond',
role='james',
cluster='marine-cluster',
health_check_config=HealthCheckConfig(
max_consecutive_failures=1, ),
task=Task(name='main',
processes=[],
resources=Resources(cpu=0.1,
ram=64 * MB,
disk=64 * MB)))
config._validate_health_check_config(AuroraConfig(base_job))
def test_update_config_passes_with_min_requirement_values():
base_job = Job(
name='hello_world',
role='john_doe',
cluster='test-cluster',
update_config=UpdateConfig(watch_secs=26),
health_check_config=HealthCheckConfig(max_consecutive_failures=1),
task=Task(name='main',
processes=[],
resources=Resources(cpu=0.1, ram=64 * MB, disk=64 * MB)))
config._validate_update_config(AuroraConfig(base_job))
def test_update_config_fails_insufficient_watch_secs_equal_to_target():
base_job = Job(
name='hello_world',
role='john_doe',
cluster='test-cluster',
update_config=UpdateConfig(watch_secs=25),
health_check_config=HealthCheckConfig(max_consecutive_failures=1),
task=Task(name='main',
processes=[],
resources=Resources(cpu=0.1, ram=64 * MB, disk=64 * MB)))
with pytest.raises(SystemExit):
config._validate_update_config(AuroraConfig(base_job))
def test_update_config_fails_insufficient_watch_secs_less_than_target():
base_job = Job(name='hello_world',
role='john_doe',
cluster='test-cluster',
update_config=UpdateConfig(watch_secs=10),
task=Task(name='main',
processes=[],
resources=Resources(cpu=0.1,
ram=64 * MB,
disk=64 * MB)))
with pytest.raises(SystemExit):
config._validate_update_config(AuroraConfig(base_job))
def test_environment_names():
BAD = ('Prod', ' prod', 'prod ', 'tEst', 'production', 'staging 2',
'stagingA')
GOOD = ('prod', 'devel', 'test', 'staging', 'staging001', 'staging1',
'staging1234')
base_job = Job(name='hello_world',
role='john_doe',
cluster='test-cluster',
task=Task(name='main',
processes=[],
resources=Resources(cpu=0.1,
ram=64 * MB,
disk=64 * MB)))
with pytest.raises(ValueError):
config._validate_environment_name(AuroraConfig(base_job))
for env_name in GOOD:
config._validate_environment_name(
AuroraConfig(base_job(environment=env_name)))
for env_name in BAD:
with pytest.raises(ValueError):
config._validate_environment_name(
AuroraConfig(base_job(environment=env_name)))
def get_job_config(self):
return AuroraConfig(job=Job(
cluster='west',
role='bozo',
environment='test',
name='the_job',
service=False,
task=Task(name='task',
processes=[Process(cmdline='ls -la', name='process')],
resources=Resources(
cpu=1.0, ram=1024 * MB, disk=1024 * MB)),
contact='*****@*****.**',
instances=3,
cron_schedule='* * * * *'))
def test_health_check_config_shell_no_command():
# If we chose shell config, we must define shell_command.
base_job = Job(
name='hello_bond',
role='james',
cluster='marine-cluster',
health_check_config=HealthCheckConfig(
max_consecutive_failures=1,
health_checker=HealthCheckerConfig(shell=ShellHealthChecker())),
task=Task(name='main',
processes=[],
resources=Resources(cpu=0.1, ram=64 * MB, disk=64 * MB)))
with pytest.raises(SystemExit):
config._validate_health_check_config(AuroraConfig(base_job))
def get_job_config(self, is_cron=False):
return AuroraConfig(job=Job(
cluster='west',
role='bozo',
environment='test',
name='the_job',
service=True if not is_cron else False,
cron_schedule='* * * * *' if is_cron else Empty,
task=Task(name='task',
processes=[Process(cmdline='ls -la', name='process')],
resources=Resources(
cpu=1.0, ram=1024 * MB, disk=1024 * MB)),
instances=3,
))
def test_docker_binding_throws(self, mock_resolve):
mock_resolve.side_effect = Exception('mock resolve failure')
binding_helper.unregister_all()
BindingHelper.register(DockerBindingHelper())
with temporary_file() as fp:
fp.write(DOCKER_BINDING_CONFIG)
fp.flush()
with CLUSTERS.patch(TEST_CLUSTERS):
cfg = AuroraConfig.load(fp.name)
with pytest.raises(Exception):
binding_helper.apply_all(cfg)
assert mock_resolve.mock_calls == [call(TEST_CLUSTER, 'some/name', 'some.tag')]
def make_config(announce, *ports):
process = Process(name='hello',
cmdline=' '.join('{{thermos.ports[%s]}}' % port
for port in ports))
return AuroraConfig(
Job(name='hello_world',
environment='staging42',
role='john_doe',
cluster='smf1-test',
announce=announce,
task=Task(name='main',
processes=[process],
resources=Resources(cpu=0.1,
ram=64 * 1048576,
disk=64 * 1048576))))
def test_validate_deprecated_config_adds_warning_for_production():
job = Job(name='hello_world',
role='john_doe',
cluster='test-cluster',
environment='test',
task=Task(name='main',
processes=[
Process(cmdline='echo {{_unbound_}}', name='eco')
],
resources=Resources(cpu=0.1, ram=64 * MB,
disk=64 * MB)),
production='true')
with mock.patch(
'apache.aurora.client.config.deprecation_warning') as mock_warning:
config._validate_deprecated_config(AuroraConfig(job))
mock_warning.assert_called_once_with(PRODUCTION_DEPRECATED_WARNING)
def test_docker_binding_throws(self, mock_resolve):
mock_resolve.side_effect = Exception('mock resolve failure')
binding_helper.unregister_all()
BindingHelper.register(DockerBindingHelper())
with temporary_file() as fp:
fp.write(DOCKER_BINDING_CONFIG)
fp.flush()
with CLUSTERS.patch(TEST_CLUSTERS):
cfg = AuroraConfig.load(fp.name)
with pytest.raises(Exception):
binding_helper.apply_all(cfg)
assert mock_resolve.mock_calls == [
call(TEST_CLUSTER, 'some/name', 'some.tag')
]
def test_docker_binding(self, mock_resolve):
image_reference = 'registry.example.com/some/repo@some:digest'
mock_resolve.return_value = image_reference
binding_helper.unregister_all()
BindingHelper.register(DockerBindingHelper())
with temporary_file() as fp:
fp.write(DOCKER_BINDING_CONFIG)
fp.flush()
with CLUSTERS.patch(TEST_CLUSTERS):
cfg = AuroraConfig.load(fp.name)
binding_helper.apply_all(cfg)
assert cfg.job().taskConfig.container.docker.image == image_reference
assert mock_resolve.mock_calls == [call(TEST_CLUSTER, 'some/name', 'some.tag')]
def test_validate_deprecated_config_adds_no_warning_when_tier_is_set():
job = Job(name='hello_world',
role='john_doe',
cluster='test-cluster',
environment='test',
task=Task(name='main',
processes=[
Process(cmdline='echo {{_unbound_}}', name='eco')
],
resources=Resources(cpu=0.1, ram=64 * MB,
disk=64 * MB)),
production='true',
tier='preferred')
with mock.patch(
'apache.aurora.client.config.deprecation_warning') as mock_warning:
config._validate_deprecated_config(AuroraConfig(job))
assert mock_warning.call_count == 0
def test_docker_binding(self, mock_resolve):
image_reference = 'registry.example.com/some/repo@some:digest'
mock_resolve.return_value = image_reference
binding_helper.unregister_all()
BindingHelper.register(DockerBindingHelper())
with temporary_file() as fp:
fp.write(DOCKER_BINDING_CONFIG)
fp.flush()
with CLUSTERS.patch(TEST_CLUSTERS):
cfg = AuroraConfig.load(fp.name)
binding_helper.apply_all(cfg)
assert cfg.job(
).taskConfig.container.docker.image == image_reference
assert mock_resolve.mock_calls == [
call(TEST_CLUSTER, 'some/name', 'some.tag')
]
def test_health_check_config_deprecate_message(monkeypatch):
base_job = Job(name='hello_bond',
role='james',
cluster='marine-cluster',
health_check_config=HealthCheckConfig(
max_consecutive_failures=1,
endpoint='/to_be_deprecated'),
task=Task(name='main',
processes=[],
resources=Resources(cpu=0.1,
ram=64 * MB,
disk=64 * MB)))
log_items = []
def capture_log(msg):
log_items.append(msg)
monkeypatch.setattr(log, 'warn', capture_log)
config._validate_health_check_config(AuroraConfig(base_job))
assert log_items == [HTTP_DEPRECATION_WARNING]
def _job_diff(self, api, context, config_content):
# return true if jobs are diff, false if not
config = AuroraConfig.loads_json(config_content)
role, env, name = config.role(), config.environment(), config.name()
resp = api.query(api.build_query(role, name, env=env, statuses=ACTIVE_STATES))
context.log_response_and_raise(resp, err_code=EXIT_INVALID_PARAMETER,
err_msg="Could not find job to diff against")
if resp.result.scheduleStatusResult.tasks is None:
context.print_err("No tasks found for job %s" % context.options.jobspec)
return True
else:
remote_tasks = [t.assignedTask.task for t in resp.result.scheduleStatusResult.tasks]
resp = api.populate_job_config(config)
context.log_response_and_raise(resp, err_code=EXIT_INVALID_CONFIGURATION,
err_msg="Error loading configuration")
local_tasks = [resp.result.populateJobResult.taskConfig] * config.instances()
if len(remote_tasks) != len(local_tasks):
return True
for task1, task2 in zip(remote_tasks, local_tasks):
if task1 != task2:
return True
return False
def test_inject_default_environment():
base_job = Job(name='hello_world',
role='john_doe',
cluster='test-cluster',
task=Task(name='main',
processes=[],
resources=Resources(cpu=0.1,
ram=64 * MB,
disk=64 * MB)))
no_env_config = AuroraConfig(base_job)
config._inject_default_environment(no_env_config)
assert no_env_config.environment() == DEFAULT_ENVIRONMENT
test_env_config = AuroraConfig(base_job(environment='test'))
config._inject_default_environment(test_env_config)
assert test_env_config.environment() == 'test'
def job_command(cmdline):
return AuroraConfig(HELLO_WORLD(task=SimpleTask('hello_world', cmdline))).raw()
def test_empty_config():
with pytest.raises(AuroraConfig.InvalidConfig):
with temporary_file() as fp:
fp.write(UNDERSPECIFIED_MESOS_CONFIG)
fp.flush()
AuroraConfig.load(fp.name)
def write_and_load_config(role):
with temporary_file() as fp:
fp.write(GENERIC_CONFIG)
fp.flush()
return AuroraConfig.load(fp.name, name='hello_world', select_role=role)
def test_empty_config():
with pytest.raises(AuroraConfig.InvalidConfig):
with temporary_file() as fp:
fp.write(UNDERSPECIFIED_MESOS_CONFIG)
fp.flush()
AuroraConfig.load(fp.name)
def write_and_load_config(role):
with temporary_file() as fp:
fp.write(GENERIC_CONFIG)
fp.flush()
return AuroraConfig.load(fp.name, name='hello_world', select_role=role)