def test_patch_cleanup_on_error():
clusters = Clusters([Cluster(name='original')])
with pytest.raises(RuntimeError):
with clusters.patch([Cluster(name='replacement')]):
assert list(clusters) == ['replacement']
raise RuntimeError("exit contextmanager scope")
assert list(clusters) == ['original']
def test_patch_cleanup_on_error():
clusters = Clusters([Cluster(name="original")])
with pytest.raises(RuntimeError):
with clusters.patch([Cluster(name="replacement")]):
assert list(clusters) == ["replacement"]
raise RuntimeError("exit contextmanager scope")
assert list(clusters) == ["original"]
def test_load_json():
with temporary_dir() as td:
clusters_json = os.path.join(td, 'clusters.json')
# as dict
with open(clusters_json, 'w') as fp:
fp.write(json.dumps(CLUSTERS))
validate_loaded_clusters(Clusters.from_file(clusters_json))
# as list
with open(clusters_json, 'w') as fp:
fp.write(json.dumps(CLUSTERS.values()))
validate_loaded_clusters(Clusters.from_file(clusters_json))
def test_load_json():
with temporary_dir() as td:
clusters_json = os.path.join(td, 'clusters.json')
# as dict
with open(clusters_json, 'w') as fp:
fp.write(json.dumps(CLUSTERS))
validate_loaded_clusters(Clusters.from_file(clusters_json))
# as list
with open(clusters_json, 'w') as fp:
fp.write(json.dumps(CLUSTERS.values()))
validate_loaded_clusters(Clusters.from_file(clusters_json))
def test_load_invalid_syntax():
with temporary_dir() as td:
# bad json
clusters_json = os.path.join(td, 'clusters.json')
with open(clusters_json, 'w') as fp:
fp.write('This is not json')
with pytest.raises(Clusters.ParseError):
Clusters.from_file(clusters_json)
# bad yaml
clusters_yml = os.path.join(td, 'clusters.yml')
with open(clusters_yml, 'w') as fp:
fp.write('L{}L')
with pytest.raises(Clusters.ParseError):
Clusters.from_file(clusters_yml)
# bad layout
clusters_yml = os.path.join(td, 'clusters.yml')
with open(clusters_yml, 'w') as fp:
fp.write('just a string')
with pytest.raises(Clusters.ParseError):
Clusters.from_file(clusters_yml)
# not a dict
clusters_json = os.path.join(td, 'clusters.json')
with open(clusters_json, 'w') as fp:
fp.write(json.dumps({'cluster1': ['not', 'cluster', 'values']}))
with pytest.raises(Clusters.ParseError):
Clusters.from_file(clusters_json)
def test_load_invalid_syntax():
with temporary_dir() as td:
# bad json
clusters_json = os.path.join(td, 'clusters.json')
with open(clusters_json, 'w') as fp:
fp.write('This is not json')
with pytest.raises(Clusters.ParseError):
Clusters.from_file(clusters_json)
# bad yaml
clusters_yml = os.path.join(td, 'clusters.yml')
with open(clusters_yml, 'w') as fp:
fp.write('L{}L')
with pytest.raises(Clusters.ParseError):
Clusters.from_file(clusters_yml)
# bad layout
clusters_yml = os.path.join(td, 'clusters.yml')
with open(clusters_yml, 'w') as fp:
fp.write('just a string')
with pytest.raises(Clusters.ParseError):
Clusters.from_file(clusters_yml)
# not a dict
clusters_json = os.path.join(td, 'clusters.json')
with open(clusters_json, 'w') as fp:
fp.write(json.dumps({'cluster1': ['not', 'cluster', 'values']}))
with pytest.raises(Clusters.ParseError):
Clusters.from_file(clusters_json)
def test_load_invalid_syntax():
with temporary_dir() as td:
# bad json
clusters_json = os.path.join(td, "clusters.json")
with open(clusters_json, "w") as fp:
fp.write("This is not json")
with pytest.raises(Clusters.ParseError):
Clusters.from_file(clusters_json)
# not a dict
clusters_json = os.path.join(td, "clusters.json")
with open(clusters_json, "w") as fp:
fp.write(json.dumps({"cluster1": ["not", "cluster", "values"]}))
with pytest.raises(Clusters.ParseError):
Clusters.from_file(clusters_json)
class TestAdminUtil(unittest.TestCase):
TEST_CLUSTER_NAME = 'west'
TEST_CLUSTER = Cluster(name=TEST_CLUSTER_NAME,
zk='zookeeper.example.com',
scheduler_zk_path='/foo/bar',
auth_mechanism='UNAUTHENTICATED')
TEST_CLUSTERS = Clusters([TEST_CLUSTER])
@mock.patch("apache.aurora.admin.admin_util.subprocess", spec=subprocess)
def test_parse_script(self, mock_subprocess):
with temporary_file() as fp:
mock_popen = mock.Mock()
mock_popen.wait.return_value = 0
mock_subprocess.Popen.return_value = mock_popen
parse_script(fp.name)('h1')
assert mock_popen.wait.call_count == 1
def test_parse_script_invalid_filename(self):
self.assertRaises(SystemExit, parse_script, "invalid filename")
def test_make_admin_client_cluster_string(self):
with mock.patch('apache.aurora.admin.admin_util.CLUSTERS',
new=self.TEST_CLUSTERS):
self.assertIsNotNone(make_admin_client(self.TEST_CLUSTER_NAME))
def test_make_admin_client_cluster_object(self):
with mock.patch('apache.aurora.admin.admin_util.CLUSTERS',
new=self.TEST_CLUSTERS):
self.assertIsNotNone(make_admin_client(self.TEST_CLUSTER))
def test_make_admin_client_cluster_unknown(self):
with mock.patch('apache.aurora.admin.admin_util.CLUSTERS',
new=self.TEST_CLUSTERS):
self.assertRaises(SystemExit, make_admin_client, 'east')
class AuroraClientCommandTest(unittest.TestCase):
@classmethod
def create_blank_response(cls, code, msg):
# TODO(wfarner): Don't use a mock here.
response = create_autospec(spec=Response, instance=True)
response.responseCode = code
response.result = create_autospec(spec=Result, instance=True)
response.details = [ResponseDetail(message=msg)]
return response
@classmethod
def create_simple_success_response(cls):
return cls.create_blank_response(ResponseCode.OK, 'OK')
@classmethod
def create_mock_api(cls):
"""Builds up a mock API object, with a mock SchedulerProxy"""
mock_scheduler_proxy = create_autospec(spec=SchedulerProxyApiSpec,
spec_set=False,
instance=True)
mock_scheduler_proxy.url = "http://something_or_other"
mock_scheduler_proxy.scheduler_client.return_value = mock_scheduler_proxy
mock_api = create_autospec(spec=HookedAuroraClientAPI)
mock_api.scheduler_proxy = mock_scheduler_proxy
return mock_api, mock_scheduler_proxy
TEST_CLUSTER = 'west'
TEST_CLUSTERS = Clusters([
Cluster(name='west',
zk='zookeeper.example.com',
scheduler_zk_path='/foo/bar',
auth_mechanism='UNAUTHENTICATED')
])
@classmethod
def create_mock_probe_hosts_vector(cls, side_effects):
mock_vector = create_autospec(spec=DomainUpTimeSlaVector,
instance=True)
mock_vector.probe_hosts.side_effect = side_effects
return mock_vector
@classmethod
def create_probe_hosts(cls, hostname, predicted, safe, safe_in):
hosts = defaultdict(list)
job = AuroraJobKey.from_path('west/role/env/job-%s' % hostname)
hosts[hostname].append(JobUpTimeDetails(job, predicted, safe, safe_in))
return [hosts]
# TODO(wfarner): Remove this, force tests to call out their flags.
@classmethod
def setup_mock_options(cls):
mock_options = create_autospec(spec=['verbosity'], instance=True)
mock_options.verbosity = 'verbose'
mock_options.bypass_leader_redirect = False
return mock_options
def test_load_yaml():
with temporary_dir() as td:
clusters_yml = os.path.join(td, 'clusters.yml')
with open(clusters_yml, 'w') as fp:
fp.write(TEST_YAML)
validate_loaded_clusters(Clusters.from_file(clusters_yml))
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from optparse import OptionParser
import pytest
from apache.aurora.common.cluster import Cluster
from apache.aurora.common.cluster_option import ClusterOption
from apache.aurora.common.clusters import Clusters
CLUSTER_LIST = Clusters((
Cluster(name='smf1'),
Cluster(name='smf1-test'),
))
def cluster_provider(name):
return CLUSTER_LIST[name]
def test_constructors():
ClusterOption('--test', '-t', help="Test cluster.", clusters=CLUSTER_LIST)
ClusterOption('--test', '-t', help="Test cluster.", cluster_provider=cluster_provider)
with pytest.raises(ValueError):
ClusterOption()
with pytest.raises(ValueError):
class AuroraClientCommandTest(unittest.TestCase):
@classmethod
def create_blank_response(cls, code, msg):
response = Mock(spec=Response)
response.responseCode = code
response.messageDEPRECATED = msg
response.result = Mock(spec=Result)
return response
@classmethod
def create_simple_success_response(cls):
return cls.create_blank_response(ResponseCode.OK, 'OK')
@classmethod
def create_error_response(cls):
return cls.create_blank_response(ResponseCode.ERROR, 'Damn')
@classmethod
def create_mock_api(cls):
"""Builds up a mock API object, with a mock SchedulerProxy"""
# This looks strange, but we set up the same object to use as both
# the SchedulerProxy and the SchedulerClient. These tests want to observe
# what API calls get made against the scheduler, and both of these objects
# delegate calls to the scheduler. It doesn't matter which one is used:
# what we care about is that the right API calls get made.
mock_scheduler_proxy = Mock()
mock_scheduler_proxy.url = "http://something_or_other"
mock_scheduler_proxy.scheduler_client.return_value = mock_scheduler_proxy
mock_api = Mock(spec=HookedAuroraClientAPI)
mock_api.scheduler_proxy = mock_scheduler_proxy
return (mock_api, mock_scheduler_proxy)
@classmethod
def create_mock_api_factory(cls):
"""Create a collection of mocks for a test that wants to mock out the client API
by patching the api factory."""
mock_api, mock_scheduler_proxy = cls.create_mock_api()
mock_api_factory = Mock()
mock_api_factory.return_value = mock_api
return mock_api_factory, mock_scheduler_proxy
FAKE_TIME = 42131
@classmethod
def fake_time(cls, ignored):
"""Utility function used for faking time to speed up tests."""
cls.FAKE_TIME += 2
return cls.FAKE_TIME
CONFIG_BASE = """
HELLO_WORLD = Job(
name = '%(job)s',
role = '%(role)s',
cluster = '%(cluster)s',
environment = '%(env)s',
instances = 20,
%(inner)s
update_config = UpdateConfig(
batch_size = 1,
restart_threshold = 60,
watch_secs = 45,
max_per_shard_failures = 2,
),
task = Task(
name = 'test',
processes = [Process(name = 'hello_world', cmdline = 'echo {{thermos.ports[http]}}')],
resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB),
)
)
jobs = [HELLO_WORLD]
"""
TEST_ROLE = 'mchucarroll'
TEST_ENV = 'test'
TEST_JOB = 'hello'
TEST_CLUSTER = 'west'
TEST_JOBSPEC = 'west/mchucarroll/test/hello'
TEST_CLUSTERS = Clusters([
Cluster(name='west',
packer_copy_command='copying {{package}}',
zk='zookeeper.example.com',
scheduler_zk_path='/foo/bar',
auth_mechanism='UNAUTHENTICATED')
])
@classmethod
def get_test_config(cls, cluster, role, env, job, filler=''):
"""Create a config from the template"""
return cls.CONFIG_BASE % {
'job': job,
'role': role,
'env': env,
'cluster': cluster,
'inner': filler
}
@classmethod
def get_valid_config(cls):
return cls.get_test_config(cls.TEST_CLUSTER, cls.TEST_ROLE,
cls.TEST_ENV, cls.TEST_JOB)
@classmethod
def get_invalid_config(cls, bad_clause):
return cls.get_test_config(cls.TEST_CLUSTER, cls.TEST_ROLE,
cls.TEST_ENV, cls.TEST_JOB, bad_clause)
@classmethod
def create_mock_probe_hosts_vector(cls, side_effects):
mock_vector = Mock(spec=DomainUpTimeSlaVector)
mock_vector.probe_hosts.side_effect = side_effects
return mock_vector
@classmethod
def create_probe_hosts(cls, hostname, predicted, safe, safe_in):
hosts = defaultdict(list)
job = AuroraJobKey.from_path('west/role/env/job-%s' % hostname)
hosts[hostname].append(JobUpTimeDetails(job, predicted, safe, safe_in))
return [hosts]
@classmethod
def setup_mock_options(cls):
mock_options = Mock(spec=['verbosity'])
mock_options.verbosity = 'verbose'
return mock_options
class AuroraClientCommandTest(unittest.TestCase):
FAKE_TIME = 42131
@classmethod
def create_blank_response(cls, code, msg):
response = Mock(spec=Response)
response.responseCode = code
mock_msg = Mock()
mock_msg.message = msg
response.details = [mock_msg]
response.messageDEPRECATED = msg
response.result = Mock(spec=Result)
return response
@classmethod
def create_simple_success_response(cls):
return cls.create_blank_response(ResponseCode.OK, 'OK')
@classmethod
def create_error_response(cls):
return cls.create_blank_response(ResponseCode.ERROR, 'Damn')
@classmethod
def create_mock_api(cls):
"""Builds up a mock API object, with a mock SchedulerProxy"""
mock_api = Mock(spec=HookedAuroraClientAPI)
mock_scheduler = Mock()
mock_scheduler.url = "http://something_or_other"
mock_scheduler_client = Mock()
mock_scheduler_client.scheduler.return_value = mock_scheduler
mock_scheduler_client.url = "http://something_or_other"
mock_api = Mock(spec=HookedAuroraClientAPI)
mock_api.scheduler_proxy = mock_scheduler_client
return (mock_api, mock_scheduler_client)
@classmethod
def create_mock_api_factory(cls):
"""Create a collection of mocks for a test that wants to mock out the client API
by patching the api factory."""
mock_api, mock_scheduler_client = cls.create_mock_api()
mock_api_factory = Mock()
mock_api_factory.return_value = mock_api
return mock_api_factory, mock_scheduler_client
@classmethod
def create_status_call_result(cls, mock_task=None):
status_response = cls.create_simple_success_response()
schedule_status = Mock(spec=ScheduleStatusResult)
status_response.result.scheduleStatusResult = schedule_status
# This should be a list of ScheduledTask's.
schedule_status.tasks = []
if mock_task is None:
for i in range(20):
schedule_status.tasks.append(cls.create_mock_task(i))
else:
schedule_status.tasks.append(mock_task)
return status_response
@classmethod
def create_mock_task(cls, instance_id, status=ScheduleStatus.RUNNING):
mock_task = Mock(spec=ScheduledTask)
mock_task.assignedTask = Mock(spec=AssignedTask)
mock_task.assignedTask.instanceId = instance_id
mock_task.assignedTask.taskId = "Task%s" % instance_id
mock_task.assignedTask.slaveId = "Slave%s" % instance_id
mock_task.assignedTask.task = Mock(spec=TaskConfig)
mock_task.slaveHost = "Slave%s" % instance_id
mock_task.status = status
mock_task_event = Mock(spec=TaskEvent)
mock_task_event.timestamp = 1000
mock_task.taskEvents = [mock_task_event]
return mock_task
@classmethod
def setup_get_tasks_status_calls(cls, scheduler):
status_response = cls.create_status_call_result()
scheduler.getTasksWithoutConfigs.return_value = status_response
@classmethod
def fake_time(cls, ignored):
"""Utility function used for faking time to speed up tests."""
cls.FAKE_TIME += 2
return cls.FAKE_TIME
CONFIG_BASE = """
HELLO_WORLD = Job(
name = '%(job)s',
role = '%(role)s',
cluster = '%(cluster)s',
environment = '%(env)s',
instances = 20,
%(inner)s
update_config = UpdateConfig(
batch_size = 1,
restart_threshold = 60,
watch_secs = 45,
max_per_shard_failures = 2,
),
task = Task(
name = 'test',
processes = [Process(name = 'hello_world', cmdline = 'echo {{thermos.ports[http]}}')],
resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB),
)
)
jobs = [HELLO_WORLD]
"""
UNBOUND_CONFIG = textwrap.dedent("""\
HELLO_WORLD = Job(
name = '%(job)s',
role = '%(role)s',
cluster = '{{cluster_binding}}',
environment = '%(env)s',
instances = '{{instances_binding}}',
update_config = UpdateConfig(
batch_size = 1,
restart_threshold = 60,
watch_secs = 45,
max_per_shard_failures = 2,
),
task = Task(
name = 'test',
processes = [Process(name = 'hello_world', cmdline = 'echo {{thermos.ports[http]}}')],
resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB),
)
)
jobs = [HELLO_WORLD]
""")
TEST_ROLE = 'bozo'
TEST_ENV = 'test'
TEST_JOB = 'hello'
TEST_CLUSTER = 'west'
TEST_JOBSPEC = 'west/bozo/test/hello'
TEST_CLUSTERS = Clusters([
Cluster(name='west',
packer_copy_command='copying {{package}}',
zk='zookeeper.example.com',
scheduler_zk_path='/foo/bar',
auth_mechanism='UNAUTHENTICATED')
])
@classmethod
def get_test_config(cls, cluster, role, env, job, filler=''):
"""Create a config from the template"""
return cls.CONFIG_BASE % {
'job': job,
'role': role,
'env': env,
'cluster': cluster,
'inner': filler
}
@classmethod
def get_unbound_test_config(cls, role=None, env=None, job=None):
result = cls.UNBOUND_CONFIG % {
'job': job or cls.TEST_JOB,
'role': role or cls.TEST_ROLE,
'env': env or cls.TEST_ENV
}
print(
"CONFIG:===========================\n%s\n============================="
% result)
return result
@classmethod
def get_valid_config(cls):
return cls.get_test_config(cls.TEST_CLUSTER, cls.TEST_ROLE,
cls.TEST_ENV, cls.TEST_JOB)
@classmethod
def get_invalid_config(cls, bad_clause):
return cls.get_test_config(cls.TEST_CLUSTER, cls.TEST_ROLE,
cls.TEST_ENV, cls.TEST_JOB, bad_clause)
def test_load():
with temporary_dir() as td:
clusters_json = os.path.join(td, 'clusters.json')
with open(clusters_json, 'w') as fp:
fp.write(CLUSTERS)
validate_loaded_clusters(Clusters.from_file(clusters_json))
class AuroraClientCommandTest(unittest.TestCase):
FAKE_TIME = 42131
@classmethod
def create_blank_response(cls, code, msg):
return Response(responseCode=code,
details=[ResponseDetail(message=msg)])
@classmethod
def create_simple_success_response(cls):
return cls.create_blank_response(ResponseCode.OK, 'OK')
@classmethod
def create_error_response(cls):
return cls.create_blank_response(ResponseCode.ERROR, 'Damn')
@classmethod
def create_mock_api(cls):
"""Builds up a mock API object, with a mock SchedulerProxy"""
mock_scheduler = create_autospec(spec=SchedulerThriftApiSpec,
instance=True)
mock_scheduler.url = "http://something_or_other"
mock_scheduler_client = create_autospec(spec=SchedulerProxyApiSpec,
instance=True)
mock_scheduler_client.url = "http://something_or_other"
mock_api = create_autospec(spec=HookedAuroraClientAPI, instance=True)
mock_api.scheduler_proxy = mock_scheduler_client
return mock_api, mock_scheduler_client
@classmethod
def create_mock_api_factory(cls):
"""Create a collection of mocks for a test that wants to mock out the client API
by patching the api factory."""
mock_api, mock_scheduler_client = cls.create_mock_api()
mock_api_factory = lambda: mock_api
return mock_api_factory, mock_scheduler_client
@classmethod
def create_query_call_result(cls, task=None):
status_response = cls.create_empty_task_result()
if task is None:
for i in range(20):
status_response.result.scheduleStatusResult.tasks.append(
cls.create_scheduled_task(i))
else:
status_response.result.scheduleStatusResult.tasks.append(task)
return status_response
@classmethod
def create_empty_task_result(cls):
status_response = cls.create_simple_success_response()
status_response.result = Result(
scheduleStatusResult=ScheduleStatusResult(tasks=[]))
return status_response
@classmethod
def create_scheduled_task(cls,
instance_id,
status=ScheduleStatus.RUNNING,
task_id=None,
initial_time=None):
task = ScheduledTask(
status=status,
assignedTask=AssignedTask(instanceId=instance_id,
taskId=task_id or "Task%s" % instance_id,
slaveId="Slave%s" % instance_id,
slaveHost="Slave%s" % instance_id,
task=TaskConfig()),
taskEvents=[TaskEvent(timestamp=initial_time or 1000)])
return task
@classmethod
def create_scheduled_tasks(cls):
tasks = []
for name in ['foo', 'bar', 'baz']:
task = ScheduledTask(
failureCount=0,
assignedTask=AssignedTask(
taskId=1287391823,
slaveHost='slavehost',
task=TaskConfig(
maxTaskFailures=1,
executorConfig=ExecutorConfig(data='fake data'),
metadata=[],
job=JobKey(role=cls.TEST_ROLE,
environment=cls.TEST_ENV,
name=name),
owner=Identity(role=cls.TEST_ROLE),
environment=cls.TEST_ENV,
jobName=name,
numCpus=2,
ramMb=2,
diskMb=2),
instanceId=4237894,
assignedPorts={}),
status=ScheduleStatus.RUNNING,
taskEvents=[
TaskEvent(timestamp=28234726395,
status=ScheduleStatus.RUNNING,
message="Hi there")
])
tasks.append(task)
return tasks
@classmethod
def setup_get_tasks_status_calls(cls, scheduler):
status_response = cls.create_query_call_result()
scheduler.getTasksWithoutConfigs.return_value = status_response
@classmethod
def fake_time(cls, ignored):
"""Utility function used for faking time to speed up tests."""
cls.FAKE_TIME += 2
return cls.FAKE_TIME
CONFIG_BASE = """
HELLO_WORLD = Job(
name = '%(job)s',
role = '%(role)s',
cluster = '%(cluster)s',
environment = '%(env)s',
instances = 20,
%(inner)s
update_config = UpdateConfig(
batch_size = 1,
restart_threshold = 60,
watch_secs = 45,
max_per_shard_failures = 2,
),
task = Task(
name = 'test',
processes = [Process(name = 'hello_world', cmdline = 'echo {{thermos.ports[http]}}')],
resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB),
)
)
jobs = [HELLO_WORLD]
"""
CRON_CONFIG_BASE = """
HELLO_WORLD = Job(
name = '%(job)s',
role = '%(role)s',
cluster = '%(cluster)s',
environment = '%(env)s',
cron_schedule = '*/5 * * * *',
%(inner)s
task = SimpleTask('test', 'echo test')
)
jobs = [HELLO_WORLD]
"""
UNBOUND_CONFIG = textwrap.dedent("""\
HELLO_WORLD = Job(
name = '%(job)s',
role = '%(role)s',
cluster = '{{cluster_binding}}',
environment = '%(env)s',
instances = '{{instances_binding}}',
update_config = UpdateConfig(
batch_size = "{{TEST_BATCH}}",
restart_threshold = 60,
watch_secs = 45,
max_per_shard_failures = 2,
),
task = Task(
name = 'test',
processes = [Process(name = 'hello_world', cmdline = 'echo {{thermos.ports[http]}}')],
resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB),
)
)
jobs = [HELLO_WORLD]
""")
TEST_ROLE = 'bozo'
TEST_ENV = 'test'
TEST_JOB = 'hello'
TEST_CLUSTER = 'west'
TEST_JOBSPEC = 'west/bozo/test/hello'
TEST_JOBKEY = AuroraJobKey('west', 'bozo', 'test', 'hello')
TEST_CLUSTERS = Clusters([
Cluster(name='west',
zk='zookeeper.example.com',
scheduler_zk_path='/foo/bar',
auth_mechanism='UNAUTHENTICATED')
])
@classmethod
def get_instance_spec(cls, instances_spec):
"""Create a job instance spec string"""
return '%s/%s' % (cls.TEST_JOBSPEC, instances_spec)
@classmethod
def get_test_config(cls, base, cluster, role, env, job, inner=''):
"""Create a config from the template"""
return base % {
'job': job,
'role': role,
'env': env,
'cluster': cluster,
'inner': inner
}
@classmethod
def get_unbound_test_config(cls, role=None, env=None, job=None):
result = cls.UNBOUND_CONFIG % {
'job': job or cls.TEST_JOB,
'role': role or cls.TEST_ROLE,
'env': env or cls.TEST_ENV
}
return result
@classmethod
def get_valid_config(cls):
return cls.get_test_config(cls.CONFIG_BASE, cls.TEST_CLUSTER,
cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB)
@classmethod
def get_valid_cron_config(cls):
return cls.get_test_config(cls.CRON_CONFIG_BASE, cls.TEST_CLUSTER,
cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB)
@classmethod
def get_invalid_config(cls, bad_clause):
return cls.get_test_config(cls.CONFIG_BASE, cls.TEST_CLUSTER,
cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB,
bad_clause)
@classmethod
def get_invalid_cron_config(cls, bad_clause):
return cls.get_test_config(cls.CRON_CONFIG_BASE, cls.TEST_CLUSTER,
cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB,
bad_clause)
@classmethod
def assert_lock_message(cls, context):
assert context.get_err()[2] == "\t%s" % context.LOCK_ERROR_MSG
def test_load_yaml():
with temporary_dir() as td:
clusters_yml = os.path.join(td, 'clusters.yml')
with open(clusters_yml, 'w') as fp:
fp.write(TEST_YAML)
validate_loaded_clusters(Clusters.from_file(clusters_yml))
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from apache.aurora.common.cluster import Cluster
from apache.aurora.common.clusters import Clusters
TEST_CLUSTER = 'west'
TEST_CLUSTERS = Clusters([
Cluster(name=TEST_CLUSTER,
zk='zookeeper.example.com',
scheduler_zk_path='/foo/bar',
auth_mechanism='UNAUTHENTICATED')
])
class AuroraClientCommandTest(unittest.TestCase):
FAKE_TIME = 42131
def setUp(self):
patcher = patch('webbrowser.open_new_tab')
self.mock_webbrowser = patcher.start()
self.addCleanup(patcher.stop)
def run(self, result=None):
# Since CLUSTERS is a global value that evaluates code on import this is the best way to
# ensure it does not pollute any tests.
with CLUSTERS.patch(self.TEST_CLUSTERS._clusters.values()):
super(AuroraClientCommandTest, self).run(result)
@classmethod
def create_blank_response(cls, code, msg):
return Response(responseCode=code,
details=[ResponseDetail(message=msg)])
@classmethod
def create_simple_success_response(cls):
return cls.create_blank_response(ResponseCode.OK, 'OK')
@classmethod
def create_error_response(cls):
return cls.create_blank_response(ResponseCode.ERROR, 'Whoops')
@classmethod
def create_mock_api(cls):
"""Builds up a mock API object, with a mock SchedulerProxy"""
mock_scheduler = create_autospec(spec=SchedulerThriftApiSpec,
instance=True)
mock_scheduler.url = "http://something_or_other"
mock_scheduler_client = create_autospec(spec=SchedulerProxyApiSpec,
instance=True)
mock_scheduler_client.url = "http://something_or_other"
mock_api = create_autospec(spec=HookedAuroraClientAPI, instance=True)
mock_api.scheduler_proxy = mock_scheduler_client
return mock_api, mock_scheduler_client
@classmethod
def create_mock_api_factory(cls):
"""Create a collection of mocks for a test that wants to mock out the client API
by patching the api factory."""
mock_api, mock_scheduler_client = cls.create_mock_api()
mock_api_factory = lambda: mock_api
return mock_api_factory, mock_scheduler_client
@classmethod
def create_query_call_result(cls, task=None):
status_response = cls.create_empty_task_result()
if task is None:
for i in range(20):
status_response.result.scheduleStatusResult.tasks.append(
cls.create_scheduled_task(i))
else:
status_response.result.scheduleStatusResult.tasks.append(task)
return status_response
@classmethod
def create_start_job_update_result(cls, code, msg, key, metadata):
resp = cls.create_blank_response(code, msg)
resp.result = Result(startJobUpdateResult=StartJobUpdateResult(
key=key, updateSummary=JobUpdateSummary(metadata=metadata)))
return resp
@classmethod
def create_empty_task_result(cls):
status_response = cls.create_simple_success_response()
status_response.result = Result(
scheduleStatusResult=ScheduleStatusResult(tasks=[]))
return status_response
@classmethod
def create_scheduled_task(cls,
instance_id,
status=ScheduleStatus.RUNNING,
task_id=None,
initial_time=None):
task = ScheduledTask(
status=status,
assignedTask=AssignedTask(instanceId=instance_id,
taskId=task_id or "Task%s" % instance_id,
slaveId="Slave%s" % instance_id,
slaveHost="Slave%s" % instance_id,
task=TaskConfig()),
taskEvents=[TaskEvent(timestamp=initial_time or 1000)])
return task
@classmethod
def create_task_config(cls, name):
return TaskConfig(
maxTaskFailures=1,
executorConfig=ExecutorConfig(data='{"fake": "data"}'),
metadata=[],
job=JobKey(role=cls.TEST_ROLE, environment=cls.TEST_ENV,
name=name),
numCpus=2,
ramMb=2,
diskMb=2)
@classmethod
def create_scheduled_tasks(cls):
tasks = []
for name in ['foo', 'bar', 'baz']:
task = ScheduledTask(failureCount=0,
assignedTask=AssignedTask(
taskId=1287391823,
slaveHost='slavehost',
task=cls.create_task_config(name),
instanceId=4237894,
assignedPorts={}),
status=ScheduleStatus.RUNNING,
taskEvents=[
TaskEvent(timestamp=28234726395,
status=ScheduleStatus.RUNNING,
message="Hi there")
])
tasks.append(task)
return tasks
@classmethod
def setup_get_tasks_status_calls(cls, scheduler):
status_response = cls.create_query_call_result()
scheduler.getTasksWithoutConfigs.return_value = status_response
@classmethod
def fake_time(cls, ignored):
"""Utility function used for faking time to speed up tests."""
cls.FAKE_TIME += 2
return cls.FAKE_TIME
CONFIG_BASE = """
HELLO_WORLD = Job(
name = '%(job)s',
role = '%(role)s',
cluster = '%(cluster)s',
environment = '%(env)s',
instances = 20,
%(inner)s
update_config = UpdateConfig(
batch_size = 1,
watch_secs = 45,
max_per_shard_failures = 2,
),
task = Task(
name = 'test',
processes = [Process(name = 'hello_world', cmdline = 'echo {{thermos.ports[http]}}')],
resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB),
)
)
jobs = [HELLO_WORLD]
"""
CRON_CONFIG_BASE = """
HELLO_WORLD = Job(
name = '%(job)s',
role = '%(role)s',
cluster = '%(cluster)s',
environment = '%(env)s',
cron_schedule = '*/5 * * * *',
%(inner)s
task = SimpleTask('test', 'echo test')
)
jobs = [HELLO_WORLD]
"""
UNBOUND_CONFIG = textwrap.dedent("""\
HELLO_WORLD = Job(
name = '%(job)s',
role = '%(role)s',
cluster = '{{cluster_binding}}',
environment = '%(env)s',
instances = '{{instances_binding}}',
update_config = UpdateConfig(
batch_size = "{{TEST_BATCH}}",
watch_secs = 45,
max_per_shard_failures = 2,
),
task = Task(
name = 'test',
processes = [Process(
name = 'hello_world',
cmdline = 'echo {{thermos.ports[http]}} {{flags_binding}}'
)],
resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB),
)
)
jobs = [HELLO_WORLD]
""")
TEST_ROLE = 'bozo'
TEST_ENV = 'test'
TEST_JOB = 'hello'
TEST_CLUSTER = 'west'
TEST_JOBSPEC = 'west/bozo/test/hello'
TEST_JOBKEY = AuroraJobKey('west', 'bozo', 'test', 'hello')
TEST_CLUSTERS = Clusters([
Cluster(name=TEST_CLUSTER,
zk='zookeeper.example.com',
scheduler_zk_path='/foo/bar',
auth_mechanism='UNAUTHENTICATED')
])
@classmethod
def get_instance_spec(cls, instances_spec):
"""Create a job instance spec string"""
return '%s/%s' % (cls.TEST_JOBSPEC, instances_spec)
@classmethod
def get_test_config(cls, base, cluster, role, env, job, inner=''):
"""Create a config from the template"""
return base % {
'job': job,
'role': role,
'env': env,
'cluster': cluster,
'inner': inner
}
@classmethod
def get_unbound_test_config(cls, role=None, env=None, job=None):
result = cls.UNBOUND_CONFIG % {
'job': job or cls.TEST_JOB,
'role': role or cls.TEST_ROLE,
'env': env or cls.TEST_ENV
}
return result
@classmethod
def get_valid_config(cls):
return cls.get_test_config(cls.CONFIG_BASE, cls.TEST_CLUSTER,
cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB)
@classmethod
def get_valid_cron_config(cls):
return cls.get_test_config(cls.CRON_CONFIG_BASE, cls.TEST_CLUSTER,
cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB)
@classmethod
def get_invalid_config(cls, bad_clause):
return cls.get_test_config(cls.CONFIG_BASE, cls.TEST_CLUSTER,
cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB,
bad_clause)
@classmethod
def get_invalid_cron_config(cls, bad_clause):
return cls.get_test_config(cls.CRON_CONFIG_BASE, cls.TEST_CLUSTER,
cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB,
bad_clause)
@classmethod
def assert_lock_message(cls, context):
assert [
line for line in context.get_err()
if line == "\t%s" % context.LOCK_ERROR_MSG
]
PREFERRED_TIER = TierConfig(name='preferred',
settings={
'preemptible': 'false',
'revocable': 'false'
})
PREEMPTIBLE_TIER = TierConfig(name='preemptible',
settings={
'preemptible': 'true',
'revocable': 'false'
})
REVOCABLE_TIER = TierConfig(name='revocable',
settings={
'preemptible': 'true',
'revocable': 'true'
})
@classmethod
def get_mock_tier_configurations(cls):
response = cls.create_simple_success_response()
response.result = Result(getTierConfigResult=GetTierConfigResult(
defaultTierName=cls.PREEMPTIBLE_TIER.name,
tiers=frozenset([
cls.PREFERRED_TIER, cls.PREEMPTIBLE_TIER, cls.REVOCABLE_TIER
])))
return response
def test_load():
with temporary_dir() as td:
clusters_json = os.path.join(td, "clusters.json")
with open(clusters_json, "w") as fp:
fp.write(CLUSTERS)
validate_loaded_clusters(Clusters.from_file(clusters_json))