def test_patch_cleanup_on_error():
clusters = Clusters([Cluster(name='original')])
with pytest.raises(RuntimeError):
with clusters.patch([Cluster(name='replacement')]):
assert list(clusters) == ['replacement']
raise RuntimeError("exit contextmanager scope")
assert list(clusters) == ['original']
class TestAdminUtil(unittest.TestCase):
TEST_CLUSTER_NAME = 'west'
TEST_CLUSTER = Cluster(name=TEST_CLUSTER_NAME,
zk='zookeeper.example.com',
scheduler_zk_path='/foo/bar',
auth_mechanism='UNAUTHENTICATED')
TEST_CLUSTERS = Clusters([TEST_CLUSTER])
@mock.patch("apache.aurora.admin.admin_util.subprocess", spec=subprocess)
def test_parse_script(self, mock_subprocess):
with temporary_file() as fp:
mock_popen = mock.Mock()
mock_popen.wait.return_value = 0
mock_subprocess.Popen.return_value = mock_popen
parse_script(fp.name)('h1')
assert mock_popen.wait.call_count == 1
def test_parse_script_invalid_filename(self):
self.assertRaises(SystemExit, parse_script, "invalid filename")
def test_make_admin_client_cluster_string(self):
with mock.patch('apache.aurora.admin.admin_util.CLUSTERS',
new=self.TEST_CLUSTERS):
self.assertIsNotNone(make_admin_client(self.TEST_CLUSTER_NAME))
def test_make_admin_client_cluster_object(self):
with mock.patch('apache.aurora.admin.admin_util.CLUSTERS',
new=self.TEST_CLUSTERS):
self.assertIsNotNone(make_admin_client(self.TEST_CLUSTER))
def test_make_admin_client_cluster_unknown(self):
with mock.patch('apache.aurora.admin.admin_util.CLUSTERS',
new=self.TEST_CLUSTERS):
self.assertRaises(SystemExit, make_admin_client, 'east')
def test_no_zk_or_scheduler_uri(self):
cluster = None
with self.assertRaises(TypeError):
scheduler_client.SchedulerClient.get(cluster)
cluster = Cluster()
with self.assertRaises(ValueError):
scheduler_client.SchedulerClient.get(cluster)
def test_transient_error(_, client):
mock_scheduler_client = mock.create_autospec(
spec=scheduler_client.SchedulerClient, spec_set=False, instance=True)
mock_thrift_client = mock.create_autospec(spec=AuroraAdmin.Client,
instance=True)
mock_thrift_client.killTasks.side_effect = [
Response(responseCode=ResponseCode.ERROR_TRANSIENT,
details=[
ResponseDetail(message="message1"),
ResponseDetail(message="message2")
],
serverInfo=DEFAULT_RESPONSE.serverInfo),
Response(responseCode=ResponseCode.ERROR_TRANSIENT,
serverInfo=DEFAULT_RESPONSE.serverInfo),
Response(responseCode=ResponseCode.OK,
serverInfo=DEFAULT_RESPONSE.serverInfo)
]
mock_scheduler_client.get_thrift_client.return_value = mock_thrift_client
client.get.return_value = mock_scheduler_client
proxy = TestSchedulerProxy(Cluster(name='local'))
proxy.killTasks(TaskQuery(), None)
assert mock_thrift_client.killTasks.call_count == 3
def mock_api(cls):
api = AuroraClientAPI(Cluster(name="foo"), 'test-client')
mock_proxy = create_autospec(spec=SchedulerProxyApiSpec,
spec_set=True,
instance=True)
api._scheduler_proxy = mock_proxy
return api, mock_proxy
def make_mock_client(proxy_url):
client = scheduler_client.ZookeeperSchedulerClient(
Cluster(proxy_url=proxy_url))
client.get_scheduler_serverset = mock.MagicMock(
return_value=(mock_zk, service_endpoints))
client.SERVERSET_TIMEOUT = Amount(0, Time.SECONDS)
client._connect_scheduler = mock.MagicMock()
return client
def setUp(self): self._scheduler = Mock() self._sla = Sla(self._scheduler) self._cluster = Cluster(name='cl') self._role = 'mesos' self._name = 'job' self._env = 'test' self._job_key = AuroraJobKey(self._cluster.name, self._role, self._env, self._name) self._min_count = 1
class AuroraClientCommandTest(unittest.TestCase):
@classmethod
def create_blank_response(cls, code, msg):
# TODO(wfarner): Don't use a mock here.
response = create_autospec(spec=Response, instance=True)
response.responseCode = code
response.result = create_autospec(spec=Result, instance=True)
response.details = [ResponseDetail(message=msg)]
return response
@classmethod
def create_simple_success_response(cls):
return cls.create_blank_response(ResponseCode.OK, 'OK')
@classmethod
def create_mock_api(cls):
"""Builds up a mock API object, with a mock SchedulerProxy"""
mock_scheduler_proxy = create_autospec(spec=SchedulerProxyApiSpec,
spec_set=False,
instance=True)
mock_scheduler_proxy.url = "http://something_or_other"
mock_scheduler_proxy.scheduler_client.return_value = mock_scheduler_proxy
mock_api = create_autospec(spec=HookedAuroraClientAPI)
mock_api.scheduler_proxy = mock_scheduler_proxy
return mock_api, mock_scheduler_proxy
TEST_CLUSTER = 'west'
TEST_CLUSTERS = Clusters([
Cluster(name='west',
zk='zookeeper.example.com',
scheduler_zk_path='/foo/bar',
auth_mechanism='UNAUTHENTICATED')
])
@classmethod
def create_mock_probe_hosts_vector(cls, side_effects):
mock_vector = create_autospec(spec=DomainUpTimeSlaVector,
instance=True)
mock_vector.probe_hosts.side_effect = side_effects
return mock_vector
@classmethod
def create_probe_hosts(cls, hostname, predicted, safe, safe_in):
hosts = defaultdict(list)
job = AuroraJobKey.from_path('west/role/env/job-%s' % hostname)
hosts[hostname].append(JobUpTimeDetails(job, predicted, safe, safe_in))
return [hosts]
# TODO(wfarner): Remove this, force tests to call out their flags.
@classmethod
def setup_mock_options(cls):
mock_options = create_autospec(spec=['verbosity'], instance=True)
mock_options.verbosity = 'verbose'
mock_options.bypass_leader_redirect = False
return mock_options
def make_mock_client(proxy_url):
client = scheduler_client.ZookeeperSchedulerClient(
Cluster(proxy_url=proxy_url),
user_agent='Some-User-Agent',
_deadline=lambda x, **kws: x())
client.get_scheduler_serverset = mock.MagicMock(
return_value=(mock_zk, service_endpoints))
client.SERVERSET_TIMEOUT = Amount(0, Time.SECONDS)
client._connect_scheduler = mock.MagicMock()
return client
def setUp(self):
super(TestRestarter, self).setUp()
self.mock_scheduler = self.mox.CreateMock(scheduler_client)
self.mock_instance_watcher = self.mox.CreateMock(InstanceWatcher)
self.restarter = Restarter(
JOB,
RESTART_SETTINGS,
FakeSchedulerProxy(Cluster(name=CLUSTER), self.mock_scheduler),
self.mock_instance_watcher)
def test_simple():
class AudubonTrait(Cluster.Trait):
master_role = String
slave_role = Default(String, 'slave')
version = Required(Integer)
west = Cluster(name = 'west',
master_role = 'west.master',
slave_role = 'west.slave',
version = 10)
east = Cluster(name = 'east', version = 11)
assert east.name == 'east'
with pytest.raises(AttributeError):
east.slave_role
assert east.with_traits(AudubonTrait).slave_role == 'slave'
assert west.with_traits(AudubonTrait).slave_role == 'west.slave'
assert east.with_traits(AudubonTrait).master_role is None
with pytest.raises(TypeError):
# requires version at least
Cluster(name = 'east').with_traits(AudubonTrait)
def test_unknown_connection_error(self, client):
mock_scheduler_client = mock.create_autospec(spec=scheduler_client.SchedulerClient,
instance=True)
client.get.return_value = mock_scheduler_client
proxy = scheduler_client.SchedulerProxy(Cluster(name='local'))
# unknown, transient connection error
mock_scheduler_client.get_thrift_client.side_effect = RuntimeError
with pytest.raises(Exception):
proxy.client()
# successful connection on re-attempt
mock_scheduler_client.get_thrift_client.side_effect = None
assert proxy.client() is not None
def setUp(self):
super(TestRestarter, self).setUp()
self.mock_scheduler = self.mox.CreateMock(scheduler_client)
self.mock_instance_watcher = self.mox.CreateMock(InstanceWatcher)
self.lock = None
self.restarter = Restarter(
JOB,
UPDATER_CONFIG,
HEALTH_CHECK_INTERVAL_SECONDS,
FakeSchedulerProxy(
Cluster(name=CLUSTER),
self.mock_scheduler,
SESSION_KEY),
self.mock_instance_watcher)
def test_performBackup_transport_exception(self, mock_wait, mock_client):
mock_scheduler_client = mock.create_autospec(
spec=scheduler_client.SchedulerClient,
spec_set=False,
instance=True)
mock_thrift_client = mock.create_autospec(spec=AuroraAdmin.Client, instance=True)
mock_thrift_client.performBackup.side_effect = TTransport.TTransportException('error')
mock_scheduler_client.get_thrift_client.return_value = mock_thrift_client
mock_client.get.return_value = mock_scheduler_client
proxy = scheduler_client.SchedulerProxy(Cluster(name='local'))
with pytest.raises(scheduler_client.SchedulerProxy.NotRetriableError):
proxy.performBackup()
assert mock_thrift_client.performBackup.call_count == 1
assert not mock_wait.called
def setUp(self):
self._role = 'mesos'
self._name = 'jimbob'
self._env = 'test'
self._job_key = JobKey(name=self._name, environment=self._env, role=self._role)
self._session_key = 'test_session'
self._lock = 'test_lock'
self._instance_watcher = MockObject(InstanceWatcher)
self._job_monitor = MockObject(JobMonitor)
self._scheduler_mux = FakeSchedulerMux()
self._scheduler = MockObject(scheduler_client)
self._scheduler_proxy = FakeSchedulerProxy(Cluster(name='test-cluster'),
self._scheduler,
self._session_key)
self._quota_check = MockObject(QuotaCheck)
self.init_updater(deepcopy(self.UPDATE_CONFIG))
self._num_cpus = 1.0
self._num_ram = 1
self._num_disk = 1
def test_getTierConfigs_transport_exception(self, mock_wait, mock_client):
mock_scheduler_client = mock.create_autospec(
spec=scheduler_client.SchedulerClient,
spec_set=False,
instance=True)
mock_thrift_client = mock.create_autospec(spec=AuroraAdmin.Client, instance=True)
mock_thrift_client.getTierConfigs.side_effect = [
TTransport.TTransportException('error'),
Response(responseCode=ResponseCode.OK)
]
mock_scheduler_client.get_thrift_client.return_value = mock_thrift_client
mock_client.get.return_value = mock_scheduler_client
proxy = scheduler_client.SchedulerProxy(Cluster(name='local'))
proxy.getTierConfigs(retry=True)
assert mock_thrift_client.getTierConfigs.call_count == 2
assert mock_wait.call_count == 1
def test_performBackup_retriable_errors(self, mock_wait, mock_client):
mock_scheduler_client = mock.create_autospec(
spec=scheduler_client.SchedulerClient,
spec_set=False,
instance=True)
mock_thrift_client = mock.create_autospec(spec=AuroraAdmin.Client, instance=True)
mock_thrift_client.performBackup.side_effect = [
Response(responseCode=ResponseCode.ERROR_TRANSIENT),
scheduler_client.SchedulerProxy.TimeoutError,
Response(responseCode=ResponseCode.OK)]
mock_scheduler_client.get_thrift_client.return_value = mock_thrift_client
mock_client.get.return_value = mock_scheduler_client
proxy = scheduler_client.SchedulerProxy(Cluster(name='local'))
proxy.performBackup()
assert mock_thrift_client.performBackup.call_count == 3
assert mock_wait.call_count == 2
def test_simple():
class AudubonTrait(Cluster.Trait):
master_role = String # noqa
slave_role = Default(String, 'slave') # noqa
version = Required(Integer) # noqa
west = Cluster(name='west',
master_role='west.master',
slave_role='west.slave',
version=10)
east = Cluster(name='east', version=11)
assert east.name == 'east'
with pytest.raises(AttributeError):
east.slave_role
assert east.with_traits(AudubonTrait).slave_role == 'slave'
assert west.with_traits(AudubonTrait).slave_role == 'west.slave'
assert east.with_traits(AudubonTrait).master_role is None
with pytest.raises(TypeError):
# requires version at least
Cluster(name='east').with_traits(AudubonTrait)
def test_connect_direct_client_with_auth(self, mock_transport):
mock_transport.return_value.open.side_effect = [TTransport.TTransportException, True]
mock_time = mock.create_autospec(spec=time, instance=True)
user_agent = 'Some-User-Agent'
uri = 'https://scheduler.example.com:1337'
auth = mock_auth()
cluster = Cluster(scheduler_uri='uri')
def auth_factory(_):
return auth
client = scheduler_client.SchedulerClient.get(
cluster,
auth_factory=auth_factory,
user_agent=user_agent)
client._connect_scheduler(uri, mock_time)
mock_transport.assert_called_once_with(uri, auth=auth.auth(), user_agent=user_agent)
def test_connect_direct_scheduler_with_user_agent(self, mock_transport):
mock_transport.return_value.open.side_effect = [
TTransport.TTransportException, True
]
mock_time = mock.create_autospec(spec=time, instance=True)
auth = mock_auth()
user_agent = 'Some-User-Agent'
uri = 'https://scheduler.example.com:1337'
cluster = Cluster(scheduler_uri=uri)
client = scheduler_client.DirectSchedulerClient(cluster,
auth=auth,
verbose=True,
user_agent=user_agent)
client._connect_scheduler(uri, mock_time)
mock_transport.assert_called_once_with(uri,
auth=auth.auth(),
user_agent=user_agent,
session_factory=mock.ANY)
class AuroraClientCommandTest(unittest.TestCase):
FAKE_TIME = 42131
def setUp(self):
patcher = patch('webbrowser.open_new_tab')
self.mock_webbrowser = patcher.start()
self.addCleanup(patcher.stop)
def run(self, result=None):
# Since CLUSTERS is a global value that evaluates code on import this is the best way to
# ensure it does not pollute any tests.
with CLUSTERS.patch(self.TEST_CLUSTERS._clusters.values()):
super(AuroraClientCommandTest, self).run(result)
@classmethod
def create_blank_response(cls, code, msg):
return Response(responseCode=code,
details=[ResponseDetail(message=msg)])
@classmethod
def create_simple_success_response(cls):
return cls.create_blank_response(ResponseCode.OK, 'OK')
@classmethod
def create_error_response(cls):
return cls.create_blank_response(ResponseCode.ERROR, 'Whoops')
@classmethod
def create_mock_api(cls):
"""Builds up a mock API object, with a mock SchedulerProxy"""
mock_scheduler = create_autospec(spec=SchedulerThriftApiSpec,
instance=True)
mock_scheduler.url = "http://something_or_other"
mock_scheduler_client = create_autospec(spec=SchedulerProxyApiSpec,
instance=True)
mock_scheduler_client.url = "http://something_or_other"
mock_api = create_autospec(spec=HookedAuroraClientAPI, instance=True)
mock_api.scheduler_proxy = mock_scheduler_client
return mock_api, mock_scheduler_client
@classmethod
def create_mock_api_factory(cls):
"""Create a collection of mocks for a test that wants to mock out the client API
by patching the api factory."""
mock_api, mock_scheduler_client = cls.create_mock_api()
mock_api_factory = lambda: mock_api
return mock_api_factory, mock_scheduler_client
@classmethod
def create_query_call_result(cls, task=None):
status_response = cls.create_empty_task_result()
if task is None:
for i in range(20):
status_response.result.scheduleStatusResult.tasks.append(
cls.create_scheduled_task(i))
else:
status_response.result.scheduleStatusResult.tasks.append(task)
return status_response
@classmethod
def create_start_job_update_result(cls, code, msg, key, metadata):
resp = cls.create_blank_response(code, msg)
resp.result = Result(startJobUpdateResult=StartJobUpdateResult(
key=key, updateSummary=JobUpdateSummary(metadata=metadata)))
return resp
@classmethod
def create_empty_task_result(cls):
status_response = cls.create_simple_success_response()
status_response.result = Result(
scheduleStatusResult=ScheduleStatusResult(tasks=[]))
return status_response
@classmethod
def create_scheduled_task(cls,
instance_id,
status=ScheduleStatus.RUNNING,
task_id=None,
initial_time=None):
task = ScheduledTask(
status=status,
assignedTask=AssignedTask(instanceId=instance_id,
taskId=task_id or "Task%s" % instance_id,
slaveId="Slave%s" % instance_id,
slaveHost="Slave%s" % instance_id,
task=TaskConfig()),
taskEvents=[TaskEvent(timestamp=initial_time or 1000)])
return task
@classmethod
def create_task_config(cls, name):
return TaskConfig(
maxTaskFailures=1,
executorConfig=ExecutorConfig(data='{"fake": "data"}'),
metadata=[],
job=JobKey(role=cls.TEST_ROLE, environment=cls.TEST_ENV,
name=name),
numCpus=2,
ramMb=2,
diskMb=2)
@classmethod
def create_scheduled_tasks(cls):
tasks = []
for name in ['foo', 'bar', 'baz']:
task = ScheduledTask(failureCount=0,
assignedTask=AssignedTask(
taskId=1287391823,
slaveHost='slavehost',
task=cls.create_task_config(name),
instanceId=4237894,
assignedPorts={}),
status=ScheduleStatus.RUNNING,
taskEvents=[
TaskEvent(timestamp=28234726395,
status=ScheduleStatus.RUNNING,
message="Hi there")
])
tasks.append(task)
return tasks
@classmethod
def setup_get_tasks_status_calls(cls, scheduler):
status_response = cls.create_query_call_result()
scheduler.getTasksWithoutConfigs.return_value = status_response
@classmethod
def fake_time(cls, ignored):
"""Utility function used for faking time to speed up tests."""
cls.FAKE_TIME += 2
return cls.FAKE_TIME
CONFIG_BASE = """
HELLO_WORLD = Job(
name = '%(job)s',
role = '%(role)s',
cluster = '%(cluster)s',
environment = '%(env)s',
instances = 20,
%(inner)s
update_config = UpdateConfig(
batch_size = 1,
watch_secs = 45,
max_per_shard_failures = 2,
),
task = Task(
name = 'test',
processes = [Process(name = 'hello_world', cmdline = 'echo {{thermos.ports[http]}}')],
resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB),
)
)
jobs = [HELLO_WORLD]
"""
CRON_CONFIG_BASE = """
HELLO_WORLD = Job(
name = '%(job)s',
role = '%(role)s',
cluster = '%(cluster)s',
environment = '%(env)s',
cron_schedule = '*/5 * * * *',
%(inner)s
task = SimpleTask('test', 'echo test')
)
jobs = [HELLO_WORLD]
"""
UNBOUND_CONFIG = textwrap.dedent("""\
HELLO_WORLD = Job(
name = '%(job)s',
role = '%(role)s',
cluster = '{{cluster_binding}}',
environment = '%(env)s',
instances = '{{instances_binding}}',
update_config = UpdateConfig(
batch_size = "{{TEST_BATCH}}",
watch_secs = 45,
max_per_shard_failures = 2,
),
task = Task(
name = 'test',
processes = [Process(
name = 'hello_world',
cmdline = 'echo {{thermos.ports[http]}} {{flags_binding}}'
)],
resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB),
)
)
jobs = [HELLO_WORLD]
""")
TEST_ROLE = 'bozo'
TEST_ENV = 'test'
TEST_JOB = 'hello'
TEST_CLUSTER = 'west'
TEST_JOBSPEC = 'west/bozo/test/hello'
TEST_JOBKEY = AuroraJobKey('west', 'bozo', 'test', 'hello')
TEST_CLUSTERS = Clusters([
Cluster(name=TEST_CLUSTER,
zk='zookeeper.example.com',
scheduler_zk_path='/foo/bar',
auth_mechanism='UNAUTHENTICATED')
])
@classmethod
def get_instance_spec(cls, instances_spec):
"""Create a job instance spec string"""
return '%s/%s' % (cls.TEST_JOBSPEC, instances_spec)
@classmethod
def get_test_config(cls, base, cluster, role, env, job, inner=''):
"""Create a config from the template"""
return base % {
'job': job,
'role': role,
'env': env,
'cluster': cluster,
'inner': inner
}
@classmethod
def get_unbound_test_config(cls, role=None, env=None, job=None):
result = cls.UNBOUND_CONFIG % {
'job': job or cls.TEST_JOB,
'role': role or cls.TEST_ROLE,
'env': env or cls.TEST_ENV
}
return result
@classmethod
def get_valid_config(cls):
return cls.get_test_config(cls.CONFIG_BASE, cls.TEST_CLUSTER,
cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB)
@classmethod
def get_valid_cron_config(cls):
return cls.get_test_config(cls.CRON_CONFIG_BASE, cls.TEST_CLUSTER,
cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB)
@classmethod
def get_invalid_config(cls, bad_clause):
return cls.get_test_config(cls.CONFIG_BASE, cls.TEST_CLUSTER,
cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB,
bad_clause)
@classmethod
def get_invalid_cron_config(cls, bad_clause):
return cls.get_test_config(cls.CRON_CONFIG_BASE, cls.TEST_CLUSTER,
cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB,
bad_clause)
@classmethod
def assert_lock_message(cls, context):
assert [
line for line in context.get_err()
if line == "\t%s" % context.LOCK_ERROR_MSG
]
PREFERRED_TIER = TierConfig(name='preferred',
settings={
'preemptible': 'false',
'revocable': 'false'
})
PREEMPTIBLE_TIER = TierConfig(name='preemptible',
settings={
'preemptible': 'true',
'revocable': 'false'
})
REVOCABLE_TIER = TierConfig(name='revocable',
settings={
'preemptible': 'true',
'revocable': 'true'
})
@classmethod
def get_mock_tier_configurations(cls):
response = cls.create_simple_success_response()
response.result = Result(getTierConfigResult=GetTierConfigResult(
defaultTierName=cls.PREEMPTIBLE_TIER.name,
tiers=frozenset([
cls.PREFERRED_TIER, cls.PREEMPTIBLE_TIER, cls.REVOCABLE_TIER
])))
return response
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from optparse import OptionParser
import pytest
from apache.aurora.common.cluster import Cluster
from apache.aurora.common.cluster_option import ClusterOption
from apache.aurora.common.clusters import Clusters
CLUSTER_LIST = Clusters((
Cluster(name='smf1'),
Cluster(name='smf1-test'),
))
def cluster_provider(name):
return CLUSTER_LIST[name]
def test_constructors():
ClusterOption('--test', '-t', help="Test cluster.", clusters=CLUSTER_LIST)
ClusterOption('--test', '-t', help="Test cluster.", cluster_provider=cluster_provider)
with pytest.raises(ValueError):
ClusterOption()
def mock_api(cls):
api = AuroraClientAPI(Cluster(name="foo"))
mock_proxy = Mock()
api._scheduler_proxy = mock_proxy
return api, mock_proxy
def make_scheduler_proxy(self): return scheduler_client.SchedulerProxy(Cluster(name='local'))
class AuroraClientCommandTest(unittest.TestCase):
FAKE_TIME = 42131
@classmethod
def create_blank_response(cls, code, msg):
return Response(responseCode=code,
details=[ResponseDetail(message=msg)])
@classmethod
def create_simple_success_response(cls):
return cls.create_blank_response(ResponseCode.OK, 'OK')
@classmethod
def create_error_response(cls):
return cls.create_blank_response(ResponseCode.ERROR, 'Damn')
@classmethod
def create_mock_api(cls):
"""Builds up a mock API object, with a mock SchedulerProxy"""
mock_scheduler = create_autospec(spec=SchedulerThriftApiSpec,
instance=True)
mock_scheduler.url = "http://something_or_other"
mock_scheduler_client = create_autospec(spec=SchedulerProxyApiSpec,
instance=True)
mock_scheduler_client.url = "http://something_or_other"
mock_api = create_autospec(spec=HookedAuroraClientAPI, instance=True)
mock_api.scheduler_proxy = mock_scheduler_client
return mock_api, mock_scheduler_client
@classmethod
def create_mock_api_factory(cls):
"""Create a collection of mocks for a test that wants to mock out the client API
by patching the api factory."""
mock_api, mock_scheduler_client = cls.create_mock_api()
mock_api_factory = lambda: mock_api
return mock_api_factory, mock_scheduler_client
@classmethod
def create_query_call_result(cls, task=None):
status_response = cls.create_empty_task_result()
if task is None:
for i in range(20):
status_response.result.scheduleStatusResult.tasks.append(
cls.create_scheduled_task(i))
else:
status_response.result.scheduleStatusResult.tasks.append(task)
return status_response
@classmethod
def create_empty_task_result(cls):
status_response = cls.create_simple_success_response()
status_response.result = Result(
scheduleStatusResult=ScheduleStatusResult(tasks=[]))
return status_response
@classmethod
def create_scheduled_task(cls,
instance_id,
status=ScheduleStatus.RUNNING,
task_id=None,
initial_time=None):
task = ScheduledTask(
status=status,
assignedTask=AssignedTask(instanceId=instance_id,
taskId=task_id or "Task%s" % instance_id,
slaveId="Slave%s" % instance_id,
slaveHost="Slave%s" % instance_id,
task=TaskConfig()),
taskEvents=[TaskEvent(timestamp=initial_time or 1000)])
return task
@classmethod
def create_scheduled_tasks(cls):
tasks = []
for name in ['foo', 'bar', 'baz']:
task = ScheduledTask(
failureCount=0,
assignedTask=AssignedTask(
taskId=1287391823,
slaveHost='slavehost',
task=TaskConfig(
maxTaskFailures=1,
executorConfig=ExecutorConfig(data='fake data'),
metadata=[],
job=JobKey(role=cls.TEST_ROLE,
environment=cls.TEST_ENV,
name=name),
owner=Identity(role=cls.TEST_ROLE),
environment=cls.TEST_ENV,
jobName=name,
numCpus=2,
ramMb=2,
diskMb=2),
instanceId=4237894,
assignedPorts={}),
status=ScheduleStatus.RUNNING,
taskEvents=[
TaskEvent(timestamp=28234726395,
status=ScheduleStatus.RUNNING,
message="Hi there")
])
tasks.append(task)
return tasks
@classmethod
def setup_get_tasks_status_calls(cls, scheduler):
status_response = cls.create_query_call_result()
scheduler.getTasksWithoutConfigs.return_value = status_response
@classmethod
def fake_time(cls, ignored):
"""Utility function used for faking time to speed up tests."""
cls.FAKE_TIME += 2
return cls.FAKE_TIME
CONFIG_BASE = """
HELLO_WORLD = Job(
name = '%(job)s',
role = '%(role)s',
cluster = '%(cluster)s',
environment = '%(env)s',
instances = 20,
%(inner)s
update_config = UpdateConfig(
batch_size = 1,
restart_threshold = 60,
watch_secs = 45,
max_per_shard_failures = 2,
),
task = Task(
name = 'test',
processes = [Process(name = 'hello_world', cmdline = 'echo {{thermos.ports[http]}}')],
resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB),
)
)
jobs = [HELLO_WORLD]
"""
CRON_CONFIG_BASE = """
HELLO_WORLD = Job(
name = '%(job)s',
role = '%(role)s',
cluster = '%(cluster)s',
environment = '%(env)s',
cron_schedule = '*/5 * * * *',
%(inner)s
task = SimpleTask('test', 'echo test')
)
jobs = [HELLO_WORLD]
"""
UNBOUND_CONFIG = textwrap.dedent("""\
HELLO_WORLD = Job(
name = '%(job)s',
role = '%(role)s',
cluster = '{{cluster_binding}}',
environment = '%(env)s',
instances = '{{instances_binding}}',
update_config = UpdateConfig(
batch_size = "{{TEST_BATCH}}",
restart_threshold = 60,
watch_secs = 45,
max_per_shard_failures = 2,
),
task = Task(
name = 'test',
processes = [Process(name = 'hello_world', cmdline = 'echo {{thermos.ports[http]}}')],
resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB),
)
)
jobs = [HELLO_WORLD]
""")
TEST_ROLE = 'bozo'
TEST_ENV = 'test'
TEST_JOB = 'hello'
TEST_CLUSTER = 'west'
TEST_JOBSPEC = 'west/bozo/test/hello'
TEST_JOBKEY = AuroraJobKey('west', 'bozo', 'test', 'hello')
TEST_CLUSTERS = Clusters([
Cluster(name='west',
zk='zookeeper.example.com',
scheduler_zk_path='/foo/bar',
auth_mechanism='UNAUTHENTICATED')
])
@classmethod
def get_instance_spec(cls, instances_spec):
"""Create a job instance spec string"""
return '%s/%s' % (cls.TEST_JOBSPEC, instances_spec)
@classmethod
def get_test_config(cls, base, cluster, role, env, job, inner=''):
"""Create a config from the template"""
return base % {
'job': job,
'role': role,
'env': env,
'cluster': cluster,
'inner': inner
}
@classmethod
def get_unbound_test_config(cls, role=None, env=None, job=None):
result = cls.UNBOUND_CONFIG % {
'job': job or cls.TEST_JOB,
'role': role or cls.TEST_ROLE,
'env': env or cls.TEST_ENV
}
return result
@classmethod
def get_valid_config(cls):
return cls.get_test_config(cls.CONFIG_BASE, cls.TEST_CLUSTER,
cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB)
@classmethod
def get_valid_cron_config(cls):
return cls.get_test_config(cls.CRON_CONFIG_BASE, cls.TEST_CLUSTER,
cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB)
@classmethod
def get_invalid_config(cls, bad_clause):
return cls.get_test_config(cls.CONFIG_BASE, cls.TEST_CLUSTER,
cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB,
bad_clause)
@classmethod
def get_invalid_cron_config(cls, bad_clause):
return cls.get_test_config(cls.CRON_CONFIG_BASE, cls.TEST_CLUSTER,
cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB,
bad_clause)
@classmethod
def assert_lock_message(cls, context):
assert context.get_err()[2] == "\t%s" % context.LOCK_ERROR_MSG
#
import mock
import pytest
from apache.aurora.client.api import AuroraClientAPI, SchedulerProxy
from apache.aurora.client.cli import EXIT_AUTH_ERROR, Context
from apache.aurora.client.cli.context import AuroraCommandContext
from apache.aurora.client.hooks.hooked_api import HookedAuroraClientAPI
from apache.aurora.common.aurora_job_key import AuroraJobKey
from apache.aurora.common.cluster import Cluster
from apache.aurora.common.clusters import CLUSTERS
from ...api_util import SchedulerProxyApiSpec
TEST_CLUSTER = Cluster(name='some-cluster',
auth_mechanism='nothing',
scheduler_uri='nowhere')
def test_get_api_defaults_to_hooks_enabled():
with CLUSTERS.patch([TEST_CLUSTER]):
api = AuroraCommandContext().get_api(TEST_CLUSTER.name)
assert isinstance(api, HookedAuroraClientAPI)
assert api._cluster == TEST_CLUSTER
def test_get_api_forwards_hooks_disabled():
with CLUSTERS.patch([TEST_CLUSTER]):
api = AuroraCommandContext().get_api(TEST_CLUSTER.name,
enable_hooks=False)
assert isinstance(api, AuroraClientAPI)
DOCKER_BINDING_CONFIG = """
jobs = [Job(
name = 'hello_world',
role = 'john_doe',
environment = 'staging42',
cluster = 'smf1-test',
task = Task(
name = 'main',
processes = [Process(name='command', cmdline="echo helloworld")],
resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB),
),
container = Docker(image='{{docker.image[some/name][some.tag]}}')
)]
"""
TEST_CLUSTER = Cluster(name='smf1-test',
docker_registry='some.registry.domain.com')
TEST_CLUSTERS = [TEST_CLUSTER]
class TestDockerBindingHelper(unittest.TestCase):
@patch.object(DockerRegistryClient, 'resolve')
def test_docker_binding(self, mock_resolve):
image_reference = 'registry.example.com/some/repo@some:digest'
mock_resolve.return_value = image_reference
binding_helper.unregister_all()
BindingHelper.register(DockerBindingHelper())
with temporary_file() as fp:
fp.write(DOCKER_BINDING_CONFIG)
from apache.aurora.client.base import add_grouping, remove_grouping
from apache.aurora.common.cluster import Cluster
from gen.apache.aurora.api.ttypes import (
Hosts,
HostStatus,
MaintenanceMode,
MaintenanceStatusResult,
Response,
ResponseCode,
Result,
StartMaintenanceResult
)
DEFAULT_CLUSTER = Cluster(
name='us-west',
scheduler_uri='us-west-234.example.com:8888',
)
TEST_HOSTNAMES = [
'us-west-001.example.com',
'us-west-002.example.com',
'us-west-003.example.com']
class TestHostMaintenance(unittest.TestCase):
@mock.patch("apache.aurora.client.api.AuroraClientAPI.maintenance_status",
spec=AuroraClientAPI.maintenance_status)
@mock.patch("apache.aurora.client.api.AuroraClientAPI.drain_hosts",
spec=AuroraClientAPI.drain_hosts)
def test_drain_hosts(self, mock_drain_hosts, mock_maintenance_status):
fake_maintenance_status_response = [
Response(result=Result(maintenanceStatusResult=MaintenanceStatusResult(set([
class AuroraClientCommandTest(unittest.TestCase):
FAKE_TIME = 42131
@classmethod
def create_blank_response(cls, code, msg):
response = Mock(spec=Response)
response.responseCode = code
mock_msg = Mock()
mock_msg.message = msg
response.details = [mock_msg]
response.messageDEPRECATED = msg
response.result = Mock(spec=Result)
return response
@classmethod
def create_simple_success_response(cls):
return cls.create_blank_response(ResponseCode.OK, 'OK')
@classmethod
def create_error_response(cls):
return cls.create_blank_response(ResponseCode.ERROR, 'Damn')
@classmethod
def create_mock_api(cls):
"""Builds up a mock API object, with a mock SchedulerProxy"""
mock_api = Mock(spec=HookedAuroraClientAPI)
mock_scheduler = Mock()
mock_scheduler.url = "http://something_or_other"
mock_scheduler_client = Mock()
mock_scheduler_client.scheduler.return_value = mock_scheduler
mock_scheduler_client.url = "http://something_or_other"
mock_api = Mock(spec=HookedAuroraClientAPI)
mock_api.scheduler_proxy = mock_scheduler_client
return (mock_api, mock_scheduler_client)
@classmethod
def create_mock_api_factory(cls):
"""Create a collection of mocks for a test that wants to mock out the client API
by patching the api factory."""
mock_api, mock_scheduler_client = cls.create_mock_api()
mock_api_factory = Mock()
mock_api_factory.return_value = mock_api
return mock_api_factory, mock_scheduler_client
@classmethod
def create_status_call_result(cls, mock_task=None):
status_response = cls.create_simple_success_response()
schedule_status = Mock(spec=ScheduleStatusResult)
status_response.result.scheduleStatusResult = schedule_status
# This should be a list of ScheduledTask's.
schedule_status.tasks = []
if mock_task is None:
for i in range(20):
schedule_status.tasks.append(cls.create_mock_task(i))
else:
schedule_status.tasks.append(mock_task)
return status_response
@classmethod
def create_mock_task(cls, instance_id, status=ScheduleStatus.RUNNING):
mock_task = Mock(spec=ScheduledTask)
mock_task.assignedTask = Mock(spec=AssignedTask)
mock_task.assignedTask.instanceId = instance_id
mock_task.assignedTask.taskId = "Task%s" % instance_id
mock_task.assignedTask.slaveId = "Slave%s" % instance_id
mock_task.assignedTask.task = Mock(spec=TaskConfig)
mock_task.slaveHost = "Slave%s" % instance_id
mock_task.status = status
mock_task_event = Mock(spec=TaskEvent)
mock_task_event.timestamp = 1000
mock_task.taskEvents = [mock_task_event]
return mock_task
@classmethod
def setup_get_tasks_status_calls(cls, scheduler):
status_response = cls.create_status_call_result()
scheduler.getTasksWithoutConfigs.return_value = status_response
@classmethod
def fake_time(cls, ignored):
"""Utility function used for faking time to speed up tests."""
cls.FAKE_TIME += 2
return cls.FAKE_TIME
CONFIG_BASE = """
HELLO_WORLD = Job(
name = '%(job)s',
role = '%(role)s',
cluster = '%(cluster)s',
environment = '%(env)s',
instances = 20,
%(inner)s
update_config = UpdateConfig(
batch_size = 1,
restart_threshold = 60,
watch_secs = 45,
max_per_shard_failures = 2,
),
task = Task(
name = 'test',
processes = [Process(name = 'hello_world', cmdline = 'echo {{thermos.ports[http]}}')],
resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB),
)
)
jobs = [HELLO_WORLD]
"""
UNBOUND_CONFIG = textwrap.dedent("""\
HELLO_WORLD = Job(
name = '%(job)s',
role = '%(role)s',
cluster = '{{cluster_binding}}',
environment = '%(env)s',
instances = '{{instances_binding}}',
update_config = UpdateConfig(
batch_size = 1,
restart_threshold = 60,
watch_secs = 45,
max_per_shard_failures = 2,
),
task = Task(
name = 'test',
processes = [Process(name = 'hello_world', cmdline = 'echo {{thermos.ports[http]}}')],
resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB),
)
)
jobs = [HELLO_WORLD]
""")
TEST_ROLE = 'bozo'
TEST_ENV = 'test'
TEST_JOB = 'hello'
TEST_CLUSTER = 'west'
TEST_JOBSPEC = 'west/bozo/test/hello'
TEST_CLUSTERS = Clusters([
Cluster(name='west',
packer_copy_command='copying {{package}}',
zk='zookeeper.example.com',
scheduler_zk_path='/foo/bar',
auth_mechanism='UNAUTHENTICATED')
])
@classmethod
def get_test_config(cls, cluster, role, env, job, filler=''):
"""Create a config from the template"""
return cls.CONFIG_BASE % {
'job': job,
'role': role,
'env': env,
'cluster': cluster,
'inner': filler
}
@classmethod
def get_unbound_test_config(cls, role=None, env=None, job=None):
result = cls.UNBOUND_CONFIG % {
'job': job or cls.TEST_JOB,
'role': role or cls.TEST_ROLE,
'env': env or cls.TEST_ENV
}
print(
"CONFIG:===========================\n%s\n============================="
% result)
return result
@classmethod
def get_valid_config(cls):
return cls.get_test_config(cls.TEST_CLUSTER, cls.TEST_ROLE,
cls.TEST_ENV, cls.TEST_JOB)
@classmethod
def get_invalid_config(cls, bad_clause):
return cls.get_test_config(cls.TEST_CLUSTER, cls.TEST_ROLE,
cls.TEST_ENV, cls.TEST_JOB, bad_clause)
def make_scheduler_proxy(self):
return TestSchedulerProxy(Cluster(name='local'))
#
import functools
import mox
import pytest
from apache.aurora.client.api import AuroraClientAPI
from apache.aurora.client.api.disambiguator import LiveJobDisambiguator
from apache.aurora.common.aurora_job_key import AuroraJobKey
from apache.aurora.common.cluster import Cluster
from gen.apache.aurora.api.constants import ResponseCode
from gen.apache.aurora.api.ttypes import GetJobsResult, JobConfiguration, JobKey, Response, Result
TEST_CLUSTER = Cluster(name='smf1')
class LiveJobDisambiguatorTest(mox.MoxTestBase):
CLUSTER = TEST_CLUSTER
ROLE = 'mesos'
ENV = 'test'
NAME = 'labrat'
JOB_PATH = 'smf1/mesos/test/labrat'
CONFIG_FILE = 'abc.aurora'
def setUp(self):
super(LiveJobDisambiguatorTest, self).setUp()
self._api = self.mox.CreateMock(AuroraClientAPI)
self._api.cluster = self.CLUSTER
self._options = self.mox.CreateMockAnything()
