def test_patch_cleanup_on_error(): clusters = Clusters([Cluster(name='original')]) with pytest.raises(RuntimeError): with clusters.patch([Cluster(name='replacement')]): assert list(clusters) == ['replacement'] raise RuntimeError("exit contextmanager scope") assert list(clusters) == ['original']
class TestAdminUtil(unittest.TestCase): TEST_CLUSTER_NAME = 'west' TEST_CLUSTER = Cluster(name=TEST_CLUSTER_NAME, zk='zookeeper.example.com', scheduler_zk_path='/foo/bar', auth_mechanism='UNAUTHENTICATED') TEST_CLUSTERS = Clusters([TEST_CLUSTER]) @mock.patch("apache.aurora.admin.admin_util.subprocess", spec=subprocess) def test_parse_script(self, mock_subprocess): with temporary_file() as fp: mock_popen = mock.Mock() mock_popen.wait.return_value = 0 mock_subprocess.Popen.return_value = mock_popen parse_script(fp.name)('h1') assert mock_popen.wait.call_count == 1 def test_parse_script_invalid_filename(self): self.assertRaises(SystemExit, parse_script, "invalid filename") def test_make_admin_client_cluster_string(self): with mock.patch('apache.aurora.admin.admin_util.CLUSTERS', new=self.TEST_CLUSTERS): self.assertIsNotNone(make_admin_client(self.TEST_CLUSTER_NAME)) def test_make_admin_client_cluster_object(self): with mock.patch('apache.aurora.admin.admin_util.CLUSTERS', new=self.TEST_CLUSTERS): self.assertIsNotNone(make_admin_client(self.TEST_CLUSTER)) def test_make_admin_client_cluster_unknown(self): with mock.patch('apache.aurora.admin.admin_util.CLUSTERS', new=self.TEST_CLUSTERS): self.assertRaises(SystemExit, make_admin_client, 'east')
def test_no_zk_or_scheduler_uri(self): cluster = None with self.assertRaises(TypeError): scheduler_client.SchedulerClient.get(cluster) cluster = Cluster() with self.assertRaises(ValueError): scheduler_client.SchedulerClient.get(cluster)
def test_transient_error(_, client): mock_scheduler_client = mock.create_autospec( spec=scheduler_client.SchedulerClient, spec_set=False, instance=True) mock_thrift_client = mock.create_autospec(spec=AuroraAdmin.Client, instance=True) mock_thrift_client.killTasks.side_effect = [ Response(responseCode=ResponseCode.ERROR_TRANSIENT, details=[ ResponseDetail(message="message1"), ResponseDetail(message="message2") ], serverInfo=DEFAULT_RESPONSE.serverInfo), Response(responseCode=ResponseCode.ERROR_TRANSIENT, serverInfo=DEFAULT_RESPONSE.serverInfo), Response(responseCode=ResponseCode.OK, serverInfo=DEFAULT_RESPONSE.serverInfo) ] mock_scheduler_client.get_thrift_client.return_value = mock_thrift_client client.get.return_value = mock_scheduler_client proxy = TestSchedulerProxy(Cluster(name='local')) proxy.killTasks(TaskQuery(), None) assert mock_thrift_client.killTasks.call_count == 3
def mock_api(cls): api = AuroraClientAPI(Cluster(name="foo"), 'test-client') mock_proxy = create_autospec(spec=SchedulerProxyApiSpec, spec_set=True, instance=True) api._scheduler_proxy = mock_proxy return api, mock_proxy
def make_mock_client(proxy_url): client = scheduler_client.ZookeeperSchedulerClient( Cluster(proxy_url=proxy_url)) client.get_scheduler_serverset = mock.MagicMock( return_value=(mock_zk, service_endpoints)) client.SERVERSET_TIMEOUT = Amount(0, Time.SECONDS) client._connect_scheduler = mock.MagicMock() return client
def setUp(self): self._scheduler = Mock() self._sla = Sla(self._scheduler) self._cluster = Cluster(name='cl') self._role = 'mesos' self._name = 'job' self._env = 'test' self._job_key = AuroraJobKey(self._cluster.name, self._role, self._env, self._name) self._min_count = 1
class AuroraClientCommandTest(unittest.TestCase): @classmethod def create_blank_response(cls, code, msg): # TODO(wfarner): Don't use a mock here. response = create_autospec(spec=Response, instance=True) response.responseCode = code response.result = create_autospec(spec=Result, instance=True) response.details = [ResponseDetail(message=msg)] return response @classmethod def create_simple_success_response(cls): return cls.create_blank_response(ResponseCode.OK, 'OK') @classmethod def create_mock_api(cls): """Builds up a mock API object, with a mock SchedulerProxy""" mock_scheduler_proxy = create_autospec(spec=SchedulerProxyApiSpec, spec_set=False, instance=True) mock_scheduler_proxy.url = "http://something_or_other" mock_scheduler_proxy.scheduler_client.return_value = mock_scheduler_proxy mock_api = create_autospec(spec=HookedAuroraClientAPI) mock_api.scheduler_proxy = mock_scheduler_proxy return mock_api, mock_scheduler_proxy TEST_CLUSTER = 'west' TEST_CLUSTERS = Clusters([ Cluster(name='west', zk='zookeeper.example.com', scheduler_zk_path='/foo/bar', auth_mechanism='UNAUTHENTICATED') ]) @classmethod def create_mock_probe_hosts_vector(cls, side_effects): mock_vector = create_autospec(spec=DomainUpTimeSlaVector, instance=True) mock_vector.probe_hosts.side_effect = side_effects return mock_vector @classmethod def create_probe_hosts(cls, hostname, predicted, safe, safe_in): hosts = defaultdict(list) job = AuroraJobKey.from_path('west/role/env/job-%s' % hostname) hosts[hostname].append(JobUpTimeDetails(job, predicted, safe, safe_in)) return [hosts] # TODO(wfarner): Remove this, force tests to call out their flags. @classmethod def setup_mock_options(cls): mock_options = create_autospec(spec=['verbosity'], instance=True) mock_options.verbosity = 'verbose' mock_options.bypass_leader_redirect = False return mock_options
def make_mock_client(proxy_url): client = scheduler_client.ZookeeperSchedulerClient( Cluster(proxy_url=proxy_url), user_agent='Some-User-Agent', _deadline=lambda x, **kws: x()) client.get_scheduler_serverset = mock.MagicMock( return_value=(mock_zk, service_endpoints)) client.SERVERSET_TIMEOUT = Amount(0, Time.SECONDS) client._connect_scheduler = mock.MagicMock() return client
def setUp(self): super(TestRestarter, self).setUp() self.mock_scheduler = self.mox.CreateMock(scheduler_client) self.mock_instance_watcher = self.mox.CreateMock(InstanceWatcher) self.restarter = Restarter( JOB, RESTART_SETTINGS, FakeSchedulerProxy(Cluster(name=CLUSTER), self.mock_scheduler), self.mock_instance_watcher)
def test_simple(): class AudubonTrait(Cluster.Trait): master_role = String slave_role = Default(String, 'slave') version = Required(Integer) west = Cluster(name = 'west', master_role = 'west.master', slave_role = 'west.slave', version = 10) east = Cluster(name = 'east', version = 11) assert east.name == 'east' with pytest.raises(AttributeError): east.slave_role assert east.with_traits(AudubonTrait).slave_role == 'slave' assert west.with_traits(AudubonTrait).slave_role == 'west.slave' assert east.with_traits(AudubonTrait).master_role is None with pytest.raises(TypeError): # requires version at least Cluster(name = 'east').with_traits(AudubonTrait)
def test_unknown_connection_error(self, client): mock_scheduler_client = mock.create_autospec(spec=scheduler_client.SchedulerClient, instance=True) client.get.return_value = mock_scheduler_client proxy = scheduler_client.SchedulerProxy(Cluster(name='local')) # unknown, transient connection error mock_scheduler_client.get_thrift_client.side_effect = RuntimeError with pytest.raises(Exception): proxy.client() # successful connection on re-attempt mock_scheduler_client.get_thrift_client.side_effect = None assert proxy.client() is not None
def setUp(self): super(TestRestarter, self).setUp() self.mock_scheduler = self.mox.CreateMock(scheduler_client) self.mock_instance_watcher = self.mox.CreateMock(InstanceWatcher) self.lock = None self.restarter = Restarter( JOB, UPDATER_CONFIG, HEALTH_CHECK_INTERVAL_SECONDS, FakeSchedulerProxy( Cluster(name=CLUSTER), self.mock_scheduler, SESSION_KEY), self.mock_instance_watcher)
def test_performBackup_transport_exception(self, mock_wait, mock_client): mock_scheduler_client = mock.create_autospec( spec=scheduler_client.SchedulerClient, spec_set=False, instance=True) mock_thrift_client = mock.create_autospec(spec=AuroraAdmin.Client, instance=True) mock_thrift_client.performBackup.side_effect = TTransport.TTransportException('error') mock_scheduler_client.get_thrift_client.return_value = mock_thrift_client mock_client.get.return_value = mock_scheduler_client proxy = scheduler_client.SchedulerProxy(Cluster(name='local')) with pytest.raises(scheduler_client.SchedulerProxy.NotRetriableError): proxy.performBackup() assert mock_thrift_client.performBackup.call_count == 1 assert not mock_wait.called
def setUp(self): self._role = 'mesos' self._name = 'jimbob' self._env = 'test' self._job_key = JobKey(name=self._name, environment=self._env, role=self._role) self._session_key = 'test_session' self._lock = 'test_lock' self._instance_watcher = MockObject(InstanceWatcher) self._job_monitor = MockObject(JobMonitor) self._scheduler_mux = FakeSchedulerMux() self._scheduler = MockObject(scheduler_client) self._scheduler_proxy = FakeSchedulerProxy(Cluster(name='test-cluster'), self._scheduler, self._session_key) self._quota_check = MockObject(QuotaCheck) self.init_updater(deepcopy(self.UPDATE_CONFIG)) self._num_cpus = 1.0 self._num_ram = 1 self._num_disk = 1
def test_getTierConfigs_transport_exception(self, mock_wait, mock_client): mock_scheduler_client = mock.create_autospec( spec=scheduler_client.SchedulerClient, spec_set=False, instance=True) mock_thrift_client = mock.create_autospec(spec=AuroraAdmin.Client, instance=True) mock_thrift_client.getTierConfigs.side_effect = [ TTransport.TTransportException('error'), Response(responseCode=ResponseCode.OK) ] mock_scheduler_client.get_thrift_client.return_value = mock_thrift_client mock_client.get.return_value = mock_scheduler_client proxy = scheduler_client.SchedulerProxy(Cluster(name='local')) proxy.getTierConfigs(retry=True) assert mock_thrift_client.getTierConfigs.call_count == 2 assert mock_wait.call_count == 1
def test_performBackup_retriable_errors(self, mock_wait, mock_client): mock_scheduler_client = mock.create_autospec( spec=scheduler_client.SchedulerClient, spec_set=False, instance=True) mock_thrift_client = mock.create_autospec(spec=AuroraAdmin.Client, instance=True) mock_thrift_client.performBackup.side_effect = [ Response(responseCode=ResponseCode.ERROR_TRANSIENT), scheduler_client.SchedulerProxy.TimeoutError, Response(responseCode=ResponseCode.OK)] mock_scheduler_client.get_thrift_client.return_value = mock_thrift_client mock_client.get.return_value = mock_scheduler_client proxy = scheduler_client.SchedulerProxy(Cluster(name='local')) proxy.performBackup() assert mock_thrift_client.performBackup.call_count == 3 assert mock_wait.call_count == 2
def test_simple(): class AudubonTrait(Cluster.Trait): master_role = String # noqa slave_role = Default(String, 'slave') # noqa version = Required(Integer) # noqa west = Cluster(name='west', master_role='west.master', slave_role='west.slave', version=10) east = Cluster(name='east', version=11) assert east.name == 'east' with pytest.raises(AttributeError): east.slave_role assert east.with_traits(AudubonTrait).slave_role == 'slave' assert west.with_traits(AudubonTrait).slave_role == 'west.slave' assert east.with_traits(AudubonTrait).master_role is None with pytest.raises(TypeError): # requires version at least Cluster(name='east').with_traits(AudubonTrait)
def test_connect_direct_client_with_auth(self, mock_transport): mock_transport.return_value.open.side_effect = [TTransport.TTransportException, True] mock_time = mock.create_autospec(spec=time, instance=True) user_agent = 'Some-User-Agent' uri = 'https://scheduler.example.com:1337' auth = mock_auth() cluster = Cluster(scheduler_uri='uri') def auth_factory(_): return auth client = scheduler_client.SchedulerClient.get( cluster, auth_factory=auth_factory, user_agent=user_agent) client._connect_scheduler(uri, mock_time) mock_transport.assert_called_once_with(uri, auth=auth.auth(), user_agent=user_agent)
def test_connect_direct_scheduler_with_user_agent(self, mock_transport): mock_transport.return_value.open.side_effect = [ TTransport.TTransportException, True ] mock_time = mock.create_autospec(spec=time, instance=True) auth = mock_auth() user_agent = 'Some-User-Agent' uri = 'https://scheduler.example.com:1337' cluster = Cluster(scheduler_uri=uri) client = scheduler_client.DirectSchedulerClient(cluster, auth=auth, verbose=True, user_agent=user_agent) client._connect_scheduler(uri, mock_time) mock_transport.assert_called_once_with(uri, auth=auth.auth(), user_agent=user_agent, session_factory=mock.ANY)
class AuroraClientCommandTest(unittest.TestCase): FAKE_TIME = 42131 def setUp(self): patcher = patch('webbrowser.open_new_tab') self.mock_webbrowser = patcher.start() self.addCleanup(patcher.stop) def run(self, result=None): # Since CLUSTERS is a global value that evaluates code on import this is the best way to # ensure it does not pollute any tests. with CLUSTERS.patch(self.TEST_CLUSTERS._clusters.values()): super(AuroraClientCommandTest, self).run(result) @classmethod def create_blank_response(cls, code, msg): return Response(responseCode=code, details=[ResponseDetail(message=msg)]) @classmethod def create_simple_success_response(cls): return cls.create_blank_response(ResponseCode.OK, 'OK') @classmethod def create_error_response(cls): return cls.create_blank_response(ResponseCode.ERROR, 'Whoops') @classmethod def create_mock_api(cls): """Builds up a mock API object, with a mock SchedulerProxy""" mock_scheduler = create_autospec(spec=SchedulerThriftApiSpec, instance=True) mock_scheduler.url = "http://something_or_other" mock_scheduler_client = create_autospec(spec=SchedulerProxyApiSpec, instance=True) mock_scheduler_client.url = "http://something_or_other" mock_api = create_autospec(spec=HookedAuroraClientAPI, instance=True) mock_api.scheduler_proxy = mock_scheduler_client return mock_api, mock_scheduler_client @classmethod def create_mock_api_factory(cls): """Create a collection of mocks for a test that wants to mock out the client API by patching the api factory.""" mock_api, mock_scheduler_client = cls.create_mock_api() mock_api_factory = lambda: mock_api return mock_api_factory, mock_scheduler_client @classmethod def create_query_call_result(cls, task=None): status_response = cls.create_empty_task_result() if task is None: for i in range(20): status_response.result.scheduleStatusResult.tasks.append( cls.create_scheduled_task(i)) else: status_response.result.scheduleStatusResult.tasks.append(task) return status_response @classmethod def create_start_job_update_result(cls, code, msg, key, metadata): resp = cls.create_blank_response(code, msg) resp.result = Result(startJobUpdateResult=StartJobUpdateResult( key=key, updateSummary=JobUpdateSummary(metadata=metadata))) return resp @classmethod def create_empty_task_result(cls): status_response = cls.create_simple_success_response() status_response.result = Result( scheduleStatusResult=ScheduleStatusResult(tasks=[])) return status_response @classmethod def create_scheduled_task(cls, instance_id, status=ScheduleStatus.RUNNING, task_id=None, initial_time=None): task = ScheduledTask( status=status, assignedTask=AssignedTask(instanceId=instance_id, taskId=task_id or "Task%s" % instance_id, slaveId="Slave%s" % instance_id, slaveHost="Slave%s" % instance_id, task=TaskConfig()), taskEvents=[TaskEvent(timestamp=initial_time or 1000)]) return task @classmethod def create_task_config(cls, name): return TaskConfig( maxTaskFailures=1, executorConfig=ExecutorConfig(data='{"fake": "data"}'), metadata=[], job=JobKey(role=cls.TEST_ROLE, environment=cls.TEST_ENV, name=name), numCpus=2, ramMb=2, diskMb=2) @classmethod def create_scheduled_tasks(cls): tasks = [] for name in ['foo', 'bar', 'baz']: task = ScheduledTask(failureCount=0, assignedTask=AssignedTask( taskId=1287391823, slaveHost='slavehost', task=cls.create_task_config(name), instanceId=4237894, assignedPorts={}), status=ScheduleStatus.RUNNING, taskEvents=[ TaskEvent(timestamp=28234726395, status=ScheduleStatus.RUNNING, message="Hi there") ]) tasks.append(task) return tasks @classmethod def setup_get_tasks_status_calls(cls, scheduler): status_response = cls.create_query_call_result() scheduler.getTasksWithoutConfigs.return_value = status_response @classmethod def fake_time(cls, ignored): """Utility function used for faking time to speed up tests.""" cls.FAKE_TIME += 2 return cls.FAKE_TIME CONFIG_BASE = """ HELLO_WORLD = Job( name = '%(job)s', role = '%(role)s', cluster = '%(cluster)s', environment = '%(env)s', instances = 20, %(inner)s update_config = UpdateConfig( batch_size = 1, watch_secs = 45, max_per_shard_failures = 2, ), task = Task( name = 'test', processes = [Process(name = 'hello_world', cmdline = 'echo {{thermos.ports[http]}}')], resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB), ) ) jobs = [HELLO_WORLD] """ CRON_CONFIG_BASE = """ HELLO_WORLD = Job( name = '%(job)s', role = '%(role)s', cluster = '%(cluster)s', environment = '%(env)s', cron_schedule = '*/5 * * * *', %(inner)s task = SimpleTask('test', 'echo test') ) jobs = [HELLO_WORLD] """ UNBOUND_CONFIG = textwrap.dedent("""\ HELLO_WORLD = Job( name = '%(job)s', role = '%(role)s', cluster = '{{cluster_binding}}', environment = '%(env)s', instances = '{{instances_binding}}', update_config = UpdateConfig( batch_size = "{{TEST_BATCH}}", watch_secs = 45, max_per_shard_failures = 2, ), task = Task( name = 'test', processes = [Process( name = 'hello_world', cmdline = 'echo {{thermos.ports[http]}} {{flags_binding}}' )], resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB), ) ) jobs = [HELLO_WORLD] """) TEST_ROLE = 'bozo' TEST_ENV = 'test' TEST_JOB = 'hello' TEST_CLUSTER = 'west' TEST_JOBSPEC = 'west/bozo/test/hello' TEST_JOBKEY = AuroraJobKey('west', 'bozo', 'test', 'hello') TEST_CLUSTERS = Clusters([ Cluster(name=TEST_CLUSTER, zk='zookeeper.example.com', scheduler_zk_path='/foo/bar', auth_mechanism='UNAUTHENTICATED') ]) @classmethod def get_instance_spec(cls, instances_spec): """Create a job instance spec string""" return '%s/%s' % (cls.TEST_JOBSPEC, instances_spec) @classmethod def get_test_config(cls, base, cluster, role, env, job, inner=''): """Create a config from the template""" return base % { 'job': job, 'role': role, 'env': env, 'cluster': cluster, 'inner': inner } @classmethod def get_unbound_test_config(cls, role=None, env=None, job=None): result = cls.UNBOUND_CONFIG % { 'job': job or cls.TEST_JOB, 'role': role or cls.TEST_ROLE, 'env': env or cls.TEST_ENV } return result @classmethod def get_valid_config(cls): return cls.get_test_config(cls.CONFIG_BASE, cls.TEST_CLUSTER, cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB) @classmethod def get_valid_cron_config(cls): return cls.get_test_config(cls.CRON_CONFIG_BASE, cls.TEST_CLUSTER, cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB) @classmethod def get_invalid_config(cls, bad_clause): return cls.get_test_config(cls.CONFIG_BASE, cls.TEST_CLUSTER, cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB, bad_clause) @classmethod def get_invalid_cron_config(cls, bad_clause): return cls.get_test_config(cls.CRON_CONFIG_BASE, cls.TEST_CLUSTER, cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB, bad_clause) @classmethod def assert_lock_message(cls, context): assert [ line for line in context.get_err() if line == "\t%s" % context.LOCK_ERROR_MSG ] PREFERRED_TIER = TierConfig(name='preferred', settings={ 'preemptible': 'false', 'revocable': 'false' }) PREEMPTIBLE_TIER = TierConfig(name='preemptible', settings={ 'preemptible': 'true', 'revocable': 'false' }) REVOCABLE_TIER = TierConfig(name='revocable', settings={ 'preemptible': 'true', 'revocable': 'true' }) @classmethod def get_mock_tier_configurations(cls): response = cls.create_simple_success_response() response.result = Result(getTierConfigResult=GetTierConfigResult( defaultTierName=cls.PREEMPTIBLE_TIER.name, tiers=frozenset([ cls.PREFERRED_TIER, cls.PREEMPTIBLE_TIER, cls.REVOCABLE_TIER ]))) return response
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from optparse import OptionParser import pytest from apache.aurora.common.cluster import Cluster from apache.aurora.common.cluster_option import ClusterOption from apache.aurora.common.clusters import Clusters CLUSTER_LIST = Clusters(( Cluster(name='smf1'), Cluster(name='smf1-test'), )) def cluster_provider(name): return CLUSTER_LIST[name] def test_constructors(): ClusterOption('--test', '-t', help="Test cluster.", clusters=CLUSTER_LIST) ClusterOption('--test', '-t', help="Test cluster.", cluster_provider=cluster_provider) with pytest.raises(ValueError): ClusterOption()
def mock_api(cls): api = AuroraClientAPI(Cluster(name="foo")) mock_proxy = Mock() api._scheduler_proxy = mock_proxy return api, mock_proxy
def make_scheduler_proxy(self): return scheduler_client.SchedulerProxy(Cluster(name='local'))
class AuroraClientCommandTest(unittest.TestCase): FAKE_TIME = 42131 @classmethod def create_blank_response(cls, code, msg): return Response(responseCode=code, details=[ResponseDetail(message=msg)]) @classmethod def create_simple_success_response(cls): return cls.create_blank_response(ResponseCode.OK, 'OK') @classmethod def create_error_response(cls): return cls.create_blank_response(ResponseCode.ERROR, 'Damn') @classmethod def create_mock_api(cls): """Builds up a mock API object, with a mock SchedulerProxy""" mock_scheduler = create_autospec(spec=SchedulerThriftApiSpec, instance=True) mock_scheduler.url = "http://something_or_other" mock_scheduler_client = create_autospec(spec=SchedulerProxyApiSpec, instance=True) mock_scheduler_client.url = "http://something_or_other" mock_api = create_autospec(spec=HookedAuroraClientAPI, instance=True) mock_api.scheduler_proxy = mock_scheduler_client return mock_api, mock_scheduler_client @classmethod def create_mock_api_factory(cls): """Create a collection of mocks for a test that wants to mock out the client API by patching the api factory.""" mock_api, mock_scheduler_client = cls.create_mock_api() mock_api_factory = lambda: mock_api return mock_api_factory, mock_scheduler_client @classmethod def create_query_call_result(cls, task=None): status_response = cls.create_empty_task_result() if task is None: for i in range(20): status_response.result.scheduleStatusResult.tasks.append( cls.create_scheduled_task(i)) else: status_response.result.scheduleStatusResult.tasks.append(task) return status_response @classmethod def create_empty_task_result(cls): status_response = cls.create_simple_success_response() status_response.result = Result( scheduleStatusResult=ScheduleStatusResult(tasks=[])) return status_response @classmethod def create_scheduled_task(cls, instance_id, status=ScheduleStatus.RUNNING, task_id=None, initial_time=None): task = ScheduledTask( status=status, assignedTask=AssignedTask(instanceId=instance_id, taskId=task_id or "Task%s" % instance_id, slaveId="Slave%s" % instance_id, slaveHost="Slave%s" % instance_id, task=TaskConfig()), taskEvents=[TaskEvent(timestamp=initial_time or 1000)]) return task @classmethod def create_scheduled_tasks(cls): tasks = [] for name in ['foo', 'bar', 'baz']: task = ScheduledTask( failureCount=0, assignedTask=AssignedTask( taskId=1287391823, slaveHost='slavehost', task=TaskConfig( maxTaskFailures=1, executorConfig=ExecutorConfig(data='fake data'), metadata=[], job=JobKey(role=cls.TEST_ROLE, environment=cls.TEST_ENV, name=name), owner=Identity(role=cls.TEST_ROLE), environment=cls.TEST_ENV, jobName=name, numCpus=2, ramMb=2, diskMb=2), instanceId=4237894, assignedPorts={}), status=ScheduleStatus.RUNNING, taskEvents=[ TaskEvent(timestamp=28234726395, status=ScheduleStatus.RUNNING, message="Hi there") ]) tasks.append(task) return tasks @classmethod def setup_get_tasks_status_calls(cls, scheduler): status_response = cls.create_query_call_result() scheduler.getTasksWithoutConfigs.return_value = status_response @classmethod def fake_time(cls, ignored): """Utility function used for faking time to speed up tests.""" cls.FAKE_TIME += 2 return cls.FAKE_TIME CONFIG_BASE = """ HELLO_WORLD = Job( name = '%(job)s', role = '%(role)s', cluster = '%(cluster)s', environment = '%(env)s', instances = 20, %(inner)s update_config = UpdateConfig( batch_size = 1, restart_threshold = 60, watch_secs = 45, max_per_shard_failures = 2, ), task = Task( name = 'test', processes = [Process(name = 'hello_world', cmdline = 'echo {{thermos.ports[http]}}')], resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB), ) ) jobs = [HELLO_WORLD] """ CRON_CONFIG_BASE = """ HELLO_WORLD = Job( name = '%(job)s', role = '%(role)s', cluster = '%(cluster)s', environment = '%(env)s', cron_schedule = '*/5 * * * *', %(inner)s task = SimpleTask('test', 'echo test') ) jobs = [HELLO_WORLD] """ UNBOUND_CONFIG = textwrap.dedent("""\ HELLO_WORLD = Job( name = '%(job)s', role = '%(role)s', cluster = '{{cluster_binding}}', environment = '%(env)s', instances = '{{instances_binding}}', update_config = UpdateConfig( batch_size = "{{TEST_BATCH}}", restart_threshold = 60, watch_secs = 45, max_per_shard_failures = 2, ), task = Task( name = 'test', processes = [Process(name = 'hello_world', cmdline = 'echo {{thermos.ports[http]}}')], resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB), ) ) jobs = [HELLO_WORLD] """) TEST_ROLE = 'bozo' TEST_ENV = 'test' TEST_JOB = 'hello' TEST_CLUSTER = 'west' TEST_JOBSPEC = 'west/bozo/test/hello' TEST_JOBKEY = AuroraJobKey('west', 'bozo', 'test', 'hello') TEST_CLUSTERS = Clusters([ Cluster(name='west', zk='zookeeper.example.com', scheduler_zk_path='/foo/bar', auth_mechanism='UNAUTHENTICATED') ]) @classmethod def get_instance_spec(cls, instances_spec): """Create a job instance spec string""" return '%s/%s' % (cls.TEST_JOBSPEC, instances_spec) @classmethod def get_test_config(cls, base, cluster, role, env, job, inner=''): """Create a config from the template""" return base % { 'job': job, 'role': role, 'env': env, 'cluster': cluster, 'inner': inner } @classmethod def get_unbound_test_config(cls, role=None, env=None, job=None): result = cls.UNBOUND_CONFIG % { 'job': job or cls.TEST_JOB, 'role': role or cls.TEST_ROLE, 'env': env or cls.TEST_ENV } return result @classmethod def get_valid_config(cls): return cls.get_test_config(cls.CONFIG_BASE, cls.TEST_CLUSTER, cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB) @classmethod def get_valid_cron_config(cls): return cls.get_test_config(cls.CRON_CONFIG_BASE, cls.TEST_CLUSTER, cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB) @classmethod def get_invalid_config(cls, bad_clause): return cls.get_test_config(cls.CONFIG_BASE, cls.TEST_CLUSTER, cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB, bad_clause) @classmethod def get_invalid_cron_config(cls, bad_clause): return cls.get_test_config(cls.CRON_CONFIG_BASE, cls.TEST_CLUSTER, cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB, bad_clause) @classmethod def assert_lock_message(cls, context): assert context.get_err()[2] == "\t%s" % context.LOCK_ERROR_MSG
# import mock import pytest from apache.aurora.client.api import AuroraClientAPI, SchedulerProxy from apache.aurora.client.cli import EXIT_AUTH_ERROR, Context from apache.aurora.client.cli.context import AuroraCommandContext from apache.aurora.client.hooks.hooked_api import HookedAuroraClientAPI from apache.aurora.common.aurora_job_key import AuroraJobKey from apache.aurora.common.cluster import Cluster from apache.aurora.common.clusters import CLUSTERS from ...api_util import SchedulerProxyApiSpec TEST_CLUSTER = Cluster(name='some-cluster', auth_mechanism='nothing', scheduler_uri='nowhere') def test_get_api_defaults_to_hooks_enabled(): with CLUSTERS.patch([TEST_CLUSTER]): api = AuroraCommandContext().get_api(TEST_CLUSTER.name) assert isinstance(api, HookedAuroraClientAPI) assert api._cluster == TEST_CLUSTER def test_get_api_forwards_hooks_disabled(): with CLUSTERS.patch([TEST_CLUSTER]): api = AuroraCommandContext().get_api(TEST_CLUSTER.name, enable_hooks=False) assert isinstance(api, AuroraClientAPI)
DOCKER_BINDING_CONFIG = """ jobs = [Job( name = 'hello_world', role = 'john_doe', environment = 'staging42', cluster = 'smf1-test', task = Task( name = 'main', processes = [Process(name='command', cmdline="echo helloworld")], resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB), ), container = Docker(image='{{docker.image[some/name][some.tag]}}') )] """ TEST_CLUSTER = Cluster(name='smf1-test', docker_registry='some.registry.domain.com') TEST_CLUSTERS = [TEST_CLUSTER] class TestDockerBindingHelper(unittest.TestCase): @patch.object(DockerRegistryClient, 'resolve') def test_docker_binding(self, mock_resolve): image_reference = 'registry.example.com/some/repo@some:digest' mock_resolve.return_value = image_reference binding_helper.unregister_all() BindingHelper.register(DockerBindingHelper()) with temporary_file() as fp: fp.write(DOCKER_BINDING_CONFIG)
from apache.aurora.client.base import add_grouping, remove_grouping from apache.aurora.common.cluster import Cluster from gen.apache.aurora.api.ttypes import ( Hosts, HostStatus, MaintenanceMode, MaintenanceStatusResult, Response, ResponseCode, Result, StartMaintenanceResult ) DEFAULT_CLUSTER = Cluster( name='us-west', scheduler_uri='us-west-234.example.com:8888', ) TEST_HOSTNAMES = [ 'us-west-001.example.com', 'us-west-002.example.com', 'us-west-003.example.com'] class TestHostMaintenance(unittest.TestCase): @mock.patch("apache.aurora.client.api.AuroraClientAPI.maintenance_status", spec=AuroraClientAPI.maintenance_status) @mock.patch("apache.aurora.client.api.AuroraClientAPI.drain_hosts", spec=AuroraClientAPI.drain_hosts) def test_drain_hosts(self, mock_drain_hosts, mock_maintenance_status): fake_maintenance_status_response = [ Response(result=Result(maintenanceStatusResult=MaintenanceStatusResult(set([
class AuroraClientCommandTest(unittest.TestCase): FAKE_TIME = 42131 @classmethod def create_blank_response(cls, code, msg): response = Mock(spec=Response) response.responseCode = code mock_msg = Mock() mock_msg.message = msg response.details = [mock_msg] response.messageDEPRECATED = msg response.result = Mock(spec=Result) return response @classmethod def create_simple_success_response(cls): return cls.create_blank_response(ResponseCode.OK, 'OK') @classmethod def create_error_response(cls): return cls.create_blank_response(ResponseCode.ERROR, 'Damn') @classmethod def create_mock_api(cls): """Builds up a mock API object, with a mock SchedulerProxy""" mock_api = Mock(spec=HookedAuroraClientAPI) mock_scheduler = Mock() mock_scheduler.url = "http://something_or_other" mock_scheduler_client = Mock() mock_scheduler_client.scheduler.return_value = mock_scheduler mock_scheduler_client.url = "http://something_or_other" mock_api = Mock(spec=HookedAuroraClientAPI) mock_api.scheduler_proxy = mock_scheduler_client return (mock_api, mock_scheduler_client) @classmethod def create_mock_api_factory(cls): """Create a collection of mocks for a test that wants to mock out the client API by patching the api factory.""" mock_api, mock_scheduler_client = cls.create_mock_api() mock_api_factory = Mock() mock_api_factory.return_value = mock_api return mock_api_factory, mock_scheduler_client @classmethod def create_status_call_result(cls, mock_task=None): status_response = cls.create_simple_success_response() schedule_status = Mock(spec=ScheduleStatusResult) status_response.result.scheduleStatusResult = schedule_status # This should be a list of ScheduledTask's. schedule_status.tasks = [] if mock_task is None: for i in range(20): schedule_status.tasks.append(cls.create_mock_task(i)) else: schedule_status.tasks.append(mock_task) return status_response @classmethod def create_mock_task(cls, instance_id, status=ScheduleStatus.RUNNING): mock_task = Mock(spec=ScheduledTask) mock_task.assignedTask = Mock(spec=AssignedTask) mock_task.assignedTask.instanceId = instance_id mock_task.assignedTask.taskId = "Task%s" % instance_id mock_task.assignedTask.slaveId = "Slave%s" % instance_id mock_task.assignedTask.task = Mock(spec=TaskConfig) mock_task.slaveHost = "Slave%s" % instance_id mock_task.status = status mock_task_event = Mock(spec=TaskEvent) mock_task_event.timestamp = 1000 mock_task.taskEvents = [mock_task_event] return mock_task @classmethod def setup_get_tasks_status_calls(cls, scheduler): status_response = cls.create_status_call_result() scheduler.getTasksWithoutConfigs.return_value = status_response @classmethod def fake_time(cls, ignored): """Utility function used for faking time to speed up tests.""" cls.FAKE_TIME += 2 return cls.FAKE_TIME CONFIG_BASE = """ HELLO_WORLD = Job( name = '%(job)s', role = '%(role)s', cluster = '%(cluster)s', environment = '%(env)s', instances = 20, %(inner)s update_config = UpdateConfig( batch_size = 1, restart_threshold = 60, watch_secs = 45, max_per_shard_failures = 2, ), task = Task( name = 'test', processes = [Process(name = 'hello_world', cmdline = 'echo {{thermos.ports[http]}}')], resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB), ) ) jobs = [HELLO_WORLD] """ UNBOUND_CONFIG = textwrap.dedent("""\ HELLO_WORLD = Job( name = '%(job)s', role = '%(role)s', cluster = '{{cluster_binding}}', environment = '%(env)s', instances = '{{instances_binding}}', update_config = UpdateConfig( batch_size = 1, restart_threshold = 60, watch_secs = 45, max_per_shard_failures = 2, ), task = Task( name = 'test', processes = [Process(name = 'hello_world', cmdline = 'echo {{thermos.ports[http]}}')], resources = Resources(cpu = 0.1, ram = 64 * MB, disk = 64 * MB), ) ) jobs = [HELLO_WORLD] """) TEST_ROLE = 'bozo' TEST_ENV = 'test' TEST_JOB = 'hello' TEST_CLUSTER = 'west' TEST_JOBSPEC = 'west/bozo/test/hello' TEST_CLUSTERS = Clusters([ Cluster(name='west', packer_copy_command='copying {{package}}', zk='zookeeper.example.com', scheduler_zk_path='/foo/bar', auth_mechanism='UNAUTHENTICATED') ]) @classmethod def get_test_config(cls, cluster, role, env, job, filler=''): """Create a config from the template""" return cls.CONFIG_BASE % { 'job': job, 'role': role, 'env': env, 'cluster': cluster, 'inner': filler } @classmethod def get_unbound_test_config(cls, role=None, env=None, job=None): result = cls.UNBOUND_CONFIG % { 'job': job or cls.TEST_JOB, 'role': role or cls.TEST_ROLE, 'env': env or cls.TEST_ENV } print( "CONFIG:===========================\n%s\n=============================" % result) return result @classmethod def get_valid_config(cls): return cls.get_test_config(cls.TEST_CLUSTER, cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB) @classmethod def get_invalid_config(cls, bad_clause): return cls.get_test_config(cls.TEST_CLUSTER, cls.TEST_ROLE, cls.TEST_ENV, cls.TEST_JOB, bad_clause)
def make_scheduler_proxy(self): return TestSchedulerProxy(Cluster(name='local'))
# import functools import mox import pytest from apache.aurora.client.api import AuroraClientAPI from apache.aurora.client.api.disambiguator import LiveJobDisambiguator from apache.aurora.common.aurora_job_key import AuroraJobKey from apache.aurora.common.cluster import Cluster from gen.apache.aurora.api.constants import ResponseCode from gen.apache.aurora.api.ttypes import GetJobsResult, JobConfiguration, JobKey, Response, Result TEST_CLUSTER = Cluster(name='smf1') class LiveJobDisambiguatorTest(mox.MoxTestBase): CLUSTER = TEST_CLUSTER ROLE = 'mesos' ENV = 'test' NAME = 'labrat' JOB_PATH = 'smf1/mesos/test/labrat' CONFIG_FILE = 'abc.aurora' def setUp(self): super(LiveJobDisambiguatorTest, self).setUp() self._api = self.mox.CreateMock(AuroraClientAPI) self._api.cluster = self.CLUSTER self._options = self.mox.CreateMockAnything()