def test_deserialize_thermos_task_unbound_refs(): # test unbound {{standard}} refs task_config = TaskConfig(executorConfig=ExecutorConfig( name='thermos', data=MESOS_JOB(task=HELLO_WORLD_UNBOUND).json_dumps())) assigned_task = AssignedTask(task=task_config, instanceId=0) with pytest.raises(TaskInfoError) as execinfo: mesos_task_instance_from_assigned_task(assigned_task) assert "Unexpected unbound refs: {{unbound_cmd}} {{unbound}}" in execinfo.value.message # test bound unscoped refs, valid case. task = BASE_TASK(name='task_name', processes=[ Process(name='process_name', cmdline='echo {{thermos.ports[health]}}') ]) task_config = TaskConfig(executorConfig=ExecutorConfig( name='thermos', data=MESOS_JOB(task=task).json_dumps())) assigned_task = AssignedTask(task=task_config, instanceId=0) assert mesos_task_instance_from_assigned_task(assigned_task) is not None # test unbound unscoped refs for cmdline in ('echo {{hello_{{thermos.ports[health]}}}}', 'echo {{hello_{{thermos.user_id}}}}'): task = BASE_TASK( name='task_name', processes=[Process(name='process_name', cmdline=cmdline)]) task_config = TaskConfig(executorConfig=ExecutorConfig( name='thermos', data=MESOS_JOB(task=task).json_dumps())) assigned_task = AssignedTask(task=task_config, instanceId=0) with pytest.raises(UnexpectedUnboundRefsError): mesos_task_instance_from_assigned_task(assigned_task)
def test_deserialize_thermos_task(): task_config = TaskConfig(executorConfig=ExecutorConfig( name='thermos', data=MESOS_JOB(task=HELLO_WORLD).json_dumps())) assigned_task = AssignedTask(task=task_config, instanceId=0) assert mesos_task_instance_from_assigned_task(assigned_task) == BASE_MTI( task=HELLO_WORLD) task_config = TaskConfig(executorConfig=ExecutorConfig( name='thermos', data=HELLO_WORLD_MTI.json_dumps())) assigned_task = AssignedTask(task=task_config, instanceId=0) assert mesos_task_instance_from_assigned_task(assigned_task) == BASE_MTI( task=HELLO_WORLD)
def assigned_task_from_mesos_task(task): """Deserialize AssignedTask from a launchTask task protocol buffer.""" try: assigned_task = thrift_deserialize(AssignedTask(), task.data) except (EOFError, TException) as e: raise TaskInfoError('Could not deserialize task! %s' % e) return assigned_task
def test_from_assigned_task_shell(self): interval_secs = 17 initial_interval_secs = 3 max_consecutive_failures = 2 timeout_secs = 5 shell_config = ShellHealthChecker(shell_command='failed command') task_config = TaskConfig(executorConfig=ExecutorConfig( name='thermos-generic', data=MESOS_JOB( task=HELLO_WORLD, health_check_config=HealthCheckConfig( health_checker=HealthCheckerConfig(shell=shell_config), interval_secs=interval_secs, initial_interval_secs=initial_interval_secs, max_consecutive_failures=max_consecutive_failures, timeout_secs=timeout_secs, )).json_dumps())) assigned_task = AssignedTask(task=task_config, instanceId=1) execconfig_data = json.loads(assigned_task.task.executorConfig.data) assert execconfig_data['health_check_config']['health_checker'][ 'shell']['shell_command'] == 'failed command' health_checker = HealthCheckerProvider().from_assigned_task( assigned_task, None) assert health_checker.threaded_health_checker.interval == interval_secs assert health_checker.threaded_health_checker.initial_interval == initial_interval_secs hct_max_fail = health_checker.threaded_health_checker.max_consecutive_failures assert hct_max_fail == max_consecutive_failures
def create_scheduled_task(instance, start_time): task = ScheduledTask() task.assignedTask = AssignedTask() task.assignedTask.taskId = "task_%s" % instance task.assignedTask.slaveId = "random_machine_id" task.assignedTask.slaveHost = "junk.nothing" task.assignedTask.task = TaskConfig() task.assignedTask.task.owner = Identity(role="nobody") task.assignedTask.task.environment = "prod" task.assignedTask.task.jobName = "flibber" task.assignedTask.task.isService = False task.assignedTask.task.numCpus = 2 task.assignedTask.task.ramMb = 2048 task.assignedTask.task.diskMb = 4096 task.assignedTask.task.priority = 7 task.assignedTask.task.maxTaskFailures = 3 task.assignedTask.task.production = False task.assignedTask.task.requestedPorts = ["http"] task.assignedTask.assignedPorts = {"http": 1001} task.assignedTask.instanceId = 0 task.status = 2 task.failureCount = instance + 4 task.taskEvents = create_task_events(start_time) task.ancestorId = "random_task_ancestor%s" % instance return task
def create_scheduled_tasks(cls): tasks = [] for name in ['foo', 'bar', 'baz']: task = ScheduledTask( failureCount=0, assignedTask=AssignedTask( taskId=1287391823, slaveHost='slavehost', task=TaskConfig( maxTaskFailures=1, executorConfig=ExecutorConfig(data='fake data'), metadata=[], job=JobKey(role=cls.TEST_ROLE, environment=cls.TEST_ENV, name=name), owner=Identity(role=cls.TEST_ROLE), environment=cls.TEST_ENV, jobName=name, numCpus=2, ramMb=2, diskMb=2), instanceId=4237894, assignedPorts={}), status=ScheduleStatus.RUNNING, taskEvents=[ TaskEvent(timestamp=28234726395, status=ScheduleStatus.RUNNING, message="Hi there") ]) tasks.append(task) return tasks
def create_scheduled_task(instance, start_time): task = ScheduledTask(assignedTask=AssignedTask( taskId="task_%s" % instance, slaveId="random_machine_id", slaveHost="junk.nothing", task=TaskConfig(job=JobKey(role="nobody", environment="prod", name='flibber'), isService=False, resources=frozenset([ Resource(numCpus=2), Resource(ramMb=2048), Resource(diskMb=4096) ]), priority=7, maxTaskFailures=3, production=False), assignedPorts={"http": 1001}, instanceId=instance), status=2, failureCount=instance + 4, taskEvents=create_task_events(start_time), ancestorId="random_task_ancestor%s" % instance) return task
def test_interpolate_cmd(self): """Making sure thermos.ports[foo] gets correctly substituted with assignedPorts info.""" interval_secs = 17 initial_interval_secs = 3 max_consecutive_failures = 2 timeout_secs = 5 shell_cmd = 'FOO_PORT={{thermos.ports[foo]}} failed command' shell_config = ShellHealthChecker(shell_command=shell_cmd) task_config = TaskConfig(executorConfig=ExecutorConfig( name='thermos-generic', data=MESOS_JOB( task=HELLO_WORLD, health_check_config=HealthCheckConfig( health_checker=HealthCheckerConfig(shell=shell_config), interval_secs=interval_secs, initial_interval_secs=initial_interval_secs, max_consecutive_failures=max_consecutive_failures, timeout_secs=timeout_secs, )).json_dumps())) assigned_task = AssignedTask(task=task_config, instanceId=1, assignedPorts={'foo': 9001}) interpolated_cmd = HealthCheckerProvider.interpolate_cmd(assigned_task, cmd=shell_cmd) assert interpolated_cmd == 'FOO_PORT=9001 failed command'
def test_from_assigned_task_http_endpoint_style_config(self): interval_secs = 17 initial_interval_secs = 3 max_consecutive_failures = 2 http_config = HttpHealthChecker(endpoint='/foo', expected_response='bar', expected_response_code=201) task_config = TaskConfig(executorConfig=ExecutorConfig( name='thermos', data=MESOS_JOB( task=HELLO_WORLD, health_check_config=HealthCheckConfig( health_checker=HealthCheckerConfig(http=http_config), interval_secs=interval_secs, initial_interval_secs=initial_interval_secs, max_consecutive_failures=max_consecutive_failures, timeout_secs=7)).json_dumps())) assigned_task = AssignedTask(task=task_config, instanceId=1, assignedPorts={'health': 9001}) execconfig_data = json.loads(assigned_task.task.executorConfig.data) http_exec_config = execconfig_data['health_check_config'][ 'health_checker']['http'] assert http_exec_config['endpoint'] == '/foo' assert http_exec_config['expected_response'] == 'bar' assert http_exec_config['expected_response_code'] == 201 health_checker = HealthCheckerProvider().from_assigned_task( assigned_task, None) assert health_checker.threaded_health_checker.interval == interval_secs assert health_checker.threaded_health_checker.initial_interval == initial_interval_secs
def test_from_assigned_task_shell_no_demotion(self, mock_getpwnam): interval_secs = 17 initial_interval_secs = 3 max_consecutive_failures = 2 timeout_secs = 5 shell_config = ShellHealthChecker(shell_command='failed command') task_config = TaskConfig( job=JobKey(role='role', environment='env', name='name'), executorConfig=ExecutorConfig( name='thermos-generic', data=MESOS_JOB( task=HELLO_WORLD, health_check_config=HealthCheckConfig( health_checker=HealthCheckerConfig(shell=shell_config), interval_secs=interval_secs, initial_interval_secs=initial_interval_secs, max_consecutive_failures=max_consecutive_failures, timeout_secs=timeout_secs, )).json_dumps())) assigned_task = AssignedTask(task=task_config, instanceId=1, assignedPorts={'foo': 9001}) execconfig_data = json.loads(assigned_task.task.executorConfig.data) assert execconfig_data['health_check_config']['health_checker'][ 'shell']['shell_command'] == 'failed command' health_checker = HealthCheckerProvider( nosetuid_health_checks=True).from_assigned_task( assigned_task, None) assert health_checker.threaded_health_checker.interval == interval_secs assert health_checker.threaded_health_checker.initial_interval == initial_interval_secs hct_max_fail = health_checker.threaded_health_checker.max_consecutive_failures assert hct_max_fail == max_consecutive_failures # Should not be trying to access role's user info. assert not mock_getpwnam.called
def make_assigned_task(thermos_config, assigned_ports=None): from gen.apache.aurora.api.constants import AURORA_EXECUTOR_NAME from gen.apache.aurora.api.ttypes import ( AssignedTask, ExecutorConfig, Identity, JobKey, TaskConfig ) assigned_ports = assigned_ports or {} executor_config = ExecutorConfig(name=AURORA_EXECUTOR_NAME, data=thermos_config.json_dumps()) task_config = TaskConfig( job=JobKey( role=thermos_config.role().get(), environment="prod", name=thermos_config.name().get()), owner=Identity(role=thermos_config.role().get(), user=thermos_config.role().get()), environment=thermos_config.environment().get(), jobName=thermos_config.name().get(), executorConfig=executor_config) return AssignedTask( instanceId=12345, task=task_config, assignedPorts=assigned_ports, slaveHost='test-host')
def create_task(cls): return [ ScheduledTask(assignedTask=AssignedTask( instanceId=0, task=TaskConfig( job=JobKey(role='role', environment='test', name='job'))), status=ScheduleStatus.RUNNING) ]
def test_deserialize_thermos_task_unbound_refs(): task_config = TaskConfig(executorConfig=ExecutorConfig( name='thermos', data=MESOS_JOB(task=HELLO_WORLD_UNBOUND).json_dumps())) assigned_task = AssignedTask(task=task_config, instanceId=0) with pytest.raises(ValueError) as execinfo: mesos_task_instance_from_assigned_task(assigned_task) assert execinfo.value.message == "Unexpected unbound refs: {{unbound_cmd}} {{unbound}}"
def create_task(cls): return [ ScheduledTask(assignedTask=AssignedTask( instanceId=0, task=TaskConfig(owner=Identity(role='test_role'), jobName='test_job')), status=ScheduleStatus.RUNNING) ]
def mock_status_active_tasks(self, instance_ids): tasks = [] for i in instance_ids: tasks.append(ScheduledTask( status=ScheduleStatus.RUNNING, assignedTask=AssignedTask(task=TaskConfig(), instanceId=i) )) response = make_response(result=Result(scheduleStatusResult=ScheduleStatusResult(tasks=tasks))) self.mock_scheduler.getTasksWithoutConfigs(IgnoreArg()).AndReturn(response)
def expect_get_tasks(self, tasks, ignore_ids=None, response_code=ResponseCode.OK): scheduled = [] for index, task in enumerate(tasks): if not ignore_ids or index not in ignore_ids: scheduled.append(ScheduledTask(assignedTask=AssignedTask(task=task, instanceId=index))) response = make_response(response_code) response.result = Result(scheduleStatusResult=ScheduleStatusResult(tasks=scheduled)) query = TaskQuery(jobKeys=[self._job_key], statuses=ACTIVE_STATES) self._scheduler.getTasksStatus(query).AndReturn(response)
def create_scheduled_task(cls, instance_id, status=ScheduleStatus.RUNNING, task_id=None, initial_time=None): task = ScheduledTask( status=status, assignedTask=AssignedTask(instanceId=instance_id, taskId=task_id or "Task%s" % instance_id, slaveId="Slave%s" % instance_id, slaveHost="Slave%s" % instance_id, task=TaskConfig()), taskEvents=[TaskEvent(timestamp=initial_time or 1000)]) return task
def create_task(self, duration, id, host=None, name=None, prod=None): return ScheduledTask(assignedTask=AssignedTask( instanceId=id, slaveHost=host, task=TaskConfig(production=prod if prod is not None else True, job=JobKey(role=self._role, environment=self._env, name=name or self._name))), status=ScheduleStatus.RUNNING, taskEvents=[ TaskEvent(status=ScheduleStatus.RUNNING, timestamp=(time.time() - duration) * 1000) ])
def test_from_assigned_task_shell_filesystem_image(self, mock_getpwnam): interval_secs = 17 initial_interval_secs = 3 max_consecutive_failures = 2 timeout_secs = 5 shell_config = ShellHealthChecker(shell_command='failed command') task_config = TaskConfig( job=JobKey(role='role', environment='env', name='name'), executorConfig=ExecutorConfig( name='thermos-generic', data=MESOS_JOB( task=HELLO_WORLD, health_check_config=HealthCheckConfig( health_checker=HealthCheckerConfig(shell=shell_config), interval_secs=interval_secs, initial_interval_secs=initial_interval_secs, max_consecutive_failures=max_consecutive_failures, timeout_secs=timeout_secs, )).json_dumps())) assigned_task = AssignedTask(task=task_config, instanceId=1, assignedPorts={'foo': 9001}) execconfig_data = json.loads(assigned_task.task.executorConfig.data) assert execconfig_data['health_check_config']['health_checker'][ 'shell']['shell_command'] == 'failed command' mock_sandbox = mock.Mock(spec_set=SandboxInterface) type(mock_sandbox).root = mock.PropertyMock(return_value='/some/path') type(mock_sandbox).is_filesystem_image = mock.PropertyMock( return_value=True) with mock.patch( 'apache.aurora.executor.common.health_checker.ShellHealthCheck' ) as mock_shell: HealthCheckerProvider( nosetuid_health_checks=False, mesos_containerizer_path='/some/path/mesos-containerizer' ).from_assigned_task(assigned_task, mock_sandbox) class NotNone(object): def __eq__(self, other): return other is not None assert mock_shell.mock_calls == [ mock.call(cmd='failed command', wrapper_fn=NotNone(), preexec_fn=None, timeout_secs=5.0) ]
def make_task(thermos_config, assigned_ports={}, **kw): role = getpass.getuser() task_id = thermos_config.task().name().get() + '-001' at = AssignedTask(taskId=task_id, task=TaskConfig(executorConfig=ExecutorConfig( name=AURORA_EXECUTOR_NAME, data=thermos_config.json_dumps()), owner=Identity(role=role, user=role)), assignedPorts=assigned_ports, **kw) td = mesos_pb2.TaskInfo() td.task_id.value = task_id td.name = thermos_config.task().name().get() td.data = serialize(at) return td
def mock_status_active_tasks(self, instance_ids): tasks = [] for i in instance_ids: tasks.append( ScheduledTask(status=ScheduleStatus.RUNNING, assignedTask=AssignedTask(task=TaskConfig(), instanceId=i))) response = Response(responseCode=ResponseCode.OK, messageDEPRECATED='test') response.result = Result() response.result.scheduleStatusResult = ScheduleStatusResult( tasks=tasks) self.mock_scheduler.getTasksWithoutConfigs( IgnoreArg()).AndReturn(response)
def make_assigned_task(thermos_config, assigned_ports=None): from gen.apache.aurora.api.constants import AURORA_EXECUTOR_NAME from gen.apache.aurora.api.ttypes import AssignedTask, ExecutorConfig, JobKey, TaskConfig assigned_ports = assigned_ports or {} executor_config = ExecutorConfig(name=AURORA_EXECUTOR_NAME, data=thermos_config.json_dumps()) task_config = TaskConfig(job=JobKey(role=thermos_config.role().get(), environment='test', name=thermos_config.name().get()), executorConfig=executor_config) return AssignedTask(instanceId=12345, task=task_config, assignedPorts=assigned_ports, taskId="taskId-12345")
def expect_get_tasks(self, tasks, ignore_ids=None, response_code=None): response_code = ResponseCode.OK if response_code is None else response_code response = Response(responseCode=response_code, messageDEPRECATED='test') scheduled = [] for index, task in enumerate(tasks): if not ignore_ids or index not in ignore_ids: scheduled.append( ScheduledTask(assignedTask=AssignedTask(task=task, instanceId=index))) response.result = Result(scheduleStatusResult=ScheduleStatusResult( tasks=scheduled)) query = TaskQuery(owner=Identity(role=self._job_key.role), environment=self._job_key.environment, jobName=self._job_key.name, statuses=ACTIVE_STATES) self._scheduler.getTasksStatus(query).AndReturn(response)
def create_scheduled_tasks(cls): tasks = [] for name in ['foo', 'bar', 'baz']: task = ScheduledTask(failureCount=0, assignedTask=AssignedTask( taskId=1287391823, slaveHost='slavehost', task=cls.create_task_config(name), instanceId=4237894, assignedPorts={}), status=ScheduleStatus.RUNNING, taskEvents=[ TaskEvent(timestamp=28234726395, status=ScheduleStatus.RUNNING, message="Hi there") ]) tasks.append(task) return tasks
def test_launchTask_deserialization_fail(self): # noqa proxy_driver = ProxyDriver() role = getpass.getuser() task_info = mesos_pb2.TaskInfo() task_info.name = task_info.task_id.value = 'broken' task_info.data = serialize( AssignedTask(task=TaskConfig( job=JobKey(role=role, environment='env', name='name'), executorConfig=ExecutorConfig(name=AURORA_EXECUTOR_NAME, data='garbage')))) te = FastThermosExecutor(runner_provider=make_provider(safe_mkdtemp()), sandbox_provider=DefaultTestSandboxProvider()) te.launchTask(proxy_driver, task_info) proxy_driver.wait_stopped() updates = proxy_driver.method_calls['sendStatusUpdate'] assert len(updates) == 2 assert updates[0][0][0].state == mesos_pb2.TASK_STARTING assert updates[1][0][0].state == mesos_pb2.TASK_FAILED
def test_from_assigned_task_no_health_port(self): interval_secs = 17 initial_interval_secs = 3 max_consecutive_failures = 2 timeout_secs = 5 task_config = TaskConfig(executorConfig=ExecutorConfig( name='thermos-generic', data=MESOS_JOB( task=HELLO_WORLD, health_check_config=HealthCheckConfig( interval_secs=interval_secs, initial_interval_secs=initial_interval_secs, max_consecutive_failures=max_consecutive_failures, timeout_secs=timeout_secs, )).json_dumps())) # No health port and we don't have a shell_command. assigned_task = AssignedTask(task=task_config, instanceId=1, assignedPorts={'http': 9001}) health_checker = HealthCheckerProvider().from_assigned_task( assigned_task, None) self.assertIsNone(health_checker)
def test_from_assigned_task(self): interval_secs = 17 initial_interval_secs = 3 max_consecutive_failures = 2 task_config = TaskConfig(executorConfig=ExecutorConfig( name='thermos', data=MESOS_JOB( task=HELLO_WORLD, health_check_config=HealthCheckConfig( interval_secs=interval_secs, initial_interval_secs=initial_interval_secs, max_consecutive_failures=max_consecutive_failures, timeout_secs=7)).json_dumps())) assigned_task = AssignedTask(task=task_config, instanceId=1, assignedPorts={'health': 9001}) health_checker = HealthCheckerProvider().from_assigned_task( assigned_task, None) assert health_checker.threaded_health_checker.interval == interval_secs assert health_checker.threaded_health_checker.initial_interval == initial_interval_secs hct_max_fail = health_checker.threaded_health_checker.max_consecutive_failures assert hct_max_fail == max_consecutive_failures
def test_from_assigned_task_shell(self, mock_getpwnam): interval_secs = 17 initial_interval_secs = 3 max_consecutive_failures = 2 timeout_secs = 5 shell_config = ShellHealthChecker(shell_command='failed command') task_config = TaskConfig( job=JobKey(role='role', environment='env', name='name'), executorConfig=ExecutorConfig( name='thermos-generic', data=MESOS_JOB( task=HELLO_WORLD, health_check_config=HealthCheckConfig( health_checker=HealthCheckerConfig(shell=shell_config), interval_secs=interval_secs, initial_interval_secs=initial_interval_secs, max_consecutive_failures=max_consecutive_failures, timeout_secs=timeout_secs, )).json_dumps())) assigned_task = AssignedTask(task=task_config, instanceId=1, assignedPorts={'foo': 9001}) execconfig_data = json.loads(assigned_task.task.executorConfig.data) assert execconfig_data['health_check_config']['health_checker'][ 'shell']['shell_command'] == 'failed command' mock_sandbox = mock.Mock(spec_set=SandboxInterface) type(mock_sandbox).root = mock.PropertyMock(return_value='/some/path') type(mock_sandbox).is_filesystem_image = mock.PropertyMock( return_value=False) health_checker = HealthCheckerProvider().from_assigned_task( assigned_task, mock_sandbox) assert health_checker.threaded_health_checker.interval == interval_secs assert health_checker.threaded_health_checker.initial_interval == initial_interval_secs hct_max_fail = health_checker.threaded_health_checker.max_consecutive_failures assert hct_max_fail == max_consecutive_failures mock_getpwnam.assert_called_once_with(task_config.job.role)
def create_tasks(cls): return [ScheduledTask(assignedTask=AssignedTask(instanceId=index)) for index in cls.INSTANCES]
def create_task(self, status, id): return ScheduledTask( assignedTask=AssignedTask(instanceId=id, taskId=id), status=status, taskEvents=[TaskEvent(status=status, timestamp=10)])