def test_diff_unordered_configs(self): """Diff between two config objects with different repr but identical content works ok.""" from_config = self.make_task_configs()[0] from_config.constraints = set([ Constraint(name='value', constraint=ValueConstraint(values=set(['1', '2']))), Constraint(name='limit', constraint=TaskConstraint(limit=LimitConstraint( limit=int(10)))) ]) from_config.taskLinks = {'task1': 'link1', 'task2': 'link2'} from_config.metadata = set( [Metadata(key='k2', value='v2'), Metadata(key='k1', value='v1')]) from_config.executorConfig = ExecutorConfig(name='test', data='test data') from_config.requestedPorts = set(['3424', '142', '45235']) # Deepcopy() almost guarantees from_config != to_config due to a different sequence of # dict insertions. That in turn generates unequal json objects. The ideal here would be to # assert to_config != from_config but that would produce a flaky test as I have observed # the opposite on rare occasions as the ordering is not stable between test runs. to_config = deepcopy(from_config) diff_result = self._updater._diff_configs(from_config, to_config) assert diff_result == "", ('diff result must be empty but was: %s' % diff_result)
def create_mock_scheduled_task_with_metadata(cls): result = cls.create_scheduled_tasks() for job in result: job.assignedTask.task.metadata = [ Metadata("meta", "data"), Metadata("data", "meta") ] return result
def test_start_update_different_update_inprogress(self): mock_config = self.create_mock_config() self._fake_context.get_job_config = Mock(return_value=mock_config) update_id = 'some-mocked-uuid' update_id_2 = 'some-other-mocked-uuid' err_msg = "Active updates exist for this job." return_value = AuroraClientCommandTest.create_start_job_update_result( ResponseCode.INVALID_REQUEST, err_msg, UPDATE_KEY, {Metadata(CLIENT_UPDATE_ID, update_id)}) self._mock_api.start_job_update.return_value = return_value with patch('apache.aurora.client.cli.update.DiffFormatter'): with pytest.raises(Context.CommandError): with patch( 'apache.aurora.client.cli.update.uuid') as mock_uuid: mock_uuid.uuid4.return_value = update_id_2 self._command.execute(self._fake_context) assert self._mock_api.start_job_update.mock_calls == [ call(mock_config, None, self._mock_options.instance_spec.instance, {CLIENT_UPDATE_ID: update_id_2}) ] assert self._fake_context.get_out() == [] assert self._fake_context.get_err() == [ StartUpdate.FAILED_TO_START_UPDATE_ERROR_MSG, "\t%s" % err_msg ]
def test_start_update_already_inprogress(self): mock_config = self.create_mock_config() self._fake_context.get_job_config = Mock(return_value=mock_config) update_id = 'some-mocked-uuid' err_msg = "Active updates exist for this job." return_value = AuroraClientCommandTest.create_start_job_update_result( ResponseCode.INVALID_REQUEST, err_msg, UPDATE_KEY, {Metadata(CLIENT_UPDATE_ID, update_id)}) self._mock_api.start_job_update.return_value = return_value with patch('apache.aurora.client.cli.update.DiffFormatter'): with patch('apache.aurora.client.cli.update.uuid') as mock_uuid: mock_uuid.uuid4.return_value = update_id self._command.execute(self._fake_context) assert self._mock_api.start_job_update.mock_calls == [ call(mock_config, None, self._mock_options.instance_spec.instance, {CLIENT_UPDATE_ID: update_id}) ] assert self._fake_context.get_out() == [ StartUpdate.UPDATE_MSG_TEMPLATE % ('http://something_or_other/scheduler/bozo/test/hello/update/update_id' ), ] assert self._fake_context.get_err() == []
def get_update_details_response(cls): query_response = Response() query_response.responseCode = ResponseCode.OK query_response.result = Result() details = JobUpdateDetails( update=JobUpdate(summary=JobUpdateSummary( key=UPDATE_KEY, user="******", state=JobUpdateState(status=JobUpdateStatus.ROLLING_FORWARD, createdTimestampMs=1000, lastModifiedTimestampMs=2000), metadata={ Metadata("issue", "test"), Metadata("country", "America"), Metadata("country", "Canada") })), updateEvents=[ JobUpdateEvent(status=JobUpdateStatus.ROLLING_FORWARD, timestampMs=3000), JobUpdateEvent(status=JobUpdateStatus.ROLL_FORWARD_PAUSED, message="Investigating issues", timestampMs=4000), JobUpdateEvent(status=JobUpdateStatus.ROLLING_FORWARD, timestampMs=5000) ], instanceEvents=[ JobInstanceUpdateEvent( instanceId=1, timestampMs=6000, action=JobUpdateAction.INSTANCE_UPDATING), JobInstanceUpdateEvent( instanceId=2, timestampMs=7000, action=JobUpdateAction.INSTANCE_UPDATING), JobInstanceUpdateEvent( instanceId=1, timestampMs=8000, action=JobUpdateAction.INSTANCE_UPDATED), JobInstanceUpdateEvent(instanceId=2, timestampMs=9000, action=JobUpdateAction.INSTANCE_UPDATED) ]) query_response.result.getJobUpdateDetailsResult = GetJobUpdateDetailsResult( detailsList=[details]) return query_response
def _job_update_request(self, config, instances=None, metadata=None): try: settings = UpdaterConfig(**config.update_config().get() ).to_thrift_update_settings(instances) except ValueError as e: raise self.UpdateConfigError(str(e)) return JobUpdateRequest( instanceCount=config.instances(), settings=settings, taskConfig=config.job().taskConfig, metadata={Metadata(k, v) for k, v in metadata.items()} if metadata else None)
def convert(job, metadata=frozenset(), ports=frozenset()): """Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration.""" owner = Identity(user=getpass.getuser()) key = JobKey( role=assert_valid_field('role', fully_interpolated(job.role())), environment=assert_valid_field('environment', fully_interpolated(job.environment())), name=assert_valid_field('name', fully_interpolated(job.name()))) task_raw = job.task() MB = 1024 * 1024 task = TaskConfig() def not_empty_or(item, default): return default if item is Empty else fully_interpolated(item) # job components task.production = fully_interpolated(job.production(), bool) task.isService = select_service_bit(job) task.maxTaskFailures = fully_interpolated(job.max_task_failures()) task.priority = fully_interpolated(job.priority()) task.contactEmail = not_empty_or(job.contact(), None) task.tier = not_empty_or(job.tier(), None) if job.has_partition_policy(): task.partitionPolicy = PartitionPolicy( fully_interpolated(job.partition_policy().reschedule()), fully_interpolated(job.partition_policy().delay_secs())) # Add metadata to a task, to display in the scheduler UI. metadata_set = frozenset() if job.has_metadata(): customized_metadata = job.metadata() metadata_set |= frozenset( (str(fully_interpolated(key_value_metadata.key())), str(fully_interpolated(key_value_metadata.value()))) for key_value_metadata in customized_metadata) metadata_set |= frozenset( (str(key), str(value)) for key, value in metadata) task.metadata = frozenset( Metadata(key=key, value=value) for key, value in metadata_set) # task components if not task_raw.has_resources(): raise InvalidConfig('Task must specify resources!') if (fully_interpolated(task_raw.resources().ram()) == 0 or fully_interpolated(task_raw.resources().disk()) == 0): raise InvalidConfig( 'Must specify ram and disk resources, got ram:%r disk:%r' % (fully_interpolated(task_raw.resources().ram()), fully_interpolated(task_raw.resources().disk()))) numCpus = fully_interpolated(task_raw.resources().cpu()) ramMb = fully_interpolated(task_raw.resources().ram()) / MB diskMb = fully_interpolated(task_raw.resources().disk()) / MB if numCpus <= 0 or ramMb <= 0 or diskMb <= 0: raise InvalidConfig( 'Task has invalid resources. cpu/ramMb/diskMb must all be positive: ' 'cpu:%r ramMb:%r diskMb:%r' % (numCpus, ramMb, diskMb)) numGpus = fully_interpolated(task_raw.resources().gpu()) task.resources = frozenset([ Resource(numCpus=numCpus), Resource(ramMb=ramMb), Resource(diskMb=diskMb) ] + [Resource(namedPort=p) for p in ports] + ([Resource(numGpus=numGpus)] if numGpus else [])) task.job = key task.owner = owner task.taskLinks = {} # See AURORA-739 task.constraints = constraints_to_thrift( not_empty_or(job.constraints(), {})) task.container = create_container_config(job.container()) underlying, refs = job.interpolate() # need to fake an instance id for the sake of schema checking underlying_checked = underlying.bind(mesos={ 'instance': 31337, 'hostname': '' }) try: ThermosTaskValidator.assert_valid_task(underlying_checked.task()) except ThermosTaskValidator.InvalidTaskError as e: raise InvalidConfig('Task is invalid: %s' % e) if not underlying_checked.check().ok(): raise InvalidConfig('Job not fully specified: %s' % underlying.check().message()) unbound = [] for ref in refs: if ref in (THERMOS_TASK_ID_REF, MESOS_INSTANCE_REF, MESOS_HOSTNAME_REF) or (Ref.subscope( THERMOS_PORT_SCOPE_REF, ref)): continue unbound.append(ref) if unbound: raise InvalidConfig('Config contains unbound variables: %s' % ' '.join(map(str, unbound))) # set the executor that will be used by the Mesos task. Thermos is the default executor = job.executor_config() if fully_interpolated(executor.name()) == AURORA_EXECUTOR_NAME: task.executorConfig = ExecutorConfig( name=AURORA_EXECUTOR_NAME, data=filter_aliased_fields(underlying).json_dumps()) else: task.executorConfig = ExecutorConfig( name=fully_interpolated(executor.name()), data=fully_interpolated(executor.data())) return JobConfiguration( key=key, owner=owner, cronSchedule=not_empty_or(job.cron_schedule(), None), cronCollisionPolicy=select_cron_policy(job.cron_collision_policy()), taskConfig=task, instanceCount=fully_interpolated(job.instances()))