def test_diff_unordered_configs(self):
"""Diff between two config objects with different repr but identical content works ok."""
from_config = self.make_task_configs()[0]
from_config.constraints = set([
Constraint(name='value',
constraint=ValueConstraint(values=set(['1', '2']))),
Constraint(name='limit',
constraint=TaskConstraint(limit=LimitConstraint(
limit=int(10))))
])
from_config.taskLinks = {'task1': 'link1', 'task2': 'link2'}
from_config.metadata = set(
[Metadata(key='k2', value='v2'),
Metadata(key='k1', value='v1')])
from_config.executorConfig = ExecutorConfig(name='test',
data='test data')
from_config.requestedPorts = set(['3424', '142', '45235'])
# Deepcopy() almost guarantees from_config != to_config due to a different sequence of
# dict insertions. That in turn generates unequal json objects. The ideal here would be to
# assert to_config != from_config but that would produce a flaky test as I have observed
# the opposite on rare occasions as the ordering is not stable between test runs.
to_config = deepcopy(from_config)
diff_result = self._updater._diff_configs(from_config, to_config)
assert diff_result == "", ('diff result must be empty but was: %s' %
diff_result)
def create_mock_scheduled_task_with_metadata(cls):
result = cls.create_scheduled_tasks()
for job in result:
job.assignedTask.task.metadata = [
Metadata("meta", "data"),
Metadata("data", "meta")
]
return result
def test_start_update_different_update_inprogress(self):
mock_config = self.create_mock_config()
self._fake_context.get_job_config = Mock(return_value=mock_config)
update_id = 'some-mocked-uuid'
update_id_2 = 'some-other-mocked-uuid'
err_msg = "Active updates exist for this job."
return_value = AuroraClientCommandTest.create_start_job_update_result(
ResponseCode.INVALID_REQUEST, err_msg, UPDATE_KEY,
{Metadata(CLIENT_UPDATE_ID, update_id)})
self._mock_api.start_job_update.return_value = return_value
with patch('apache.aurora.client.cli.update.DiffFormatter'):
with pytest.raises(Context.CommandError):
with patch(
'apache.aurora.client.cli.update.uuid') as mock_uuid:
mock_uuid.uuid4.return_value = update_id_2
self._command.execute(self._fake_context)
assert self._mock_api.start_job_update.mock_calls == [
call(mock_config, None, self._mock_options.instance_spec.instance,
{CLIENT_UPDATE_ID: update_id_2})
]
assert self._fake_context.get_out() == []
assert self._fake_context.get_err() == [
StartUpdate.FAILED_TO_START_UPDATE_ERROR_MSG,
"\t%s" % err_msg
]
def test_start_update_already_inprogress(self):
mock_config = self.create_mock_config()
self._fake_context.get_job_config = Mock(return_value=mock_config)
update_id = 'some-mocked-uuid'
err_msg = "Active updates exist for this job."
return_value = AuroraClientCommandTest.create_start_job_update_result(
ResponseCode.INVALID_REQUEST, err_msg, UPDATE_KEY,
{Metadata(CLIENT_UPDATE_ID, update_id)})
self._mock_api.start_job_update.return_value = return_value
with patch('apache.aurora.client.cli.update.DiffFormatter'):
with patch('apache.aurora.client.cli.update.uuid') as mock_uuid:
mock_uuid.uuid4.return_value = update_id
self._command.execute(self._fake_context)
assert self._mock_api.start_job_update.mock_calls == [
call(mock_config, None, self._mock_options.instance_spec.instance,
{CLIENT_UPDATE_ID: update_id})
]
assert self._fake_context.get_out() == [
StartUpdate.UPDATE_MSG_TEMPLATE %
('http://something_or_other/scheduler/bozo/test/hello/update/update_id'
),
]
assert self._fake_context.get_err() == []
def get_update_details_response(cls):
query_response = Response()
query_response.responseCode = ResponseCode.OK
query_response.result = Result()
details = JobUpdateDetails(
update=JobUpdate(summary=JobUpdateSummary(
key=UPDATE_KEY,
user="******",
state=JobUpdateState(status=JobUpdateStatus.ROLLING_FORWARD,
createdTimestampMs=1000,
lastModifiedTimestampMs=2000),
metadata={
Metadata("issue", "test"),
Metadata("country", "America"),
Metadata("country", "Canada")
})),
updateEvents=[
JobUpdateEvent(status=JobUpdateStatus.ROLLING_FORWARD,
timestampMs=3000),
JobUpdateEvent(status=JobUpdateStatus.ROLL_FORWARD_PAUSED,
message="Investigating issues",
timestampMs=4000),
JobUpdateEvent(status=JobUpdateStatus.ROLLING_FORWARD,
timestampMs=5000)
],
instanceEvents=[
JobInstanceUpdateEvent(
instanceId=1,
timestampMs=6000,
action=JobUpdateAction.INSTANCE_UPDATING),
JobInstanceUpdateEvent(
instanceId=2,
timestampMs=7000,
action=JobUpdateAction.INSTANCE_UPDATING),
JobInstanceUpdateEvent(
instanceId=1,
timestampMs=8000,
action=JobUpdateAction.INSTANCE_UPDATED),
JobInstanceUpdateEvent(instanceId=2,
timestampMs=9000,
action=JobUpdateAction.INSTANCE_UPDATED)
])
query_response.result.getJobUpdateDetailsResult = GetJobUpdateDetailsResult(
detailsList=[details])
return query_response
def _job_update_request(self, config, instances=None, metadata=None):
try:
settings = UpdaterConfig(**config.update_config().get()
).to_thrift_update_settings(instances)
except ValueError as e:
raise self.UpdateConfigError(str(e))
return JobUpdateRequest(
instanceCount=config.instances(),
settings=settings,
taskConfig=config.job().taskConfig,
metadata={Metadata(k, v)
for k, v in metadata.items()} if metadata else None)
def convert(job, metadata=frozenset(), ports=frozenset()):
"""Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration."""
owner = Identity(user=getpass.getuser())
key = JobKey(
role=assert_valid_field('role', fully_interpolated(job.role())),
environment=assert_valid_field('environment',
fully_interpolated(job.environment())),
name=assert_valid_field('name', fully_interpolated(job.name())))
task_raw = job.task()
MB = 1024 * 1024
task = TaskConfig()
def not_empty_or(item, default):
return default if item is Empty else fully_interpolated(item)
# job components
task.production = fully_interpolated(job.production(), bool)
task.isService = select_service_bit(job)
task.maxTaskFailures = fully_interpolated(job.max_task_failures())
task.priority = fully_interpolated(job.priority())
task.contactEmail = not_empty_or(job.contact(), None)
task.tier = not_empty_or(job.tier(), None)
if job.has_partition_policy():
task.partitionPolicy = PartitionPolicy(
fully_interpolated(job.partition_policy().reschedule()),
fully_interpolated(job.partition_policy().delay_secs()))
# Add metadata to a task, to display in the scheduler UI.
metadata_set = frozenset()
if job.has_metadata():
customized_metadata = job.metadata()
metadata_set |= frozenset(
(str(fully_interpolated(key_value_metadata.key())),
str(fully_interpolated(key_value_metadata.value())))
for key_value_metadata in customized_metadata)
metadata_set |= frozenset(
(str(key), str(value)) for key, value in metadata)
task.metadata = frozenset(
Metadata(key=key, value=value) for key, value in metadata_set)
# task components
if not task_raw.has_resources():
raise InvalidConfig('Task must specify resources!')
if (fully_interpolated(task_raw.resources().ram()) == 0
or fully_interpolated(task_raw.resources().disk()) == 0):
raise InvalidConfig(
'Must specify ram and disk resources, got ram:%r disk:%r' %
(fully_interpolated(task_raw.resources().ram()),
fully_interpolated(task_raw.resources().disk())))
numCpus = fully_interpolated(task_raw.resources().cpu())
ramMb = fully_interpolated(task_raw.resources().ram()) / MB
diskMb = fully_interpolated(task_raw.resources().disk()) / MB
if numCpus <= 0 or ramMb <= 0 or diskMb <= 0:
raise InvalidConfig(
'Task has invalid resources. cpu/ramMb/diskMb must all be positive: '
'cpu:%r ramMb:%r diskMb:%r' % (numCpus, ramMb, diskMb))
numGpus = fully_interpolated(task_raw.resources().gpu())
task.resources = frozenset([
Resource(numCpus=numCpus),
Resource(ramMb=ramMb),
Resource(diskMb=diskMb)
] + [Resource(namedPort=p)
for p in ports] + ([Resource(numGpus=numGpus)] if numGpus else []))
task.job = key
task.owner = owner
task.taskLinks = {} # See AURORA-739
task.constraints = constraints_to_thrift(
not_empty_or(job.constraints(), {}))
task.container = create_container_config(job.container())
underlying, refs = job.interpolate()
# need to fake an instance id for the sake of schema checking
underlying_checked = underlying.bind(mesos={
'instance': 31337,
'hostname': ''
})
try:
ThermosTaskValidator.assert_valid_task(underlying_checked.task())
except ThermosTaskValidator.InvalidTaskError as e:
raise InvalidConfig('Task is invalid: %s' % e)
if not underlying_checked.check().ok():
raise InvalidConfig('Job not fully specified: %s' %
underlying.check().message())
unbound = []
for ref in refs:
if ref in (THERMOS_TASK_ID_REF, MESOS_INSTANCE_REF,
MESOS_HOSTNAME_REF) or (Ref.subscope(
THERMOS_PORT_SCOPE_REF, ref)):
continue
unbound.append(ref)
if unbound:
raise InvalidConfig('Config contains unbound variables: %s' %
' '.join(map(str, unbound)))
# set the executor that will be used by the Mesos task. Thermos is the default
executor = job.executor_config()
if fully_interpolated(executor.name()) == AURORA_EXECUTOR_NAME:
task.executorConfig = ExecutorConfig(
name=AURORA_EXECUTOR_NAME,
data=filter_aliased_fields(underlying).json_dumps())
else:
task.executorConfig = ExecutorConfig(
name=fully_interpolated(executor.name()),
data=fully_interpolated(executor.data()))
return JobConfiguration(
key=key,
owner=owner,
cronSchedule=not_empty_or(job.cron_schedule(), None),
cronCollisionPolicy=select_cron_policy(job.cron_collision_policy()),
taskConfig=task,
instanceCount=fully_interpolated(job.instances()))
