def mesos_task_instance_from_assigned_task(assigned_task):
"""Deserialize MesosTaskInstance from an AssignedTask thrift."""
thermos_task = assigned_task.task.executorConfig.data
if not thermos_task:
raise ValueError('Task did not have a thermos config!')
try:
json_blob = json.loads(thermos_task)
except (TypeError, ValueError) as e:
raise ValueError('Could not deserialize thermos config: %s' % e)
# As part of the transition for MESOS-2133, we can send either a MesosTaskInstance
# or we can be sending a MesosJob. So handle both possible cases. Once everyone
# is using MesosJob, then we can begin to leverage additional information that
# becomes available such as cluster.
if 'instance' in json_blob:
return MesosTaskInstance.json_loads(thermos_task)
# This is a MesosJob
mti, refs = task_instance_from_job(MesosJob.json_loads(thermos_task), assigned_task.instanceId)
unbound_refs = []
for ref in refs:
# If the ref is {{thermos.task_id}} or a subscope of
# {{thermos.ports}}, it currently gets bound by the Thermos Runner,
# so we must leave them unbound.
#
# {{thermos.user}} is a legacy binding which we can safely ignore.
#
# TODO(wickman) These should be rewritten by the mesos client to use
# %%style%% replacements in order to allow us to better type-check configs
# client-side.
if ref == Ref.from_address('thermos.task_id'):
continue
if Ref.subscope(Ref.from_address('thermos.ports'), ref):
continue
if ref == Ref.from_address('thermos.user'):
continue
else:
unbound_refs.append(ref)
if len(unbound_refs) != 0:
raise ValueError('Unexpected unbound refs: %s' % ' '.join(map(str, unbound_refs)))
return mti
def convert(job, metadata=frozenset(), ports=frozenset()):
"""Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration."""
owner = Identity(user=getpass.getuser())
key = JobKey(
role=assert_valid_field('role', fully_interpolated(job.role())),
environment=assert_valid_field('environment',
fully_interpolated(job.environment())),
name=assert_valid_field('name', fully_interpolated(job.name())))
task_raw = job.task()
MB = 1024 * 1024
task = TaskConfig()
def not_empty_or(item, default):
return default if item is Empty else fully_interpolated(item)
# job components
task.production = fully_interpolated(job.production(), bool)
task.isService = select_service_bit(job)
task.maxTaskFailures = fully_interpolated(job.max_task_failures())
task.priority = fully_interpolated(job.priority())
task.contactEmail = not_empty_or(job.contact(), None)
task.tier = not_empty_or(job.tier(), None)
if job.has_partition_policy():
task.partitionPolicy = PartitionPolicy(
fully_interpolated(job.partition_policy().reschedule()),
fully_interpolated(job.partition_policy().delay_secs()))
# Add metadata to a task, to display in the scheduler UI.
metadata_set = frozenset()
if job.has_metadata():
customized_metadata = job.metadata()
metadata_set |= frozenset(
(str(fully_interpolated(key_value_metadata.key())),
str(fully_interpolated(key_value_metadata.value())))
for key_value_metadata in customized_metadata)
metadata_set |= frozenset(
(str(key), str(value)) for key, value in metadata)
task.metadata = frozenset(
Metadata(key=key, value=value) for key, value in metadata_set)
# task components
if not task_raw.has_resources():
raise InvalidConfig('Task must specify resources!')
if (fully_interpolated(task_raw.resources().ram()) == 0
or fully_interpolated(task_raw.resources().disk()) == 0):
raise InvalidConfig(
'Must specify ram and disk resources, got ram:%r disk:%r' %
(fully_interpolated(task_raw.resources().ram()),
fully_interpolated(task_raw.resources().disk())))
numCpus = fully_interpolated(task_raw.resources().cpu())
ramMb = fully_interpolated(task_raw.resources().ram()) / MB
diskMb = fully_interpolated(task_raw.resources().disk()) / MB
if numCpus <= 0 or ramMb <= 0 or diskMb <= 0:
raise InvalidConfig(
'Task has invalid resources. cpu/ramMb/diskMb must all be positive: '
'cpu:%r ramMb:%r diskMb:%r' % (numCpus, ramMb, diskMb))
numGpus = fully_interpolated(task_raw.resources().gpu())
task.resources = frozenset([
Resource(numCpus=numCpus),
Resource(ramMb=ramMb),
Resource(diskMb=diskMb)
] + [Resource(namedPort=p)
for p in ports] + ([Resource(numGpus=numGpus)] if numGpus else []))
task.job = key
task.owner = owner
task.taskLinks = {} # See AURORA-739
task.constraints = constraints_to_thrift(
not_empty_or(job.constraints(), {}))
task.container = create_container_config(job.container())
underlying, refs = job.interpolate()
# need to fake an instance id for the sake of schema checking
underlying_checked = underlying.bind(mesos={
'instance': 31337,
'hostname': ''
})
try:
ThermosTaskValidator.assert_valid_task(underlying_checked.task())
except ThermosTaskValidator.InvalidTaskError as e:
raise InvalidConfig('Task is invalid: %s' % e)
if not underlying_checked.check().ok():
raise InvalidConfig('Job not fully specified: %s' %
underlying.check().message())
unbound = []
for ref in refs:
if ref in (THERMOS_TASK_ID_REF, MESOS_INSTANCE_REF,
MESOS_HOSTNAME_REF) or (Ref.subscope(
THERMOS_PORT_SCOPE_REF, ref)):
continue
unbound.append(ref)
if unbound:
raise InvalidConfig('Config contains unbound variables: %s' %
' '.join(map(str, unbound)))
# set the executor that will be used by the Mesos task. Thermos is the default
executor = job.executor_config()
if fully_interpolated(executor.name()) == AURORA_EXECUTOR_NAME:
task.executorConfig = ExecutorConfig(
name=AURORA_EXECUTOR_NAME,
data=filter_aliased_fields(underlying).json_dumps())
else:
task.executorConfig = ExecutorConfig(
name=fully_interpolated(executor.name()),
data=fully_interpolated(executor.data()))
return JobConfiguration(
key=key,
owner=owner,
cronSchedule=not_empty_or(job.cron_schedule(), None),
cronCollisionPolicy=select_cron_policy(job.cron_collision_policy()),
taskConfig=task,
instanceCount=fully_interpolated(job.instances()))
def convert(job, metadata=frozenset(), ports=frozenset()):
"""Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration."""
owner = Identity(user=getpass.getuser())
key = JobKey(
role=assert_valid_field('role', fully_interpolated(job.role())),
environment=assert_valid_field('environment', fully_interpolated(job.environment())),
name=assert_valid_field('name', fully_interpolated(job.name())))
task_raw = job.task()
MB = 1024 * 1024
task = TaskConfig()
def not_empty_or(item, default):
return default if item is Empty else fully_interpolated(item)
# job components
task.production = fully_interpolated(job.production(), bool)
task.isService = select_service_bit(job)
task.maxTaskFailures = fully_interpolated(job.max_task_failures())
task.priority = fully_interpolated(job.priority())
task.contactEmail = not_empty_or(job.contact(), None)
task.tier = not_empty_or(job.tier(), None)
# Add metadata to a task, to display in the scheduler UI.
task.metadata = frozenset(Metadata(key=str(key), value=str(value)) for key, value in metadata)
# task components
if not task_raw.has_resources():
raise InvalidConfig('Task must specify resources!')
if (fully_interpolated(task_raw.resources().ram()) == 0
or fully_interpolated(task_raw.resources().disk()) == 0):
raise InvalidConfig('Must specify ram and disk resources, got ram:%r disk:%r' % (
fully_interpolated(task_raw.resources().ram()),
fully_interpolated(task_raw.resources().disk())))
task.numCpus = fully_interpolated(task_raw.resources().cpu())
task.ramMb = fully_interpolated(task_raw.resources().ram()) / MB
task.diskMb = fully_interpolated(task_raw.resources().disk()) / MB
if task.numCpus <= 0 or task.ramMb <= 0 or task.diskMb <= 0:
raise InvalidConfig('Task has invalid resources. cpu/ramMb/diskMb must all be positive: '
'cpu:%r ramMb:%r diskMb:%r' % (task.numCpus, task.ramMb, task.diskMb))
task.job = key
task.owner = owner
task.requestedPorts = ports
task.taskLinks = {} # See AURORA-739
task.constraints = constraints_to_thrift(not_empty_or(job.constraints(), {}))
task.container = create_container_config(job.container())
underlying, refs = job.interpolate()
# need to fake an instance id for the sake of schema checking
underlying_checked = underlying.bind(mesos={'instance': 31337, 'hostname': ''})
try:
ThermosTaskValidator.assert_valid_task(underlying_checked.task())
except ThermosTaskValidator.InvalidTaskError as e:
raise InvalidConfig('Task is invalid: %s' % e)
if not underlying_checked.check().ok():
raise InvalidConfig('Job not fully specified: %s' % underlying.check().message())
unbound = []
for ref in refs:
if ref in (THERMOS_TASK_ID_REF, MESOS_INSTANCE_REF, MESOS_HOSTNAME_REF) or (
Ref.subscope(THERMOS_PORT_SCOPE_REF, ref)):
continue
unbound.append(ref)
if unbound:
raise InvalidConfig('Config contains unbound variables: %s' % ' '.join(map(str, unbound)))
task.executorConfig = ExecutorConfig(
name=AURORA_EXECUTOR_NAME,
data=filter_aliased_fields(underlying).json_dumps())
return JobConfiguration(
key=key,
owner=owner,
cronSchedule=not_empty_or(job.cron_schedule(), None),
cronCollisionPolicy=select_cron_policy(job.cron_collision_policy()),
taskConfig=task,
instanceCount=fully_interpolated(job.instances()))
def convert(job, packages=frozenset(), ports=frozenset()):
"""Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration."""
owner = Identity(role=fully_interpolated(job.role()),
user=getpass.getuser())
key = JobKey(
role=assert_valid_field('role', fully_interpolated(job.role())),
environment=assert_valid_field('environment',
fully_interpolated(job.environment())),
name=assert_valid_field('name', fully_interpolated(job.name())))
task_raw = job.task()
MB = 1024 * 1024
task = TaskConfig()
def not_empty_or(item, default):
return default if item is Empty else fully_interpolated(item)
# job components
task.jobName = fully_interpolated(job.name())
task.environment = fully_interpolated(job.environment())
task.production = fully_interpolated(job.production(), bool)
task.isService = select_service_bit(job)
task.maxTaskFailures = fully_interpolated(job.max_task_failures())
task.priority = fully_interpolated(job.priority())
task.contactEmail = not_empty_or(job.contact(), None)
# Add package tuples to a task, to display in the scheduler UI.
task.packages = frozenset(
Package(role=str(role), name=str(package_name), version=int(version))
for role, package_name, version in packages)
# task components
if not task_raw.has_resources():
raise InvalidConfig('Task must specify resources!')
if (fully_interpolated(task_raw.resources().ram()) == 0
or fully_interpolated(task_raw.resources().disk()) == 0):
raise InvalidConfig(
'Must specify ram and disk resources, got ram:%r disk:%r' %
(fully_interpolated(task_raw.resources().ram()),
fully_interpolated(task_raw.resources().disk())))
task.numCpus = fully_interpolated(task_raw.resources().cpu())
task.ramMb = fully_interpolated(task_raw.resources().ram()) / MB
task.diskMb = fully_interpolated(task_raw.resources().disk()) / MB
if task.numCpus <= 0 or task.ramMb <= 0 or task.diskMb <= 0:
raise InvalidConfig(
'Task has invalid resources. cpu/ramMb/diskMb must all be positive: '
'cpu:%r ramMb:%r diskMb:%r' %
(task.numCpus, task.ramMb, task.diskMb))
task.owner = owner
task.requestedPorts = ports
task.taskLinks = not_empty_or(job.task_links(), {})
task.constraints = constraints_to_thrift(
not_empty_or(job.constraints(), {}))
underlying, refs = job.interpolate()
# need to fake an instance id for the sake of schema checking
underlying_checked = underlying.bind(mesos={'instance': 31337})
try:
ThermosTaskValidator.assert_valid_task(underlying_checked.task())
except ThermosTaskValidator.InvalidTaskError as e:
raise InvalidConfig('Task is invalid: %s' % e)
if not underlying_checked.check().ok():
raise InvalidConfig('Job not fully specified: %s' %
underlying.check().message())
unbound = []
for ref in refs:
if ref == THERMOS_TASK_ID_REF or ref == MESOS_INSTANCE_REF or (
Ref.subscope(THERMOS_PORT_SCOPE_REF, ref)):
continue
unbound.append(ref)
if unbound:
raise InvalidConfig('Config contains unbound variables: %s' %
' '.join(map(str, unbound)))
cron_schedule = not_empty_or(job.cron_schedule(), '')
cron_policy = select_cron_policy(job.cron_policy(),
job.cron_collision_policy())
task.executorConfig = ExecutorConfig(
name=AURORA_EXECUTOR_NAME,
data=filter_aliased_fields(underlying).json_dumps())
return JobConfiguration(key=key,
owner=owner,
cronSchedule=cron_schedule,
cronCollisionPolicy=cron_policy,
taskConfig=task,
instanceCount=fully_interpolated(job.instances()))
