def mesos_task_instance_from_assigned_task(assigned_task): """Deserialize MesosTaskInstance from an AssignedTask thrift.""" thermos_task = assigned_task.task.executorConfig.data if not thermos_task: raise ValueError('Task did not have a thermos config!') try: json_blob = json.loads(thermos_task) except (TypeError, ValueError) as e: raise ValueError('Could not deserialize thermos config: %s' % e) # As part of the transition for MESOS-2133, we can send either a MesosTaskInstance # or we can be sending a MesosJob. So handle both possible cases. Once everyone # is using MesosJob, then we can begin to leverage additional information that # becomes available such as cluster. if 'instance' in json_blob: return MesosTaskInstance.json_loads(thermos_task) # This is a MesosJob mti, refs = task_instance_from_job(MesosJob.json_loads(thermos_task), assigned_task.instanceId) unbound_refs = [] for ref in refs: # If the ref is {{thermos.task_id}} or a subscope of # {{thermos.ports}}, it currently gets bound by the Thermos Runner, # so we must leave them unbound. # # {{thermos.user}} is a legacy binding which we can safely ignore. # # TODO(wickman) These should be rewritten by the mesos client to use # %%style%% replacements in order to allow us to better type-check configs # client-side. if ref == Ref.from_address('thermos.task_id'): continue if Ref.subscope(Ref.from_address('thermos.ports'), ref): continue if ref == Ref.from_address('thermos.user'): continue else: unbound_refs.append(ref) if len(unbound_refs) != 0: raise ValueError('Unexpected unbound refs: %s' % ' '.join(map(str, unbound_refs))) return mti
def convert(job, metadata=frozenset(), ports=frozenset()): """Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration.""" owner = Identity(user=getpass.getuser()) key = JobKey( role=assert_valid_field('role', fully_interpolated(job.role())), environment=assert_valid_field('environment', fully_interpolated(job.environment())), name=assert_valid_field('name', fully_interpolated(job.name()))) task_raw = job.task() MB = 1024 * 1024 task = TaskConfig() def not_empty_or(item, default): return default if item is Empty else fully_interpolated(item) # job components task.production = fully_interpolated(job.production(), bool) task.isService = select_service_bit(job) task.maxTaskFailures = fully_interpolated(job.max_task_failures()) task.priority = fully_interpolated(job.priority()) task.contactEmail = not_empty_or(job.contact(), None) task.tier = not_empty_or(job.tier(), None) if job.has_partition_policy(): task.partitionPolicy = PartitionPolicy( fully_interpolated(job.partition_policy().reschedule()), fully_interpolated(job.partition_policy().delay_secs())) # Add metadata to a task, to display in the scheduler UI. metadata_set = frozenset() if job.has_metadata(): customized_metadata = job.metadata() metadata_set |= frozenset( (str(fully_interpolated(key_value_metadata.key())), str(fully_interpolated(key_value_metadata.value()))) for key_value_metadata in customized_metadata) metadata_set |= frozenset( (str(key), str(value)) for key, value in metadata) task.metadata = frozenset( Metadata(key=key, value=value) for key, value in metadata_set) # task components if not task_raw.has_resources(): raise InvalidConfig('Task must specify resources!') if (fully_interpolated(task_raw.resources().ram()) == 0 or fully_interpolated(task_raw.resources().disk()) == 0): raise InvalidConfig( 'Must specify ram and disk resources, got ram:%r disk:%r' % (fully_interpolated(task_raw.resources().ram()), fully_interpolated(task_raw.resources().disk()))) numCpus = fully_interpolated(task_raw.resources().cpu()) ramMb = fully_interpolated(task_raw.resources().ram()) / MB diskMb = fully_interpolated(task_raw.resources().disk()) / MB if numCpus <= 0 or ramMb <= 0 or diskMb <= 0: raise InvalidConfig( 'Task has invalid resources. cpu/ramMb/diskMb must all be positive: ' 'cpu:%r ramMb:%r diskMb:%r' % (numCpus, ramMb, diskMb)) numGpus = fully_interpolated(task_raw.resources().gpu()) task.resources = frozenset([ Resource(numCpus=numCpus), Resource(ramMb=ramMb), Resource(diskMb=diskMb) ] + [Resource(namedPort=p) for p in ports] + ([Resource(numGpus=numGpus)] if numGpus else [])) task.job = key task.owner = owner task.taskLinks = {} # See AURORA-739 task.constraints = constraints_to_thrift( not_empty_or(job.constraints(), {})) task.container = create_container_config(job.container()) underlying, refs = job.interpolate() # need to fake an instance id for the sake of schema checking underlying_checked = underlying.bind(mesos={ 'instance': 31337, 'hostname': '' }) try: ThermosTaskValidator.assert_valid_task(underlying_checked.task()) except ThermosTaskValidator.InvalidTaskError as e: raise InvalidConfig('Task is invalid: %s' % e) if not underlying_checked.check().ok(): raise InvalidConfig('Job not fully specified: %s' % underlying.check().message()) unbound = [] for ref in refs: if ref in (THERMOS_TASK_ID_REF, MESOS_INSTANCE_REF, MESOS_HOSTNAME_REF) or (Ref.subscope( THERMOS_PORT_SCOPE_REF, ref)): continue unbound.append(ref) if unbound: raise InvalidConfig('Config contains unbound variables: %s' % ' '.join(map(str, unbound))) # set the executor that will be used by the Mesos task. Thermos is the default executor = job.executor_config() if fully_interpolated(executor.name()) == AURORA_EXECUTOR_NAME: task.executorConfig = ExecutorConfig( name=AURORA_EXECUTOR_NAME, data=filter_aliased_fields(underlying).json_dumps()) else: task.executorConfig = ExecutorConfig( name=fully_interpolated(executor.name()), data=fully_interpolated(executor.data())) return JobConfiguration( key=key, owner=owner, cronSchedule=not_empty_or(job.cron_schedule(), None), cronCollisionPolicy=select_cron_policy(job.cron_collision_policy()), taskConfig=task, instanceCount=fully_interpolated(job.instances()))
def convert(job, metadata=frozenset(), ports=frozenset()): """Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration.""" owner = Identity(user=getpass.getuser()) key = JobKey( role=assert_valid_field('role', fully_interpolated(job.role())), environment=assert_valid_field('environment', fully_interpolated(job.environment())), name=assert_valid_field('name', fully_interpolated(job.name()))) task_raw = job.task() MB = 1024 * 1024 task = TaskConfig() def not_empty_or(item, default): return default if item is Empty else fully_interpolated(item) # job components task.production = fully_interpolated(job.production(), bool) task.isService = select_service_bit(job) task.maxTaskFailures = fully_interpolated(job.max_task_failures()) task.priority = fully_interpolated(job.priority()) task.contactEmail = not_empty_or(job.contact(), None) task.tier = not_empty_or(job.tier(), None) # Add metadata to a task, to display in the scheduler UI. task.metadata = frozenset(Metadata(key=str(key), value=str(value)) for key, value in metadata) # task components if not task_raw.has_resources(): raise InvalidConfig('Task must specify resources!') if (fully_interpolated(task_raw.resources().ram()) == 0 or fully_interpolated(task_raw.resources().disk()) == 0): raise InvalidConfig('Must specify ram and disk resources, got ram:%r disk:%r' % ( fully_interpolated(task_raw.resources().ram()), fully_interpolated(task_raw.resources().disk()))) task.numCpus = fully_interpolated(task_raw.resources().cpu()) task.ramMb = fully_interpolated(task_raw.resources().ram()) / MB task.diskMb = fully_interpolated(task_raw.resources().disk()) / MB if task.numCpus <= 0 or task.ramMb <= 0 or task.diskMb <= 0: raise InvalidConfig('Task has invalid resources. cpu/ramMb/diskMb must all be positive: ' 'cpu:%r ramMb:%r diskMb:%r' % (task.numCpus, task.ramMb, task.diskMb)) task.job = key task.owner = owner task.requestedPorts = ports task.taskLinks = {} # See AURORA-739 task.constraints = constraints_to_thrift(not_empty_or(job.constraints(), {})) task.container = create_container_config(job.container()) underlying, refs = job.interpolate() # need to fake an instance id for the sake of schema checking underlying_checked = underlying.bind(mesos={'instance': 31337, 'hostname': ''}) try: ThermosTaskValidator.assert_valid_task(underlying_checked.task()) except ThermosTaskValidator.InvalidTaskError as e: raise InvalidConfig('Task is invalid: %s' % e) if not underlying_checked.check().ok(): raise InvalidConfig('Job not fully specified: %s' % underlying.check().message()) unbound = [] for ref in refs: if ref in (THERMOS_TASK_ID_REF, MESOS_INSTANCE_REF, MESOS_HOSTNAME_REF) or ( Ref.subscope(THERMOS_PORT_SCOPE_REF, ref)): continue unbound.append(ref) if unbound: raise InvalidConfig('Config contains unbound variables: %s' % ' '.join(map(str, unbound))) task.executorConfig = ExecutorConfig( name=AURORA_EXECUTOR_NAME, data=filter_aliased_fields(underlying).json_dumps()) return JobConfiguration( key=key, owner=owner, cronSchedule=not_empty_or(job.cron_schedule(), None), cronCollisionPolicy=select_cron_policy(job.cron_collision_policy()), taskConfig=task, instanceCount=fully_interpolated(job.instances()))
def convert(job, packages=frozenset(), ports=frozenset()): """Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration.""" owner = Identity(role=fully_interpolated(job.role()), user=getpass.getuser()) key = JobKey( role=assert_valid_field('role', fully_interpolated(job.role())), environment=assert_valid_field('environment', fully_interpolated(job.environment())), name=assert_valid_field('name', fully_interpolated(job.name()))) task_raw = job.task() MB = 1024 * 1024 task = TaskConfig() def not_empty_or(item, default): return default if item is Empty else fully_interpolated(item) # job components task.jobName = fully_interpolated(job.name()) task.environment = fully_interpolated(job.environment()) task.production = fully_interpolated(job.production(), bool) task.isService = select_service_bit(job) task.maxTaskFailures = fully_interpolated(job.max_task_failures()) task.priority = fully_interpolated(job.priority()) task.contactEmail = not_empty_or(job.contact(), None) # Add package tuples to a task, to display in the scheduler UI. task.packages = frozenset( Package(role=str(role), name=str(package_name), version=int(version)) for role, package_name, version in packages) # task components if not task_raw.has_resources(): raise InvalidConfig('Task must specify resources!') if (fully_interpolated(task_raw.resources().ram()) == 0 or fully_interpolated(task_raw.resources().disk()) == 0): raise InvalidConfig( 'Must specify ram and disk resources, got ram:%r disk:%r' % (fully_interpolated(task_raw.resources().ram()), fully_interpolated(task_raw.resources().disk()))) task.numCpus = fully_interpolated(task_raw.resources().cpu()) task.ramMb = fully_interpolated(task_raw.resources().ram()) / MB task.diskMb = fully_interpolated(task_raw.resources().disk()) / MB if task.numCpus <= 0 or task.ramMb <= 0 or task.diskMb <= 0: raise InvalidConfig( 'Task has invalid resources. cpu/ramMb/diskMb must all be positive: ' 'cpu:%r ramMb:%r diskMb:%r' % (task.numCpus, task.ramMb, task.diskMb)) task.owner = owner task.requestedPorts = ports task.taskLinks = not_empty_or(job.task_links(), {}) task.constraints = constraints_to_thrift( not_empty_or(job.constraints(), {})) underlying, refs = job.interpolate() # need to fake an instance id for the sake of schema checking underlying_checked = underlying.bind(mesos={'instance': 31337}) try: ThermosTaskValidator.assert_valid_task(underlying_checked.task()) except ThermosTaskValidator.InvalidTaskError as e: raise InvalidConfig('Task is invalid: %s' % e) if not underlying_checked.check().ok(): raise InvalidConfig('Job not fully specified: %s' % underlying.check().message()) unbound = [] for ref in refs: if ref == THERMOS_TASK_ID_REF or ref == MESOS_INSTANCE_REF or ( Ref.subscope(THERMOS_PORT_SCOPE_REF, ref)): continue unbound.append(ref) if unbound: raise InvalidConfig('Config contains unbound variables: %s' % ' '.join(map(str, unbound))) cron_schedule = not_empty_or(job.cron_schedule(), '') cron_policy = select_cron_policy(job.cron_policy(), job.cron_collision_policy()) task.executorConfig = ExecutorConfig( name=AURORA_EXECUTOR_NAME, data=filter_aliased_fields(underlying).json_dumps()) return JobConfiguration(key=key, owner=owner, cronSchedule=cron_schedule, cronCollisionPolicy=cron_policy, taskConfig=task, instanceCount=fully_interpolated(job.instances()))