def job(self):
interpolated_job = self._job % self.context()
# TODO(wickman) Once thermos is onto thrift instead of pystachio, use
# %%replacements%% instead.
#
# Typecheck against the Job, with the following free variables unwrapped at the Task level:
# - a dummy {{mesos.instance}}
# - dummy values for the {{thermos.ports}} context, to allow for their use in task_links
env = dict(mesos=Environment(instance=0))
if interpolated_job.task_links() is not Empty:
try:
dummy_ports = dict((port, 31337)
for port in PortExtractor.extract(
interpolated_job.task_links()))
except PortExtractor.InvalidPorts as err:
raise self.InvalidConfig(
'Invalid port references in task_links! %s' % err)
env.update(thermos=ThermosContext(ports=dummy_ports))
typecheck = interpolated_job.bind(Environment(env)).check()
if not typecheck.ok():
raise self.InvalidConfig(typecheck.message())
interpolated_job = interpolated_job(task_links=self.task_links())
try:
return convert_thrift(interpolated_job, self._packages,
self.ports())
except InvalidThriftConfig as e:
raise self.InvalidConfig(str(e))
def substitute_thermos(cls, command, task, cluster, **kw):
prefix_command = 'cd %s;' % cls.thermos_sandbox(cluster, **kw)
thermos_namespace = ThermosContext(
task_id=task.assignedTask.taskId,
ports=task.assignedTask.assignedPorts)
mesos_namespace = MesosContext(instance=task.assignedTask.instanceId)
command = String(prefix_command + command) % Environment(
thermos=thermos_namespace, mesos=mesos_namespace)
return command.get()
def interpolate_cmd(task, cmd):
"""
:param task: Assigned task passed from Mesos Agent
:param cmd: Command defined inside shell_command inside config.
:return: Interpolated cmd with filled in values, for example ports.
"""
thermos_namespace = ThermosContext(task_id=task.taskId,
ports=task.assignedPorts)
mesos_namespace = MesosContext(instance=task.instanceId)
command = String(cmd) % Environment(thermos=thermos_namespace,
mesos=mesos_namespace)
return command.get()
def _build_path(context, target):
(task_instance,
path) = ScpCommand._extract_task_instance_and_path(context, target)
# No jobkey is specified therefore we are using a local path.
if (task_instance is None):
return path
# Jobkey specified, we want to convert to the user@host:file scp format
(cluster, role, env, name) = task_instance.jobkey
instance = set([task_instance.instance])
api = context.get_api(cluster)
resp = api.query(
api.build_query(role, name, env=env, instances=instance))
context.log_response_and_raise(
resp,
err_msg=('Unable to get information about instance: %s' %
combine_messages(resp)))
if (resp.result.scheduleStatusResult.tasks is None
or len(resp.result.scheduleStatusResult.tasks) == 0):
raise context.CommandError(
EXIT_INVALID_PARAMETER, ScpCommand.JOB_NOT_FOUND_ERROR_MSG %
(task_instance.jobkey, task_instance.instance))
first_task = resp.result.scheduleStatusResult.tasks[0]
assigned = first_task.assignedTask
role = assigned.task.job.role
slave_host = assigned.slaveHost
# If path is absolute, use that. Else if it is a tilde expansion, throw an error.
# Otherwise, use sandbox as relative root.
normalized_input_path = os.path.normpath(path)
if (os.path.isabs(normalized_input_path)):
final_path = normalized_input_path
elif (normalized_input_path.startswith('~/')
or normalized_input_path == '~'):
raise context.CommandError(EXIT_INVALID_PARAMETER,
ScpCommand.TILDE_USAGE_ERROR_MSG % path)
else:
sandbox_path_pre_format = DistributedCommandRunner.thermos_sandbox(
api.cluster, executor_sandbox=context.options.executor_sandbox)
thermos_namespace = ThermosContext(task_id=assigned.taskId,
ports=assigned.assignedPorts)
sandbox_path = String(sandbox_path_pre_format) % Environment(
thermos=thermos_namespace)
# Join the individual folders to the sandbox path to build safely
final_path = os.path.join(str(sandbox_path),
*normalized_input_path.split(os.sep))
return '%s@%s:%s' % (role, slave_host, final_path)
def _read_task(self, memoized={}):
"""Read the corresponding task from disk and return a ThermosTask. Memoizes already-read tasks.
"""
if self._task_id not in memoized:
path = self._pathspec.given(task_id=self._task_id, state=self.type).getpath('task_path')
if os.path.exists(path):
task = ThermosTaskWrapper.from_file(path)
if task is None:
log.error('Error reading ThermosTask from %s in observer.' % path)
else:
context = self.context(self._task_id)
if not context:
log.warning('Task not yet available: %s' % self._task_id)
task = task.task() % Environment(thermos=context)
memoized[self._task_id] = task
return memoized.get(self._task_id, None)
def __init__(self,
task,
checkpoint_root,
sandbox,
log_dir=None,
task_id=None,
portmap=None,
user=None,
chroot=False,
clock=time,
universal_handler=None,
planner_class=TaskPlanner,
hostname=None,
process_logger_destination=None,
process_logger_mode=None,
rotate_log_size_mb=None,
rotate_log_backups=None,
preserve_env=False):
"""
required:
task (config.Task) = the task to run
checkpoint_root (path) = the checkpoint root
sandbox (path) = the sandbox in which the path will be run
[if None, cwd will be assumed, but garbage collection will be
disabled for this task.]
optional:
log_dir (string) = directory to house stdout/stderr logs. If not specified, logs will be
written into the sandbox directory under .logs/
task_id (string) = bind to this task id. if not specified, will synthesize an id based
upon task.name()
portmap (dict) = a map (string => integer) from name to port, e.g. { 'http': 80 }
user (string) = the user to run the task as. if not current user, requires setuid
privileges.
chroot (boolean) = whether or not to chroot into the sandbox prior to exec.
clock (time interface) = the clock to use throughout
universal_handler = checkpoint record handler (only used for testing)
planner_class (TaskPlanner class) = TaskPlanner class to use for constructing the task
planning policy.
process_logger_destination (string) = The destination of logger to use for all processes.
process_logger_mode (string) = The mode of logger to use for all processes.
rotate_log_size_mb (integer) = The maximum size of the rotated stdout/stderr logs in MiB.
rotate_log_backups (integer) = The maximum number of rotated stdout/stderr log backups.
preserve_env (boolean) = whether or not env variables for the runner should be in the
env for the task being run
"""
if not issubclass(planner_class, TaskPlanner):
raise TypeError('planner_class must be a TaskPlanner.')
self._clock = clock
launch_time = self._clock.time()
launch_time_ms = '%06d' % int(
(launch_time - int(launch_time)) * (10**6))
if not task_id:
self._task_id = '%s-%s.%s' % (
task.name(),
time.strftime('%Y%m%d-%H%M%S',
time.localtime(launch_time)), launch_time_ms)
else:
self._task_id = task_id
current_user = TaskRunnerHelper.get_actual_user()
self._user = user or current_user
# TODO(wickman) This should be delegated to the ProcessPlatform / Helper
if self._user != current_user:
if os.geteuid() != 0:
raise ValueError(
'task specifies user as %s, but %s does not have setuid permission!'
% (self._user, current_user))
self._portmap = portmap or {}
self._launch_time = launch_time
self._log_dir = log_dir or os.path.join(sandbox, '.logs')
self._process_logger_destination = process_logger_destination
self._process_logger_mode = process_logger_mode
self._rotate_log_size_mb = rotate_log_size_mb
self._rotate_log_backups = rotate_log_backups
self._pathspec = TaskPath(root=checkpoint_root,
task_id=self._task_id,
log_dir=self._log_dir)
self._hostname = hostname or socket.gethostname()
try:
ThermosTaskValidator.assert_valid_task(task)
ThermosTaskValidator.assert_valid_ports(task, self._portmap)
except ThermosTaskValidator.InvalidTaskError as e:
raise self.InvalidTask('Invalid task: %s' % e)
context = ThermosContext(task_id=self._task_id,
ports=self._portmap,
user=self._user)
self._task, uninterp = (task %
Environment(thermos=context)).interpolate()
if len(uninterp) > 0:
raise self.InvalidTask('Failed to interpolate task, missing: %s' %
', '.join(str(ref) for ref in uninterp))
try:
ThermosTaskValidator.assert_same_task(self._pathspec, self._task)
except ThermosTaskValidator.InvalidTaskError as e:
raise self.InvalidTask('Invalid task: %s' % e)
self._plan = None # plan currently being executed (updated by Handlers)
self._regular_plan = planner_class(
self._task,
clock=clock,
process_filter=lambda proc: proc.final().get() is False)
self._finalizing_plan = planner_class(
self._task,
clock=clock,
process_filter=lambda proc: proc.final().get() is True)
self._chroot = chroot
self._sandbox = sandbox
self._terminal_state = None
self._ckpt = None
self._process_map = dict(
(p.name().get(), p) for p in self._task.processes())
self._task_processes = {}
self._stages = dict(
(state, stage(self)) for state, stage in self.STAGES.items())
self._finalization_start = None
self._preemption_deadline = None
self._watcher = ProcessMuxer(self._pathspec)
self._state = RunnerState(processes={})
self._preserve_env = preserve_env
# create runner state
universal_handler = universal_handler or TaskRunnerUniversalHandler
self._dispatcher = CheckpointDispatcher()
self._dispatcher.register_handler(universal_handler(self))
self._dispatcher.register_handler(TaskRunnerProcessHandler(self))
self._dispatcher.register_handler(TaskRunnerTaskHandler(self))
# recover checkpointed runner state and update plan
self._recovery = True
self._replay_runner_ckpt()
def context(self, instance=None): context = dict(instance=instance) # Filter unspecified values return Environment(mesos=MesosContext(dict((k, v) for k, v in context.items() if v)))