def execute(self, context):
# TODO(mchucarroll): add options to specify which instances to run on (AURORA-198)
cluster_name, role, env, name = context.options.jobspec
cluster = CLUSTERS[cluster_name]
dcr = DistributedCommandRunner(cluster, role, env, [name], context.options.ssh_user)
dcr.run(context.options.cmd, parallelism=context.options.num_threads,
executor_sandbox=context.options.executor_sandbox)
def run(args, options):
"""usage: run cluster/role/env/job cmd
Runs a shell command on all machines currently hosting shards of a single job.
This feature supports the same command line wildcards that are used to
populate a job's commands.
This means anything in the {{mesos.*}} and {{thermos.*}} namespaces.
"""
# TODO(William Farner): Add support for invoking on individual shards.
# TODO(Kevin Sweeney): Restore the ability to run across jobs with globs (See MESOS-3010).
if not args:
die("job path is required")
job_path = args.pop(0)
new_cmd = ["task", "run"]
instances_spec = job_path
if options.num_threads != 1:
new_cmd.append("--threads=%s" % options.num_threads)
if options.ssh_user is not None:
new_cmd.append("--ssh-user=%s" % options.ssh_user)
if options.executor_sandbox:
new_cmd.append("--executor-sandbox")
new_cmd.append('"%s"' % " ".join(args))
v1_deprecation_warning("ssh", new_cmd)
try:
cluster_name, role, env, name = AuroraJobKey.from_path(job_path)
except AuroraJobKey.Error as e:
die('Invalid job path "%s": %s' % (job_path, e))
command = " ".join(args)
cluster = CLUSTERS[cluster_name]
dcr = DistributedCommandRunner(cluster, role, env, [name], options.ssh_user)
dcr.run(command, parallelism=options.num_threads, executor_sandbox=options.executor_sandbox)
def execute(self, context):
(cluster, role, env, name) = context.options.task_instance.jobkey
instance = context.options.task_instance.instance
api = context.get_api(cluster)
resp = api.query(api.build_query(role, name, set([int(instance)]), env=env))
context.check_and_log_response(
resp, err_msg=("Unable to get information about instance: %s" % resp.messageDEPRECATED)
)
first_task = resp.result.scheduleStatusResult.tasks[0]
remote_cmd = context.options.command or "bash"
command = DistributedCommandRunner.substitute(
remote_cmd, first_task, api.cluster, executor_sandbox=context.options.executor_sandbox
)
ssh_command = ["ssh", "-t"]
role = first_task.assignedTask.task.owner.role
slave_host = first_task.assignedTask.slaveHost
for tunnel in context.options.tunnels:
try:
port, name = tunnel.split(":")
port = int(port)
except ValueError:
raise context.CommandError(
EXIT_INVALID_PARAMETER, "Could not parse tunnel: %s. Must be of form PORT:NAME" % tunnel
)
if name not in first_task.assignedTask.assignedPorts:
raise context.CommandError(
EXIT_INVALID_PARAMETER, "Task %s has no port named %s" % (first_task.assignedTask.taskId, name)
)
ssh_command += ["-L", "%d:%s:%d" % (port, slave_host, first_task.assignedTask.assignedPorts[name])]
ssh_command += ["%s@%s" % (context.options.ssh_user or role, slave_host), command]
return subprocess.call(ssh_command)
def execute(self, context):
(cluster, role, env, name) = context.options.task_instance.jobkey
instance = context.options.task_instance.instance
api = context.get_api(cluster)
resp = api.query(api.build_query(role, name, set([int(instance)]), env=env))
if resp.responseCode != ResponseCode.OK:
raise context.CommandError('Unable to get information about instance: %s' % resp.message)
first_task = resp.result.scheduleStatusResult.tasks[0]
remote_cmd = context.options.command or 'bash'
command = DistributedCommandRunner.substitute(remote_cmd, first_task,
api.cluster, executor_sandbox=context.options.executor_sandbox)
ssh_command = ['ssh', '-t']
role = first_task.assignedTask.task.owner.role
slave_host = first_task.assignedTask.slaveHost
for tunnel in context.options.tunnels:
try:
port, name = tunnel.split(':')
port = int(port)
except ValueError:
die('Could not parse tunnel: %s. Must be of form PORT:NAME' % tunnel)
if name not in first_task.assignedTask.assignedPorts:
die('Task %s has no port named %s' % (first_task.assignedTask.taskId, name))
ssh_command += [
'-L', '%d:%s:%d' % (port, slave_host, first_task.assignedTask.assignedPorts[name])]
ssh_command += ['%s@%s' % (context.options.ssh_user or role, slave_host), command]
return subprocess.call(ssh_command)
def ssh(args, options):
"""usage: ssh cluster/role/env/job shard [args...]
Initiate an SSH session on the machine that a shard is running on.
"""
if not args:
die('Job path is required')
job_path = args.pop(0)
try:
cluster_name, role, env, name = AuroraJobKey.from_path(job_path)
except AuroraJobKey.Error as e:
die('Invalid job path "%s": %s' % (job_path, e))
if not args:
die('Shard is required')
try:
shard = int(args.pop(0))
except ValueError:
die('Shard must be an integer')
newcmd = ["task", "ssh", "%s/%s" % (job_path, shard)]
if len(options.tunnels) > 0:
newcmd.append("--tunnels=%s" % options.tunnels)
if options.ssh_user is not None:
newcmd.append("--ssh-user=%s" % options.ssh_user)
if options.executor_sandbox:
newcmd.append("--executor-sandbox")
if len(args) > 0:
newcmd.append("--command=\"%s\"" % " ".join(args))
v1_deprecation_warning("ssh", newcmd)
api = make_client(cluster_name)
resp = api.query(api.build_query(role, name, set([int(shard)]), env=env))
check_and_log_response(resp)
first_task = resp.result.scheduleStatusResult.tasks[0]
remote_cmd = 'bash' if not args else ' '.join(args)
command = DistributedCommandRunner.substitute(remote_cmd, first_task,
api.cluster, executor_sandbox=options.executor_sandbox)
ssh_command = ['ssh', '-t']
role = first_task.assignedTask.task.owner.role
slave_host = first_task.assignedTask.slaveHost
for tunnel in options.tunnels:
try:
port, name = tunnel.split(':')
port = int(port)
except ValueError:
die('Could not parse tunnel: %s. Must be of form PORT:NAME' % tunnel)
if name not in first_task.assignedTask.assignedPorts:
die('Task %s has no port named %s' % (first_task.assignedTask.taskId, name))
ssh_command += [
'-L', '%d:%s:%d' % (port, slave_host, first_task.assignedTask.assignedPorts[name])]
ssh_command += ['%s@%s' % (options.ssh_user or role, slave_host), command]
return subprocess.call(ssh_command)
def execute(self, context):
(cluster, role, env, name) = context.options.instance_spec.jobkey
instance = (None
if context.options.instance_spec.instance == ALL_INSTANCES
else set(context.options.instance_spec.instance))
if instance is None and context.options.command:
raise context.CommandError(
EXIT_INVALID_PARAMETER,
'INSTANCE must be specified when --command option is given')
api = context.get_api(cluster)
resp = api.query(
api.build_query(role, name, env=env, instances=instance))
context.log_response_and_raise(
resp,
err_msg=('Unable to get information about instance: %s' %
combine_messages(resp)))
if (resp.result.scheduleStatusResult.tasks is None
or len(resp.result.scheduleStatusResult.tasks) == 0):
raise context.CommandError(
EXIT_INVALID_PARAMETER,
"Job %s not found" % context.options.instance_spec.jobkey)
first_task = resp.result.scheduleStatusResult.tasks[0]
remote_cmd = context.options.command or 'bash'
command = DistributedCommandRunner.substitute(
remote_cmd,
first_task,
api.cluster,
executor_sandbox=context.options.executor_sandbox)
ssh_command = ['ssh', '-t']
ssh_command += context.options.ssh_options if context.options.ssh_options else []
assigned = first_task.assignedTask
role = assigned.task.job.role
slave_host = assigned.slaveHost
for tunnel in context.options.tunnels:
try:
port, name = tunnel.split(':')
port = int(port)
except ValueError:
raise context.CommandError(
EXIT_INVALID_PARAMETER,
'Could not parse tunnel: %s. Must be of form PORT:NAME' %
tunnel)
if name not in assigned.assignedPorts:
raise context.CommandError(
EXIT_INVALID_PARAMETER,
'Task %s has no port named %s' % (assigned.taskId, name))
ssh_command += [
'-L',
'%d:%s:%d' % (port, slave_host, assigned.assignedPorts[name])
]
ssh_command += [
'%s@%s' % (context.options.ssh_user or role, slave_host), command
]
return subprocess.call(ssh_command)
def execute(self, context):
(cluster, role, env, name) = context.options.task_instance.jobkey
instance = context.options.task_instance.instance
api = context.get_api(cluster)
resp = api.query(
api.build_query(role, name, set([int(instance)]), env=env))
context.check_and_log_response(
resp,
err_msg=('Unable to get information about instance: %s' %
resp.messageDEPRECATED))
if (resp.result.scheduleStatusResult.tasks is None
or len(resp.result.scheduleStatusResult.tasks) == 0):
raise context.CommandError(
EXIT_INVALID_PARAMETER,
"Job %s not found" % context.options.task_instance.jobkey)
first_task = resp.result.scheduleStatusResult.tasks[0]
remote_cmd = context.options.command or 'bash'
command = DistributedCommandRunner.substitute(
remote_cmd,
first_task,
api.cluster,
executor_sandbox=context.options.executor_sandbox)
ssh_command = ['ssh', '-t']
role = first_task.assignedTask.task.owner.role
slave_host = first_task.assignedTask.slaveHost
for tunnel in context.options.tunnels:
try:
port, name = tunnel.split(':')
port = int(port)
except ValueError:
raise context.CommandError(
EXIT_INVALID_PARAMETER,
'Could not parse tunnel: %s. Must be of form PORT:NAME' %
tunnel)
if name not in first_task.assignedTask.assignedPorts:
raise context.CommandError(
EXIT_INVALID_PARAMETER, 'Task %s has no port named %s' %
(first_task.assignedTask.taskId, name))
ssh_command += [
'-L',
'%d:%s:%d' %
(port, slave_host, first_task.assignedTask.assignedPorts[name])
]
ssh_command += [
'%s@%s' % (context.options.ssh_user or role, slave_host), command
]
return subprocess.call(ssh_command)
def run(args, options):
"""usage: run cluster/role/env/job cmd
Runs a shell command on all machines currently hosting shards of a single job.
This feature supports the same command line wildcards that are used to
populate a job's commands.
This means anything in the {{mesos.*}} and {{thermos.*}} namespaces.
"""
# TODO(William Farner): Add support for invoking on individual shards.
# TODO(Kevin Sweeney): Restore the ability to run across jobs with globs (See MESOS-3010).
if not args:
die('job path is required')
job_path = args.pop(0)
new_cmd = ["task", "run"]
instances_spec = job_path
if options.num_threads != 1:
new_cmd.append("--threads=%s" % options.num_threads)
if options.ssh_user is not None:
new_cmd.append("--ssh-user=%s" % options.ssh_user)
if options.executor_sandbox:
new_cmd.append("--executor-sandbox")
new_cmd.append("\"%s\"" % " ".join(args))
v1_deprecation_warning("ssh", new_cmd)
try:
cluster_name, role, env, name = AuroraJobKey.from_path(job_path)
except AuroraJobKey.Error as e:
die('Invalid job path "%s": %s' % (job_path, e))
command = ' '.join(args)
cluster = CLUSTERS[cluster_name]
dcr = DistributedCommandRunner(cluster, role, env, [name],
options.ssh_user)
dcr.run(command,
parallelism=options.num_threads,
executor_sandbox=options.executor_sandbox)
def execute(self, context):
(cluster, role, env, name) = context.options.instance_spec.jobkey
instance = (None if context.options.instance_spec.instance == ALL_INSTANCES else
set(context.options.instance_spec.instance))
if instance is None and context.options.command:
raise context.CommandError(EXIT_INVALID_PARAMETER,
'INSTANCE must be specified when --command option is given')
api = context.get_api(cluster)
resp = api.query(api.build_query(role, name, env=env, instances=instance))
context.log_response_and_raise(resp,
err_msg=('Unable to get information about instance: %s' % combine_messages(resp)))
if (resp.result.scheduleStatusResult.tasks is None or
len(resp.result.scheduleStatusResult.tasks) == 0):
raise context.CommandError(EXIT_INVALID_PARAMETER,
"Job %s not found" % context.options.instance_spec.jobkey)
first_task = resp.result.scheduleStatusResult.tasks[0]
remote_cmd = context.options.command or 'bash'
command = DistributedCommandRunner.substitute(
remote_cmd,
first_task,
api.cluster,
executor_sandbox=context.options.executor_sandbox)
ssh_command = ['ssh', '-t']
ssh_command += context.options.ssh_options if context.options.ssh_options else []
assigned = first_task.assignedTask
role = assigned.task.job.role
slave_host = assigned.slaveHost
for tunnel in context.options.tunnels:
try:
port, name = tunnel.split(':')
port = int(port)
except ValueError:
raise context.CommandError(EXIT_INVALID_PARAMETER,
'Could not parse tunnel: %s. Must be of form PORT:NAME' % tunnel)
if name not in assigned.assignedPorts:
raise context.CommandError(EXIT_INVALID_PARAMETER,
'Task %s has no port named %s' % (assigned.taskId, name))
ssh_command += [
'-L', '%d:%s:%d' % (port, slave_host, assigned.assignedPorts[name])]
ssh_command += ['%s@%s' % (context.options.ssh_user or role, slave_host), command]
process = subprocess.Popen(ssh_command)
if context.options.pid_file:
with open(context.options.pid_file, "w") as f:
f.write(str(process.pid))
return process.wait()
def _build_path(context, target):
(task_instance,
path) = ScpCommand._extract_task_instance_and_path(context, target)
# No jobkey is specified therefore we are using a local path.
if (task_instance is None):
return path
# Jobkey specified, we want to convert to the user@host:file scp format
(cluster, role, env, name) = task_instance.jobkey
instance = set([task_instance.instance])
api = context.get_api(cluster)
resp = api.query(
api.build_query(role, name, env=env, instances=instance))
context.log_response_and_raise(
resp,
err_msg=('Unable to get information about instance: %s' %
combine_messages(resp)))
if (resp.result.scheduleStatusResult.tasks is None
or len(resp.result.scheduleStatusResult.tasks) == 0):
raise context.CommandError(
EXIT_INVALID_PARAMETER, ScpCommand.JOB_NOT_FOUND_ERROR_MSG %
(task_instance.jobkey, task_instance.instance))
first_task = resp.result.scheduleStatusResult.tasks[0]
assigned = first_task.assignedTask
role = assigned.task.job.role
slave_host = assigned.slaveHost
# If path is absolute, use that. Else if it is a tilde expansion, throw an error.
# Otherwise, use sandbox as relative root.
normalized_input_path = os.path.normpath(path)
if (os.path.isabs(normalized_input_path)):
final_path = normalized_input_path
elif (normalized_input_path.startswith('~/')
or normalized_input_path == '~'):
raise context.CommandError(EXIT_INVALID_PARAMETER,
ScpCommand.TILDE_USAGE_ERROR_MSG % path)
else:
sandbox_path_pre_format = DistributedCommandRunner.thermos_sandbox(
api.cluster, executor_sandbox=context.options.executor_sandbox)
thermos_namespace = ThermosContext(task_id=assigned.taskId,
ports=assigned.assignedPorts)
sandbox_path = String(sandbox_path_pre_format) % Environment(
thermos=thermos_namespace)
# Join the individual folders to the sandbox path to build safely
final_path = os.path.join(str(sandbox_path),
*normalized_input_path.split(os.sep))
return '%s@%s:%s' % (role, slave_host, final_path)
def run(args, options):
"""usage: run cluster/role/env/job cmd
Runs a shell command on all machines currently hosting shards of a single job.
This feature supports the same command line wildcards that are used to
populate a job's commands.
This means anything in the {{mesos.*}} and {{thermos.*}} namespaces.
"""
# TODO(William Farner): Add support for invoking on individual shards.
# TODO(Kevin Sweeney): Restore the ability to run across jobs with globs (See MESOS-3010).
if not args:
die('job path is required')
job_path = args.pop(0)
try:
cluster_name, role, env, name = AuroraJobKey.from_path(job_path)
except AuroraJobKey.Error as e:
die('Invalid job path "%s": %s' % (job_path, e))
command = ' '.join(args)
cluster = CLUSTERS[cluster_name]
dcr = DistributedCommandRunner(cluster, role, env, [name], options.ssh_user)
dcr.run(command, parallelism=options.num_threads, executor_sandbox=options.executor_sandbox)
def _build_path(context, target):
(task_instance, path) = ScpCommand._extract_task_instance_and_path(context, target)
# No jobkey is specified therefore we are using a local path.
if (task_instance is None):
return path
# Jobkey specified, we want to convert to the user@host:file scp format
(cluster, role, env, name) = task_instance.jobkey
instance = set([task_instance.instance])
api = context.get_api(cluster)
resp = api.query(api.build_query(role, name, env=env, instances=instance))
context.log_response_and_raise(resp,
err_msg=('Unable to get information about instance: %s' % combine_messages(resp)))
if (resp.result.scheduleStatusResult.tasks is None or
len(resp.result.scheduleStatusResult.tasks) == 0):
raise context.CommandError(EXIT_INVALID_PARAMETER,
ScpCommand.JOB_NOT_FOUND_ERROR_MSG % (task_instance.jobkey, task_instance.instance))
first_task = resp.result.scheduleStatusResult.tasks[0]
assigned = first_task.assignedTask
role = assigned.task.job.role
slave_host = assigned.slaveHost
# If path is absolute, use that. Else if it is a tilde expansion, throw an error.
# Otherwise, use sandbox as relative root.
normalized_input_path = os.path.normpath(path)
if (os.path.isabs(normalized_input_path)):
final_path = normalized_input_path
elif (normalized_input_path.startswith('~/') or normalized_input_path == '~'):
raise context.CommandError(EXIT_INVALID_PARAMETER, ScpCommand.TILDE_USAGE_ERROR_MSG % path)
else:
sandbox_path_pre_format = DistributedCommandRunner.thermos_sandbox(
api.cluster,
executor_sandbox=context.options.executor_sandbox)
thermos_namespace = ThermosContext(
task_id=assigned.taskId,
ports=assigned.assignedPorts)
sandbox_path = String(sandbox_path_pre_format) % Environment(thermos=thermos_namespace)
# Join the individual folders to the sandbox path to build safely
final_path = os.path.join(str(sandbox_path), *normalized_input_path.split(os.sep))
return '%s@%s:%s' % (role, slave_host, final_path)
def execute(self, context):
(cluster, role, env, name) = context.options.task_instance.jobkey
instance = context.options.task_instance.instance
api = context.get_api(cluster)
resp = api.query(api.build_query(role, name, env=env, instances=set([int(instance)])))
context.log_response_and_raise(resp,
err_msg=('Unable to get information about instance: %s' % combine_messages(resp)))
if (resp.result.scheduleStatusResult.tasks is None or
len(resp.result.scheduleStatusResult.tasks) == 0):
raise context.CommandError(EXIT_INVALID_PARAMETER,
"Job %s not found" % context.options.task_instance.jobkey)
first_task = resp.result.scheduleStatusResult.tasks[0]
remote_cmd = context.options.command or 'bash'
command = DistributedCommandRunner.substitute(remote_cmd, first_task,
api.cluster, executor_sandbox=context.options.executor_sandbox)
ssh_command = ['ssh', '-t']
assigned = first_task.assignedTask
role = assigned.task.job.role if assigned.task.job else assigned.task.owner.role
slave_host = assigned.slaveHost
for tunnel in context.options.tunnels:
try:
port, name = tunnel.split(':')
port = int(port)
except ValueError:
raise context.CommandError(EXIT_INVALID_PARAMETER,
'Could not parse tunnel: %s. Must be of form PORT:NAME' % tunnel)
if name not in assigned.assignedPorts:
raise context.CommandError(EXIT_INVALID_PARAMETER,
'Task %s has no port named %s' % (assigned.taskId, name))
ssh_command += [
'-L', '%d:%s:%d' % (port, slave_host, assigned.assignedPorts[name])]
ssh_command += ['%s@%s' % (context.options.ssh_user or role, slave_host), command]
return subprocess.call(ssh_command)
def ssh(args, options):
"""usage: ssh cluster/role/env/job shard [args...]
Initiate an SSH session on the machine that a shard is running on.
"""
if not args:
die('Job path is required')
job_path = args.pop(0)
try:
cluster_name, role, env, name = AuroraJobKey.from_path(job_path)
except AuroraJobKey.Error as e:
die('Invalid job path "%s": %s' % (job_path, e))
if not args:
die('Shard is required')
try:
shard = int(args.pop(0))
except ValueError:
die('Shard must be an integer')
newcmd = ["task", "ssh", "%s/%s" % (job_path, shard)]
if len(options.tunnels) > 0:
newcmd.append("--tunnels=%s" % options.tunnels)
if options.ssh_user is not None:
newcmd.append("--ssh-user=%s" % options.ssh_user)
if options.executor_sandbox:
newcmd.append("--executor-sandbox")
if len(args) > 0:
newcmd.append("--command=\"%s\"" % " ".join(args))
v1_deprecation_warning("ssh", newcmd)
api = make_client(cluster_name)
resp = api.query(api.build_query(role, name, set([int(shard)]), env=env))
check_and_log_response(resp)
if (resp.result.scheduleStatusResult.tasks is None or
len(resp.result.scheduleStatusResult.tasks) == 0):
die("Job %s not found" % job_path)
first_task = resp.result.scheduleStatusResult.tasks[0]
remote_cmd = 'bash' if not args else ' '.join(args)
command = DistributedCommandRunner.substitute(remote_cmd, first_task,
api.cluster, executor_sandbox=options.executor_sandbox)
ssh_command = ['ssh', '-t']
role = first_task.assignedTask.task.owner.role
slave_host = first_task.assignedTask.slaveHost
for tunnel in options.tunnels:
try:
port, name = tunnel.split(':')
port = int(port)
except ValueError:
die('Could not parse tunnel: %s. Must be of form PORT:NAME' % tunnel)
if name not in first_task.assignedTask.assignedPorts:
die('Task %s has no port named %s' % (first_task.assignedTask.taskId, name))
ssh_command += [
'-L', '%d:%s:%d' % (port, slave_host, first_task.assignedTask.assignedPorts[name])]
ssh_command += ['%s@%s' % (options.ssh_user or role, slave_host), command]
return subprocess.call(ssh_command)