def ssh_to_instance(job,
instance,
sandbox_dir_fn,
cluster,
command_to_run=None):
"""
When using Mesos, attempts to ssh (using os.execlp) to the Mesos agent corresponding to the given instance.
When using Kubernetes, calls the exec command of the kubectl cli.
"""
print_info(
f'Attempting ssh for job instance {terminal.bold(instance["task_id"])}...'
)
compute_cluster = instance["compute-cluster"]
compute_cluster_type = compute_cluster["type"]
compute_cluster_name = compute_cluster["name"]
if compute_cluster_type == "kubernetes":
kubectl_exec_to_instance_fn = plugins.get_fn(
'kubectl-exec-to-instance', kubectl_exec_to_instance)
compute_cluster_config = get_compute_cluster_config(
cluster, compute_cluster_name)
kubectl_exec_to_instance_fn(job["user"], instance["task_id"],
compute_cluster_config, command_to_run)
else:
command_to_run = command_to_run or ['bash']
sandbox_dir = sandbox_dir_fn()
command = os.environ.get('CS_SSH', 'ssh')
logging.info(f'using ssh command: {command}')
hostname = instance['hostname']
print_info(f'Executing ssh to {terminal.bold(hostname)}.')
args = ['ssh', '-t', hostname, 'cd', sandbox_dir, ';'] + command_to_run
os.execlp(command, *args)
def cat_using_download_file(instance, sandbox_dir_fn, path):
retrieve_fn = plugins.get_fn('download-job-instance-file', download_file)
download = retrieve_fn(instance, sandbox_dir_fn, path)
try:
for data in download(chunk_size=4096):
if data:
sys.stdout.buffer.write(data)
except BrokenPipeError as bpe:
sys.stderr.close()
logging.exception(bpe)
def tail_using_read_file(instance, sandbox_dir_fn, path, num_lines_to_print,
follow, follow_sleep_seconds):
retrieve_fn = plugins.get_fn('read-job-instance-file', read_file)
read = partial(retrieve_fn,
instance=instance,
sandbox_dir_fn=sandbox_dir_fn,
path=path)
file_size = read()['offset']
tail_backwards(file_size, read, num_lines_to_print)
if follow:
tail_follow(file_size, read, follow_sleep_seconds)
def cat_for_instance(instance, sandbox_dir, path):
"""Outputs the contents of the Mesos sandbox path for the given instance."""
retrieve_fn = plugins.get_fn('download-job-instance-file', download_file)
download = retrieve_fn(instance, sandbox_dir, path)
try:
for data in download(chunk_size=4096):
if data:
sys.stdout.buffer.write(data)
except BrokenPipeError as bpe:
sys.stderr.close()
logging.exception(bpe)
def cat_for_instance(_, instance, sandbox_dir_fn, __, path):
"""
Outputs the contents of the Mesos sandbox path for the given instance.
When using Kubernetes, calls the exec command of the kubectl cli.
"""
compute_cluster = instance["compute-cluster"]
compute_cluster_type = compute_cluster["type"]
if compute_cluster_type == "kubernetes" and ("end_time" not in instance or instance["end_time"] is None):
kubernetes_cat_instance_file_fn = plugins.get_fn('kubernetes-cat-for-instance', kubectl_cat_instance_file)
kubernetes_cat_instance_file_fn(instance["task_id"], path)
else:
cat_using_download_file(instance, sandbox_dir_fn, path)
def tail_for_instance(instance, sandbox_dir, path, num_lines_to_print, follow,
follow_sleep_seconds):
"""
Tails the contents of the Mesos sandbox path for the given instance. If follow is truthy, it will
try and read more data from the file until the user terminates. This assumes files will not shrink.
"""
retrieve_fn = plugins.get_fn('read-job-instance-file', read_file)
read = partial(retrieve_fn,
instance=instance,
sandbox_dir=sandbox_dir,
path=path)
file_size = read()['offset']
tail_backwards(file_size, read, num_lines_to_print)
if follow:
tail_follow(file_size, read, follow_sleep_seconds)
def ls_for_instance(_, instance, sandbox_dir_fn, __, path, long_format,
as_json):
"""
Lists contents of the Mesos sandbox path for the given instance.
When using Kubernetes, calls the exec command of the kubectl cli.
"""
compute_cluster = instance["compute-cluster"]
compute_cluster_type = compute_cluster["type"]
if compute_cluster_type == "kubernetes" and ("end_time" not in instance or
instance["end_time"] is None):
kubernetes_ls_for_instance_fn = plugins.get_fn(
'kubernetes-ls-for-instance', kubectl_ls_for_instance)
kubernetes_ls_for_instance_fn(instance["task_id"], path, long_format,
as_json)
else:
ls_for_instance_from_mesos(instance, sandbox_dir_fn, path, long_format,
as_json)
def ls_for_instance(instance, sandbox_dir, path, long_format, as_json):
"""Lists contents of the Mesos sandbox path for the given instance"""
retrieve_fn = plugins.get_fn('retrieve-job-instance-files',
retrieve_entries_from_mesos)
entries = retrieve_fn(instance, sandbox_dir, path)
if as_json:
print(json.dumps(entries))
else:
if len(entries) > 0:
if long_format:
rows = [directory_entry_to_row(e) for e in entries]
table = tabulate(rows, tablefmt='plain')
print(table)
else:
print('\n'.join(
terminal.wrap(' '.join([format_path(e)
for e in entries]))))
else:
logging.info('the directory is empty')
def tail_for_instance(_, instance, sandbox_dir_fn, __, path,
num_lines_to_print, follow, follow_sleep_seconds):
"""
Tails the contents of the Mesos sandbox path for the given instance. If follow is truthy, it will
try and read more data from the file until the user terminates. This assumes files will not shrink.
When using Kubernetes, calls the exec command of the kubectl cli.
"""
compute_cluster = instance["compute-cluster"]
compute_cluster_type = compute_cluster["type"]
if compute_cluster_type == "kubernetes" and ("end_time" not in instance or
instance["end_time"] is None):
kubernetes_tail_instance_file_fn = plugins.get_fn(
'kubernetes-tail-instance-file', kubectl_tail_instance_file)
kubernetes_tail_instance_file_fn(instance["task_id"], path,
num_lines_to_print, follow,
follow_sleep_seconds)
else:
tail_using_read_file(instance, sandbox_dir_fn, path,
num_lines_to_print, follow, follow_sleep_seconds)
def ssh_to_instance(job,
instance,
sandbox_dir_fn,
cluster,
command_to_run=None):
"""
When using Mesos, attempts to ssh (using os.execlp) to the Mesos agent corresponding to the given instance.
When using Kubernetes, calls the exec command of the kubectl cli.
"""
compute_cluster = instance['compute-cluster']
compute_cluster_type = compute_cluster['type']
instance_status = instance['status']
instance_uuid = instance['task_id']
if compute_cluster_type == 'kubernetes':
if instance_status == 'unknown':
print_info(
f'Job instance {terminal.bold(instance_uuid)} is not yet running.'
)
return
elif instance_status == 'success' or instance_status == 'failed':
cs_command = 'cs'
print_info(
f'Job instance {terminal.bold(instance_uuid)} already completed, so you cannot ssh to it.'
)
print_info('')
print_info(
'To inspect individual files, e.g. stdout, try one of these:')
print_info('')
print_info(f'{cs_command} cat {instance_uuid} stdout')
print_info(f'{cs_command} tail {instance_uuid} stdout')
print_info('')
print_info('To retrieve the entire output directory, try:')
print_info('')
print_info(f'{cs_command} download {instance_uuid}')
print_info('')
print_info(f'Here are the results of running {cs_command} ls:')
print_info('')
print_info(f'{cs_command} ls -l {instance_uuid}')
args = {
'json': False,
'literal': False,
'long_format': True,
'path': None,
'uuid': [instance_uuid]
}
ls([cluster], args, _=None)
return
print_info(
f'Attempting ssh for job instance {terminal.bold(instance_uuid)}...')
compute_cluster_name = compute_cluster['name']
if compute_cluster_type == 'kubernetes':
kubectl_exec_to_instance_fn = plugins.get_fn(
'kubectl-exec-to-instance', kubectl_exec_to_instance)
compute_cluster_config = get_compute_cluster_config(
cluster, compute_cluster_name)
kubectl_exec_to_instance_fn(job['user'], instance_uuid,
compute_cluster_config, command_to_run)
else:
command_to_run = command_to_run or ['bash']
sandbox_dir = sandbox_dir_fn()
command = os.environ.get('CS_SSH', 'ssh')
logging.info(f'using ssh command: {command}')
hostname = instance['hostname']
print_info(f'Executing ssh to {terminal.bold(hostname)}.')
args = ['ssh', '-t', hostname, 'cd', sandbox_dir, ';'] + command_to_run
os.execlp(command, *args)
