def __init__(self, argv=None):
self.user = None # i.e. owner; name it user for consistency with CLI
self.host = None
self.ssh_login_shell = None
self.ssh_cylc = None
self.argv = argv or sys.argv
cylc.flow.flags.verbose = '-v' in self.argv or '--verbose' in self.argv
# Detect and replace host and user options
argv = self.argv[1:]
self.args = []
while argv:
arg = argv.pop(0)
if arg.startswith(tuple([opt + '=' for opt in self.OPT_ARG_OPTS])):
# e.g. if arg is '--host=HOST' here set self.host to HOST
opt_with_dashes, opt_arg = arg.split('=', 1)
setattr(self, opt_with_dashes.strip('--'), opt_arg)
elif arg in self.OPT_ARG_OPTS: # if opt arg provided after a space
# e.g. if arg is '--host' set self.host to next element in argv
setattr(self, arg.strip('--'), argv.pop(0))
elif arg == '--login':
self.ssh_login_shell = True
elif arg == '--no-login':
self.ssh_login_shell = False
else:
self.args.append(arg)
if self.user is None and self.host is None:
self.is_remote = False
else:
self.is_remote = is_remote(self.host, self.user)
def __init__(self, argv=None):
self.user = None # i.e. owner; name it user for consistency with CLI
self.host = None
self.ssh_login_shell = None
self.ssh_cylc = None
self.argv = argv or sys.argv
cylc.flow.flags.verbose = '-v' in self.argv or '--verbose' in self.argv
# Detect and replace host and user options
argv = self.argv[1:]
self.args = []
while argv:
arg = argv.pop(0)
if arg.startswith(tuple([opt + '=' for opt in self.OPT_ARG_OPTS])):
# e.g. if arg is '--host=HOST' here set self.host to HOST
opt_with_dashes, opt_arg = arg.split('=', 1)
setattr(self, opt_with_dashes.strip('--'), opt_arg)
elif arg in self.OPT_ARG_OPTS: # if opt arg provided after a space
# e.g. if arg is '--host' set self.host to next element in argv
setattr(self, arg.strip('--'), argv.pop(0))
elif arg == '--login':
self.ssh_login_shell = True
elif arg == '--no-login':
self.ssh_login_shell = False
else:
self.args.append(arg)
if self.user is None and self.host is None:
self.is_remote = False
else:
self.is_remote = is_remote(self.host, self.user)
def _is_local_auth_ok(reg, owner, host):
"""Return True if it is OK to use local passphrase file.
Use values in ~/cylc-run/REG/.service/contact to make a judgement.
"""
if is_remote(host, owner):
fname = os.path.join(get_suite_srv_dir(reg),
SuiteFiles.Service.CONTACT)
data = {}
try:
for line in open(fname):
key, value = ([item.strip() for item in line.split("=", 1)])
data[key] = value
except (IOError, ValueError):
# No contact file
return False
else:
# Contact file exists, check values match
if owner is None:
owner = get_user()
if host is None:
host = get_host()
host_value = data.get(ContactFileFields.HOST, "")
return (reg == data.get(ContactFileFields.NAME)
and owner == data.get(ContactFileFields.OWNER)
and (host == host_value
or host == host_value.split(".", 1)[0] # no domain
))
else:
return True
def _is_local_auth_ok(self, reg, owner, host):
"""Return True if it is OK to use local passphrase file.
Use values in ~/cylc-run/REG/.service/contact to make a judgement.
Cache results in self.can_use_load_auths.
"""
if (reg, owner, host) not in self.can_use_load_auths:
if is_remote(host, owner):
fname = os.path.join(
self.get_suite_srv_dir(reg), self.FILE_BASE_CONTACT)
data = {}
try:
for line in open(fname):
key, value = (
[item.strip() for item in line.split("=", 1)])
data[key] = value
except (IOError, ValueError):
# No contact file
self.can_use_load_auths[(reg, owner, host)] = False
else:
# Contact file exists, check values match
if owner is None:
owner = get_user()
if host is None:
host = get_host()
host_value = data.get(self.KEY_HOST, "")
self.can_use_load_auths[(reg, owner, host)] = (
reg == data.get(self.KEY_NAME) and
owner == data.get(self.KEY_OWNER) and
(
host == host_value or
host == host_value.split(".", 1)[0] # no domain
)
)
else:
self.can_use_load_auths[(reg, owner, host)] = True
return self.can_use_load_auths[(reg, owner, host)]
def _is_local_auth_ok(self, reg, owner, host):
"""Return True if it is OK to use local passphrase file.
Use values in ~/cylc-run/REG/.service/contact to make a judgement.
Cache results in self.can_use_load_auths.
"""
if (reg, owner, host) not in self.can_use_load_auths:
if is_remote(host, owner):
fname = os.path.join(
self.get_suite_srv_dir(reg), self.FILE_BASE_CONTACT)
data = {}
try:
for line in open(fname):
key, value = (
[item.strip() for item in line.split("=", 1)])
data[key] = value
except (IOError, ValueError):
# No contact file
self.can_use_load_auths[(reg, owner, host)] = False
else:
# Contact file exists, check values match
if owner is None:
owner = get_user()
if host is None:
host = get_host()
host_value = data.get(self.KEY_HOST, "")
self.can_use_load_auths[(reg, owner, host)] = (
reg == data.get(self.KEY_NAME) and
owner == data.get(self.KEY_OWNER) and
(
host == host_value or
host == host_value.split(".", 1)[0] # no domain
)
)
else:
self.can_use_load_auths[(reg, owner, host)] = True
return self.can_use_load_auths[(reg, owner, host)]
def main(parser, options, *args, color=False):
"""Implement cylc cat-log CLI.
Determine log path, user@host, batchview_cmd, and action (print, dir-list,
cat, edit, or tail), and then if the log path is:
a) local: perform action on log path, or
b) remote: re-invoke cylc cat-log as a) on the remote account
"""
if options.remote_args:
# Invoked on job hosts for job logs only, as a wrapper to view_log().
# Tail and batchview commands come from global config on suite host).
logpath, mode, tail_tmpl = options.remote_args[0:3]
if logpath.startswith('$'):
logpath = os.path.expandvars(logpath)
elif logpath.startswith('~'):
logpath = os.path.expanduser(logpath)
try:
batchview_cmd = options.remote_args[3]
except IndexError:
batchview_cmd = None
res = view_log(logpath,
mode,
tail_tmpl,
batchview_cmd,
remote=True,
color=color)
if res == 1:
sys.exit(res)
return
suite_name = args[0]
# Get long-format mode.
try:
mode = MODES[options.mode]
except KeyError:
mode = options.mode
if len(args) == 1:
# Cat suite logs, local only.
if options.filename is not None:
raise UserInputError("The '-f' option is for job logs only.")
logpath = get_suite_run_log_name(suite_name)
if options.rotation_num:
logs = glob('%s.*' % logpath)
logs.sort(key=os.path.getmtime, reverse=True)
try:
logpath = logs[int(options.rotation_num)]
except IndexError:
raise UserInputError("max rotation %d" % (len(logs) - 1))
tail_tmpl = str(glbl_cfg().get_host_item("tail command template"))
out = view_log(logpath, mode, tail_tmpl, color=color)
if out == 1:
sys.exit(1)
if mode == 'edit':
tmpfile_edit(out, options.geditor)
return
if len(args) == 2:
# Cat task job logs, may be on suite or job host.
if options.rotation_num is not None:
raise UserInputError("only suite (not job) logs get rotated")
task_id = args[1]
try:
task, point = TaskID.split(task_id)
except ValueError:
parser.error("Illegal task ID: %s" % task_id)
if options.submit_num != NN:
try:
options.submit_num = "%02d" % int(options.submit_num)
except ValueError:
parser.error("Illegal submit number: %s" % options.submit_num)
if options.filename is None:
options.filename = JOB_LOG_OUT
else:
# Convert short filename args to long (e.g. 'o' to 'job.out').
try:
options.filename = JOB_LOG_OPTS[options.filename]
except KeyError:
# Is already long form (standard log, or custom).
pass
user_at_host, batch_sys_name, live_job_id = get_task_job_attrs(
suite_name, point, task, options.submit_num)
user, host = split_user_at_host(user_at_host)
batchview_cmd = None
if live_job_id is not None:
# Job is currently running. Get special batch system log view
# command (e.g. qcat) if one exists, and the log is out or err.
conf_key = None
if options.filename == JOB_LOG_OUT:
if mode == 'cat':
conf_key = "out viewer"
elif mode == 'tail':
conf_key = "out tailer"
elif options.filename == JOB_LOG_ERR:
if mode == 'cat':
conf_key = "err viewer"
elif mode == 'tail':
conf_key = "err tailer"
if conf_key is not None:
conf = glbl_cfg().get_host_item("batch systems", host, user)
batchview_cmd_tmpl = None
try:
batchview_cmd_tmpl = conf[batch_sys_name][conf_key]
except KeyError:
pass
if batchview_cmd_tmpl is not None:
batchview_cmd = batchview_cmd_tmpl % {
"job_id": str(live_job_id)
}
log_is_remote = (is_remote(host, user)
and (options.filename not in JOB_LOGS_LOCAL))
log_is_retrieved = (glbl_cfg().get_host_item('retrieve job logs', host)
and live_job_id is None)
if log_is_remote and (not log_is_retrieved or options.force_remote):
logpath = os.path.normpath(
get_remote_suite_run_job_dir(host, user, suite_name, point,
task, options.submit_num,
options.filename))
tail_tmpl = str(glbl_cfg().get_host_item("tail command template",
host, user))
# Reinvoke the cat-log command on the remote account.
cmd = ['cat-log']
if cylc.flow.flags.debug:
cmd.append('--debug')
for item in [logpath, mode, tail_tmpl]:
cmd.append('--remote-arg=%s' % quote(item))
if batchview_cmd:
cmd.append('--remote-arg=%s' % quote(batchview_cmd))
cmd.append(suite_name)
is_edit_mode = (mode == 'edit')
try:
proc = remote_cylc_cmd(cmd,
user,
host,
capture_process=is_edit_mode,
manage=(mode == 'tail'))
except KeyboardInterrupt:
# Ctrl-C while tailing.
pass
else:
if is_edit_mode:
# Write remote stdout to a temp file for viewing in editor.
# Only BUFSIZE bytes at a time in case huge stdout volume.
out = NamedTemporaryFile()
data = proc.stdout.read(BUFSIZE)
while data:
out.write(data)
data = proc.stdout.read(BUFSIZE)
os.chmod(out.name, S_IRUSR)
out.seek(0, 0)
else:
# Local task job or local job log.
logpath = os.path.normpath(
get_suite_run_job_dir(suite_name, point, task,
options.submit_num, options.filename))
tail_tmpl = str(glbl_cfg().get_host_item("tail command template"))
out = view_log(logpath,
mode,
tail_tmpl,
batchview_cmd,
color=color)
if mode != 'edit':
sys.exit(out)
if mode == 'edit':
tmpfile_edit(out, options.geditor)
def _load_remote_item(self, item, reg, owner, host):
"""Load content of service item from remote [owner@]host via SSH."""
if not is_remote(host, owner):
return
if host is None:
host = 'localhost'
if owner is None:
owner = get_user()
if item == self.FILE_BASE_CONTACT and not is_remote_host(host):
# Attempt to read suite contact file via the local filesystem.
path = r'%(run_d)s/%(srv_base)s' % {
'run_d': get_remote_suite_run_dir('localhost', owner, reg),
'srv_base': self.DIR_BASE_SRV,
}
content = self._load_local_item(item, path)
if content is not None:
return content
# Else drop through and attempt via ssh to the suite account.
# Prefix STDOUT to ensure returned content is relevant
prefix = r'[CYLC-AUTH] %(suite)s' % {'suite': reg}
# Attempt to cat passphrase file under suite service directory
script = (r"""echo '%(prefix)s'; """
r'''cat "%(run_d)s/%(srv_base)s/%(item)s"''') % {
'prefix': prefix,
'run_d': get_remote_suite_run_dir(host, owner, reg),
'srv_base': self.DIR_BASE_SRV,
'item': item
}
import shlex
command = shlex.split(glbl_cfg().get_host_item('ssh command', host,
owner))
command += ['-n', owner + '@' + host, script]
from subprocess import Popen, PIPE, DEVNULL # nosec
try:
proc = Popen(command, stdin=DEVNULL, stdout=PIPE,
stderr=PIPE) # nosec
except OSError:
if cylc.flow.flags.debug:
import traceback
traceback.print_exc()
return
out, err = (f.decode() for f in proc.communicate())
ret_code = proc.wait()
# Extract passphrase from STDOUT
# It should live in the line with the correct prefix
content = ""
can_read = False
for line in out.splitlines(True):
if can_read:
content += line
elif line.strip() == prefix:
can_read = True
if not content or ret_code:
LOG.debug(
'$ %(command)s # code=%(ret_code)s\n%(err)s',
{
'command': command,
# STDOUT may contain passphrase, so not safe to print
# 'out': out,
'err': err,
'ret_code': ret_code,
})
return
return content
def remote_init(self, host, owner):
"""Initialise a remote [owner@]host if necessary.
Create UUID file on suite host ".service/uuid" for remotes to identify
shared file system with suite host.
Call "cylc remote-init" to install suite items to remote:
".service/contact": For TCP task communication
".service/passphrase": For TCP task communication
"python/": if source exists
Return:
REMOTE_INIT_NOT_REQUIRED:
If remote init is not required, e.g. not remote
REMOTE_INIT_DONE:
If remote init done.
REMOTE_INIT_FAILED:
If init of the remote failed.
Note: this will reset to None to allow retry.
None:
If waiting for remote init command to complete
"""
if self.single_task_mode or not is_remote(host, owner):
return REMOTE_INIT_NOT_REQUIRED
try:
status = self.remote_init_map[(host, owner)]
except KeyError:
pass # Not yet initialised
else:
if status == REMOTE_INIT_FAILED:
del self.remote_init_map[(host, owner)] # reset to allow retry
return status
# Determine what items to install
comm_meth = glbl_cfg().get_host_item('task communication method', host,
owner)
owner_at_host = 'localhost'
if host:
owner_at_host = host
if owner:
owner_at_host = owner + '@' + owner_at_host
LOG.debug('comm_meth[%s]=%s' % (owner_at_host, comm_meth))
items = self._remote_init_items(comm_meth)
# No item to install
if not items:
self.remote_init_map[(host, owner)] = REMOTE_INIT_NOT_REQUIRED
return self.remote_init_map[(host, owner)]
# Create a TAR archive with the service files,
# so they can be sent later via SSH's STDIN to the task remote.
tmphandle = self.proc_pool.get_temporary_file()
tarhandle = tarfile.open(fileobj=tmphandle, mode='w')
for path, arcname in items:
tarhandle.add(path, arcname=arcname)
tarhandle.close()
tmphandle.seek(0)
# UUID file - for remote to identify shared file system with suite host
uuid_fname = os.path.join(get_suite_srv_dir(self.suite),
FILE_BASE_UUID)
if not os.path.exists(uuid_fname):
open(uuid_fname, 'wb').write(str(self.uuid_str).encode())
# Build the command
cmd = ['cylc', 'remote-init']
if is_remote_host(host):
cmd.append('--host=%s' % host)
if is_remote_user(owner):
cmd.append('--user=%s' % owner)
if cylc.flow.flags.debug:
cmd.append('--debug')
if comm_meth in ['ssh']:
cmd.append('--indirect-comm=%s' % comm_meth)
cmd.append(str(self.uuid_str))
cmd.append(get_remote_suite_run_dir(host, owner, self.suite))
self.proc_pool.put_command(
SubProcContext('remote-init', cmd, stdin_files=[tmphandle]),
self._remote_init_callback, [host, owner, tmphandle])
# None status: Waiting for command to finish
self.remote_init_map[(host, owner)] = None
return self.remote_init_map[(host, owner)]
def remote_init(self, host, owner):
"""Initialise a remote [owner@]host if necessary.
Create UUID file on suite host ".service/uuid" for remotes to identify
shared file system with suite host.
Call "cylc remote-init" to install suite items to remote:
".service/contact": For TCP task communication
".service/passphrase": For TCP task communication
"python/": if source exists
Return:
REMOTE_INIT_NOT_REQUIRED:
If remote init is not required, e.g. not remote
REMOTE_INIT_DONE:
If remote init done.
REMOTE_INIT_FAILED:
If init of the remote failed.
Note: this will reset to None to allow retry.
None:
If waiting for remote init command to complete
"""
if self.single_task_mode or not is_remote(host, owner):
return REMOTE_INIT_NOT_REQUIRED
try:
status = self.remote_init_map[(host, owner)]
except KeyError:
pass # Not yet initialised
else:
if status == REMOTE_INIT_FAILED:
del self.remote_init_map[(host, owner)] # reset to allow retry
return status
# Determine what items to install
comm_meth = glbl_cfg().get_host_item(
'task communication method', host, owner)
owner_at_host = 'localhost'
if host:
owner_at_host = host
if owner:
owner_at_host = owner + '@' + owner_at_host
LOG.debug('comm_meth[%s]=%s' % (owner_at_host, comm_meth))
items = self._remote_init_items(comm_meth)
# No item to install
if not items:
self.remote_init_map[(host, owner)] = REMOTE_INIT_NOT_REQUIRED
return self.remote_init_map[(host, owner)]
# Create a TAR archive with the service files,
# so they can be sent later via SSH's STDIN to the task remote.
tmphandle = self.proc_pool.get_temporary_file()
tarhandle = tarfile.open(fileobj=tmphandle, mode='w')
for path, arcname in items:
tarhandle.add(path, arcname=arcname)
tarhandle.close()
tmphandle.seek(0)
# UUID file - for remote to identify shared file system with suite host
uuid_fname = os.path.join(
self.suite_srv_files_mgr.get_suite_srv_dir(self.suite),
FILE_BASE_UUID)
if not os.path.exists(uuid_fname):
open(uuid_fname, 'wb').write(str(self.uuid_str).encode())
# Build the command
cmd = ['cylc', 'remote-init']
if is_remote_host(host):
cmd.append('--host=%s' % host)
if is_remote_user(owner):
cmd.append('--user=%s' % owner)
if cylc.flow.flags.debug:
cmd.append('--debug')
if comm_meth in ['ssh']:
cmd.append('--indirect-comm=%s' % comm_meth)
cmd.append(str(self.uuid_str))
cmd.append(glbl_cfg().get_derived_host_item(
self.suite, 'suite run directory', host, owner))
self.proc_pool.put_command(
SubProcContext('remote-init', cmd, stdin_files=[tmphandle]),
self._remote_init_callback,
[host, owner, tmphandle])
# None status: Waiting for command to finish
self.remote_init_map[(host, owner)] = None
return self.remote_init_map[(host, owner)]
def _load_remote_item(self, item, reg, owner, host):
"""Load content of service item from remote [owner@]host via SSH."""
if not is_remote(host, owner):
return
if host is None:
host = 'localhost'
if owner is None:
owner = get_user()
if item == self.FILE_BASE_CONTACT and not is_remote_host(host):
# Attempt to read suite contact file via the local filesystem.
path = r'%(run_d)s/%(srv_base)s' % {
'run_d': glbl_cfg().get_derived_host_item(
reg, 'suite run directory', 'localhost', owner,
replace_home=False),
'srv_base': self.DIR_BASE_SRV,
}
content = self._load_local_item(item, path)
if content is not None:
return content
# Else drop through and attempt via ssh to the suite account.
# Prefix STDOUT to ensure returned content is relevant
prefix = r'[CYLC-AUTH] %(suite)s' % {'suite': reg}
# Attempt to cat passphrase file under suite service directory
script = (
r"""echo '%(prefix)s'; """
r'''cat "%(run_d)s/%(srv_base)s/%(item)s"'''
) % {
'prefix': prefix,
'run_d': glbl_cfg().get_derived_host_item(
reg, 'suite run directory', host, owner),
'srv_base': self.DIR_BASE_SRV,
'item': item
}
import shlex
command = shlex.split(
glbl_cfg().get_host_item('ssh command', host, owner))
command += ['-n', owner + '@' + host, script]
from subprocess import Popen, PIPE
try:
proc = Popen(
command, stdin=open(os.devnull), stdout=PIPE, stderr=PIPE)
except OSError:
if cylc.flow.flags.debug:
import traceback
traceback.print_exc()
return
out, err = (f.decode() for f in proc.communicate())
ret_code = proc.wait()
# Extract passphrase from STDOUT
# It should live in the line with the correct prefix
content = ""
can_read = False
for line in out.splitlines(True):
if can_read:
content += line
elif line.strip() == prefix:
can_read = True
if not content or ret_code:
LOG.debug(
'$ %(command)s # code=%(ret_code)s\n%(err)s',
{
'command': command,
# STDOUT may contain passphrase, so not safe to print
# 'out': out,
'err': err,
'ret_code': ret_code,
})
return
return content