def parse_args(self, api_args, remove_opts=None):
"""Parse options and arguments, overrides OptionParser.parse_args.
Args:
api_args (list):
Command line options if passed via Python as opposed to
sys.argv
remove_opts (list):
List of standard options to remove before parsing.
"""
if self.auto_add:
# Add common options after command-specific options.
self.add_std_options()
if remove_opts:
for opt in remove_opts:
try:
self.remove_option(opt)
except ValueError:
pass
(options, args) = OptionParser.parse_args(self, api_args)
if len(args) < self.n_compulsory_args:
self.error("Wrong number of arguments (too few)")
elif not self.unlimited_args and \
len(args) > self.n_compulsory_args + self.n_optional_args:
self.error("Wrong number of arguments (too many)")
if self.jset:
if options.templatevars_file:
options.templatevars_file = os.path.abspath(
os.path.expanduser(options.templatevars_file))
cylc.flow.flags.verbose = options.verbose
cylc.flow.flags.debug = options.debug
# Set up stream logging for CLI. Note:
# 1. On choosing STDERR: Log messages are diagnostics, so STDERR is the
# better choice for the logging stream. This allows us to use STDOUT
# for verbosity agnostic outputs.
# 2. Suite server programs will remove this handler when it becomes a
# daemon.
if options.debug or options.verbose:
LOG.setLevel(logging.DEBUG)
else:
LOG.setLevel(logging.INFO)
# Remove NullHandler before add the StreamHandler
RSYNC_LOG.setLevel(logging.INFO)
while LOG.handlers:
LOG.handlers[0].close()
LOG.removeHandler(LOG.handlers[0])
errhandler = logging.StreamHandler(sys.stderr)
errhandler.setFormatter(
CylcLogFormatter(timestamp=options.log_timestamp))
LOG.addHandler(errhandler)
return (options, args)
def _file_install_callback(self, ctx, install_target):
"""Callback when file installation exits.
Sets remote_init_map to REMOTE_FILE_INSTALL_DONE on success and to
REMOTE_FILE_INSTALL_FAILED on error.
"""
if ctx.out:
RSYNC_LOG.info('File installation information for '
f'{install_target}:\n{ctx.out}')
if ctx.ret_code == 0:
# Both file installation and remote init success
LOG.debug(ctx)
LOG.debug(f"File installation complete for {install_target}")
self.remote_init_map[install_target] = REMOTE_FILE_INSTALL_DONE
self.ready = True
return
else:
self.remote_init_map[install_target] = REMOTE_FILE_INSTALL_FAILED
LOG.error(
TaskRemoteMgmtError(TaskRemoteMgmtError.MSG_INIT,
install_target,
' '.join(quote(item) for item in ctx.cmd),
ctx.ret_code, ctx.out, ctx.err))
LOG.error(ctx)
self.ready = True
def _open_logs(id_, no_detach):
"""Open Cylc log handlers for a flow run."""
if not no_detach:
while LOG.handlers:
LOG.handlers[0].close()
LOG.removeHandler(LOG.handlers[0])
log_path = get_workflow_run_log_name(id_)
LOG.addHandler(TimestampRotatingFileHandler(log_path, no_detach))
# Add file installation log
file_install_log_path = get_workflow_file_install_log_name(id_)
RSYNC_LOG.addHandler(
TimestampRotatingFileHandler(file_install_log_path, no_detach))
def _open_logs(reg, no_detach):
"""Open Cylc log handlers for a flow run."""
if not no_detach:
while LOG.handlers:
LOG.handlers[0].close()
LOG.removeHandler(LOG.handlers[0])
suite_log_handler = get_suite_run_log_name(reg)
LOG.addHandler(TimestampRotatingFileHandler(suite_log_handler, no_detach))
# Add file installation log
file_install_log_path = get_suite_file_install_log_name(reg)
handler = TimestampRotatingFileHandler(file_install_log_path, no_detach)
RSYNC_LOG.addHandler(handler)
def _file_install_callback(self, ctx, platform, install_target):
"""Callback when file installation exits.
Sets remote_init_map to REMOTE_FILE_INSTALL_DONE on success and to
REMOTE_FILE_INSTALL_FAILED on error.
"""
if ctx.out:
RSYNC_LOG.info('File installation information for '
f'{install_target}:\n{ctx.out}')
if ctx.ret_code == 0:
# Both file installation and remote init success
LOG.debug(f"File installation complete for {install_target}")
self.remote_init_map[install_target] = REMOTE_FILE_INSTALL_DONE
self.ready = True
return
else:
self.remote_init_map[install_target] = REMOTE_FILE_INSTALL_FAILED
LOG.error(
PlatformError(
PlatformError.MSG_INIT,
platform['name'],
ctx=ctx,
))
self.ready = True
def parse_args(self, api_args, remove_opts=None):
"""Parse options and arguments, overrides OptionParser.parse_args.
Args:
api_args (list):
Command line options if passed via Python as opposed to
sys.argv
remove_opts (list):
List of standard options to remove before parsing.
"""
if self.auto_add:
# Add common options after command-specific options.
self.add_std_options()
if remove_opts:
for opt in remove_opts:
with suppress(ValueError):
self.remove_option(opt)
(options, args) = OptionParser.parse_args(self, api_args)
if len(args) < self.n_compulsory_args:
self.error("Wrong number of arguments (too few)")
elif (not self.unlimited_args
and len(args) > self.n_compulsory_args + self.n_optional_args):
self.error("Wrong number of arguments (too many)")
if self.jset and options.templatevars_file:
options.templatevars_file = os.path.abspath(
os.path.expanduser(options.templatevars_file))
cylc.flow.flags.verbosity = options.verbosity
# Set up stream logging for CLI. Note:
# 1. On choosing STDERR: Log messages are diagnostics, so STDERR is the
# better choice for the logging stream. This allows us to use STDOUT
# for verbosity agnostic outputs.
# 2. Scheduler will remove this handler when it becomes a daemon.
if options.verbosity < 0:
LOG.setLevel(logging.WARNING)
elif options.verbosity > 0:
LOG.setLevel(logging.DEBUG)
else:
LOG.setLevel(logging.INFO)
RSYNC_LOG.setLevel(logging.INFO)
# Remove NullHandler before add the StreamHandler
for log in (LOG, RSYNC_LOG):
while log.handlers:
log.handlers[0].close()
log.removeHandler(log.handlers[0])
log_handler = logging.StreamHandler(sys.stderr)
log_handler.setFormatter(
CylcLogFormatter(timestamp=options.log_timestamp,
dev_info=(options.verbosity > 2)))
LOG.addHandler(log_handler)
if self.segregated_log:
setup_segregated_log_streams(LOG, log_handler)
return (options, args)
def _remote_init_callback(
self, proc_ctx, platform, tmphandle,
curve_auth, client_pub_key_dir):
"""Callback when "cylc remote-init" exits"""
self.ready = True
try:
tmphandle.close()
except OSError: # E.g. ignore bad unlink, etc
pass
self.install_target = platform['install target']
if proc_ctx.ret_code == 0:
if REMOTE_INIT_DONE in proc_ctx.out:
src_path = get_suite_run_dir(self.suite)
dst_path = get_remote_suite_run_dir(platform, self.suite)
try:
process = procopen(construct_rsync_over_ssh_cmd(
src_path,
dst_path,
platform,
self.rsync_includes),
stdoutpipe=True,
stderrpipe=True,
universal_newlines=True)
out, err = process.communicate(timeout=600)
install_target = platform['install target']
if out:
RSYNC_LOG.info(
'File installation information for '
f'{install_target}:\n {out}')
if err:
LOG.error(
'File installation error on '
f'{install_target}:\n {err}')
except Exception as ex:
LOG.error(f"Problem during rsync: {ex}")
self.remote_init_map[self.install_target] = (
REMOTE_INIT_FAILED)
return
if "KEYSTART" in proc_ctx.out:
regex_result = re.search(
'KEYSTART((.|\n|\r)*)KEYEND', proc_ctx.out)
key = regex_result.group(1)
suite_srv_dir = get_suite_srv_dir(self.suite)
public_key = KeyInfo(
KeyType.PUBLIC,
KeyOwner.CLIENT,
suite_srv_dir=suite_srv_dir,
install_target=self.install_target
)
old_umask = os.umask(0o177)
with open(
public_key.full_key_path,
'w', encoding='utf8') as text_file:
text_file.write(key)
os.umask(old_umask)
# configure_curve must be called every time certificates are
# added or removed, in order to update the Authenticator's
# state.
curve_auth.configure_curve(
domain='*', location=(client_pub_key_dir))
for status in (REMOTE_INIT_DONE, REMOTE_INIT_NOT_REQUIRED):
if status in proc_ctx.out:
# Good status
LOG.debug(proc_ctx)
self.remote_init_map[self.install_target] = status
return
# Bad status
LOG.error(TaskRemoteMgmtError(
TaskRemoteMgmtError.MSG_INIT,
platform['install target'], ' '.join(
quote(item) for item in proc_ctx.cmd),
proc_ctx.ret_code, proc_ctx.out, proc_ctx.err))
LOG.error(proc_ctx)
self.remote_init_map[platform['install target']] = REMOTE_INIT_FAILED