def main(parser, options, reg):
suite, suiterc = parse_suite_arg(options, reg)
if options.markup:
prefix = '!cylc!'
else:
prefix = ''
config = SuiteConfig(
suite, suiterc, options,
load_template_vars(options.templatevars, options.templatevars_file))
if options.tasks:
for task in config.get_task_name_list():
print(prefix + task)
elif options.alltasks:
for task in config.get_task_name_list():
items = ['[runtime][' + task + ']' + i for i in options.item]
print(prefix + task, end=' ')
config.pcfg.idump(items,
options.sparse,
options.pnative,
prefix,
options.oneline,
none_str=options.none_str)
else:
config.pcfg.idump(options.item,
options.sparse,
options.pnative,
prefix,
options.oneline,
none_str=options.none_str)
def main(parser, options, *args):
suite1, suite1rc = parse_suite_arg(options, args[0])
suite2, suite2rc = parse_suite_arg(options, args[1])
if suite1 == suite2:
parser.error("You can't diff a single suite.")
print("Parsing %s (%s)" % (suite1, suite1rc))
template_vars = load_template_vars(
options.templatevars, options.templatevars_file)
config1 = SuiteConfig(suite1, suite1rc, options, template_vars).cfg
print("Parsing %s (%s)" % (suite2, suite2rc))
config2 = SuiteConfig(
suite2, suite2rc, options, template_vars, is_reload=True).cfg
if config1 == config2:
print("Suite definitions %s and %s are identical" % (suite1, suite2))
sys.exit(0)
print("Suite definitions %s and %s differ" % (suite1, suite2))
suite1_only = {}
suite2_only = {}
diff_1_2 = {}
diffdict(config1, config2, suite1_only, suite2_only, diff_1_2)
if n_oone > 0:
print()
msg = str(n_oone) + ' items only in ' + suite1 + ' (<)'
print(msg)
prdict(suite1_only, '<', nested=options.nested)
if n_otwo > 0:
print()
msg = str(n_otwo) + ' items only in ' + suite2 + ' (>)'
print(msg)
prdict(suite2_only, '>', nested=options.nested)
if n_diff > 0:
print()
msg = (str(n_diff) + ' common items differ ' +
suite1 + '(<) ' + suite2 + '(>)')
print(msg)
prdict(diff_1_2, '', diff=True, nested=options.nested)
def main(parser, options, *args):
suite1_name, suite1_config = parse_suite_arg(options, args[0])
suite2_name, suite2_config = parse_suite_arg(options, args[1])
if suite1_name == suite2_name:
parser.error("You can't diff a single suite.")
print(f"Parsing {suite1_name} ({suite1_config})")
template_vars = load_template_vars(options.templatevars,
options.templatevars_file)
config1 = SuiteConfig(suite1_name, suite1_config, options,
template_vars).cfg
print(f"Parsing {suite2_name} ({suite2_config})")
config2 = SuiteConfig(suite2_name,
suite2_config,
options,
template_vars,
is_reload=True).cfg
if config1 == config2:
print(f"Suite definitions {suite1_name} and {suite2_name} are "
f"identical")
sys.exit(0)
print(f"Suite definitions {suite1_name} and {suite2_name} differ")
suite1_only = {}
suite2_only = {}
diff_1_2 = {}
diffdict(config1, config2, suite1_only, suite2_only, diff_1_2)
if n_oone > 0:
print(f'\n{n_oone} items only in {suite1_name} (<)')
prdict(suite1_only, '<', nested=options.nested)
if n_otwo > 0:
print(f'\n{n_otwo} items only in {suite2_name} (>)')
prdict(suite2_only, '>', nested=options.nested)
if n_diff > 0:
print(f'\n{n_diff} common items differ {suite1_name}(<) '
f'{suite2_name}(>)')
prdict(diff_1_2, '', diff=True, nested=options.nested)
def main(_, options, *args):
# suite name or file path
suite, flow_file = parse_suite_arg(options, args[0])
# extract task host platforms from the suite
config = SuiteConfig(
suite, flow_file, options,
load_template_vars(options.templatevars, options.templatevars_file))
platforms = {
config.get_config(['runtime', name, 'platform'])
for name in config.get_namespace_list('all tasks')
} - {None, 'localhost'}
# When "suite run hosts" are formalised as "flow platforms"
# we can substitute `localhost` for this, in the mean time
# we will have to assume that flow hosts are configured correctly.
if not platforms:
sys.exit(0)
verbose = cylc.flow.flags.verbose
# get the cylc version on each platform
versions = {}
for platform_name in sorted(platforms):
platform = get_platform(platform_name)
cmd = construct_platform_ssh_cmd(['version'], platform)
if verbose:
print(cmd)
proc = procopen(cmd, stdin=DEVNULL, stdout=PIPE, stderr=PIPE)
out, err = proc.communicate()
out = out.decode()
err = err.decode()
if proc.wait() == 0:
if verbose:
print(" %s" % out)
versions[platform_name] = out.strip()
else:
versions[platform_name] = f'ERROR: {err.strip()}'
# report results
max_len = max((len(platform_name) for platform_name in platforms))
print(f'{"platform".rjust(max_len)}: cylc version')
print('-' * (max_len + 14))
for platform_name, result in versions.items():
print(f'{platform_name.rjust(max_len)}: {result}')
if all((version == CYLC_VERSION for version in versions.values())):
exit = 0
elif options.error:
exit = 1
else:
exit = 0
sys.exit(exit)
def main(parser, options, reg=None):
if options.print_hierarchy:
print("\n".join(get_config_file_hierarchy(reg)))
return
if reg is None:
glbl_cfg().idump(options.item,
sparse=options.sparse,
oneline=options.oneline,
none_str=options.none_str)
return
suite, flow_file = parse_suite_arg(options, reg)
config = SuiteConfig(
suite, flow_file, options,
load_template_vars(options.templatevars, options.templatevars_file))
config.pcfg.idump(options.item,
options.sparse,
oneline=options.oneline,
none_str=options.none_str)
def main(parser, options, reg):
suite, suiterc = parse_suite_arg(options, reg)
if options.geditor:
editor = glbl_cfg().get(['editors', 'gui'])
else:
editor = glbl_cfg().get(['editors', 'terminal'])
# read in the suite.rc file
viewcfg = {
'mark':
options.mark,
'single':
options.single,
'label':
options.label,
'empy':
options.empy or options.process,
'jinja2':
options.jinja2 or options.process,
'contin':
options.cat or options.process,
'inline': (options.inline or options.jinja2 or options.empy
or options.process),
}
lines = read_and_proc(suiterc,
load_template_vars(options.templatevars,
options.templatevars_file),
viewcfg=viewcfg,
asedit=options.asedit)
if options.stdout:
for line in lines:
print(line)
sys.exit(0)
# write to a temporary file
viewfile = NamedTemporaryFile(
suffix=".suite.rc",
prefix=suite.replace('/', '_') + '.',
)
for line in lines:
viewfile.write((line + '\n').encode())
viewfile.seek(0, 0)
# set the file to be read only
os.chmod(viewfile.name, 0o400)
# capture the temp file's mod time in case the user edits it
# and overrides the readonly mode.
modtime1 = os.stat(viewfile.name).st_mtime
# in case editor has options, e.g. 'emacs -nw':
command_list = shlex.split(editor)
command_list.append(viewfile.name)
command = ' '.join(command_list)
# THIS BLOCKS UNTIL THE COMMAND COMPLETES
retcode = call(command_list)
if retcode != 0:
# the command returned non-zero exist status
raise CylcError(f'{command} failed: {retcode}')
# !!!VIEWING FINISHED!!!
# Did the user edit the file
modtime2 = os.stat(viewfile.name).st_mtime
if modtime2 > modtime1:
print()
print('WARNING: YOU HAVE EDITED A TEMPORARY READ-ONLY SUITE COPY:',
file=sys.stderr)
print(viewfile.name, file=sys.stderr)
print('In future use \'cylc [prep] edit\' to edit a suite.',
file=sys.stderr)
print()
# DONE
viewfile.close()
def main(_, options, reg):
"""cylc validate CLI."""
profiler = Profiler(None, options.profile_mode)
profiler.start()
if not cylc.flow.flags.debug:
# for readability omit timestamps from logging unless in debug mode
for handler in LOG.handlers:
if isinstance(handler.formatter, CylcLogFormatter):
handler.formatter.configure(timestamp=False)
suite, flow_file = parse_suite_arg(options, reg)
cfg = SuiteConfig(
suite,
flow_file,
options,
load_template_vars(options.templatevars, options.templatevars_file),
output_fname=options.output, mem_log_func=profiler.log_memory)
# Check bounds of sequences
out_of_bounds = [str(seq) for seq in cfg.sequences
if seq.get_first_point(cfg.start_point) is None]
if out_of_bounds:
if len(out_of_bounds) > 1:
# avoid spamming users with multiple warnings
msg = ('multiple sequences out of bounds for initial cycle point '
'%s:\n%s' % (
cfg.start_point,
'\n'.join(textwrap.wrap(', '.join(out_of_bounds), 70))))
else:
msg = '%s: sequence out of bounds for initial cycle point %s' % (
out_of_bounds[0], cfg.start_point)
if options.strict:
LOG.warning(msg)
elif cylc.flow.flags.verbose:
sys.stderr.write(' + %s\n' % msg)
# Instantiate tasks and force evaluation of trigger expressions.
# (Taken from config.py to avoid circular import problems.)
# TODO - This is not exhaustive, it only uses the initial cycle point.
if cylc.flow.flags.verbose:
print('Instantiating tasks to check trigger expressions')
flow_label = FlowLabelMgr().get_new_label()
for name, taskdef in cfg.taskdefs.items():
try:
itask = TaskProxy(taskdef, cfg.start_point, flow_label)
except TaskProxySequenceBoundsError:
# Should already failed above in strict mode.
mesg = 'Task out of bounds for %s: %s\n' % (cfg.start_point, name)
if cylc.flow.flags.verbose:
sys.stderr.write(' + %s\n' % mesg)
continue
except Exception as exc:
raise SuiteConfigError(
'failed to instantiate task %s: %s' % (name, exc))
# force trigger evaluation now
try:
itask.state.prerequisites_eval_all()
except TriggerExpressionError as exc:
err = str(exc)
if '@' in err:
print(f"ERROR, {name}: xtriggers can't be in conditional"
f" expressions: {err}",
file=sys.stderr)
else:
print('ERROR, %s: bad trigger: %s' % (name, err),
file=sys.stderr)
raise SuiteConfigError("ERROR: bad trigger")
except Exception as exc:
print(str(exc), file=sys.stderr)
raise SuiteConfigError(
'%s: failed to evaluate triggers.' % name)
if cylc.flow.flags.verbose:
print(' + %s ok' % itask.identity)
print(cparse('<green>Valid for cylc-%s</green>' % CYLC_VERSION))
profiler.stop()
def main(parser, options, reg):
suite, suiterc = parse_suite_arg(options, reg)
if options.all_tasks and options.all_namespaces:
parser.error("Choose either -a or -n")
if options.all_tasks:
which = "all tasks"
elif options.all_namespaces:
which = "all namespaces"
elif options.crange:
which = "crange"
try:
tr_start, tr_stop = options.crange.split(',')
except ValueError:
tr_start = tr_stop = options.crange
else:
which = "graphed tasks"
if options.tree:
if os.environ['LANG'] == 'C' and options.box:
print("WARNING, ignoring -t/--tree: $LANG=C", file=sys.stderr)
options.tree = False
if options.titles and options.mro:
parser.error("Please choose --mro or --title, not both")
if options.tree and any(
[options.all_tasks, options.all_namespaces, options.mro]):
print("WARNING: -t chosen, ignoring non-tree options.",
file=sys.stderr)
config = SuiteConfig(
suite, suiterc, options,
load_template_vars(options.templatevars, options.templatevars_file))
if options.tree:
config.print_first_parent_tree(pretty=options.box,
titles=options.titles)
elif options.crange:
for node in sorted(config.get_node_labels(tr_start, tr_stop)):
print(node)
else:
result = config.get_namespace_list(which)
namespaces = list(result)
namespaces.sort()
if (options.mro or options.titles):
# compute padding
maxlen = 0
for ns in namespaces:
if len(ns) > maxlen:
maxlen = len(ns)
padding = maxlen * ' '
for ns in namespaces:
if options.mro:
print(ns, padding[0:len(padding) - len(ns)], end=' ')
print(' '.join(config.get_mro(ns)))
elif options.titles:
print(ns, padding[0:len(padding) - len(ns)], end=' ')
print(result[ns])
else:
print(ns)
def main(parser, options, *args):
flow_file = parse_suite_arg(options, args[0])[1]
if options.geditor:
editor = glbl_cfg().get(['editors', 'gui'])
else:
editor = glbl_cfg().get(['editors', 'terminal'])
suitedir = os.path.dirname(flow_file)
if options.cleanup:
# remove backup files left by inlined editing sessions
cleanup(suitedir)
sys.exit(0)
if not options.inline:
# plain old editing.
# move to suite def dir
os.chdir(suitedir)
# edit the flow.cylc file
if not os.path.isfile(flow_file):
raise UserInputError(f'file not found: {flow_file}')
# in case editor has options, e.g. 'emacs -nw':
command_list = re.split(' ', editor)
command_list.append(flow_file)
command = ' '.join(command_list)
# THIS BLOCKS UNTIL THE COMMAND COMPLETES
retcode = call(command_list)
if retcode != 0:
# the command returned non-zero exist status
raise CylcError(f'{command} failed: {retcode}')
# !!!EDITING FINISHED!!!
sys.exit(0)
# read the flow.cylc file
if os.path.isfile(flow_file):
# back up the original
backup(flow_file)
# record original modtime
modtimes[flow_file] = os.stat(flow_file).st_mtime
# read the file
h = open(flow_file, 'r')
lines0 = h.readlines()
h.close()
if lines0[0].startswith('# !WARNING! CYLC EDIT INLINED'):
print('WARNING: RECOVERING A PREVIOUSLY INLINED FILE')
recovery = True
lines = lines0
else:
recovery = False
lines = inline(lines0, suitedir, flow_file, for_edit=True)
else:
parser.error(f"File not found: {flow_file}")
lines = [i.rstrip() for i in lines]
# overwrite the (now backed up) original with the inlined file:
h = open(flow_file, 'wb')
for line in lines:
h.write((line + '\n').encode())
h.close()
print('PRE-EDIT BACKUPS:')
for file in backups:
src = re.sub(suitedir + '/', '', file)
dst = re.sub(suitedir + '/', '', backups[file])
print(' + ' + src + ' ---> ' + dst)
# in case editor has options, e.g. 'emacs -nw':
command_list = re.split(' ', editor)
command_list.append(flow_file)
command = ' '.join(command_list)
# THIS BLOCKS UNTIL THE COMMAND COMPLETES
retcode = call(command_list)
if retcode != 0:
# the command returned non-zero exist status
raise CylcError(f'{command} failed: {retcode}')
print('EDITING DONE')
# Now back up the inlined file in case of absolute disaster, so as the
# user or his editor corrupting the inlined-include-file marker lines.
inlined_flow_file_backup = (
suitedir + '/flow.cylc.INLINED.EDIT.' +
get_current_time_string(override_use_utc=True, use_basic_format=True))
copy(flow_file, inlined_flow_file_backup)
# read in the edited inlined file
h = open(flow_file, 'r')
lines = h.readlines()
h.close()
# split it back into separate files
split_file(suitedir, lines, flow_file, recovery)
print(f' + edited: {flow_file}')
print(f' + backup: {inlined_flow_file_backup}')
print('INCLUDE-FILES WRITTEN:')
for file in newfiles:
f = re.sub(suitedir + '/', '', file)
if re.search(r'\.EDIT\.NEW\.', f):
print(' + ' + f + ' (!!! WARNING: original changed on disk !!!)')
else:
print(' + ' + f)
def main(parser, options, reg, *patterns):
suite, flow_file = parse_suite_arg(options, reg)
# cylc search SUITE PATTERN
pattern = '|'.join(patterns)
suitedir = os.path.dirname(flow_file)
if os.path.isfile(flow_file):
h = open(flow_file, 'r')
lines = h.readlines()
h.close()
lines = inline(lines, suitedir, flow_file, for_grep=True)
else:
parser.error(f"File not found: {flow_file}")
sections = deque(['(top)'])
line_count = 1
inc_file = None
in_include_file = False
prev_section_key = None
prev_file = None
for line in lines:
m = re.match(r'^#\+\+\+\+ START INLINED INCLUDE FILE ([\w/\.\-]+)',
line)
if m:
inc_file = m.groups()[0]
in_include_file = True
inc_line_count = 0
continue
if not in_include_file:
line_count += 1
else:
inc_line_count += 1
m = re.match(r'^#\+\+\+\+ END INLINED INCLUDE FILE ' + inc_file,
line)
if m:
in_include_file = False
inc_file = None
continue
m = re.match(r'\s*(\[+\s*.+\s*\]+)', line)
if m:
# new section heading detected
heading = m.groups()[0]
level = section_level(heading)
# unwind to the current section level
while len(sections) > level - 1:
sections.pop()
sections.append(heading)
continue
if re.search(pattern, line):
# Found a pattern match.
# Print the file name
if in_include_file:
curr_file = os.path.join(suitedir, inc_file)
line_no = inc_line_count
else:
curr_file = flow_file
line_no = line_count
if curr_file != prev_file:
prev_file = curr_file
print("\nFILE:", curr_file)
# Print the nested section headings
section_key = '->'.join(sections)
if section_key != prev_section_key:
prev_section_key = section_key
print(' SECTION:', section_key)
# Print the pattern match, with line number
print(' (' + str(line_no) + '):', line.rstrip('\n'))
if not options.search_bin:
sys.exit(0)
# search files in suite bin directory
bin_ = os.path.join(suitedir, 'bin')
if not os.path.isdir(bin_):
print("\nSuite " + suite + " has no bin directory", file=sys.stderr)
sys.exit(0)
for name in os.listdir(bin_):
if name.startswith('.'):
# skip hidden dot-files
# (e.g. vim editor temporary files)
continue
new_file = True
try:
h = open(os.path.join(bin_, name), 'r')
except IOError as exc:
# e.g. there's a sub-directory under bin; ignore it.
print('Unable to open file ' + os.path.join(bin_, name),
file=sys.stderr)
print(exc, file=sys.stderr)
continue
contents = h.readlines()
h.close()
count = 0
for line in contents:
line = line.rstrip('\n')
count += 1
if re.search(pattern, line):
if new_file:
print('\nFILE:', os.path.join(bin_, name))
new_file = False
print(' (' + str(count) + '): ' + line)
def main(_, options, *args):
# suite name or file path
suite, suiterc = parse_suite_arg(options, args[0])
# extract task host accounts from the suite
config = SuiteConfig(
suite, suiterc, options,
load_template_vars(options.templatevars, options.templatevars_file))
account_set = set()
for name in config.get_namespace_list('all tasks'):
account_set.add(
(config.get_config(['runtime', name, 'remote', 'owner']),
config.get_config(['runtime', name, 'remote', 'host'])))
task_remote_mgr = TaskRemoteMgr(suite, SubProcPool())
for _, host_str in account_set:
task_remote_mgr.remote_host_select(host_str)
accounts = []
while account_set:
for user, host_str in account_set.copy():
res = task_remote_mgr.remote_host_select(host_str)
if res:
account_set.remove((user, host_str))
accounts.append((user, res))
if account_set:
task_remote_mgr.proc_pool.process()
sleep(1.0)
# Interrogate the each remote account with CYLC_VERSION set to our version.
# Post backward compatibility concerns to do this we can just run:
# cylc version --host=HOST --user=USER
# but this command only exists for version > 6.3.0.
# So for the moment generate an actual remote invocation command string for
# "cylc --version".
# (save verbose flag as gets reset in remrun)
verbose = cylc.flow.flags.verbose
warn = {}
contacted = 0
for user, host in sorted(accounts):
argv = ["cylc", "version"]
if user and host:
argv += ["--user=%s" % user, "--host=%s" % host]
user_at_host = "%s@%s" % (user, host)
elif user:
argv += ["--user=%s" % user]
user_at_host = "%s@localhost" % user
elif host:
argv += ["--host=%s" % host]
user_at_host = host
if verbose:
print("%s: %s" % (user_at_host, ' '.join(argv)))
proc = procopen(argv,
stdin=open(os.devnull),
stdoutpipe=True,
stderrpipe=True)
out, err = proc.communicate()
out = out.decode()
err = err.decode()
if proc.wait() == 0:
if verbose:
print(" %s" % out)
contacted += 1
out = out.strip()
if out != CYLC_VERSION:
warn[user_at_host] = out
else:
print('ERROR ' + user_at_host + ':', file=sys.stderr)
print(err, file=sys.stderr)
# report results
if not warn:
if contacted:
print("All", contacted, "accounts have cylc-" + CYLC_VERSION)
else:
print("WARNING: failed to invoke cylc-%s on %d accounts:" %
(CYLC_VERSION, len(warn)))
m = max(len(ac) for ac in warn)
for ac, warning in warn.items():
print(' ', ac.ljust(m), warning)
if options.error:
sys.exit(1)
def get_config(flow, opts, template_vars=None):
"""Return a SuiteConfig object for the provided reg / path."""
flow, flow_file = parse_suite_arg(opts, flow)
return SuiteConfig(flow, flow_file, opts, template_vars=template_vars)
