def main(parser: COP, options: 'Values', workflow_id1: str, workflow_id2: str):
workflow_id_1, _, workflow_file_1_ = parse_id(
workflow_id1,
src=True,
constraint='workflows',
)
workflow_id_2, _, workflow_file_2_ = parse_id(
workflow_id2,
src=True,
constraint='workflows',
)
if workflow_file_1_ == workflow_file_2_:
parser.error("You can't diff a single workflow.")
print(f"Parsing {workflow_id_1} ({workflow_file_1_})")
template_vars = load_template_vars(
options.templatevars, options.templatevars_file
)
config1 = WorkflowConfig(
workflow_id_1, workflow_file_1_, options, template_vars
).cfg
print(f"Parsing {workflow_id_2} ({workflow_file_2_})")
config2 = WorkflowConfig(
workflow_id_2, workflow_file_2_, options, template_vars,
is_reload=True
).cfg
if config1 == config2:
print(
f"Workflow definitions {workflow_id_1} and {workflow_id_2} are "
f"identical"
)
sys.exit(0)
print(f"Workflow definitions {workflow_id_1} and {workflow_id_2} differ")
workflow1_only = {} # type: ignore
workflow2_only = {} # type: ignore
diff_1_2 = {} # type: ignore
# TODO: this whole file could do wih refactoring at some point
diffdict(config1, config2, workflow1_only, workflow2_only, diff_1_2)
if n_oone > 0:
print(f'\n{n_oone} items only in {workflow_id_1} (<)')
prdict(workflow1_only, '<', nested=options.nested)
if n_otwo > 0:
print(f'\n{n_otwo} items only in {workflow_id_2} (>)')
prdict(workflow2_only, '>', nested=options.nested)
if n_diff > 0:
print(f'\n{n_diff} common items differ {workflow_id_1}(<) '
f'{workflow_id_2}(>)')
prdict(diff_1_2, '', diff=True, nested=options.nested)
def main(parser: COP, options: 'Values', workflow_id: str) -> None:
workflow_id, *_ = parse_id(
workflow_id,
constraint='workflows',
)
output_options = [
options.show_raw, options.show_summary, options.html_summary
]
if output_options.count(True) > 1:
parser.error('Cannot combine output formats (choose one)')
if not any(output_options):
# No output specified - choose summary by default
options.show_summary = True
run_db = _get_dao(workflow_id)
row_buf = format_rows(*run_db.select_task_times())
with smart_open(options.output_filename) as output:
if options.show_raw:
output.write(row_buf.getvalue())
else:
summary: TimingSummary
if options.show_summary:
summary = TextTimingSummary(row_buf)
elif options.html_summary:
summary = HTMLTimingSummary(row_buf)
summary.write_summary(output)
def get_platforms_from_task_jobs(flow: str, cyclepoint: str) -> Dict[str, Any]:
"""Access flow database. Return platform for task at fixed cycle point
Uses the workflow database - designed to be used with tasks where jobs
have been submitted. We assume that we want the most recent submission.
Args:
flow: The name of the Cylc flow to be queried.
cyclepoint: The CyclePoint at which to query the job.
task: The name of the task to be queried.
Returns:
Platform Dictionary.
"""
_, _, flow_file = parse_id(flow, constraint='workflows', src=True)
dbfilepath = Path(flow_file).parent / '.service/db'
dao = CylcWorkflowDAO(dbfilepath)
task_platform_map: Dict = {}
stmt = ('SELECT "name", "platform_name", "submit_num" '
'FROM task_jobs WHERE cycle=?')
for row in dao.connect().execute(stmt, [cyclepoint]):
task, platform_n, submit_num = row
platform = get_platform(platform_n)
if ((task in task_platform_map
and task_platform_map[task][0] < submit_num)
or task not in task_platform_map):
task_platform_map[task] = [submit_num, platform]
# get rid of the submit number, we don't want it
task_platform_map = {
key: value[1]
for key, value in task_platform_map.items()
}
return task_platform_map
def get_platform_from_task_def(flow: str, task: str) -> Dict[str, Any]:
"""Return the platform dictionary for a particular task.
Uses the flow definition - designed to be used with tasks
with unsubmitted jobs. Evaluates platform/host defined as subshell.
Args:
flow: The name of the Cylc flow to be queried.
task: The name of the task to be queried.
Returns:
Platform Dictionary.
"""
_, _, flow_file = parse_id(flow, constraint='workflows', src=True)
config = WorkflowConfig(flow, flow_file, Values())
# Get entire task spec to allow Cylc 7 platform from host guessing.
task_spec = config.pcfg.get(['runtime', task])
# check for subshell and evaluate
if (task_spec.get('platform')
and is_platform_definition_subshell(task_spec['platform'])):
task_spec['platform'] = eval_subshell(task_spec['platform'])
elif (task_spec.get('remote', {}).get('host')
and HOST_REC_COMMAND.match(task_spec['remote']['host'])):
task_spec['remote']['host'] = eval_subshell(
task_spec['remote']['host'])
platform = get_platform(task_spec)
return platform
def main(_, options: 'Values', *ids) -> None:
workflow_id, _, flow_file = parse_id(
*ids,
src=True,
constraint='workflows',
)
# extract task host platforms from the workflow_id
config = WorkflowConfig(
workflow_id, flow_file, options,
load_template_vars(options.templatevars, options.templatevars_file))
platforms = {
config.get_config(['runtime', name, 'platform'])
for name in config.get_namespace_list('all tasks')
} - {None, 'localhost'}
# When "workflow run hosts" are formalised as "flow platforms"
# we can substitute `localhost` for this, in the mean time
# we will have to assume that flow hosts are configured correctly.
if not platforms:
sys.exit(0)
verbose = cylc.flow.flags.verbosity > 0
# get the cylc version on each platform
versions = {}
for platform_name in sorted(platforms):
platform = get_platform(platform_name)
host = get_host_from_platform(platform, bad_hosts=None)
cmd = construct_ssh_cmd(['version'], platform, host)
if verbose:
print(cmd)
proc = procopen(cmd, stdin=DEVNULL, stdout=PIPE, stderr=PIPE)
out, err = proc.communicate()
out = out.decode()
err = err.decode()
if proc.wait() == 0:
if verbose:
print(" %s" % out)
versions[platform_name] = out.strip()
else:
versions[platform_name] = f'ERROR: {err.strip()}'
# report results
max_len = max((len(platform_name) for platform_name in platforms))
print(f'{"platform".rjust(max_len)}: cylc version')
print('-' * (max_len + 14))
for platform_name, result in versions.items():
print(f'{platform_name.rjust(max_len)}: {result}')
if all((version == CYLC_VERSION for version in versions.values())):
ret_code = 0
elif options.error:
ret_code = 1
else:
ret_code = 0
sys.exit(ret_code)
def main(parser: COP, options: 'Values', *args: str) -> None:
"""CLI."""
if not args:
return parser.error('No message supplied')
if len(args) <= 2:
# BACK COMPAT: args <= 2
# from:
# 7.6?
# remove at:
# 9.0?
# (As of Dec 2020 some functional tests still use the classic
# two arg interface)
workflow_id = os.getenv('CYLC_WORKFLOW_ID')
task_job = os.getenv('CYLC_TASK_JOB')
message_strs = list(args)
else:
workflow_id, task_job, *message_strs = args
workflow_id, *_ = parse_id(
workflow_id,
constraint='workflows',
)
# Read messages from STDIN
if '-' in message_strs:
current_message_str = ''
while True: # Note: `for line in sys.stdin:` can hang
message_str = sys.stdin.readline()
if message_str.strip():
# non-empty line
current_message_str += message_str
elif message_str:
# empty line, start next message
if current_message_str:
message_strs.append(current_message_str)
current_message_str = '' # reset
else:
# end of file
if current_message_str:
message_strs.append(current_message_str)
break
# Separate "severity: message"
messages = [] # [(severity, message_str), ...]
for message_str in message_strs:
if message_str == '-':
pass
elif ':' in message_str:
valid, err_msg = TaskMessageValidator.validate(message_str)
if not valid:
raise UserInputError(
f'Invalid task message "{message_str}" - {err_msg}')
messages.append(
[item.strip() for item in message_str.split(':', 1)])
elif options.severity:
messages.append([options.severity, message_str.strip()])
else:
messages.append([getLevelName(INFO), message_str.strip()])
record_messages(workflow_id, task_job, messages)
def main(parser: COP, opts: 'Values', args: Optional[str] = None) -> None:
run_dir: Optional[Path]
workflow_id: str
if args is None:
try:
workflow_id = str(Path.cwd().relative_to(
Path(get_cylc_run_dir()).resolve()))
except ValueError:
raise WorkflowFilesError(
"The current working directory is not a workflow run directory"
)
else:
workflow_id, *_ = parse_id(
args,
constraint='workflows',
)
run_dir = Path(get_workflow_run_dir(workflow_id))
if not run_dir.is_dir():
raise WorkflowFilesError(
f'"{workflow_id}" is not an installed workflow.')
source, source_symlink = get_workflow_source_dir(run_dir)
if not source:
raise WorkflowFilesError(
f'"{workflow_id}" was not installed with cylc install.')
if not Path(source).is_dir():
raise WorkflowFilesError(
f'Workflow source dir is not accessible: "{source}".\n'
f'Restore the source or modify the "{source_symlink}"'
' symlink to continue.')
for entry_point in iter_entry_points('cylc.pre_configure'):
try:
entry_point.resolve()(srcdir=source, opts=opts)
except Exception as exc:
# NOTE: except Exception (purposefully vague)
# this is to separate plugin from core Cylc errors
raise PluginError('cylc.pre_configure', entry_point.name,
exc) from None
reinstall_workflow(
named_run=workflow_id,
rundir=run_dir,
source=source,
dry_run=False # TODO: ready for dry run implementation
)
for entry_point in iter_entry_points('cylc.post_install'):
try:
entry_point.resolve()(srcdir=source,
opts=opts,
rundir=str(run_dir))
except Exception as exc:
# NOTE: except Exception (purposefully vague)
# this is to separate plugin from core Cylc errors
raise PluginError('cylc.post_install', entry_point.name,
exc) from None
def get_config(workflow_id: str, opts: 'Values') -> WorkflowConfig:
"""Return a WorkflowConfig object for the provided reg / path."""
workflow_id, _, flow_file = parse_id(
workflow_id,
src=True,
constraint='workflows',
)
template_vars = get_template_vars(opts)
return WorkflowConfig(workflow_id,
flow_file,
opts,
template_vars=template_vars)
def main(parser: COP, options: 'Values', workflow_id: str) -> None:
"""CLI for "cylc get-workflow-contact"."""
workflow_id, *_ = parse_id(
workflow_id,
constraint='workflows',
)
try:
data = load_contact_file(workflow_id)
except ServiceFileError:
raise CylcError(
f"{workflow_id}: cannot get contact info, workflow not running?")
else:
for key, value in sorted(data.items()):
print("%s=%s" % (key, value))
def main(
parser: COP,
options: 'Values',
*ids,
) -> None:
if options.print_platform_names and options.print_platforms:
options.print_platform_names = False
if options.print_platform_names or options.print_platforms:
# Get platform information:
if ids:
raise UserInputError(
"Workflow IDs are incompatible with --platform options.")
glbl_cfg().platform_dump(options.print_platform_names,
options.print_platforms)
return
if not ids:
if options.print_hierarchy:
print("\n".join(get_config_file_hierarchy()))
return
glbl_cfg().idump(options.item,
not options.defaults,
oneline=options.oneline,
none_str=options.none_str)
return
workflow_id, _, flow_file = parse_id(
*ids,
src=True,
constraint='workflows',
)
if options.print_hierarchy:
print("\n".join(get_config_file_hierarchy(workflow_id)))
return
config = WorkflowConfig(workflow_id, flow_file, options,
get_template_vars(options))
config.pcfg.idump(options.item,
not options.defaults,
oneline=options.oneline,
none_str=options.none_str)
def main(_, options: 'Values', workflow_id: str, func: str) -> None:
workflow_id, *_ = parse_id(
workflow_id,
constraint='workflows',
)
pclient = WorkflowRuntimeClient(workflow_id, timeout=options.comms_timeout)
if options.no_input:
kwargs = {}
else:
kwargs = json.load(sys.stdin)
sys.stdin.close()
res = pclient(func, kwargs)
if func in PB_METHOD_MAP:
if 'element_type' in kwargs:
pb_msg = PB_METHOD_MAP[func][kwargs['element_type']]()
else:
pb_msg = PB_METHOD_MAP[func]()
pb_msg.ParseFromString(res)
res_msg = MessageToDict(pb_msg)
else:
res_msg = res
sys.stdout.write(json.dumps(res_msg, indent=4) + '\n')
def main(_, options: 'Values', workflow_id: str) -> None:
workflow_id, *_ = parse_id(
workflow_id,
constraint='workflows',
)
screen = None
if options.display == 'html':
TREE_EXPAND_DEPTH[0] = -1 # expand tree fully
screen = html_fragment.HtmlGenerator()
screen.set_terminal_properties(256)
screen.register_palette(TuiApp.palette)
html_fragment.screenshot_init(
[tuple(map(int, options.v_term_size.split(',')))], [])
try:
TuiApp(workflow_id, screen=screen).main()
if options.display == 'html':
for fragment in html_fragment.screenshot_collect():
print(fragment)
except KeyboardInterrupt:
pass
def get_platform_from_task_def(flow: str, task: str) -> Dict[str, Any]:
"""Return the platform dictionary for a particular task.
Uses the flow definition - designed to be used with tasks
with unsubmitted jobs.
Args:
flow: The name of the Cylc flow to be queried.
task: The name of the task to be queried.
Returns:
Platform Dictionary.
"""
_, _, flow_file = parse_id(flow, constraint='workflows', src=True)
config = WorkflowConfig(flow, flow_file, Values())
# Get entire task spec to allow Cylc 7 platform from host guessing.
task_spec = config.pcfg.get(['runtime', task])
platform = get_platform(task_spec)
if platform is None:
raise PlatformLookupError(
'Platform lookup failed; platform is a subshell to be evaluated: '
f' Task: {task}, platform: {task_spec["platform"]}.')
return platform
def main(parser: COP, options: 'Values', workflow_id: str, event_msg: str,
event_id: str) -> None:
workflow_id, *_ = parse_id(
workflow_id,
constraint='workflows',
)
LOG.info('Send to workflow %s: "%s" (%s)', workflow_id, event_msg,
event_id)
pclient = get_client(workflow_id, timeout=options.comms_timeout)
max_n_tries = int(options.max_n_tries)
retry_intvl_secs = float(options.retry_intvl_secs)
mutation_kwargs = {
'request_string': MUTATION,
'variables': {
'wFlows': [workflow_id],
'eventMsg': event_msg,
'eventId': event_id,
}
}
for i_try in range(max_n_tries):
try:
pclient('graphql', mutation_kwargs)
except ClientError as exc:
LOG.exception(exc)
LOG.info(MSG_SEND_FAILED, i_try + 1, max_n_tries)
if i_try == max_n_tries - 1: # final attempt
raise CylcError('send failed')
LOG.info(MSG_SEND_RETRY, retry_intvl_secs, options.comms_timeout)
sleep(retry_intvl_secs)
else:
if i_try > 0:
LOG.info(MSG_SEND_SUCCEED, i_try + 1, max_n_tries)
break
def main(parser: COP, options: 'Values', workflow_id: str) -> None:
workflow_id, _, flow_file = parse_id(
workflow_id,
src=True,
constraint='workflows',
)
if options.geditor:
editor = glbl_cfg().get(['editors', 'gui'])
else:
editor = glbl_cfg().get(['editors', 'terminal'])
# read in the flow.cylc file
viewcfg = {
'mark':
options.mark,
'single':
options.single,
'label':
options.label,
'empy':
options.empy or options.process,
'jinja2':
options.jinja2 or options.process,
'contin':
options.cat or options.process,
'inline': (options.inline or options.jinja2 or options.empy
or options.process),
}
lines = read_and_proc(flow_file,
load_template_vars(options.templatevars,
options.templatevars_file),
viewcfg=viewcfg)
if options.stdout:
for line in lines:
print(line)
sys.exit(0)
# write to a temporary file
viewfile = NamedTemporaryFile(
suffix=".flow.cylc",
prefix=workflow_id.replace('/', '_') + '.',
)
for line in lines:
viewfile.write((line + '\n').encode())
viewfile.seek(0, 0)
# set the file to be read only
os.chmod(viewfile.name, 0o400)
# capture the temp file's mod time in case the user edits it
# and overrides the readonly mode.
modtime1 = os.stat(viewfile.name).st_mtime
# in case editor has options, e.g. 'emacs -nw':
command_list = shlex.split(editor)
command_list.append(viewfile.name)
command = ' '.join(command_list)
# THIS BLOCKS UNTIL THE COMMAND COMPLETES
retcode = call(command_list) # nosec (editor command is user configurable)
if retcode != 0:
# the command returned non-zero exist status
raise CylcError(f'{command} failed: {retcode}')
# !!!VIEWING FINISHED!!!
# Did the user edit the file
modtime2 = os.stat(viewfile.name).st_mtime
if modtime2 > modtime1:
print(
"\nWARNING: YOU HAVE EDITED A TEMPORARY READ-ONLY COPY "
f"OF THE WORKFLOW:\n {viewfile.name}\n",
file=sys.stderr)
# DONE
viewfile.close()
def main(parser: COP, options: 'Values', workflow_id: str) -> None:
workflow_id, _, flow_file = parse_id(
workflow_id,
src=True,
constraint='workflows',
)
template_vars = get_template_vars(options)
if options.all_tasks and options.all_namespaces:
parser.error("Choose either -a or -n")
if options.all_tasks:
which = "all tasks"
elif options.all_namespaces:
which = "all namespaces"
elif options.prange:
which = "prange"
if options.prange == ",":
tr_start = None
tr_stop = None
elif options.prange.endswith(","):
tr_start = options.prange[:-1]
tr_stop = None
elif options.prange.startswith(","):
tr_start = None
tr_stop = options.prange[1:]
else:
tr_start, tr_stop = options.prange.split(',')
else:
which = "graphed tasks"
if options.tree and os.environ['LANG'] == 'C' and options.box:
print("WARNING, ignoring -t/--tree: $LANG=C", file=sys.stderr)
options.tree = False
if options.titles and options.mro:
parser.error("Please choose --mro or --title, not both")
if options.tree and any(
[options.all_tasks, options.all_namespaces, options.mro]):
print("WARNING: -t chosen, ignoring non-tree options.",
file=sys.stderr)
config = WorkflowConfig(workflow_id, flow_file, options, template_vars)
if options.tree:
config.print_first_parent_tree(pretty=options.box,
titles=options.titles)
elif options.prange:
for node in sorted(config.get_node_labels(tr_start, tr_stop)):
print(node)
else:
result = config.get_namespace_list(which)
namespaces = list(result)
namespaces.sort()
if (options.mro or options.titles):
# compute padding
maxlen = 0
for ns in namespaces:
if len(ns) > maxlen:
maxlen = len(ns)
padding = maxlen * ' '
for ns in namespaces:
if options.mro:
print(ns, padding[0:len(padding) - len(ns)], end=' ')
print(' '.join(config.get_mro(ns)))
elif options.titles:
print(ns, padding[0:len(padding) - len(ns)], end=' ')
print(result[ns])
else:
print(ns)
def main(parser: COP, options: 'Values', workflow_id: str) -> None:
"""cylc validate CLI."""
profiler = Profiler(None, options.profile_mode)
profiler.start()
if cylc.flow.flags.verbosity < 2:
disable_timestamps(LOG)
workflow_id, _, flow_file = parse_id(
workflow_id,
src=True,
constraint='workflows',
)
cfg = WorkflowConfig(workflow_id,
flow_file,
options,
get_template_vars(options),
output_fname=options.output,
mem_log_func=profiler.log_memory)
# Instantiate tasks and force evaluation of trigger expressions.
# (Taken from config.py to avoid circular import problems.)
# TODO - This is not exhaustive, it only uses the initial cycle point.
if cylc.flow.flags.verbosity > 0:
print('Instantiating tasks to check trigger expressions')
for name, taskdef in cfg.taskdefs.items():
try:
itask = TaskProxy(taskdef, cfg.start_point)
except TaskProxySequenceBoundsError:
# Should already failed above
mesg = 'Task out of bounds for %s: %s\n' % (cfg.start_point, name)
if cylc.flow.flags.verbosity > 0:
sys.stderr.write(' + %s\n' % mesg)
continue
except Exception as exc:
raise WorkflowConfigError('failed to instantiate task %s: %s' %
(name, exc))
# force trigger evaluation now
try:
itask.state.prerequisites_eval_all()
except TriggerExpressionError as exc:
err = str(exc)
if '@' in err:
print(
f"ERROR, {name}: xtriggers can't be in conditional"
f" expressions: {err}",
file=sys.stderr)
else:
print('ERROR, %s: bad trigger: %s' % (name, err),
file=sys.stderr)
raise WorkflowConfigError("ERROR: bad trigger")
except Exception as exc:
print(str(exc), file=sys.stderr)
raise WorkflowConfigError('%s: failed to evaluate triggers.' %
name)
if cylc.flow.flags.verbosity > 0:
print(' + %s ok' % itask.identity)
print(cparse('<green>Valid for cylc-%s</green>' % CYLC_VERSION))
profiler.stop()
def main(parser: COP, options: 'Values', workflow_id: str) -> None:
workflow_id, *_ = parse_id(
workflow_id,
constraint='workflows',
)
if options.use_task_point and options.cycle:
raise UserInputError(
"cannot specify a cycle point and use environment variable")
if options.use_task_point:
if "CYLC_TASK_CYCLE_POINT" not in os.environ:
raise UserInputError("CYLC_TASK_CYCLE_POINT is not defined")
options.cycle = os.environ["CYLC_TASK_CYCLE_POINT"]
if options.offset and not options.cycle:
raise UserInputError("You must target a cycle point to use an offset")
# Attempt to apply specified offset to the targeted cycle
if options.offset:
options.cycle = str(add_offset(options.cycle, options.offset))
# Exit if both task state and message are to being polled
if options.status and options.msg:
raise UserInputError("cannot poll both status and custom output")
if options.msg and not options.task and not options.cycle:
raise UserInputError("need a taskname and cyclepoint")
# Exit if an invalid status is requested
if (options.status and options.status not in TASK_STATUSES_ORDERED
and options.status not in CylcWorkflowDBChecker.STATE_ALIASES):
raise UserInputError(f"invalid status '{options.status}'")
# this only runs locally
if options.run_dir:
run_dir = expand_path(options.run_dir)
else:
run_dir = get_cylc_run_dir()
pollargs = {
'workflow_id': workflow_id,
'run_dir': run_dir,
'task': options.task,
'cycle': options.cycle,
'status': options.status,
'message': options.msg,
}
spoller = WorkflowPoller(
"requested state",
options.interval,
options.max_polls,
args=pollargs,
)
connected, formatted_pt = spoller.connect()
if not connected:
raise CylcError("cannot connect to the workflow_id DB")
if options.status and options.task and options.cycle:
# check a task status
spoller.condition = options.status
if not asyncio.run(spoller.poll()):
sys.exit(1)
elif options.msg:
# Check for a custom task output
spoller.condition = "output: %s" % options.msg
if not asyncio.run(spoller.poll()):
sys.exit(1)
else:
# just display query results
spoller.checker.display_maps(
spoller.checker.workflow_state_query(task=options.task,
cycle=formatted_pt,
status=options.status))
def main(_, options: 'Values', workflow_id: str) -> None:
workflow_id, *_ = parse_id(
workflow_id,
constraint='workflows',
)
pclient = get_client(workflow_id, timeout=options.comms_timeout)
if options.sort_by_cycle:
sort_args = {'keys': ['cyclePoint', 'name']}
else:
sort_args = {'keys': ['name', 'cyclePoint']}
if options.disp_form == "raw":
query = f'''
{TASK_SUMMARY_FRAGMENT}
{FAMILY_SUMMARY_FRAGMENT}
{WORKFLOW_SUMMARY_FRAGMENT}
query ($wFlows: [ID]!, $sortBy: SortArgs) {{
workflows (ids: $wFlows, stripNull: false) {{
...wFlow
taskProxies (sort: $sortBy) {{
...tProxy
}}
familyProxies (sort: $sortBy) {{
...fProxy
}}
}}
}}'''
elif options.disp_form != "tasks":
query = f'''
{WORKFLOW_SUMMARY_FRAGMENT}
query ($wFlows: [ID]!) {{
workflows (ids: $wFlows, stripNull: false) {{
...wFlow
}}
}}'''
else:
query = f'''
{TASK_SUMMARY_FRAGMENT}
query ($wFlows: [ID]!, $sortBy: SortArgs) {{
workflows (ids: $wFlows, stripNull: false) {{
taskProxies (sort: $sortBy) {{
...tProxy
}}
}}
}}'''
query_kwargs = {
'request_string': query,
'variables': {
'wFlows': [workflow_id],
'sortBy': sort_args
}
}
workflows = pclient('graphql', query_kwargs)
try:
for summary in workflows['workflows']:
if options.disp_form == "raw":
if options.pretty:
sys.stdout.write(json.dumps(summary, indent=4) + '\n')
else:
print(summary)
else:
if options.disp_form != "tasks":
node_urls = {
node['name']: node['meta']['URL']
for node in summary['tasks'] + summary['families']
}
summary['workflowUrls'] = {
node_name: node_urls[node_name]
for node_name in summary['namespaceDefinitionOrder']
if node_name in node_urls
}
summary['workflowUrls']['workflow_id'] = (
summary['meta']['URL'])
del summary['tasks']
del summary['families']
del summary['meta']
for key, value in sorted(summary.items()):
print(
f'{to_snake_case(key).replace("_", " ")}={value}')
else:
for item in summary['taskProxies']:
if options.sort_by_cycle:
values = [
item['cyclePoint'], item['name'], item['state']
]
else:
values = [
item['name'], item['cyclePoint'], item['state']
]
values.append('held' if item['isHeld'] else 'not-held')
values.append(
'queued' if item['isQueued'] else 'not-queued')
values.append('runahead' if item['isRunahead'] else
'not-runahead')
if options.show_flows:
values.append(item['flowNums'])
print(', '.join(values))
except Exception as exc:
raise CylcError(
json.dumps(workflows, indent=4) + '\n' + str(exc) + '\n')
def main(parser: COP, options: 'Values', *ids, color: bool = False) -> None:
"""Implement cylc cat-log CLI.
Determine log path, user@host, batchview_cmd, and action (print, dir-list,
cat, edit, or tail), and then if the log path is:
a) local: perform action on log path, or
b) remote: re-invoke cylc cat-log as a) on the remote account
"""
if options.remote_args:
# Invoked on job hosts for job logs only, as a wrapper to view_log().
# Tail and batchview commands from global config on workflow host).
logpath, mode, tail_tmpl = options.remote_args[0:3]
logpath = expand_path(logpath)
tail_tmpl = expand_path(tail_tmpl)
try:
batchview_cmd = options.remote_args[3]
except IndexError:
batchview_cmd = None
res = view_log(logpath,
mode,
tail_tmpl,
batchview_cmd,
remote=True,
color=color)
if res == 1:
sys.exit(res)
return
workflow_id, tokens, _ = parse_id(*ids, constraint='mixed')
# Get long-format mode.
try:
mode = MODES[options.mode]
except KeyError:
mode = options.mode
if not tokens or not tokens.get('task'):
# Cat workflow logs, local only.
if options.filename is not None:
raise UserInputError("The '-f' option is for job logs only.")
logpath = get_workflow_run_log_name(workflow_id)
if options.rotation_num:
logs = glob('%s.*' % logpath)
logs.sort(key=os.path.getmtime, reverse=True)
try:
logpath = logs[int(options.rotation_num)]
except IndexError:
raise UserInputError("max rotation %d" % (len(logs) - 1))
tail_tmpl = os.path.expandvars(get_platform()["tail command template"])
out = view_log(logpath, mode, tail_tmpl, color=color)
if out == 1:
sys.exit(1)
if mode == 'edit':
tmpfile_edit(out, options.geditor)
return
else:
# Cat task job logs, may be on workflow or job host.
if options.rotation_num is not None:
raise UserInputError("only workflow (not job) logs get rotated")
task = tokens['task']
point = tokens['cycle']
if options.submit_num != NN:
try:
options.submit_num = "%02d" % int(options.submit_num)
except ValueError:
parser.error("Illegal submit number: %s" % options.submit_num)
if options.filename is None:
options.filename = JOB_LOG_OUT
else:
# Convert short filename args to long (e.g. 'o' to 'job.out').
with suppress(KeyError):
options.filename = JOB_LOG_OPTS[options.filename]
# KeyError: Is already long form (standard log, or custom).
platform_name, job_runner_name, live_job_id = get_task_job_attrs(
workflow_id, point, task, options.submit_num)
platform = get_platform(platform_name)
batchview_cmd = None
if live_job_id is not None:
# Job is currently running. Get special job runner log view
# command (e.g. qcat) if one exists, and the log is out or err.
conf_key = None
if options.filename == JOB_LOG_OUT:
if mode == 'cat':
conf_key = "out viewer"
elif mode == 'tail':
conf_key = "out tailer"
elif options.filename == JOB_LOG_ERR:
if mode == 'cat':
conf_key = "err viewer"
elif mode == 'tail':
conf_key = "err tailer"
if conf_key is not None:
batchview_cmd_tmpl = None
with suppress(KeyError):
batchview_cmd_tmpl = platform[conf_key]
if batchview_cmd_tmpl is not None:
batchview_cmd = batchview_cmd_tmpl % {
"job_id": str(live_job_id)
}
log_is_remote = (is_remote_platform(platform)
and (options.filename != JOB_LOG_ACTIVITY))
log_is_retrieved = (platform['retrieve job logs']
and live_job_id is None)
if log_is_remote and (not log_is_retrieved or options.force_remote):
logpath = os.path.normpath(
get_remote_workflow_run_job_dir(workflow_id, point, task,
options.submit_num,
options.filename))
tail_tmpl = platform["tail command template"]
# Reinvoke the cat-log command on the remote account.
cmd = ['cat-log', *verbosity_to_opts(cylc.flow.flags.verbosity)]
for item in [logpath, mode, tail_tmpl]:
cmd.append('--remote-arg=%s' % shlex.quote(item))
if batchview_cmd:
cmd.append('--remote-arg=%s' % shlex.quote(batchview_cmd))
cmd.append(workflow_id)
is_edit_mode = (mode == 'edit')
# TODO: Add Intelligent Host selection to this
try:
proc = remote_cylc_cmd(cmd,
platform,
capture_process=is_edit_mode,
manage=(mode == 'tail'))
except KeyboardInterrupt:
# Ctrl-C while tailing.
pass
else:
if is_edit_mode:
# Write remote stdout to a temp file for viewing in editor.
# Only BUFSIZE bytes at a time in case huge stdout volume.
out = NamedTemporaryFile()
data = proc.stdout.read(BUFSIZE)
while data:
out.write(data)
data = proc.stdout.read(BUFSIZE)
os.chmod(out.name, S_IRUSR)
out.seek(0, 0)
else:
# Local task job or local job log.
logpath = os.path.normpath(
get_workflow_run_job_dir(workflow_id, point, task,
options.submit_num, options.filename))
tail_tmpl = os.path.expandvars(platform["tail command template"])
out = view_log(logpath,
mode,
tail_tmpl,
batchview_cmd,
color=color)
if mode != 'edit':
sys.exit(out)
if mode == 'edit':
tmpfile_edit(out, options.geditor)