def main(parser: COP, options: 'Values', workflow: str, event_msg: str, event_id: str) -> None: workflow, _ = parse_reg(workflow) LOG.info('Send to workflow %s: "%s" (%s)', workflow, event_msg, event_id) pclient = get_client(workflow, timeout=options.comms_timeout) max_n_tries = int(options.max_n_tries) retry_intvl_secs = float(options.retry_intvl_secs) mutation_kwargs = { 'request_string': MUTATION, 'variables': { 'wFlows': [workflow], 'eventMsg': event_msg, 'eventId': event_id, } } for i_try in range(max_n_tries): try: pclient('graphql', mutation_kwargs) except ClientError as exc: LOG.exception(exc) LOG.info(MSG_SEND_FAILED, i_try + 1, max_n_tries) if i_try == max_n_tries - 1: # final attempt raise CylcError('send failed') LOG.info(MSG_SEND_RETRY, retry_intvl_secs, options.comms_timeout) sleep(retry_intvl_secs) else: if i_try > 0: LOG.info(MSG_SEND_SUCCEED, i_try + 1, max_n_tries) break
def get_config(workflow: str, opts: 'Values') -> WorkflowConfig: """Return a WorkflowConfig object for the provided reg / path.""" workflow, flow_file = parse_reg(workflow, src=True) template_vars = get_template_vars(opts) return WorkflowConfig(workflow, flow_file, opts, template_vars=template_vars)
def main(parser: COP, options: 'Values', workflow1: str, workflow2: str): workflow1_name, workflow1_fpath = parse_reg(workflow1, src=True) workflow2_name, workflow2_fpath = parse_reg(workflow2, src=True) if workflow1_fpath == workflow2_fpath: parser.error("You can't diff a single workflow.") print(f"Parsing {workflow1_name} ({workflow1_fpath})") template_vars = load_template_vars(options.templatevars, options.templatevars_file) config1 = WorkflowConfig(workflow1_name, workflow1_fpath, options, template_vars).cfg print(f"Parsing {workflow2_name} ({workflow2_fpath})") config2 = WorkflowConfig(workflow2_name, workflow2_fpath, options, template_vars, is_reload=True).cfg if config1 == config2: print( f"Workflow definitions {workflow1_name} and {workflow2_name} are " f"identical") sys.exit(0) print(f"Workflow definitions {workflow1_name} and {workflow2_name} differ") workflow1_only = {} # type: ignore workflow2_only = {} # type: ignore diff_1_2 = {} # type: ignore # TODO: this whole file could do wih refactoring at some point diffdict(config1, config2, workflow1_only, workflow2_only, diff_1_2) if n_oone > 0: print(f'\n{n_oone} items only in {workflow1_name} (<)') prdict(workflow1_only, '<', nested=options.nested) if n_otwo > 0: print(f'\n{n_otwo} items only in {workflow2_name} (>)') prdict(workflow2_only, '>', nested=options.nested) if n_diff > 0: print(f'\n{n_diff} common items differ {workflow1_name}(<) ' f'{workflow2_name}(>)') prdict(diff_1_2, '', diff=True, nested=options.nested)
def main(parser: COP, options: 'Values', reg: str) -> None: reg, _ = parse_reg(reg) pclient = get_client(reg, timeout=options.comms_timeout) query_kwargs = {'request_string': QUERY, 'variables': {'wFlows': [reg]}} result = pclient('graphql', query_kwargs) for workflow in result['workflows']: print(workflow['cylcVersion'])
def main(parser: COP, options: 'Values', reg: str, shutdown_arg: Optional[str] = None) -> None: if shutdown_arg is not None and options.kill: parser.error("ERROR: --kill is not compatible with [STOP]") if options.kill and options.now: parser.error("ERROR: --kill is not compatible with --now") if options.flow_label and int(options.max_polls) > 0: parser.error("ERROR: --flow is not compatible with --max-polls") reg, _ = parse_reg(reg) pclient = get_client(reg, timeout=options.comms_timeout) if int(options.max_polls) > 0: # (test to avoid the "nothing to do" warning for # --max-polls=0) spoller = StopPoller(pclient, "workflow stopped", options.interval, options.max_polls) # mode defaults to 'Clean' mode = None task = None cycle_point = None if shutdown_arg is not None and TaskID.is_valid_id(shutdown_arg): # STOP argument detected task = shutdown_arg elif shutdown_arg is not None: # not a task ID, may be a cycle point cycle_point = shutdown_arg elif options.kill: mode = WorkflowStopMode.Kill.name elif options.now > 1: mode = WorkflowStopMode.NowNow.name elif options.now: mode = WorkflowStopMode.Now.name mutation_kwargs = { 'request_string': MUTATION, 'variables': { 'wFlows': [reg], 'stopMode': mode, 'cyclePoint': cycle_point, 'clockTime': options.wall_clock, 'task': task, 'flowLabel': options.flow_label, } } pclient('graphql', mutation_kwargs) if int(options.max_polls) > 0 and not spoller.poll(): # (test to avoid the "nothing to do" warning for # --max-polls=0) sys.exit(1)
def main(parser: COP, options: 'Values', reg: str) -> None: """CLI for "cylc get-workflow-contact".""" reg, _ = parse_reg(reg) try: data = load_contact_file(reg) except ServiceFileError: raise CylcError( f"{reg}: cannot get contact info, workflow not running?") else: for key, value in sorted(data.items()): print("%s=%s" % (key, value))
def main(_, options: 'Values', reg: str) -> None: workflow, flow_file = parse_reg(reg, src=True) # extract task host platforms from the workflow config = WorkflowConfig( workflow, flow_file, options, load_template_vars(options.templatevars, options.templatevars_file)) platforms = { config.get_config(['runtime', name, 'platform']) for name in config.get_namespace_list('all tasks') } - {None, 'localhost'} # When "workflow run hosts" are formalised as "flow platforms" # we can substitute `localhost` for this, in the mean time # we will have to assume that flow hosts are configured correctly. if not platforms: sys.exit(0) verbose = cylc.flow.flags.verbosity > 0 # get the cylc version on each platform versions = {} for platform_name in sorted(platforms): platform = get_platform(platform_name) host = get_host_from_platform(platform, bad_hosts=None) cmd = construct_ssh_cmd(['version'], platform, host) if verbose: print(cmd) proc = procopen(cmd, stdin=DEVNULL, stdout=PIPE, stderr=PIPE) out, err = proc.communicate() out = out.decode() err = err.decode() if proc.wait() == 0: if verbose: print(" %s" % out) versions[platform_name] = out.strip() else: versions[platform_name] = f'ERROR: {err.strip()}' # report results max_len = max((len(platform_name) for platform_name in platforms)) print(f'{"platform".rjust(max_len)}: cylc version') print('-' * (max_len + 14)) for platform_name, result in versions.items(): print(f'{platform_name.rjust(max_len)}: {result}') if all((version == CYLC_VERSION for version in versions.values())): ret_code = 0 elif options.error: ret_code = 1 else: ret_code = 0 sys.exit(ret_code)
def main(parser: COP, options: 'Values', workflow: str) -> None: workflow, _ = parse_reg(workflow) pclient = WorkflowRuntimeClient(workflow, timeout=options.comms_timeout) mutation_kwargs = { 'request_string': MUTATION, 'variables': { 'wFlows': [workflow], } } pclient('graphql', mutation_kwargs)
def main(parser: COP, options: 'Values', *args: str) -> None: """CLI.""" if not args: return parser.error('No message supplied') if len(args) <= 2: # BACK COMPAT: args <= 2 # from: # 7.6? # remove at: # 9.0? # (As of Dec 2020 some functional tests still use the classic # two arg interface) workflow = os.getenv('CYLC_WORKFLOW_NAME') task_job = os.getenv('CYLC_TASK_JOB') message_strs = list(args) else: workflow, task_job, *message_strs = args workflow, _ = parse_reg(workflow) # Read messages from STDIN if '-' in message_strs: current_message_str = '' while True: # Note: `for line in sys.stdin:` can hang message_str = sys.stdin.readline() if message_str.strip(): # non-empty line current_message_str += message_str elif message_str: # empty line, start next message if current_message_str: message_strs.append(current_message_str) current_message_str = '' # reset else: # end of file if current_message_str: message_strs.append(current_message_str) break # Separate "severity: message" messages = [] # [(severity, message_str), ...] for message_str in message_strs: if message_str == '-': pass elif ':' in message_str: valid, err_msg = TaskMessageValidator.validate(message_str) if not valid: raise UserInputError( f'Invalid task message "{message_str}" - {err_msg}') messages.append( [item.strip() for item in message_str.split(':', 1)]) elif options.severity: messages.append([options.severity, message_str.strip()]) else: messages.append([getLevelName(INFO), message_str.strip()]) record_messages(workflow, task_job, messages)
def main(parser: COP, options: 'Values', workflow: str, *task_globs: str): workflow, _ = parse_reg(workflow) pclient = get_client(workflow, timeout=options.comms_timeout) mutation_kwargs = { 'request_string': MUTATION, 'variables': { 'wFlows': [workflow], 'tasks': list(task_globs), } } pclient('graphql', mutation_kwargs)
def main(parser: COP, opts: 'Values', reg: Optional[str] = None) -> None: run_dir: Optional[Path] if reg is None: try: reg = str(Path.cwd().relative_to( Path(get_cylc_run_dir()).resolve())) except ValueError: raise WorkflowFilesError( "The current working directory is not a workflow run directory" ) else: reg, _ = parse_reg(reg) run_dir = Path(get_workflow_run_dir(reg)) if not run_dir.is_dir(): raise WorkflowFilesError(f'"{reg}" is not an installed workflow.') source, source_symlink = get_workflow_source_dir(run_dir) if not source: raise WorkflowFilesError( f'"{reg}" was not installed with cylc install.') if not Path(source).is_dir(): raise WorkflowFilesError( f'Workflow source dir is not accessible: "{source}".\n' f'Restore the source or modify the "{source_symlink}"' ' symlink to continue.') for entry_point in iter_entry_points('cylc.pre_configure'): try: entry_point.resolve()(srcdir=source, opts=opts) except Exception as exc: # NOTE: except Exception (purposefully vague) # this is to separate plugin from core Cylc errors raise PluginError('cylc.pre_configure', entry_point.name, exc) from None reinstall_workflow( named_run=reg, rundir=run_dir, source=source, dry_run=False # TODO: ready for dry run implementation ) for entry_point in iter_entry_points('cylc.post_install'): try: entry_point.resolve()(srcdir=source, opts=opts, rundir=str(run_dir)) except Exception as exc: # NOTE: except Exception (purposefully vague) # this is to separate plugin from core Cylc errors raise PluginError('cylc.post_install', entry_point.name, exc) from None
def main(parser: COP, options: 'Values', reg: str, *task_globs: str) -> None: reg, _ = parse_reg(reg) pclient = get_client(reg, timeout=options.comms_timeout) mutation_kwargs = { 'request_string': MUTATION, 'variables': { 'wFlows': [reg], 'tasks': list(task_globs), 'outputs': options.outputs, } } pclient('graphql', mutation_kwargs)
def main( parser: COP, options: 'Values', workflow: str, task_id: Optional[str] = None ) -> None: workflow, _ = parse_reg(workflow) pclient = get_client(workflow, timeout=options.comms_timeout) if task_id and not TaskID.is_valid_id(task_id): raise UserInputError("Invalid task ID: %s" % task_id) flow_kwargs = { 'request_string': FLOW_QUERY, 'variables': {'wFlows': [workflow]} } task_kwargs: Dict[str, Any] = { 'request_string': TASK_QUERY, } # cylc ping WORKFLOW result = pclient('graphql', flow_kwargs) msg = "" for flow in result['workflows']: w_name = flow['name'] w_port = flow['port'] w_pub_port = flow['pubPort'] if cylc.flow.flags.verbosity > 0: sys.stdout.write( f'{w_name} running on ' f'{pclient.host}:{w_port} {w_pub_port}\n' ) # cylc ping WORKFLOW TASKID if task_id: task, point = TaskID.split(task_id) w_id = flow['id'] task_kwargs['variables'] = { 'tProxy': f'{w_id}{ID_DELIM}{point}{ID_DELIM}{task}' } task_result = pclient('graphql', task_kwargs) if not task_result.get('taskProxy'): msg = "task not found" elif task_result['taskProxy']['state'] != TASK_STATUS_RUNNING: msg = f"task not {TASK_STATUS_RUNNING}" if msg: print(cparse(f'<red>{msg}</red>')) sys.exit(1)
def main(parser: COP, options: 'Values', workflow: str, *task_globs: str): """CLI for "cylc trigger".""" if options.flow_descr and not options.reflow: parser.error("--meta requires --reflow") workflow, _ = parse_reg(workflow) pclient = get_client(workflow, timeout=options.comms_timeout) mutation_kwargs = { 'request_string': MUTATION, 'variables': { 'wFlows': [workflow], 'tasks': list(task_globs), 'reflow': options.reflow, 'flowDescr': options.flow_descr, } } pclient('graphql', mutation_kwargs)
def main(parser: COP, options: 'Values', reg: str, severity_str: str) -> None: try: severity = LOG_LEVELS[severity_str] except KeyError: parser.error("Illegal logging level, %s" % severity_str) reg, _ = parse_reg(reg) pclient = get_client(reg, timeout=options.comms_timeout) mutation_kwargs = { 'request_string': MUTATION, 'variables': { 'wFlows': [reg], 'level': severity, } } pclient('graphql', mutation_kwargs)
def main(_, options: 'Values', workflow: str, func: str) -> None: workflow, _ = parse_reg(workflow) pclient = WorkflowRuntimeClient(workflow, timeout=options.comms_timeout) if options.no_input: kwargs = {} else: kwargs = json.load(sys.stdin) sys.stdin.close() res = pclient(func, kwargs) if func in PB_METHOD_MAP: if 'element_type' in kwargs: pb_msg = PB_METHOD_MAP[func][kwargs['element_type']]() else: pb_msg = PB_METHOD_MAP[func]() pb_msg.ParseFromString(res) res_msg = MessageToDict(pb_msg) else: res_msg = res sys.stdout.write(json.dumps(res_msg, indent=4) + '\n')
def main(_, options: 'Values', reg: str) -> None: reg, _ = parse_reg(reg) screen = None if options.display == 'html': TREE_EXPAND_DEPTH[0] = -1 # expand tree fully screen = html_fragment.HtmlGenerator() screen.set_terminal_properties(256) screen.register_palette(TuiApp.palette) html_fragment.screenshot_init( [tuple(map(int, options.v_term_size.split(',')))], []) try: TuiApp(reg, screen=screen).main() if options.display == 'html': for fragment in html_fragment.screenshot_collect(): print(fragment) except KeyboardInterrupt: pass
def main(parser: COP, options: 'Values', reg: Optional[str] = None) -> None: if options.print_hierarchy: print("\n".join(get_config_file_hierarchy(reg))) return if reg is None: glbl_cfg().idump(options.item, not options.defaults, oneline=options.oneline, none_str=options.none_str) return workflow, flow_file = parse_reg(reg, src=True) config = WorkflowConfig(workflow, flow_file, options, get_template_vars(options, flow_file)) config.pcfg.idump(options.item, not options.defaults, oneline=options.oneline, none_str=options.none_str)
def main(parser: COP, options: 'Values', workflow: str, *task_globs: str): _validate(options, *task_globs) workflow, _ = parse_reg(workflow) pclient = get_client(workflow, timeout=options.comms_timeout) if options.release_all: mutation = RELEASE_HOLD_POINT_MUTATION args = {} else: mutation = RELEASE_MUTATION args = {'tasks': list(task_globs)} mutation_kwargs = { 'request_string': mutation, 'variables': { 'wFlows': [workflow], **args } } pclient('graphql', mutation_kwargs)
def main(parser: COP, options: 'Values', workflow: str) -> None: workflow, _ = parse_reg(workflow) output_options = [ options.show_raw, options.show_summary, options.html_summary ] if output_options.count(True) > 1: parser.error('Cannot combine output formats (choose one)') if not any(output_options): # No output specified - choose summary by default options.show_summary = True run_db = _get_dao(workflow) row_buf = format_rows(*run_db.select_task_times()) with smart_open(options.output_filename) as output: if options.show_raw: output.write(row_buf.getvalue()) else: summary: TimingSummary if options.show_summary: summary = TextTimingSummary(row_buf) elif options.html_summary: summary = HTMLTimingSummary(row_buf) summary.write_summary(output)
def main(_, options: 'Values', workflow: str) -> None: workflow, _ = parse_reg(workflow) pclient = get_client(workflow, timeout=options.comms_timeout) if options.sort_by_cycle: sort_args = {'keys': ['cyclePoint', 'name']} else: sort_args = {'keys': ['name', 'cyclePoint']} if options.disp_form == "raw": query = f''' {TASK_SUMMARY_FRAGMENT} {FAMILY_SUMMARY_FRAGMENT} {WORKFLOW_SUMMARY_FRAGMENT} query ($wFlows: [ID]!, $sortBy: SortArgs) {{ workflows (ids: $wFlows, stripNull: false) {{ ...wFlow taskProxies (sort: $sortBy) {{ ...tProxy }} familyProxies (sort: $sortBy) {{ ...fProxy }} }} }}''' elif options.disp_form != "tasks": query = f''' {WORKFLOW_SUMMARY_FRAGMENT} query ($wFlows: [ID]!) {{ workflows (ids: $wFlows, stripNull: false) {{ ...wFlow }} }}''' else: query = f''' {TASK_SUMMARY_FRAGMENT} query ($wFlows: [ID]!, $sortBy: SortArgs) {{ workflows (ids: $wFlows, stripNull: false) {{ taskProxies (sort: $sortBy) {{ ...tProxy }} }} }}''' query_kwargs = { 'request_string': query, 'variables': {'wFlows': [workflow], 'sortBy': sort_args} } workflows = pclient('graphql', query_kwargs) try: for summary in workflows['workflows']: if options.disp_form == "raw": if options.pretty: sys.stdout.write(json.dumps(summary, indent=4) + '\n') else: print(summary) else: if options.disp_form != "tasks": node_urls = { node['name']: node['meta']['URL'] for node in summary['tasks'] + summary['families'] } summary['workflowUrls'] = { node_name: node_urls[node_name] for node_name in summary['namespaceDefinitionOrder'] if node_name in node_urls } summary['workflowUrls']['workflow'] = ( summary['meta']['URL']) del summary['tasks'] del summary['families'] del summary['meta'] for key, value in sorted(summary.items()): print( f'{to_snake_case(key).replace("_", " ")}={value}') else: for item in summary['taskProxies']: if options.sort_by_cycle: values = [ item['cyclePoint'], item['name'], item['state']] else: values = [ item['name'], item['cyclePoint'], item['state']] values.append('held' if item['isHeld'] else 'unheld') values.append('queued' if item['isQueued'] else 'not-queued') values.append('runahead' if item['isRunahead'] else 'not-runahead') if options.flow: values.append(item['flowLabel']) print(', '.join(values)) except Exception as exc: raise CylcError( json.dumps(workflows, indent=4) + '\n' + str(exc) + '\n')
def main(_, options: 'Values', reg: str, *task_args: str) -> None: """Implement "cylc show" CLI.""" reg, _ = parse_reg(reg) pclient = get_client(reg, timeout=options.comms_timeout) json_filter = {} if not task_args: query = WORKFLOW_META_QUERY query_kwargs = { 'request_string': query, 'variables': { 'wFlows': [reg] } } # Print workflow info. results = pclient('graphql', query_kwargs) for reg in results['workflows']: flat_data = flatten_data(reg) if options.json: json_filter.update(flat_data) else: for key, value in sorted(flat_data.items(), reverse=True): ansiprint( f'<bold>{key}:</bold> {value or "<m>(not given)</m>"}') task_names = [arg for arg in task_args if TaskID.is_valid_name(arg)] task_ids = [arg for arg in task_args if TaskID.is_valid_id_2(arg)] if task_names: tasks_query = TASK_META_QUERY tasks_kwargs = { 'request_string': tasks_query, 'variables': { 'wFlows': [reg], 'taskIds': task_names } } # Print workflow info. results = pclient('graphql', tasks_kwargs) multi = len(results['tasks']) > 1 for task in results['tasks']: flat_data = flatten_data(task['meta']) if options.json: json_filter.update({task['name']: flat_data}) else: if multi: print(f'----\nTASK NAME: {task["name"]}') for key, value in sorted(flat_data.items(), reverse=True): ansiprint( f'<bold>{key}:</bold> {value or "<m>(not given)</m>"}') if task_ids: tp_query = TASK_PREREQS_QUERY tp_kwargs = { 'request_string': tp_query, 'variables': { 'wFlows': [reg], 'taskIds': [ f'{c}{ID_DELIM}{n}' for n, c in [ TaskID.split(t_id) for t_id in task_ids if TaskID.is_valid_id(t_id) ] ] + [ f'{c}{ID_DELIM}{n}' for c, n in [ t_id.rsplit(TaskID.DELIM2, 1) for t_id in task_ids if not TaskID.is_valid_id(t_id) ] ] } } results = pclient('graphql', tp_kwargs) multi = len(results['taskProxies']) > 1 for t_proxy in results['taskProxies']: task_id = TaskID.get(t_proxy['name'], t_proxy['cyclePoint']) if options.json: json_filter.update({task_id: t_proxy}) else: if multi: print(f'----\nTASK ID: {task_id}') prereqs = [] for item in t_proxy['prerequisites']: prefix = '' multi_cond = len(item['conditions']) > 1 if multi_cond: prereqs.append([ True, '', item['expression'].replace('c', ''), item['satisfied'] ]) for cond in item['conditions']: if multi_cond and not options.list_prereqs: prefix = f'\t{cond["exprAlias"].strip("c")} = ' _, _, point, name = cond['taskId'].split(ID_DELIM) cond_id = TaskID.get(name, point) prereqs.append([ False, prefix, f'{cond_id} {cond["reqState"]}', cond['satisfied'] ]) if options.list_prereqs: for composite, _, msg, _ in prereqs: if not composite: print(msg) else: flat_meta = flatten_data(t_proxy['task']['meta']) for key, value in sorted(flat_meta.items(), reverse=True): ansiprint(f'<bold>{key}:</bold>' f' {value or "<m>(not given)</m>"}') ansiprint('\n<bold>prerequisites</bold>' ' (<red>- => not satisfied</red>):') if not prereqs: print(' (None)') for _, prefix, msg, state in prereqs: print_msg_state(f'{prefix}{msg}', state) ansiprint('\n<bold>outputs</bold>' ' (<red>- => not completed</red>):') if not t_proxy['outputs']: print(' (None)') for output in t_proxy['outputs']: info = f'{task_id} {output["label"]}' print_msg_state(info, output['satisfied']) if (t_proxy['clockTrigger']['timeString'] or t_proxy['externalTriggers'] or t_proxy['xtriggers']): ansiprint('\n<bold>other</bold>' ' (<red>- => not satisfied</red>):') if t_proxy['clockTrigger']['timeString']: state = t_proxy['clockTrigger']['satisfied'] time_str = t_proxy['clockTrigger']['timeString'] print_msg_state('Clock trigger time reached', state) print(f' o Triggers at ... {time_str}') for ext_trig in t_proxy['externalTriggers']: state = ext_trig['satisfied'] print_msg_state(f'{ext_trig["label"]} ... {state}', state) for xtrig in t_proxy['xtriggers']: state = xtrig['satisfied'] print_msg_state( f'xtrigger "{xtrig["label"]} = {xtrig["id"]}"', state) if not results['taskProxies']: ansiprint(f"<red>No matching tasks found: {task_ids}", file=sys.stderr) sys.exit(1) if options.json: print(json.dumps(json_filter, indent=4))
def main(parser: COP, options: 'Values', reg: str) -> None: """cylc validate CLI.""" profiler = Profiler(None, options.profile_mode) profiler.start() if cylc.flow.flags.verbosity < 2: # for readability omit timestamps from logging unless in debug mode for handler in LOG.handlers: if isinstance(handler.formatter, CylcLogFormatter): handler.formatter.configure(timestamp=False) workflow, flow_file = parse_reg(reg, src=True) cfg = WorkflowConfig(workflow, flow_file, options, get_template_vars(options, flow_file), output_fname=options.output, mem_log_func=profiler.log_memory) # Check bounds of sequences out_of_bounds = [ str(seq) for seq in cfg.sequences if seq.get_first_point(cfg.start_point) is None ] if out_of_bounds: if len(out_of_bounds) > 1: # avoid spamming users with multiple warnings out_of_bounds_str = '\n'.join( textwrap.wrap(', '.join(out_of_bounds), 70)) msg = ("multiple sequences out of bounds for initial cycle point " f"{cfg.start_point}:\n{out_of_bounds_str}") else: msg = (f"{out_of_bounds[0]}: sequence out of bounds for " f"initial cycle point {cfg.start_point}") LOG.warning(msg) # Instantiate tasks and force evaluation of trigger expressions. # (Taken from config.py to avoid circular import problems.) # TODO - This is not exhaustive, it only uses the initial cycle point. if cylc.flow.flags.verbosity > 0: print('Instantiating tasks to check trigger expressions') flow_label = FlowLabelMgr().get_new_label() for name, taskdef in cfg.taskdefs.items(): try: itask = TaskProxy(taskdef, cfg.start_point, flow_label) except TaskProxySequenceBoundsError: # Should already failed above mesg = 'Task out of bounds for %s: %s\n' % (cfg.start_point, name) if cylc.flow.flags.verbosity > 0: sys.stderr.write(' + %s\n' % mesg) continue except Exception as exc: raise WorkflowConfigError('failed to instantiate task %s: %s' % (name, exc)) # force trigger evaluation now try: itask.state.prerequisites_eval_all() except TriggerExpressionError as exc: err = str(exc) if '@' in err: print( f"ERROR, {name}: xtriggers can't be in conditional" f" expressions: {err}", file=sys.stderr) else: print('ERROR, %s: bad trigger: %s' % (name, err), file=sys.stderr) raise WorkflowConfigError("ERROR: bad trigger") except Exception as exc: print(str(exc), file=sys.stderr) raise WorkflowConfigError('%s: failed to evaluate triggers.' % name) if cylc.flow.flags.verbosity > 0: print(' + %s ok' % itask.identity) print(cparse('<green>Valid for cylc-%s</green>' % CYLC_VERSION)) profiler.stop()
def main(parser: COP, options: 'Values', reg: str) -> None: workflow, flow_file = parse_reg(reg, src=True) template_vars = get_template_vars(options) if options.all_tasks and options.all_namespaces: parser.error("Choose either -a or -n") if options.all_tasks: which = "all tasks" elif options.all_namespaces: which = "all namespaces" elif options.prange: which = "prange" if options.prange == ",": tr_start = None tr_stop = None elif options.prange.endswith(","): tr_start = options.prange[:-1] tr_stop = None elif options.prange.startswith(","): tr_start = None tr_stop = options.prange[1:] else: tr_start, tr_stop = options.prange.split(',') else: which = "graphed tasks" if options.tree and os.environ['LANG'] == 'C' and options.box: print("WARNING, ignoring -t/--tree: $LANG=C", file=sys.stderr) options.tree = False if options.titles and options.mro: parser.error("Please choose --mro or --title, not both") if options.tree and any( [options.all_tasks, options.all_namespaces, options.mro]): print("WARNING: -t chosen, ignoring non-tree options.", file=sys.stderr) config = WorkflowConfig( workflow, flow_file, options, template_vars ) if options.tree: config.print_first_parent_tree( pretty=options.box, titles=options.titles) elif options.prange: for node in sorted(config.get_node_labels(tr_start, tr_stop)): print(node) else: result = config.get_namespace_list(which) namespaces = list(result) namespaces.sort() if (options.mro or options.titles): # compute padding maxlen = 0 for ns in namespaces: if len(ns) > maxlen: maxlen = len(ns) padding = maxlen * ' ' for ns in namespaces: if options.mro: print(ns, padding[0:len(padding) - len(ns)], end=' ') print(' '.join(config.get_mro(ns))) elif options.titles: print(ns, padding[0:len(padding) - len(ns)], end=' ') print(result[ns]) else: print(ns)
def main(parser: COP, options: 'Values', reg: str, task_id: Optional[str] = None, color: bool = False) -> None: """Implement cylc cat-log CLI. Determine log path, user@host, batchview_cmd, and action (print, dir-list, cat, edit, or tail), and then if the log path is: a) local: perform action on log path, or b) remote: re-invoke cylc cat-log as a) on the remote account """ if options.remote_args: # Invoked on job hosts for job logs only, as a wrapper to view_log(). # Tail and batchview commands from global config on workflow host). logpath, mode, tail_tmpl = options.remote_args[0:3] logpath = expand_path(logpath) tail_tmpl = expand_path(tail_tmpl) try: batchview_cmd = options.remote_args[3] except IndexError: batchview_cmd = None res = view_log(logpath, mode, tail_tmpl, batchview_cmd, remote=True, color=color) if res == 1: sys.exit(res) return workflow_name, _ = parse_reg(reg) # Get long-format mode. try: mode = MODES[options.mode] except KeyError: mode = options.mode if not task_id: # Cat workflow logs, local only. if options.filename is not None: raise UserInputError("The '-f' option is for job logs only.") logpath = get_workflow_run_log_name(workflow_name) if options.rotation_num: logs = glob('%s.*' % logpath) logs.sort(key=os.path.getmtime, reverse=True) try: logpath = logs[int(options.rotation_num)] except IndexError: raise UserInputError("max rotation %d" % (len(logs) - 1)) tail_tmpl = os.path.expandvars(get_platform()["tail command template"]) out = view_log(logpath, mode, tail_tmpl, color=color) if out == 1: sys.exit(1) if mode == 'edit': tmpfile_edit(out, options.geditor) return if task_id: # Cat task job logs, may be on workflow or job host. if options.rotation_num is not None: raise UserInputError("only workflow (not job) logs get rotated") try: task, point = TaskID.split(task_id) except ValueError: parser.error("Illegal task ID: %s" % task_id) if options.submit_num != NN: try: options.submit_num = "%02d" % int(options.submit_num) except ValueError: parser.error("Illegal submit number: %s" % options.submit_num) if options.filename is None: options.filename = JOB_LOG_OUT else: # Convert short filename args to long (e.g. 'o' to 'job.out'). with suppress(KeyError): options.filename = JOB_LOG_OPTS[options.filename] # KeyError: Is already long form (standard log, or custom). platform_name, job_runner_name, live_job_id = get_task_job_attrs( workflow_name, point, task, options.submit_num) platform = get_platform(platform_name) batchview_cmd = None if live_job_id is not None: # Job is currently running. Get special job runner log view # command (e.g. qcat) if one exists, and the log is out or err. conf_key = None if options.filename == JOB_LOG_OUT: if mode == 'cat': conf_key = "out viewer" elif mode == 'tail': conf_key = "out tailer" elif options.filename == JOB_LOG_ERR: if mode == 'cat': conf_key = "err viewer" elif mode == 'tail': conf_key = "err tailer" if conf_key is not None: batchview_cmd_tmpl = None with suppress(KeyError): batchview_cmd_tmpl = platform[conf_key] if batchview_cmd_tmpl is not None: batchview_cmd = batchview_cmd_tmpl % { "job_id": str(live_job_id) } log_is_remote = (is_remote_platform(platform) and (options.filename != JOB_LOG_ACTIVITY)) log_is_retrieved = (platform['retrieve job logs'] and live_job_id is None) if log_is_remote and (not log_is_retrieved or options.force_remote): logpath = os.path.normpath( get_remote_workflow_run_job_dir(workflow_name, point, task, options.submit_num, options.filename)) tail_tmpl = platform["tail command template"] # Reinvoke the cat-log command on the remote account. cmd = ['cat-log', *verbosity_to_opts(cylc.flow.flags.verbosity)] for item in [logpath, mode, tail_tmpl]: cmd.append('--remote-arg=%s' % shlex.quote(item)) if batchview_cmd: cmd.append('--remote-arg=%s' % shlex.quote(batchview_cmd)) cmd.append(workflow_name) is_edit_mode = (mode == 'edit') # TODO: Add Intelligent Host selection to this try: proc = remote_cylc_cmd(cmd, platform, capture_process=is_edit_mode, manage=(mode == 'tail')) except KeyboardInterrupt: # Ctrl-C while tailing. pass else: if is_edit_mode: # Write remote stdout to a temp file for viewing in editor. # Only BUFSIZE bytes at a time in case huge stdout volume. out = NamedTemporaryFile() data = proc.stdout.read(BUFSIZE) while data: out.write(data) data = proc.stdout.read(BUFSIZE) os.chmod(out.name, S_IRUSR) out.seek(0, 0) else: # Local task job or local job log. logpath = os.path.normpath( get_workflow_run_job_dir(workflow_name, point, task, options.submit_num, options.filename)) tail_tmpl = os.path.expandvars(platform["tail command template"]) out = view_log(logpath, mode, tail_tmpl, batchview_cmd, color=color) if mode != 'edit': sys.exit(out) if mode == 'edit': tmpfile_edit(out, options.geditor)
def main(_, options: 'Values', workflow: str) -> None: """Implement cylc broadcast.""" workflow, _ = parse_reg(workflow) pclient = get_client(workflow, timeout=options.comms_timeout) mutation_kwargs: Dict[str, Any] = { 'request_string': MUTATION, 'variables': { 'wFlows': [workflow], 'bMode': 'Set', 'cPoints': options.point_strings, 'nSpaces': options.namespaces, 'bSettings': options.settings, 'bCutoff': options.expire, } } query_kwargs: Dict[str, Any] = { 'request_string': QUERY, 'variables': { 'wFlows': [workflow], 'nIds': [] } } if options.show or options.showtask: if options.showtask: try: task, point = TaskID.split(options.showtask) query_kwargs['variables']['nIds'] = [ f'{point}{ID_DELIM}{task}' ] except ValueError: raise UserInputError("TASKID must be " + TaskID.SYNTAX) result = pclient('graphql', query_kwargs) for wflow in result['workflows']: settings = wflow['broadcasts'] padding = get_padding(settings) * ' ' if options.raw: print(str(settings)) else: print_tree(settings, padding, options.unicode) sys.exit(0) report_cancel = True report_set = False if options.clear: mutation_kwargs['variables']['bMode'] = 'Clear' if options.expire: mutation_kwargs['variables']['bMode'] = 'Expire' # implement namespace and cycle point defaults here namespaces = options.namespaces if not namespaces: namespaces = ["root"] point_strings = options.point_strings if not point_strings: point_strings = ["*"] if options.cancel or options.cancel_files: settings = [] for option_item in options.cancel: if "=" in option_item: raise UserInputError( "--cancel=[SEC]ITEM does not take a value") option_item = option_item.strip() setting = get_rdict(option_item) settings.append(setting) files_to_settings(settings, options.cancel_files, options.cancel) mutation_kwargs['variables'].update({ 'bMode': 'Clear', 'cPoints': point_strings, 'nSpaces': namespaces, 'bSettings': settings, }) if options.settings or options.setting_files: settings = [] for option_item in options.settings: if "=" not in option_item: raise UserInputError("--set=[SEC]ITEM=VALUE requires a value") lhs, rhs = [s.strip() for s in option_item.split("=", 1)] setting = get_rdict(lhs, rhs) settings.append(setting) files_to_settings(settings, options.setting_files) mutation_kwargs['variables'].update({ 'bMode': 'Set', 'cPoints': point_strings, 'nSpaces': namespaces, 'bSettings': settings, }) report_cancel = False report_set = True results = pclient('graphql', mutation_kwargs) for result in results['broadcast']['result']: modified_settings = result['response'][0] bad_options = result['response'][1] if modified_settings: print( get_broadcast_change_report(modified_settings, is_cancel=report_cancel)) sys.exit(report_bad_options(bad_options, is_set=report_set))
def main(parser: COP, options: 'Values', reg: str) -> None: workflow, flow_file = parse_reg(reg, src=True) if options.geditor: editor = glbl_cfg().get(['editors', 'gui']) else: editor = glbl_cfg().get(['editors', 'terminal']) # read in the flow.cylc file viewcfg = { 'mark': options.mark, 'single': options.single, 'label': options.label, 'empy': options.empy or options.process, 'jinja2': options.jinja2 or options.process, 'contin': options.cat or options.process, 'inline': (options.inline or options.jinja2 or options.empy or options.process), } lines = read_and_proc(flow_file, load_template_vars(options.templatevars, options.templatevars_file), viewcfg=viewcfg) if options.stdout: for line in lines: print(line) sys.exit(0) # write to a temporary file viewfile = NamedTemporaryFile( suffix=".flow.cylc", prefix=workflow.replace('/', '_') + '.', ) for line in lines: viewfile.write((line + '\n').encode()) viewfile.seek(0, 0) # set the file to be read only os.chmod(viewfile.name, 0o400) # capture the temp file's mod time in case the user edits it # and overrides the readonly mode. modtime1 = os.stat(viewfile.name).st_mtime # in case editor has options, e.g. 'emacs -nw': command_list = shlex.split(editor) command_list.append(viewfile.name) command = ' '.join(command_list) # THIS BLOCKS UNTIL THE COMMAND COMPLETES retcode = call(command_list) # nosec (editor command is user configurable) if retcode != 0: # the command returned non-zero exist status raise CylcError(f'{command} failed: {retcode}') # !!!VIEWING FINISHED!!! # Did the user edit the file modtime2 = os.stat(viewfile.name).st_mtime if modtime2 > modtime1: print( "\nWARNING: YOU HAVE EDITED A TEMPORARY READ-ONLY COPY " f"OF THE WORKFLOW:\n {viewfile.name}\n", file=sys.stderr) # DONE viewfile.close()
def main(parser: COP, options: 'Values', workflow: str) -> None: workflow, _ = parse_reg(workflow) if options.use_task_point and options.cycle: raise UserInputError( "cannot specify a cycle point and use environment variable") if options.use_task_point: if "CYLC_TASK_CYCLE_POINT" not in os.environ: raise UserInputError("CYLC_TASK_CYCLE_POINT is not defined") options.cycle = os.environ["CYLC_TASK_CYCLE_POINT"] if options.offset and not options.cycle: raise UserInputError( "You must target a cycle point to use an offset") # Attempt to apply specified offset to the targeted cycle if options.offset: options.cycle = str(add_offset(options.cycle, options.offset)) # Exit if both task state and message are to being polled if options.status and options.msg: raise UserInputError("cannot poll both status and custom output") if options.msg and not options.task and not options.cycle: raise UserInputError("need a taskname and cyclepoint") # Exit if an invalid status is requested if (options.status and options.status not in TASK_STATUSES_ORDERED and options.status not in CylcWorkflowDBChecker.STATE_ALIASES): raise UserInputError(f"invalid status '{options.status}'") # this only runs locally if options.run_dir: run_dir = expand_path(options.run_dir) else: run_dir = get_cylc_run_dir() pollargs = { 'workflow': workflow, 'run_dir': run_dir, 'task': options.task, 'cycle': options.cycle, 'status': options.status, 'message': options.msg, } spoller = WorkflowPoller("requested state", options.interval, options.max_polls, args=pollargs) connected, formatted_pt = spoller.connect() if not connected: raise CylcError("cannot connect to the workflow DB") if options.status and options.task and options.cycle: # check a task status spoller.condition = options.status if not spoller.poll(): sys.exit(1) elif options.msg: # Check for a custom task output spoller.condition = "output: %s" % options.msg if not spoller.poll(): sys.exit(1) else: # just display query results spoller.checker.display_maps( spoller.checker.workflow_state_query( task=options.task, cycle=formatted_pt, status=options.status))
def scheduler_cli(options: 'Values', reg: str) -> None: """Run the workflow. This function should contain all of the command line facing functionality of the Scheduler, exit codes, logging, etc. The Scheduler itself should be a Python object you can import and run in a regular Python session so cannot contain this kind of functionality. """ # Parse workflow name but delay Cylc 7 suiter.rc deprecation warning # until after the start-up splash is printed. reg, _ = parse_reg(reg, warn_depr=False) try: detect_old_contact_file(reg) except ServiceFileError as exc: print(f"Resuming already-running workflow\n\n{exc}") pclient = WorkflowRuntimeClient(reg, timeout=options.comms_timeout) mutation_kwargs = { 'request_string': RESUME_MUTATION, 'variables': { 'wFlows': [reg] } } pclient('graphql', mutation_kwargs) sys.exit(0) # re-execute on another host if required _distribute(options.host) # print the start message if (cylc.flow.flags.verbosity > -1 and (options.no_detach or options.format == 'plain')): print(cparse(cylc_header())) if cylc.flow.flags.cylc7_back_compat: LOG.warning(SUITERC_DEPR_MSG) # setup the scheduler # NOTE: asyncio.run opens an event loop, runs your coro, # then shutdown async generators and closes the event loop scheduler = Scheduler(reg, options) asyncio.run(_setup(scheduler)) # daemonize if requested # NOTE: asyncio event loops cannot persist across daemonization # ensure you have tidied up all threads etc before daemonizing if not options.no_detach: from cylc.flow.daemonize import daemonize daemonize(scheduler) # setup loggers _open_logs(reg, options.no_detach) # run the workflow ret = asyncio.run(_run(scheduler)) # exit # NOTE: we must clean up all asyncio / threading stuff before exiting # NOTE: any threads which include sleep statements could cause # sys.exit to hang if not shutdown properly LOG.info("DONE") _close_logs() sys.exit(ret)