def main(parser: COP, options: 'Values', workflow_id: str) -> None: workflow_id, *_ = parse_id( workflow_id, constraint='workflows', ) output_options = [ options.show_raw, options.show_summary, options.html_summary ] if output_options.count(True) > 1: parser.error('Cannot combine output formats (choose one)') if not any(output_options): # No output specified - choose summary by default options.show_summary = True run_db = _get_dao(workflow_id) row_buf = format_rows(*run_db.select_task_times()) with smart_open(options.output_filename) as output: if options.show_raw: output.write(row_buf.getvalue()) else: summary: TimingSummary if options.show_summary: summary = TextTimingSummary(row_buf) elif options.html_summary: summary = HTMLTimingSummary(row_buf) summary.write_summary(output)
def install( parser: COP, opts: 'Values', reg: Optional[str] = None ) -> None: if opts.no_run_name and opts.run_name: parser.error( "options --no-run-name and --run-name are mutually exclusive.") if reg is None: source = opts.source else: if opts.source: parser.error("REG and --directory are mutually exclusive.") source = search_install_source_dirs(reg) flow_name = opts.flow_name or reg for entry_point in iter_entry_points( 'cylc.pre_configure' ): try: if source: entry_point.resolve()(srcdir=source, opts=opts) else: from pathlib import Path entry_point.resolve()(srcdir=Path().cwd(), opts=opts) except Exception as exc: # NOTE: except Exception (purposefully vague) # this is to separate plugin from core Cylc errors raise PluginError( 'cylc.pre_configure', entry_point.name, exc ) from None source_dir, rundir, _flow_name = install_workflow( flow_name=flow_name, source=source, run_name=opts.run_name, no_run_name=opts.no_run_name, no_symlinks=opts.no_symlinks ) for entry_point in iter_entry_points( 'cylc.post_install' ): try: entry_point.resolve()( srcdir=source_dir, opts=opts, rundir=str(rundir) ) except Exception as exc: # NOTE: except Exception (purposefully vague) # this is to separate plugin from core Cylc errors raise PluginError( 'cylc.post_install', entry_point.name, exc ) from None
def main(parser: COP, options: 'Values', workflow_id1: str, workflow_id2: str): workflow_id_1, _, workflow_file_1_ = parse_id( workflow_id1, src=True, constraint='workflows', ) workflow_id_2, _, workflow_file_2_ = parse_id( workflow_id2, src=True, constraint='workflows', ) if workflow_file_1_ == workflow_file_2_: parser.error("You can't diff a single workflow.") print(f"Parsing {workflow_id_1} ({workflow_file_1_})") template_vars = load_template_vars( options.templatevars, options.templatevars_file ) config1 = WorkflowConfig( workflow_id_1, workflow_file_1_, options, template_vars ).cfg print(f"Parsing {workflow_id_2} ({workflow_file_2_})") config2 = WorkflowConfig( workflow_id_2, workflow_file_2_, options, template_vars, is_reload=True ).cfg if config1 == config2: print( f"Workflow definitions {workflow_id_1} and {workflow_id_2} are " f"identical" ) sys.exit(0) print(f"Workflow definitions {workflow_id_1} and {workflow_id_2} differ") workflow1_only = {} # type: ignore workflow2_only = {} # type: ignore diff_1_2 = {} # type: ignore # TODO: this whole file could do wih refactoring at some point diffdict(config1, config2, workflow1_only, workflow2_only, diff_1_2) if n_oone > 0: print(f'\n{n_oone} items only in {workflow_id_1} (<)') prdict(workflow1_only, '<', nested=options.nested) if n_otwo > 0: print(f'\n{n_otwo} items only in {workflow_id_2} (>)') prdict(workflow2_only, '>', nested=options.nested) if n_diff > 0: print(f'\n{n_diff} common items differ {workflow_id_1}(<) ' f'{workflow_id_2}(>)') prdict(diff_1_2, '', diff=True, nested=options.nested)
def main(parser: COP, options: 'Values', reg: str, severity_str: str) -> None: try: severity = LOG_LEVELS[severity_str] except KeyError: parser.error("Illegal logging level, %s" % severity_str) reg, _ = parse_reg(reg) pclient = get_client(reg, timeout=options.comms_timeout) mutation_kwargs = { 'request_string': MUTATION, 'variables': { 'wFlows': [reg], 'level': severity, } } pclient('graphql', mutation_kwargs)
def main(parser: COP, options: 'Values', workflow: str, *task_globs: str): """CLI for "cylc trigger".""" if options.flow_descr and not options.reflow: parser.error("--meta requires --reflow") workflow, _ = parse_reg(workflow) pclient = get_client(workflow, timeout=options.comms_timeout) mutation_kwargs = { 'request_string': MUTATION, 'variables': { 'wFlows': [workflow], 'tasks': list(task_globs), 'reflow': options.reflow, 'flowDescr': options.flow_descr, } } pclient('graphql', mutation_kwargs)
def main(parser: COP, options: 'Values', *args: str) -> None: """CLI.""" if not args: return parser.error('No message supplied') if len(args) <= 2: # BACK COMPAT: args <= 2 # from: # 7.6? # remove at: # 9.0? # (As of Dec 2020 some functional tests still use the classic # two arg interface) workflow_id = os.getenv('CYLC_WORKFLOW_ID') task_job = os.getenv('CYLC_TASK_JOB') message_strs = list(args) else: workflow_id, task_job, *message_strs = args workflow_id, *_ = parse_id( workflow_id, constraint='workflows', ) # Read messages from STDIN if '-' in message_strs: current_message_str = '' while True: # Note: `for line in sys.stdin:` can hang message_str = sys.stdin.readline() if message_str.strip(): # non-empty line current_message_str += message_str elif message_str: # empty line, start next message if current_message_str: message_strs.append(current_message_str) current_message_str = '' # reset else: # end of file if current_message_str: message_strs.append(current_message_str) break # Separate "severity: message" messages = [] # [(severity, message_str), ...] for message_str in message_strs: if message_str == '-': pass elif ':' in message_str: valid, err_msg = TaskMessageValidator.validate(message_str) if not valid: raise UserInputError( f'Invalid task message "{message_str}" - {err_msg}') messages.append( [item.strip() for item in message_str.split(':', 1)]) elif options.severity: messages.append([options.severity, message_str.strip()]) else: messages.append([getLevelName(INFO), message_str.strip()]) record_messages(workflow_id, task_job, messages)
def main(parser: COP, options: 'Values', reg: str, shutdown_arg: Optional[str] = None) -> None: if shutdown_arg is not None and options.kill: parser.error("ERROR: --kill is not compatible with [STOP]") if options.kill and options.now: parser.error("ERROR: --kill is not compatible with --now") if options.flow_label and int(options.max_polls) > 0: parser.error("ERROR: --flow is not compatible with --max-polls") reg, _ = parse_reg(reg) pclient = get_client(reg, timeout=options.comms_timeout) if int(options.max_polls) > 0: # (test to avoid the "nothing to do" warning for # --max-polls=0) spoller = StopPoller(pclient, "workflow stopped", options.interval, options.max_polls) # mode defaults to 'Clean' mode = None task = None cycle_point = None if shutdown_arg is not None and TaskID.is_valid_id(shutdown_arg): # STOP argument detected task = shutdown_arg elif shutdown_arg is not None: # not a task ID, may be a cycle point cycle_point = shutdown_arg elif options.kill: mode = WorkflowStopMode.Kill.name elif options.now > 1: mode = WorkflowStopMode.NowNow.name elif options.now: mode = WorkflowStopMode.Now.name mutation_kwargs = { 'request_string': MUTATION, 'variables': { 'wFlows': [reg], 'stopMode': mode, 'cyclePoint': cycle_point, 'clockTime': options.wall_clock, 'task': task, 'flowLabel': options.flow_label, } } pclient('graphql', mutation_kwargs) if int(options.max_polls) > 0 and not spoller.poll(): # (test to avoid the "nothing to do" warning for # --max-polls=0) sys.exit(1)
def main(parser: COP, options: 'Values', reg: str, task_id: Optional[str] = None, color: bool = False) -> None: """Implement cylc cat-log CLI. Determine log path, user@host, batchview_cmd, and action (print, dir-list, cat, edit, or tail), and then if the log path is: a) local: perform action on log path, or b) remote: re-invoke cylc cat-log as a) on the remote account """ if options.remote_args: # Invoked on job hosts for job logs only, as a wrapper to view_log(). # Tail and batchview commands from global config on workflow host). logpath, mode, tail_tmpl = options.remote_args[0:3] logpath = expand_path(logpath) tail_tmpl = expand_path(tail_tmpl) try: batchview_cmd = options.remote_args[3] except IndexError: batchview_cmd = None res = view_log(logpath, mode, tail_tmpl, batchview_cmd, remote=True, color=color) if res == 1: sys.exit(res) return workflow_name, _ = parse_reg(reg) # Get long-format mode. try: mode = MODES[options.mode] except KeyError: mode = options.mode if not task_id: # Cat workflow logs, local only. if options.filename is not None: raise UserInputError("The '-f' option is for job logs only.") logpath = get_workflow_run_log_name(workflow_name) if options.rotation_num: logs = glob('%s.*' % logpath) logs.sort(key=os.path.getmtime, reverse=True) try: logpath = logs[int(options.rotation_num)] except IndexError: raise UserInputError("max rotation %d" % (len(logs) - 1)) tail_tmpl = os.path.expandvars(get_platform()["tail command template"]) out = view_log(logpath, mode, tail_tmpl, color=color) if out == 1: sys.exit(1) if mode == 'edit': tmpfile_edit(out, options.geditor) return if task_id: # Cat task job logs, may be on workflow or job host. if options.rotation_num is not None: raise UserInputError("only workflow (not job) logs get rotated") try: task, point = TaskID.split(task_id) except ValueError: parser.error("Illegal task ID: %s" % task_id) if options.submit_num != NN: try: options.submit_num = "%02d" % int(options.submit_num) except ValueError: parser.error("Illegal submit number: %s" % options.submit_num) if options.filename is None: options.filename = JOB_LOG_OUT else: # Convert short filename args to long (e.g. 'o' to 'job.out'). with suppress(KeyError): options.filename = JOB_LOG_OPTS[options.filename] # KeyError: Is already long form (standard log, or custom). platform_name, job_runner_name, live_job_id = get_task_job_attrs( workflow_name, point, task, options.submit_num) platform = get_platform(platform_name) batchview_cmd = None if live_job_id is not None: # Job is currently running. Get special job runner log view # command (e.g. qcat) if one exists, and the log is out or err. conf_key = None if options.filename == JOB_LOG_OUT: if mode == 'cat': conf_key = "out viewer" elif mode == 'tail': conf_key = "out tailer" elif options.filename == JOB_LOG_ERR: if mode == 'cat': conf_key = "err viewer" elif mode == 'tail': conf_key = "err tailer" if conf_key is not None: batchview_cmd_tmpl = None with suppress(KeyError): batchview_cmd_tmpl = platform[conf_key] if batchview_cmd_tmpl is not None: batchview_cmd = batchview_cmd_tmpl % { "job_id": str(live_job_id) } log_is_remote = (is_remote_platform(platform) and (options.filename != JOB_LOG_ACTIVITY)) log_is_retrieved = (platform['retrieve job logs'] and live_job_id is None) if log_is_remote and (not log_is_retrieved or options.force_remote): logpath = os.path.normpath( get_remote_workflow_run_job_dir(workflow_name, point, task, options.submit_num, options.filename)) tail_tmpl = platform["tail command template"] # Reinvoke the cat-log command on the remote account. cmd = ['cat-log', *verbosity_to_opts(cylc.flow.flags.verbosity)] for item in [logpath, mode, tail_tmpl]: cmd.append('--remote-arg=%s' % shlex.quote(item)) if batchview_cmd: cmd.append('--remote-arg=%s' % shlex.quote(batchview_cmd)) cmd.append(workflow_name) is_edit_mode = (mode == 'edit') # TODO: Add Intelligent Host selection to this try: proc = remote_cylc_cmd(cmd, platform, capture_process=is_edit_mode, manage=(mode == 'tail')) except KeyboardInterrupt: # Ctrl-C while tailing. pass else: if is_edit_mode: # Write remote stdout to a temp file for viewing in editor. # Only BUFSIZE bytes at a time in case huge stdout volume. out = NamedTemporaryFile() data = proc.stdout.read(BUFSIZE) while data: out.write(data) data = proc.stdout.read(BUFSIZE) os.chmod(out.name, S_IRUSR) out.seek(0, 0) else: # Local task job or local job log. logpath = os.path.normpath( get_workflow_run_job_dir(workflow_name, point, task, options.submit_num, options.filename)) tail_tmpl = os.path.expandvars(platform["tail command template"]) out = view_log(logpath, mode, tail_tmpl, batchview_cmd, color=color) if mode != 'edit': sys.exit(out) if mode == 'edit': tmpfile_edit(out, options.geditor)
def main(parser: COP, options: 'Values', reg: str) -> None: workflow, flow_file = parse_reg(reg, src=True) template_vars = get_template_vars(options) if options.all_tasks and options.all_namespaces: parser.error("Choose either -a or -n") if options.all_tasks: which = "all tasks" elif options.all_namespaces: which = "all namespaces" elif options.prange: which = "prange" if options.prange == ",": tr_start = None tr_stop = None elif options.prange.endswith(","): tr_start = options.prange[:-1] tr_stop = None elif options.prange.startswith(","): tr_start = None tr_stop = options.prange[1:] else: tr_start, tr_stop = options.prange.split(',') else: which = "graphed tasks" if options.tree and os.environ['LANG'] == 'C' and options.box: print("WARNING, ignoring -t/--tree: $LANG=C", file=sys.stderr) options.tree = False if options.titles and options.mro: parser.error("Please choose --mro or --title, not both") if options.tree and any( [options.all_tasks, options.all_namespaces, options.mro]): print("WARNING: -t chosen, ignoring non-tree options.", file=sys.stderr) config = WorkflowConfig( workflow, flow_file, options, template_vars ) if options.tree: config.print_first_parent_tree( pretty=options.box, titles=options.titles) elif options.prange: for node in sorted(config.get_node_labels(tr_start, tr_stop)): print(node) else: result = config.get_namespace_list(which) namespaces = list(result) namespaces.sort() if (options.mro or options.titles): # compute padding maxlen = 0 for ns in namespaces: if len(ns) > maxlen: maxlen = len(ns) padding = maxlen * ' ' for ns in namespaces: if options.mro: print(ns, padding[0:len(padding) - len(ns)], end=' ') print(' '.join(config.get_mro(ns))) elif options.titles: print(ns, padding[0:len(padding) - len(ns)], end=' ') print(result[ns]) else: print(ns)