def main(parser, options, reg): suite, suiterc = parse_suite_arg(options, reg) if options.markup: prefix = '!cylc!' else: prefix = '' config = SuiteConfig( suite, suiterc, options, load_template_vars(options.templatevars, options.templatevars_file)) if options.tasks: for task in config.get_task_name_list(): print(prefix + task) elif options.alltasks: for task in config.get_task_name_list(): items = ['[runtime][' + task + ']' + i for i in options.item] print(prefix + task, end=' ') config.pcfg.idump(items, options.sparse, options.pnative, prefix, options.oneline, none_str=options.none_str) else: config.pcfg.idump(options.item, options.sparse, options.pnative, prefix, options.oneline, none_str=options.none_str)
def main(parser, options, *args): suite1, suite1rc = parse_suite_arg(options, args[0]) suite2, suite2rc = parse_suite_arg(options, args[1]) if suite1 == suite2: parser.error("You can't diff a single suite.") print("Parsing %s (%s)" % (suite1, suite1rc)) template_vars = load_template_vars( options.templatevars, options.templatevars_file) config1 = SuiteConfig(suite1, suite1rc, options, template_vars).cfg print("Parsing %s (%s)" % (suite2, suite2rc)) config2 = SuiteConfig( suite2, suite2rc, options, template_vars, is_reload=True).cfg if config1 == config2: print("Suite definitions %s and %s are identical" % (suite1, suite2)) sys.exit(0) print("Suite definitions %s and %s differ" % (suite1, suite2)) suite1_only = {} suite2_only = {} diff_1_2 = {} diffdict(config1, config2, suite1_only, suite2_only, diff_1_2) if n_oone > 0: print() msg = str(n_oone) + ' items only in ' + suite1 + ' (<)' print(msg) prdict(suite1_only, '<', nested=options.nested) if n_otwo > 0: print() msg = str(n_otwo) + ' items only in ' + suite2 + ' (>)' print(msg) prdict(suite2_only, '>', nested=options.nested) if n_diff > 0: print() msg = (str(n_diff) + ' common items differ ' + suite1 + '(<) ' + suite2 + '(>)') print(msg) prdict(diff_1_2, '', diff=True, nested=options.nested)
def main(parser, options, *args): suite1_name, suite1_config = parse_suite_arg(options, args[0]) suite2_name, suite2_config = parse_suite_arg(options, args[1]) if suite1_name == suite2_name: parser.error("You can't diff a single suite.") print(f"Parsing {suite1_name} ({suite1_config})") template_vars = load_template_vars(options.templatevars, options.templatevars_file) config1 = SuiteConfig(suite1_name, suite1_config, options, template_vars).cfg print(f"Parsing {suite2_name} ({suite2_config})") config2 = SuiteConfig(suite2_name, suite2_config, options, template_vars, is_reload=True).cfg if config1 == config2: print(f"Suite definitions {suite1_name} and {suite2_name} are " f"identical") sys.exit(0) print(f"Suite definitions {suite1_name} and {suite2_name} differ") suite1_only = {} suite2_only = {} diff_1_2 = {} diffdict(config1, config2, suite1_only, suite2_only, diff_1_2) if n_oone > 0: print(f'\n{n_oone} items only in {suite1_name} (<)') prdict(suite1_only, '<', nested=options.nested) if n_otwo > 0: print(f'\n{n_otwo} items only in {suite2_name} (>)') prdict(suite2_only, '>', nested=options.nested) if n_diff > 0: print(f'\n{n_diff} common items differ {suite1_name}(<) ' f'{suite2_name}(>)') prdict(diff_1_2, '', diff=True, nested=options.nested)
def main(_, options, *args): # suite name or file path suite, flow_file = parse_suite_arg(options, args[0]) # extract task host platforms from the suite config = SuiteConfig( suite, flow_file, options, load_template_vars(options.templatevars, options.templatevars_file)) platforms = { config.get_config(['runtime', name, 'platform']) for name in config.get_namespace_list('all tasks') } - {None, 'localhost'} # When "suite run hosts" are formalised as "flow platforms" # we can substitute `localhost` for this, in the mean time # we will have to assume that flow hosts are configured correctly. if not platforms: sys.exit(0) verbose = cylc.flow.flags.verbose # get the cylc version on each platform versions = {} for platform_name in sorted(platforms): platform = get_platform(platform_name) cmd = construct_platform_ssh_cmd(['version'], platform) if verbose: print(cmd) proc = procopen(cmd, stdin=DEVNULL, stdout=PIPE, stderr=PIPE) out, err = proc.communicate() out = out.decode() err = err.decode() if proc.wait() == 0: if verbose: print(" %s" % out) versions[platform_name] = out.strip() else: versions[platform_name] = f'ERROR: {err.strip()}' # report results max_len = max((len(platform_name) for platform_name in platforms)) print(f'{"platform".rjust(max_len)}: cylc version') print('-' * (max_len + 14)) for platform_name, result in versions.items(): print(f'{platform_name.rjust(max_len)}: {result}') if all((version == CYLC_VERSION for version in versions.values())): exit = 0 elif options.error: exit = 1 else: exit = 0 sys.exit(exit)
def main(parser, options, reg=None): if options.print_hierarchy: print("\n".join(get_config_file_hierarchy(reg))) return if reg is None: glbl_cfg().idump(options.item, sparse=options.sparse, oneline=options.oneline, none_str=options.none_str) return suite, flow_file = parse_suite_arg(options, reg) config = SuiteConfig( suite, flow_file, options, load_template_vars(options.templatevars, options.templatevars_file)) config.pcfg.idump(options.item, options.sparse, oneline=options.oneline, none_str=options.none_str)
def main(parser, options, reg): suite, suiterc = parse_suite_arg(options, reg) if options.geditor: editor = glbl_cfg().get(['editors', 'gui']) else: editor = glbl_cfg().get(['editors', 'terminal']) # read in the suite.rc file viewcfg = { 'mark': options.mark, 'single': options.single, 'label': options.label, 'empy': options.empy or options.process, 'jinja2': options.jinja2 or options.process, 'contin': options.cat or options.process, 'inline': (options.inline or options.jinja2 or options.empy or options.process), } lines = read_and_proc(suiterc, load_template_vars(options.templatevars, options.templatevars_file), viewcfg=viewcfg, asedit=options.asedit) if options.stdout: for line in lines: print(line) sys.exit(0) # write to a temporary file viewfile = NamedTemporaryFile( suffix=".suite.rc", prefix=suite.replace('/', '_') + '.', ) for line in lines: viewfile.write((line + '\n').encode()) viewfile.seek(0, 0) # set the file to be read only os.chmod(viewfile.name, 0o400) # capture the temp file's mod time in case the user edits it # and overrides the readonly mode. modtime1 = os.stat(viewfile.name).st_mtime # in case editor has options, e.g. 'emacs -nw': command_list = shlex.split(editor) command_list.append(viewfile.name) command = ' '.join(command_list) # THIS BLOCKS UNTIL THE COMMAND COMPLETES retcode = call(command_list) if retcode != 0: # the command returned non-zero exist status raise CylcError(f'{command} failed: {retcode}') # !!!VIEWING FINISHED!!! # Did the user edit the file modtime2 = os.stat(viewfile.name).st_mtime if modtime2 > modtime1: print() print('WARNING: YOU HAVE EDITED A TEMPORARY READ-ONLY SUITE COPY:', file=sys.stderr) print(viewfile.name, file=sys.stderr) print('In future use \'cylc [prep] edit\' to edit a suite.', file=sys.stderr) print() # DONE viewfile.close()
def main(_, options, reg): """cylc validate CLI.""" profiler = Profiler(None, options.profile_mode) profiler.start() if not cylc.flow.flags.debug: # for readability omit timestamps from logging unless in debug mode for handler in LOG.handlers: if isinstance(handler.formatter, CylcLogFormatter): handler.formatter.configure(timestamp=False) suite, flow_file = parse_suite_arg(options, reg) cfg = SuiteConfig( suite, flow_file, options, load_template_vars(options.templatevars, options.templatevars_file), output_fname=options.output, mem_log_func=profiler.log_memory) # Check bounds of sequences out_of_bounds = [str(seq) for seq in cfg.sequences if seq.get_first_point(cfg.start_point) is None] if out_of_bounds: if len(out_of_bounds) > 1: # avoid spamming users with multiple warnings msg = ('multiple sequences out of bounds for initial cycle point ' '%s:\n%s' % ( cfg.start_point, '\n'.join(textwrap.wrap(', '.join(out_of_bounds), 70)))) else: msg = '%s: sequence out of bounds for initial cycle point %s' % ( out_of_bounds[0], cfg.start_point) if options.strict: LOG.warning(msg) elif cylc.flow.flags.verbose: sys.stderr.write(' + %s\n' % msg) # Instantiate tasks and force evaluation of trigger expressions. # (Taken from config.py to avoid circular import problems.) # TODO - This is not exhaustive, it only uses the initial cycle point. if cylc.flow.flags.verbose: print('Instantiating tasks to check trigger expressions') flow_label = FlowLabelMgr().get_new_label() for name, taskdef in cfg.taskdefs.items(): try: itask = TaskProxy(taskdef, cfg.start_point, flow_label) except TaskProxySequenceBoundsError: # Should already failed above in strict mode. mesg = 'Task out of bounds for %s: %s\n' % (cfg.start_point, name) if cylc.flow.flags.verbose: sys.stderr.write(' + %s\n' % mesg) continue except Exception as exc: raise SuiteConfigError( 'failed to instantiate task %s: %s' % (name, exc)) # force trigger evaluation now try: itask.state.prerequisites_eval_all() except TriggerExpressionError as exc: err = str(exc) if '@' in err: print(f"ERROR, {name}: xtriggers can't be in conditional" f" expressions: {err}", file=sys.stderr) else: print('ERROR, %s: bad trigger: %s' % (name, err), file=sys.stderr) raise SuiteConfigError("ERROR: bad trigger") except Exception as exc: print(str(exc), file=sys.stderr) raise SuiteConfigError( '%s: failed to evaluate triggers.' % name) if cylc.flow.flags.verbose: print(' + %s ok' % itask.identity) print(cparse('<green>Valid for cylc-%s</green>' % CYLC_VERSION)) profiler.stop()
def main(parser, options, reg): suite, suiterc = parse_suite_arg(options, reg) if options.all_tasks and options.all_namespaces: parser.error("Choose either -a or -n") if options.all_tasks: which = "all tasks" elif options.all_namespaces: which = "all namespaces" elif options.crange: which = "crange" try: tr_start, tr_stop = options.crange.split(',') except ValueError: tr_start = tr_stop = options.crange else: which = "graphed tasks" if options.tree: if os.environ['LANG'] == 'C' and options.box: print("WARNING, ignoring -t/--tree: $LANG=C", file=sys.stderr) options.tree = False if options.titles and options.mro: parser.error("Please choose --mro or --title, not both") if options.tree and any( [options.all_tasks, options.all_namespaces, options.mro]): print("WARNING: -t chosen, ignoring non-tree options.", file=sys.stderr) config = SuiteConfig( suite, suiterc, options, load_template_vars(options.templatevars, options.templatevars_file)) if options.tree: config.print_first_parent_tree(pretty=options.box, titles=options.titles) elif options.crange: for node in sorted(config.get_node_labels(tr_start, tr_stop)): print(node) else: result = config.get_namespace_list(which) namespaces = list(result) namespaces.sort() if (options.mro or options.titles): # compute padding maxlen = 0 for ns in namespaces: if len(ns) > maxlen: maxlen = len(ns) padding = maxlen * ' ' for ns in namespaces: if options.mro: print(ns, padding[0:len(padding) - len(ns)], end=' ') print(' '.join(config.get_mro(ns))) elif options.titles: print(ns, padding[0:len(padding) - len(ns)], end=' ') print(result[ns]) else: print(ns)
def main(parser, options, *args): flow_file = parse_suite_arg(options, args[0])[1] if options.geditor: editor = glbl_cfg().get(['editors', 'gui']) else: editor = glbl_cfg().get(['editors', 'terminal']) suitedir = os.path.dirname(flow_file) if options.cleanup: # remove backup files left by inlined editing sessions cleanup(suitedir) sys.exit(0) if not options.inline: # plain old editing. # move to suite def dir os.chdir(suitedir) # edit the flow.cylc file if not os.path.isfile(flow_file): raise UserInputError(f'file not found: {flow_file}') # in case editor has options, e.g. 'emacs -nw': command_list = re.split(' ', editor) command_list.append(flow_file) command = ' '.join(command_list) # THIS BLOCKS UNTIL THE COMMAND COMPLETES retcode = call(command_list) if retcode != 0: # the command returned non-zero exist status raise CylcError(f'{command} failed: {retcode}') # !!!EDITING FINISHED!!! sys.exit(0) # read the flow.cylc file if os.path.isfile(flow_file): # back up the original backup(flow_file) # record original modtime modtimes[flow_file] = os.stat(flow_file).st_mtime # read the file h = open(flow_file, 'r') lines0 = h.readlines() h.close() if lines0[0].startswith('# !WARNING! CYLC EDIT INLINED'): print('WARNING: RECOVERING A PREVIOUSLY INLINED FILE') recovery = True lines = lines0 else: recovery = False lines = inline(lines0, suitedir, flow_file, for_edit=True) else: parser.error(f"File not found: {flow_file}") lines = [i.rstrip() for i in lines] # overwrite the (now backed up) original with the inlined file: h = open(flow_file, 'wb') for line in lines: h.write((line + '\n').encode()) h.close() print('PRE-EDIT BACKUPS:') for file in backups: src = re.sub(suitedir + '/', '', file) dst = re.sub(suitedir + '/', '', backups[file]) print(' + ' + src + ' ---> ' + dst) # in case editor has options, e.g. 'emacs -nw': command_list = re.split(' ', editor) command_list.append(flow_file) command = ' '.join(command_list) # THIS BLOCKS UNTIL THE COMMAND COMPLETES retcode = call(command_list) if retcode != 0: # the command returned non-zero exist status raise CylcError(f'{command} failed: {retcode}') print('EDITING DONE') # Now back up the inlined file in case of absolute disaster, so as the # user or his editor corrupting the inlined-include-file marker lines. inlined_flow_file_backup = ( suitedir + '/flow.cylc.INLINED.EDIT.' + get_current_time_string(override_use_utc=True, use_basic_format=True)) copy(flow_file, inlined_flow_file_backup) # read in the edited inlined file h = open(flow_file, 'r') lines = h.readlines() h.close() # split it back into separate files split_file(suitedir, lines, flow_file, recovery) print(f' + edited: {flow_file}') print(f' + backup: {inlined_flow_file_backup}') print('INCLUDE-FILES WRITTEN:') for file in newfiles: f = re.sub(suitedir + '/', '', file) if re.search(r'\.EDIT\.NEW\.', f): print(' + ' + f + ' (!!! WARNING: original changed on disk !!!)') else: print(' + ' + f)
def main(parser, options, reg, *patterns): suite, flow_file = parse_suite_arg(options, reg) # cylc search SUITE PATTERN pattern = '|'.join(patterns) suitedir = os.path.dirname(flow_file) if os.path.isfile(flow_file): h = open(flow_file, 'r') lines = h.readlines() h.close() lines = inline(lines, suitedir, flow_file, for_grep=True) else: parser.error(f"File not found: {flow_file}") sections = deque(['(top)']) line_count = 1 inc_file = None in_include_file = False prev_section_key = None prev_file = None for line in lines: m = re.match(r'^#\+\+\+\+ START INLINED INCLUDE FILE ([\w/\.\-]+)', line) if m: inc_file = m.groups()[0] in_include_file = True inc_line_count = 0 continue if not in_include_file: line_count += 1 else: inc_line_count += 1 m = re.match(r'^#\+\+\+\+ END INLINED INCLUDE FILE ' + inc_file, line) if m: in_include_file = False inc_file = None continue m = re.match(r'\s*(\[+\s*.+\s*\]+)', line) if m: # new section heading detected heading = m.groups()[0] level = section_level(heading) # unwind to the current section level while len(sections) > level - 1: sections.pop() sections.append(heading) continue if re.search(pattern, line): # Found a pattern match. # Print the file name if in_include_file: curr_file = os.path.join(suitedir, inc_file) line_no = inc_line_count else: curr_file = flow_file line_no = line_count if curr_file != prev_file: prev_file = curr_file print("\nFILE:", curr_file) # Print the nested section headings section_key = '->'.join(sections) if section_key != prev_section_key: prev_section_key = section_key print(' SECTION:', section_key) # Print the pattern match, with line number print(' (' + str(line_no) + '):', line.rstrip('\n')) if not options.search_bin: sys.exit(0) # search files in suite bin directory bin_ = os.path.join(suitedir, 'bin') if not os.path.isdir(bin_): print("\nSuite " + suite + " has no bin directory", file=sys.stderr) sys.exit(0) for name in os.listdir(bin_): if name.startswith('.'): # skip hidden dot-files # (e.g. vim editor temporary files) continue new_file = True try: h = open(os.path.join(bin_, name), 'r') except IOError as exc: # e.g. there's a sub-directory under bin; ignore it. print('Unable to open file ' + os.path.join(bin_, name), file=sys.stderr) print(exc, file=sys.stderr) continue contents = h.readlines() h.close() count = 0 for line in contents: line = line.rstrip('\n') count += 1 if re.search(pattern, line): if new_file: print('\nFILE:', os.path.join(bin_, name)) new_file = False print(' (' + str(count) + '): ' + line)
def main(_, options, *args): # suite name or file path suite, suiterc = parse_suite_arg(options, args[0]) # extract task host accounts from the suite config = SuiteConfig( suite, suiterc, options, load_template_vars(options.templatevars, options.templatevars_file)) account_set = set() for name in config.get_namespace_list('all tasks'): account_set.add( (config.get_config(['runtime', name, 'remote', 'owner']), config.get_config(['runtime', name, 'remote', 'host']))) task_remote_mgr = TaskRemoteMgr(suite, SubProcPool()) for _, host_str in account_set: task_remote_mgr.remote_host_select(host_str) accounts = [] while account_set: for user, host_str in account_set.copy(): res = task_remote_mgr.remote_host_select(host_str) if res: account_set.remove((user, host_str)) accounts.append((user, res)) if account_set: task_remote_mgr.proc_pool.process() sleep(1.0) # Interrogate the each remote account with CYLC_VERSION set to our version. # Post backward compatibility concerns to do this we can just run: # cylc version --host=HOST --user=USER # but this command only exists for version > 6.3.0. # So for the moment generate an actual remote invocation command string for # "cylc --version". # (save verbose flag as gets reset in remrun) verbose = cylc.flow.flags.verbose warn = {} contacted = 0 for user, host in sorted(accounts): argv = ["cylc", "version"] if user and host: argv += ["--user=%s" % user, "--host=%s" % host] user_at_host = "%s@%s" % (user, host) elif user: argv += ["--user=%s" % user] user_at_host = "%s@localhost" % user elif host: argv += ["--host=%s" % host] user_at_host = host if verbose: print("%s: %s" % (user_at_host, ' '.join(argv))) proc = procopen(argv, stdin=open(os.devnull), stdoutpipe=True, stderrpipe=True) out, err = proc.communicate() out = out.decode() err = err.decode() if proc.wait() == 0: if verbose: print(" %s" % out) contacted += 1 out = out.strip() if out != CYLC_VERSION: warn[user_at_host] = out else: print('ERROR ' + user_at_host + ':', file=sys.stderr) print(err, file=sys.stderr) # report results if not warn: if contacted: print("All", contacted, "accounts have cylc-" + CYLC_VERSION) else: print("WARNING: failed to invoke cylc-%s on %d accounts:" % (CYLC_VERSION, len(warn))) m = max(len(ac) for ac in warn) for ac, warning in warn.items(): print(' ', ac.ljust(m), warning) if options.error: sys.exit(1)
def get_config(flow, opts, template_vars=None): """Return a SuiteConfig object for the provided reg / path.""" flow, flow_file = parse_suite_arg(opts, flow) return SuiteConfig(flow, flow_file, opts, template_vars=template_vars)