def test_merge_rose_cylc_suite_install_conf(old, new, expect): loader = ConfigLoader() old = loader.load(StringIO(old)) new = loader.load(StringIO(new)) assert (loader.load(StringIO(expect)) == merge_rose_cylc_suite_install_conf(old, new) )
def get_conf(self): """Return the site/user configuration root node.""" if self.conf is None: # base system conf path paths = [self.SYST_CONF_PATH] # add $ROSE_SITE_CONF_PATH if defined if "ROSE_SITE_CONF_PATH" in os.environ: path_str = os.environ["ROSE_SITE_CONF_PATH"].strip() if path_str: paths.append(Path(path_str)) # add user conf path paths.append(self.USER_CONF_PATH) # use $ROSE_CONF_PATH (and ignore all others) if defined if "ROSE_CONF_PATH" in os.environ: paths_str = os.getenv("ROSE_CONF_PATH").strip() if paths_str: paths = [ Path(path) for path in paths_str.split(os.pathsep) ] else: paths = [] # load and cache config self.conf = ConfigNode() config_loader = ConfigLoader() for path in paths: conffile = path / self.ROSE_CONF if conffile.is_file() and os.access(conffile, os.R_OK): config_loader.load_with_opts(str(conffile), self.conf) return self.conf
def main(): """Implement the "rose config-dump" command.""" opt_parser = RoseOptionParser() opt_parser.add_my_options("conf_dir", "files", "no_pretty_mode") opts = opt_parser.parse_args()[0] verbosity = opts.verbosity - opts.quietness report = Reporter(verbosity) fs_util = FileSystemUtil(report) if opts.conf_dir: fs_util.chdir(opts.conf_dir) file_names = [] if opts.files: file_names = opts.files else: for dirpath, _, filenames in os.walk("."): for filename in fnmatch.filter(filenames, "rose-*.conf"): path = os.path.join(dirpath, filename)[2:] # remove leading ./ file_names.append(path) for file_name in file_names: handle = NamedTemporaryFile() node = ConfigLoader()(file_name) if (not opts.no_pretty_mode and os.path.basename(file_name) != META_CONFIG_NAME): pretty_format_config(node, ignore_error=True) ConfigDumper()(node, handle) handle.seek(0) if not filecmp.cmp(handle.name, file_name, shallow=False): report(ConfigDumpEvent(file_name)) ConfigDumper()(node, file_name)
def test_get_cli_opts_node(opt_confs, defines, rose_template_vars, expect): opts = SimpleNamespace(opt_conf_keys=opt_confs, defines=defines, rose_template_vars=rose_template_vars) loader = ConfigLoader() expect = loader.load(StringIO(expect)) result = get_cli_opts_node(opts) for item in ['env', 'template variables', 'opts']: assert result[item] == expect[item]
def test_get_cli_opts_node(opt_confs, defines, define_suites, expect): opts = SimpleNamespace(opt_conf_keys=opt_confs, defines=defines, define_suites=define_suites) loader = ConfigLoader() expect = loader.load(StringIO(expect)) result = get_cli_opts_node(opts) for item in ['env', 'jinja2:suite.rc', 'opts']: assert result[item] == expect[item]
def _load_info( self, repos, sid, branch=None, revision=None, transaction=None, allow_popen_err=False, ): """Load info file from branch_path in repos @revision. Returns a ConfigNode for the "rose-suite.info" of a suite at a particular revision or transaction. Args: repos (str): The path of the repository. sid (str): The Rosie suite unique identifier, e.g. "ay327". branch (str): The branch that the info file is under. revision (int, str): A commit revision number. Cannot be used with transaction. transaction (str): A commit transaction identifer. Cannot be used with revision. allow_popen_err (bool): If True, return None if a RosePopenError occurs during svnlook command. """ if not branch: branch = self.TRUNK commit_opts = [] if transaction is not None: if revision is not None: raise ValueError( f"Cannot load transaction {transaction} and " f"revision {revision} at the same time" ) commit_opts = ["-t", transaction] if revision is not None: commit_opts = ["-r", str(revision)] info_file_path = os.path.join("/".join(sid), branch, self.INFO_FILE) t_handle = TemporaryFile() try: t_handle.write( self._svnlook( "cat", repos, info_file_path, *commit_opts ).encode() ) except UnicodeDecodeError as err: raise ConfigDecodeError(info_file_path, err) except RosePopenError as err: if allow_popen_err: return None raise err t_handle.seek(0) config_node = ConfigLoader(allow_sections=False).load(t_handle) t_handle.close() return config_node
def _get_info(self, repos, path_head, txn=None): """Return a ConfigNode for the "rose-suite.info" of a suite. The suite is located under path_head. """ opt_txn = [] if txn is not None: opt_txn = ["-t", txn] t_handle = tempfile.TemporaryFile() path = path_head + self.TRUNK_INFO_FILE t_handle.write(self._svnlook("cat", repos, path, *opt_txn).encode()) t_handle.seek(0) info_node = ConfigLoader()(t_handle) t_handle.close() return info_node
def get_conf(self): """Return the site/user configuration root node.""" if self.conf is None: paths = [self.SITE_CONF_PATH, self.USER_CONF_PATH] if "ROSE_CONF_PATH" in os.environ: paths_str = os.getenv("ROSE_CONF_PATH").strip() if paths_str: paths = paths_str.split(os.pathsep) else: paths = [] self.conf = ConfigNode() config_loader = ConfigLoader() for path in paths: name = os.path.join(path, self.ROSE_CONF) if os.path.isfile(name) and os.access(name, os.R_OK): config_loader.load_with_opts(name, self.conf) return self.conf
def test_functional_record_cylc_install_options(monkeypatch, tmp_path, opts, files, env_inserts): """It works the way the proposal says it should. TODO: Once the the dump of the final rose-suite.conf is done then this should be expanded to test that too. """ # Pin down the results of the function used to provide a timestamp. def fake(*arg, **kwargs): return '18151210T0000Z' monkeypatch.setattr(DateTimeOperator, 'process_time_point_str', fake) testdir = tmp_path / 'test' refdir = tmp_path / 'ref' # Set up existing files, should these exist: for fname, content in files.items(): path = tmp_path / fname path.parent.mkdir(parents=True, exist_ok=True) path.write_text(content) # Set any environment variables we require: for envvar, val in env_inserts.items(): monkeypatch.setenv(envvar, val) loader = ConfigLoader() # Run the entry point top-level function: rose_suite_cylc_install_node, rose_suite_opts_node = \ record_cylc_install_options( rundir=testdir, opts=opts, srcdir=testdir ) rose_fileinstall(rundir=testdir, opts=opts, srcdir=testdir) ritems = sorted([i.relative_to(refdir) for i in refdir.rglob('*')]) titems = sorted([i.relative_to(testdir) for i in testdir.rglob('*')]) assert titems == ritems for counter, item in enumerate(titems): output = testdir / item reference = refdir / ritems[counter] if output.is_file(): assert_rose_conf_full_equal(loader.load(str(output)), loader.load(str(reference)), no_ignore=False)
def main(): """Implement the "rose config-dump" command.""" opt_parser = RoseOptionParser(description=''' Re-dump Rose configuration files in the common format. Load and dump `"rose-*.conf"` files in place. Apply format-specific pretty-printing. By default, it recursively loads and dumps all `rose-*.conf` files in the current working directory. EXAMPLES rose config-dump rose config-dump -C /path/to/conf/dir rose config-dump -f /path/to/file1 -f /path/to/file2 ''') opt_parser.add_my_options("conf_dir", "files", "no_pretty_mode") opts = opt_parser.parse_args()[0] verbosity = opts.verbosity - opts.quietness report = Reporter(verbosity) fs_util = FileSystemUtil(report) if opts.conf_dir: fs_util.chdir(opts.conf_dir) file_names = [] if opts.files: file_names = opts.files else: for dirpath, _, filenames in os.walk("."): for filename in fnmatch.filter(filenames, "rose-*.conf"): path = os.path.join(dirpath, filename)[2:] # remove leading ./ file_names.append(path) for file_name in file_names: handle = NamedTemporaryFile() node = ConfigLoader()(file_name) if (not opts.no_pretty_mode and os.path.basename(file_name) != META_CONFIG_NAME): pretty_format_config(node, ignore_error=True) ConfigDumper()(node, handle) handle.seek(0) if not filecmp.cmp(handle.name, file_name, shallow=False): report(ConfigDumpEvent(file_name)) ConfigDumper()(node, file_name)
def __init__(self, *args, **kwargs): self.node_loader = ConfigLoader(*args, **kwargs)
def record_cylc_install_options( rundir=None, opts=None, srcdir=None, ): """Create/modify files recording Cylc install config options. Creates a new config based on CLI options and writes it to the workflow install location as ``rose-suite-cylc-install.conf``. If ``rose-suite-cylc-install.conf`` already exists over-writes changed items, except for ``!opts=`` which is merged and simplified. If ``!opts=`` have been changed these are appended to those that have been written in the installed ``rose-suite.conf``. Args: srcdir (pathlib.Path): Used to check whether the source directory contains a rose config. opts: Cylc option parser object - we want to extract the following values: - opt_conf_keys (list or str): Equivelent of ``rose suite-run --option KEY`` - defines (list of str): Equivelent of ``rose suite-run --define KEY=VAL`` - suite_defines (list of str): Equivelent of ``rose suite-run --define-suite KEY=VAL`` rundir (pathlib.Path): Path to dump the rose-suite-cylc-conf Returns: cli_config - Config Node which has been dumped to ``rose-suite-cylc-install.conf``. rose_suite_conf['opts'] - Opts section of the config node dumped to installed ``rose-suite.conf``. """ # Create a config based on command line options: cli_config = get_cli_opts_node(opts) # Leave now if there is nothing to do: if not cli_config: return False # Construct path objects representing our target files. (Path(rundir) / 'opt').mkdir(exist_ok=True) conf_filepath = Path(rundir) / 'opt/rose-suite-cylc-install.conf' rose_conf_filepath = Path(rundir) / 'rose-suite.conf' dumper = ConfigDumper() loader = ConfigLoader() # If file exists we need to merge with our new config, over-writing with # new items where there are duplicates. if conf_filepath.is_file(): if opts.clear_rose_install_opts: conf_filepath.unlink() else: oldconfig = loader.load(str(conf_filepath)) cli_config = merge_rose_cylc_suite_install_conf( oldconfig, cli_config) cli_config.comments = [' This file records CLI Options.'] dumper.dump(cli_config, str(conf_filepath)) # Merge the opts section of the rose-suite.conf with those set by CLI: if not rose_conf_filepath.is_file(): rose_conf_filepath.touch() rose_suite_conf = loader.load(str(rose_conf_filepath)) rose_suite_conf = add_cylc_install_to_rose_conf_node_opts( rose_suite_conf, cli_config) dumper(rose_suite_conf, rose_conf_filepath) return cli_config, rose_suite_conf
def main(): """Implement the "rose config" command.""" opt_parser = RoseOptionParser() opt_parser.add_my_options("default", "env_var_process_mode", "files", "keys", "meta", "meta_key", "no_ignore", "no_opts", "print_conf_mode") opts, args = opt_parser.parse_args() report = Reporter(opts.verbosity - opts.quietness) metomi.rose.macro.add_meta_paths() if opts.meta_key: opts.meta = True if opts.files and opts.meta_key: report(Exception("Cannot specify both a file and meta key.")) sys.exit(1) config_loader = ConfigLoader() sources = [] if opts.files: root_node = ConfigNode() for fname in opts.files: if fname == "-": sources.append(sys.stdin) else: if opts.meta: try: root_node = config_loader.load(fname) except ConfigSyntaxError as exc: report(exc) sys.exit(1) rel_path = os.sep.join(fname.split(os.sep)[:-1]) fpath = get_meta_path(root_node, rel_path) if fpath is None: report(MetadataNotFoundEvent(fname)) else: sources.append(fpath) else: sources.append(fname) elif opts.meta: root_node = ConfigNode() if opts.meta_key: root_node.set(["meta"], opts.meta_key) else: fname = os.path.join(os.getcwd(), metomi.rose.SUB_CONFIG_NAME) try: root_node = config_loader.load(fname) except ConfigSyntaxError as exc: report(exc) sys.exit(1) fpath = get_meta_path(root_node, meta_key=opts.meta_key) root_node.unset(["meta"]) if fpath is None: report(Exception("Metadata not found")) sys.exit(1) else: sources.append(fpath) else: root_node = ResourceLocator.default().get_conf() for source in sources: try: if opts.meta or opts.no_opts: config_loader.load(source, root_node) else: config_loader.load_with_opts(source, root_node) except (ConfigSyntaxError, IOError) as exc: report(exc) sys.exit(1) if source is sys.stdin: source.close() if opts.quietness: sys.exit(root_node.get(args, opts.no_ignore) is None) if opts.keys_mode: try: keys = list(root_node.get(args, opts.no_ignore).value) except AttributeError: sys.exit(1) keys.sort() for key in keys: print(key) sys.exit() conf_dump = ConfigDumper() if len(args) == 0: conf_dump(root_node, concat_mode=opts.print_conf_mode) sys.exit() node = root_node.get(args, opts.no_ignore) if node is not None and isinstance(node.value, dict): if opts.print_conf_mode: conf_dump(ConfigNode().set(args, node.value), concat_mode=True) sys.exit() keys = list(node.value) keys.sort() for key in keys: node_of_key = node.get([key], opts.no_ignore) if node_of_key: value = node_of_key.value state = node_of_key.state string = "%s%s=%s" % (state, key, value) lines = string.splitlines() print(lines[0]) i_equal = len(state + key) + 1 for line in lines[1:]: print(" " * i_equal + line) sys.exit() if node is None: if opts.default is None: sys.exit(1) value = opts.default elif opts.env_var_process_mode: value = env_var_process(node.value) else: value = node.value if opts.print_conf_mode: conf_dump(ConfigNode().set(args, value), concat_mode=True) else: print(value) sys.exit()
def run_impl(self, opts, args, uuid, work_files): # Log file, temporary if hasattr(self.event_handler, "contexts"): t_file = TemporaryFile() log_context = ReporterContext(None, self.event_handler.VV, t_file) self.event_handler.contexts[uuid] = log_context # Check suite engine specific compatibility self.suite_engine_proc.check_global_conf_compat() # Suite name from the current working directory if opts.conf_dir: self.fs_util.chdir(opts.conf_dir) opts.conf_dir = os.getcwd() # --remote=KEY=VALUE,... if opts.remote: # opts.name always set for remote. return self._run_remote(opts, opts.name) conf_tree = self.config_load(opts) self.fs_util.chdir(conf_tree.conf_dirs[0]) suite_name = opts.name if not opts.name: suite_name = os.path.basename(os.getcwd()) # Check suite.rc #! line for template scheme templ_scheme = "jinja2" if self.suite_engine_proc.SUITE_CONF in conf_tree.files: suiterc_path = os.path.join( conf_tree.files[self.suite_engine_proc.SUITE_CONF], self.suite_engine_proc.SUITE_CONF) with open(suiterc_path) as fh: line = fh.readline() if line.startswith("#!"): templ_scheme = line[2:].strip().lower() suite_section = (templ_scheme + ':' + self.suite_engine_proc.SUITE_CONF) extra_defines = [] if opts.defines_suite: for define in opts.defines_suite: extra_defines.append("[" + suite_section + "]" + define) # Automatic Rose constants # ROSE_ORIG_HOST: originating host # ROSE_VERSION: Rose version (not retained in run_mode=="reload") # Suite engine version my_rose_version = ResourceLocator.default().get_version() suite_engine_key = self.suite_engine_proc.get_version_env_name() if opts.run_mode in ["reload", "restart"]: prev_config_path = self.suite_engine_proc.get_suite_dir( suite_name, "log", "rose-suite-run.conf") prev_config = ConfigLoader()(prev_config_path) suite_engine_version = prev_config.get_value( ["env", suite_engine_key]) else: suite_engine_version =\ self.suite_engine_proc.get_version().decode() resloc = ResourceLocator.default() auto_items = [(suite_engine_key, suite_engine_version), ("ROSE_ORIG_HOST", self.host_selector.get_local_host()), ("ROSE_SITE", resloc.get_conf().get_value(['site'], '')), ("ROSE_VERSION", resloc.get_version())] for key, val in auto_items: requested_value = conf_tree.node.get_value(["env", key]) if requested_value: if key == "ROSE_VERSION" and val != requested_value: exc = VersionMismatchError(requested_value, val) raise ConfigValueError(["env", key], requested_value, exc) val = requested_value else: conf_tree.node.set(["env", key], val, state=conf_tree.node.STATE_NORMAL) extra_defines.append('[%s]%s="%s"' % (suite_section, key, val)) # Pass automatic Rose constants as suite defines self.conf_tree_loader.node_loader.load(extra_defines, conf_tree.node) # See if suite is running or not if opts.run_mode == "reload": # Check suite is running self.suite_engine_proc.get_suite_contact(suite_name) else: self.suite_engine_proc.check_suite_not_running(suite_name) # Install the suite to its run location suite_dir_rel = self._suite_dir_rel(suite_name) # Unfortunately a large try/finally block to ensure a temporary folder # created in validate only mode is cleaned up. Exceptions are not # caught here try: # Process Environment Variables environ = self.config_pm(conf_tree, "env") if opts.validate_suite_only_mode: temp_dir = mkdtemp() suite_dir = os.path.join(temp_dir, suite_dir_rel) os.makedirs(suite_dir, 0o0700) else: suite_dir = os.path.join(os.path.expanduser("~"), suite_dir_rel) suite_conf_dir = os.getcwd() locs_conf = ConfigNode() if opts.new_mode: if os.getcwd() == suite_dir: raise NewModeError("PWD", os.getcwd()) elif opts.run_mode in ["reload", "restart"]: raise NewModeError("--run", opts.run_mode) self.suite_run_cleaner.clean(suite_name) if os.getcwd() != suite_dir: if opts.run_mode == "run": self._run_init_dir(opts, suite_name, conf_tree, locs_conf=locs_conf) os.chdir(suite_dir) # Housekeep log files now_str = None if not opts.install_only_mode and not opts.local_install_only_mode: now_str = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") self._run_init_dir_log(opts, now_str) self.fs_util.makedirs("log/suite") # Rose configuration and version logs self.fs_util.makedirs("log/rose-conf") run_mode = opts.run_mode if run_mode not in ["reload", "restart", "run"]: run_mode = "run" mode = run_mode if opts.validate_suite_only_mode: mode = "validate-suite-only" elif opts.install_only_mode: mode = "install-only" elif opts.local_install_only_mode: mode = "local-install-only" prefix = "rose-conf/%s-%s" % (strftime("%Y%m%dT%H%M%S"), mode) # Dump the actual configuration as rose-suite-run.conf ConfigDumper()(conf_tree.node, "log/" + prefix + ".conf") # Install version information file write_source_vc_info(suite_conf_dir, "log/" + prefix + ".version", self.popen) # If run through rose-stem, install version information # files for each source tree if they're a working copy if hasattr(opts, 'source') and hasattr(opts, 'project'): for i, url in enumerate(opts.source): if os.path.isdir(url): write_source_vc_info( url, "log/" + opts.project[i] + "-" + str(i) + ".version", self.popen) for ext in [".conf", ".version"]: self.fs_util.symlink(prefix + ext, "log/rose-suite-run" + ext) # Move temporary log to permanent log if hasattr(self.event_handler, "contexts"): log_file_path = os.path.abspath( os.path.join("log", "rose-suite-run.log")) log_file = open(log_file_path, "ab") temp_log_file = self.event_handler.contexts[uuid].handle temp_log_file.seek(0) log_file.write(temp_log_file.read()) self.event_handler.contexts[uuid].handle = log_file temp_log_file.close() # Process Files cwd = os.getcwd() for rel_path, conf_dir in conf_tree.files.items(): if (conf_dir == cwd or any( fnmatchcase(os.sep + rel_path, exclude) for exclude in self.SYNC_EXCLUDES) or conf_tree.node.get([templ_scheme + ":" + rel_path ]) is not None): continue # No sub-directories, very slow otherwise if os.sep in rel_path: rel_path = rel_path.split(os.sep, 1)[0] target_key = self.config_pm.get_handler( "file").PREFIX + rel_path target_node = conf_tree.node.get([target_key]) if target_node is None: conf_tree.node.set([target_key]) target_node = conf_tree.node.get([target_key]) elif target_node.is_ignored(): continue source_node = target_node.get("source") if source_node is None: target_node.set(["source"], os.path.join(conf_dir, rel_path)) elif source_node.is_ignored(): continue self.config_pm(conf_tree, "file", no_overwrite_mode=opts.no_overwrite_mode) # Process suite configuration template header # (e.g. Jinja2:suite.rc, EmPy:suite.rc) self.config_pm(conf_tree, templ_scheme, environ=environ) # Ask suite engine to parse suite configuration # and determine if it is up to date (unchanged) if opts.validate_suite_only_mode: suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf( suite_dir, None, opts.strict_mode, debug_mode=True) else: suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf( suite_name, opts.run_mode, opts.strict_mode, opts.debug_mode) finally: # Ensure the temporary directory created is cleaned up regardless # of success or failure if opts.validate_suite_only_mode and os.path.exists(temp_dir): shutil.rmtree(temp_dir) # Only validating so finish now if opts.validate_suite_only_mode: return # Install share/work directories (local) for name in ["share", "share/cycle", "work"]: self._run_init_dir_work(opts, suite_name, name, conf_tree, locs_conf=locs_conf) if opts.local_install_only_mode: return # Install suite files to each remote [user@]host for name in ["", "log/", "share/", "share/cycle/", "work/"]: uuid_file = os.path.abspath(name + uuid) open(uuid_file, "w").close() work_files.append(uuid_file) # Install items to user@host auths = self.suite_engine_proc.get_tasks_auths(suite_name) proc_queue = [] # [[proc, command, "ssh"|"rsync", auth], ...] for auth in sorted(auths): host = auth if "@" in auth: host = auth.split("@", 1)[1] # Remote shell command = self.popen.get_cmd("ssh", "-n", auth) # Provide ROSE_VERSION and CYLC_VERSION in the environment shcommand = "env ROSE_VERSION=%s %s=%s" % ( my_rose_version, suite_engine_key, suite_engine_version) # Use login shell? no_login_shell = self._run_conf("remote-no-login-shell", host=host, conf_tree=conf_tree) if not no_login_shell or no_login_shell.lower() != "true": shcommand += r""" bash -l -c '"$0" "$@"'""" # Path to "rose" command, if applicable rose_bin = self._run_conf("remote-rose-bin", host=host, conf_tree=conf_tree, default="rose") # Build remote "rose suite-run" command shcommand += " %s suite-run -vv -n %s" % (rose_bin, suite_name) for key in ["new", "debug", "install-only"]: attr = key.replace("-", "_") + "_mode" if getattr(opts, attr, None) is not None: shcommand += " --%s" % key if opts.log_keep: shcommand += " --log-keep=%s" % opts.log_keep if opts.log_name: shcommand += " --log-name=%s" % opts.log_name if not opts.log_archive_mode: shcommand += " --no-log-archive" shcommand += " --run=%s" % opts.run_mode # Build --remote= option shcommand += " --remote=uuid=%s" % uuid if now_str is not None: shcommand += ",now-str=%s" % now_str host_confs = [ "root-dir", "root-dir{share}", "root-dir{share/cycle}", "root-dir{work}" ] locs_conf.set([auth]) for key in host_confs: value = self._run_conf(key, host=host, conf_tree=conf_tree) if value is not None: val = self.popen.list_to_shell_str([str(value)]) shcommand += ",%s=%s" % (key, pipes.quote(val)) locs_conf.set([auth, key], value) command.append(shcommand) proc = self.popen.run_bg(*command) proc_queue.append([proc, command, "ssh", auth]) while proc_queue: sleep(self.SLEEP_PIPE) proc, command, command_name, auth = proc_queue.pop(0) if proc.poll() is None: # put it back in proc_queue proc_queue.append([proc, command, command_name, auth]) continue ret_code = proc.wait() out, err = proc.communicate() ret_code, out, err = [ i.decode() if isinstance(i, bytes) else i for i in [ret_code, out, err] ] if ret_code: raise RosePopenError(command, ret_code, out, err) if command_name == "rsync": self.handle_event(out, level=Event.VV) continue else: self.handle_event(out, level=Event.VV, prefix="[%s] " % auth) for line in out.split("\n"): if "/" + uuid == line.strip(): locs_conf.unset([auth]) break else: filters = {"excludes": [], "includes": []} for name in ["", "log/", "share/", "share/cycle/", "work/"]: filters["excludes"].append(name + uuid) target = auth + ":" + suite_dir_rel cmd = self._get_cmd_rsync(target, **filters) proc_queue.append( [self.popen.run_bg(*cmd), cmd, "rsync", auth]) # Install ends ConfigDumper()(locs_conf, os.path.join("log", "rose-suite-run.locs")) if opts.install_only_mode: return elif opts.run_mode == "reload" and suite_conf_unchanged: conf_name = self.suite_engine_proc.SUITE_CONF self.handle_event(SkipReloadEvent(suite_name, conf_name)) return # Start the suite self.fs_util.chdir("log") self.suite_engine_proc.run(suite_name, opts.host, opts.run_mode, args) # Disconnect log file handle, so monitoring tool command will no longer # be associated with the log file. self.event_handler.contexts[uuid].handle.close() self.event_handler.contexts.pop(uuid) return 0
def main(): """Implement the "rose config" command.""" opt_parser = RoseOptionParser(description=''' Parse and print rose configuration files. With no option and no argument, print the rose site + user configuration. EXAMPLES # Print the value of OPTION in SECTION. rose config SECTION OPTION # Print the value of OPTION in SECTION in FILE. rose config --file=FILE SECTION OPTION # Print the value of OPTION in SECTION if exists, or VALUE otherwise. rose config --default=VALUE SECTION OPTION # Print the OPTION=VALUE pairs in SECTION. rose config SECTION # Print the value of a top level OPTION. rose config OPTION # Print the OPTION keys in SECTION. rose config --keys SECTION # Print the SECTION keys. rose config --keys # Exit with 0 if OPTION exists in SECTION, or 1 otherwise. rose config -q SECTION OPTION # Exit with 0 if SECTION exists, or 1 otherwise. rose config -q SECTION # Combine the configurations in FILE1 and FILE2, and dump the result. rose config --file=FILE1 --file=FILE2 # Print the value of OPTION in SECTION of the metadata associated with # the specified config FILE rose config --file=FILE --meta SECTION OPTION # Print the value of a specified metadata KEY rose config --meta-key=KEY ''', epilog=''' ENVIRONMENT VARIABLES optional ROSE_META_PATH Prepend `$ROSE_META_PATH` to the metadata search path. ''') opt_parser.add_my_options( "default", "env_var_process_mode", "files", "keys", "meta", "meta_key", "no_ignore", "no_opts", "print_conf_mode", ) # the quietness argument is non-standard for this command opt_parser.modify_option( 'quietness', help=("Exit with 0 if the specified `SECTION` and/or `OPTION` exist in" " the configuration, or 1 otherwise."), ) opts, args = opt_parser.parse_args() report = Reporter(opts.verbosity - opts.quietness) metomi.rose.macro.add_meta_paths() if opts.meta_key: opts.meta = True if opts.files and opts.meta_key: report(Exception("Cannot specify both a file and meta key.")) sys.exit(1) config_loader = ConfigLoader() sources = [] if opts.files: root_node = ConfigNode() for fname in opts.files: if fname == "-": sources.append(sys.stdin) else: if opts.meta: try: root_node = config_loader.load(fname) except ConfigSyntaxError as exc: report(exc) sys.exit(1) rel_path = os.sep.join(fname.split(os.sep)[:-1]) fpath = get_meta_path(root_node, rel_path) if fpath is None: report(MetadataNotFoundEvent(fname)) else: sources.append(fpath) else: sources.append(fname) elif opts.meta: root_node = ConfigNode() if opts.meta_key: root_node.set(["meta"], opts.meta_key) else: fname = os.path.join(os.getcwd(), metomi.rose.SUB_CONFIG_NAME) try: root_node = config_loader.load(fname) except ConfigSyntaxError as exc: report(exc) sys.exit(1) fpath = get_meta_path(root_node, meta_key=opts.meta_key) root_node.unset(["meta"]) if fpath is None: report(Exception("Metadata not found")) sys.exit(1) else: sources.append(fpath) else: root_node = ResourceLocator.default().get_conf() for source in sources: try: if opts.meta or opts.no_opts: config_loader.load(source, root_node) else: config_loader.load_with_opts(source, root_node) except (ConfigSyntaxError, IOError) as exc: report(exc) sys.exit(1) if source is sys.stdin: source.close() if opts.quietness: sys.exit(root_node.get(args, opts.no_ignore) is None) if opts.keys_mode: try: keys = list(root_node.get(args, opts.no_ignore).value) except AttributeError: sys.exit(1) keys.sort() for key in keys: print(key) sys.exit() conf_dump = ConfigDumper() if len(args) == 0: conf_dump(root_node, concat_mode=opts.print_conf_mode) sys.exit() node = root_node.get(args, opts.no_ignore) if node is not None and isinstance(node.value, dict): if opts.print_conf_mode: conf_dump(ConfigNode().set(args, node.value), concat_mode=True) sys.exit() keys = list(node.value) keys.sort() for key in keys: node_of_key = node.get([key], opts.no_ignore) if node_of_key: value = node_of_key.value state = node_of_key.state string = "%s%s=%s" % (state, key, value) lines = string.splitlines() print(lines[0]) i_equal = len(state + key) + 1 for line in lines[1:]: print(" " * i_equal + line) sys.exit() if node is None: if opts.default is None: sys.exit(1) value = opts.default elif opts.env_var_process_mode: value = env_var_process(node.value) else: value = node.value if opts.print_conf_mode: conf_dump(ConfigNode().set(args, value), concat_mode=True) else: print(value) sys.exit()
def _clean(self, suite_name, only_items=None): """Perform the cleaning operations.""" engine = self.suite_engine_proc suite_dir_rel = engine.get_suite_dir_rel(suite_name) locs_file_path = engine.get_suite_dir(suite_name, "log", "rose-suite-run.locs") locs_conf = ConfigNode().set(["localhost"], {}) try: ConfigLoader().load(locs_file_path, locs_conf) except IOError: pass items = self.CLEANABLE_PATHS + [""] if only_items: items = only_items items.sort() uuid_str = str(uuid4()) for auth, node in sorted(locs_conf.value.items(), key=cmp_to_key(self._auth_node_cmp)): locs = [] roots = set([""]) for item in items: if item: locs.append(os.path.join(suite_dir_rel, item)) else: locs.append(suite_dir_rel) if item and os.path.normpath(item) in self.CLEANABLE_PATHS: item_root = node.get_value(["root-dir{" + item + "}"]) if item_root is None: # backward compat item_root = node.get_value(["root-dir-" + item]) elif item == "": item_root = node.get_value(["root-dir"]) else: continue if item_root: loc_rel = suite_dir_rel if item: loc_rel = os.path.join(suite_dir_rel, item) locs.append(os.path.join(item_root, loc_rel)) roots.add(item_root) locs.reverse() # Invoke bash as a login shell. The root location of a path may be # in $DIR syntax, which can only be expanded correctly in a login # shell. However, profile scripts invoked on login shell may print # lots of junks. Hence we use a UUID here as a delimiter. Only # output after the UUID lines are desirable lines. command = ["bash", "-l", "-O", "extglob", "-c"] sh_command = "cd; echo '%s'" % (uuid_str, ) if not self.host_selector.is_local_host(auth): command = engine.popen.get_cmd("ssh", auth) + command sh_command += "; ls -d -r %(locs)s; rm -fr %(locs)s" % { "locs": engine.popen.list_to_shell_str(locs) } if not only_items: # Clean empty directories # Change directory to root level to avoid cleaning them as # well For cylc suites, e.g. it can clean up to an empty # "cylc-run/" directory. for root in roots: names = [] # Reverse sort to ensure that e.g. "share/cycle/" is # cleaned before "share/" for name in sorted(self.CLEANABLE_PATHS, reverse=True): names.append(os.path.join(suite_dir_rel, name)) if os.sep in suite_dir_rel: names.append(os.path.dirname(suite_dir_rel)) sh_command += ( "; " + "(cd %(root)s; " + "rmdir -p %(names)s 2>/dev/null || true)") % { "root": root, "names": engine.popen.list_to_shell_str(names), } if self.host_selector.is_local_host(auth): command.append(sh_command) else: command.append(quote(sh_command)) is_after_uuid_str = False for line in engine.popen(*command)[0].splitlines(): line = line.decode() if is_after_uuid_str: engine.handle_event( FileSystemEvent(FileSystemEvent.DELETE, auth + ":" + line.strip())) elif line == uuid_str: is_after_uuid_str = True