def _get_tests(conf_tree):
"""Return the poll tests configuration."""
poll_test = conf_tree.node.get_value(["poll", "test"])
poll_all_files_value = conf_tree.node.get_value(["poll", "all-files"])
poll_all_files = []
if poll_all_files_value:
try:
poll_all_files = shlex.split(
env_var_process(poll_all_files_value))
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError(["poll", "all-files"],
poll_all_files_value, exc)
poll_any_files_value = conf_tree.node.get_value(["poll", "any-files"])
poll_any_files = []
if poll_any_files_value:
try:
poll_any_files = shlex.split(
env_var_process(poll_any_files_value))
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError(["poll", "any-files"],
poll_any_files_value, exc)
poll_file_test = None
if poll_all_files or poll_any_files:
poll_file_test = conf_tree.node.get_value(["poll", "file-test"])
if poll_file_test and "{}" not in poll_file_test:
raise ConfigValueError(["poll", "file-test"], poll_file_test,
ConfigValueError.SYNTAX)
return poll_test, poll_file_test, poll_all_files, poll_any_files
def _run_init_dir(self,
opts,
suite_name,
conf_tree=None,
r_opts=None,
locs_conf=None):
"""Create the suite's directory."""
suite_dir_rel = self._suite_dir_rel(suite_name)
home = os.path.expanduser("~")
suite_dir_root = self._run_conf("root-dir",
conf_tree=conf_tree,
r_opts=r_opts)
if suite_dir_root:
if locs_conf is not None:
locs_conf.set(["localhost", "root-dir"], suite_dir_root)
suite_dir_root = env_var_process(suite_dir_root)
suite_dir_home = os.path.join(home, suite_dir_rel)
if (suite_dir_root and
os.path.realpath(home) != os.path.realpath(suite_dir_root)):
suite_dir_real = os.path.join(suite_dir_root, suite_dir_rel)
self.fs_util.makedirs(suite_dir_real)
self.fs_util.symlink(suite_dir_real, suite_dir_home,
opts.no_overwrite_mode)
else:
self.fs_util.makedirs(suite_dir_home)
def process(self,
conf_tree,
item,
orig_keys=None,
orig_value=None,
**kwargs):
"""Export environment variables in an [env] in "conf_tree.node"."""
env_node = conf_tree.node.get([item], no_ignore=True)
if env_node is None:
return
if "UNDEF" in os.environ:
os.environ.pop("UNDEF")
environ = {}
if env_node and not env_node.state:
for key, node in env_node.value.items():
if node.state:
continue
try:
environ[key] = env_var_process(node.value)
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError([item, key], node.value, exc)
environ[key] = os.path.expanduser(environ[key]) # ~ expansion
for key, value in sorted(environ.items()):
env_export(key, value, self.manager.event_handler)
return environ
def rose_env_cat(args, opts):
if not args:
args = ["-"]
if not opts.output_file or opts.output_file == "-":
out_handle = sys.stdout
else:
out_handle = open(opts.output_file, "w")
for arg in args:
if arg == "-":
in_handle = sys.stdin
else:
try:
in_handle = open(arg)
except FileNotFoundError as exc:
Reporter().report(exc)
if opts.debug_mode:
raise exc
return
line_num = 0
while True:
line_num += 1
line = in_handle.readline()
if not line:
break
try:
out_handle.write(
env_var_process(line, opts.unbound, opts.match_mode))
except UnboundEnvironmentVariableError as exc:
name = arg
if arg == "-":
name = "<STDIN>"
sys.exit("%s:%s: %s" % (name, line_num, str(exc)))
in_handle.close()
out_handle.close()
def main():
"""Implement "rose env-cat"."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options("match_mode", "output_file", "unbound")
opts, args = opt_parser.parse_args()
if not args:
args = ["-"]
if not opts.output_file or opts.output_file == "-":
out_handle = sys.stdout
else:
out_handle = open(opts.output_file, "wb")
for arg in args:
if arg == "-":
in_handle = sys.stdin
else:
in_handle = open(arg)
line_num = 0
while True:
line_num += 1
line = in_handle.readline()
if not line:
break
try:
out_handle.write(
env_var_process(line, opts.unbound, opts.match_mode))
except UnboundEnvironmentVariableError as exc:
name = arg
if arg == "-":
name = "<STDIN>"
sys.exit("%s:%s: %s" % (name, line_num, str(exc)))
in_handle.close()
out_handle.close()
def load_tasks(self):
"""Loads AnalysisTasks from files.
Given a list of files, return AnalysisTasks generated from those files.
This also expands environment variables in filenames, but saves the
original contents for use when writing out config files
"""
tasks = []
for task in self.config.value.keys():
if task is "env":
continue
if task.startswith("file:"):
continue
newtask = AnalysisTask()
newtask.name = task
value = self.config.get_value([task, "resultfile"])
# If the task is ignored, this will be None, so continue
# on to the next task
if value is None:
continue
if "{}" in value:
newtask.resultfile = value.replace("{}", self.args[0])
else:
newtask.resultfile = value
newtask = self._find_file("result", newtask)
newtask.extract = self.config.get_value([task, "extract"])
result = re.search(r":", newtask.extract)
if result:
newtask.subextract = ":".join(newtask.extract.split(":")[1:])
newtask.extract = newtask.extract.split(":")[0]
newtask.comparison = self.config.get_value([task, "comparison"])
newtask.tolerance = env_var_process(self.config.get_value(
[task, "tolerance"]))
newtask.warnonfail = (
self.config.get_value([task, "warnonfail"]) in ["yes", "true"])
# Allow for multiple KGO, e.g. kgo1file, kgo2file, for
# statistical comparisons of results
newtask.numkgofiles = 0
for i in range(1, MAX_KGO_FILES):
kgovar = "kgo" + str(i)
kgofilevar = kgovar + "file"
if self.config.get([task, kgofilevar]):
value = self.config.get([task, kgofilevar])[:]
if "{}" in value:
setattr(
newtask, kgofilevar,
value.replace("{}", self.args[0]))
else:
setattr(newtask, kgofilevar, value)
newtask.numkgofiles += 1
newtask = self._find_file(kgovar, newtask)
else:
break
tasks.append(newtask)
return tasks
def _run(self, dao, app_runner, config):
"""Transform and archive suite files.
This application is designed to work under "rose task-run" in a suite.
"""
compress_manager = SchemeHandlersManager(
[os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))],
"rose.apps.rose_arch_compressions",
["compress_sources"],
None, app_runner)
# Set up the targets
s_key_tails = set()
targets = []
for t_key, t_node in sorted(config.value.items()):
if t_node.is_ignored() or ":" not in t_key:
continue
s_key_head, s_key_tail = t_key.split(":", 1)
if s_key_head != self.SECTION or not s_key_tail:
continue
# Determine target path.
s_key_tail = t_key.split(":", 1)[1]
try:
s_key_tail = env_var_process(s_key_tail)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError([t_key, ""], "", exc)
# If parenthesised target is optional.
is_compulsory_target = True
if s_key_tail.startswith("(") and s_key_tail.endswith(")"):
s_key_tail = s_key_tail[1:-1]
is_compulsory_target = False
# Don't permit duplicate targets.
if s_key_tail in s_key_tails:
raise RoseArchDuplicateError([t_key], '', s_key_tail)
else:
s_key_tails.add(s_key_tail)
target = self._run_target_setup(
app_runner, compress_manager, config, t_key, s_key_tail,
t_node, is_compulsory_target)
old_target = dao.select(target.name)
if old_target is None or old_target != target:
dao.delete(target)
else:
target.status = target.ST_OLD
targets.append(target)
targets.sort(key=lambda target: target.name)
# Delete from database items that are no longer relevant
dao.delete_all(filter_targets=targets)
# Update the targets
for target in targets:
self._run_target_update(dao, app_runner, compress_manager, target)
return [target.status for target in targets].count(
RoseArchTarget.ST_BAD)
def _conf_value(conf_tree, keys, default=None):
"""Return conf setting value, with env var processed."""
value = conf_tree.node.get_value(keys, default)
if value is None:
return
try:
return env_var_process(value)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError(keys, value, exc)
def _get_conf_value(self, name, default=None):
"""Return the value of a named conf setting for this prefix."""
conf = ResourceLocator.default().get_conf()
value = conf.get_value(
["rosie-id", "prefix-%s.%s" % (name, self.prefix)], default=default
)
if value:
value = env_var_process(value)
return value
def main():
"""Implement "rose env-cat"."""
opt_parser = RoseOptionParser(usage='rose env-cat [OPTIONS] [FILE ...]',
description=r'''
Substitute environment variables in input files and print.
If no argument is specified, read from STDIN. One `FILE` argument may be
`-`, which means read from STDIN.
In `match-mode=default`, the command will look for `$NAME` or `${NAME}`
syntax and substitute them with the value of the environment variable
`NAME`. A backslash in front of the syntax, e.g. `\$NAME` or `\${NAME}`
will escape the substitution.
In `match-mode=brace`, the command will look for `${NAME}` syntax only.
EXAMPLES
rose env-cat [OPTIONS] [FILE ...]
''')
opt_parser.add_my_options("match_mode", "output_file", "unbound")
opt_parser.modify_option(
'output_file',
help=("Specify an output file."
"\nIf no output file is specified or if `FILE`"
"is `-`, write output to STDOUT."),
)
opts, args = opt_parser.parse_args()
if not args:
args = ["-"]
if not opts.output_file or opts.output_file == "-":
out_handle = sys.stdout
else:
out_handle = open(opts.output_file, "wb")
for arg in args:
if arg == "-":
in_handle = sys.stdin
else:
in_handle = open(arg)
line_num = 0
while True:
line_num += 1
line = in_handle.readline()
if not line:
break
try:
out_handle.write(
env_var_process(line, opts.unbound, opts.match_mode))
except UnboundEnvironmentVariableError as exc:
name = arg
if arg == "-":
name = "<STDIN>"
sys.exit("%s:%s: %s" % (name, line_num, str(exc)))
in_handle.close()
out_handle.close()
def _get_conf(self, r_node, t_node, key, compulsory=False, default=None):
"""Return the value of a configuration."""
value = t_node.get_value([key],
r_node.get_value([self.SECTION, key],
default=default))
if compulsory and not value:
raise CompulsoryConfigValueError([key], None, KeyError(key))
if value:
try:
value = env_var_process(value)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError([key], value, exc)
return value
def _find_file(self, var, task):
"""Finds a file given a variable name containing the filename.
Given a variable name and task object, this returns the filename it
points to, including expanding any * characters with glob.
"""
filevar = var + "file"
if hasattr(task, filevar):
configvar = var + "fileconfig"
setattr(task, configvar, getattr(task, filevar))
filenames = glob.glob(env_var_process(getattr(task, filevar)))
if len(filenames) > 0:
setattr(task, filevar, os.path.abspath(filenames[0]))
return task
def _run_init_dir_work(self,
opts,
suite_name,
name,
conf_tree=None,
r_opts=None,
locs_conf=None):
"""Create a named suite's directory."""
item_path = os.path.realpath(name)
item_path_source = item_path
key = "root-dir{" + name + "}"
item_root = self._run_conf(key, conf_tree=conf_tree, r_opts=r_opts)
if item_root is None: # backward compat
item_root = self._run_conf("root-dir-" + name,
conf_tree=conf_tree,
r_opts=r_opts)
if item_root:
if locs_conf is not None:
locs_conf.set(["localhost", key], item_root)
item_root = env_var_process(item_root)
suite_dir_rel = self._suite_dir_rel(suite_name)
if os.path.isabs(item_root):
item_path_source = os.path.join(item_root, suite_dir_rel, name)
else:
item_path_source = item_root
item_path_source = os.path.realpath(item_path_source)
if item_path == item_path_source:
if opts.new_mode:
self.fs_util.delete(name)
self.fs_util.makedirs(name)
else:
if opts.new_mode:
self.fs_util.delete(item_path_source)
self.fs_util.makedirs(item_path_source)
if os.sep in name:
dirname_of_name = os.path.dirname(name)
self.fs_util.makedirs(dirname_of_name)
item_path_source_rel = os.path.relpath(
item_path_source, os.path.realpath(dirname_of_name))
else:
item_path_source_rel = os.path.relpath(item_path_source)
if len(item_path_source_rel) < len(item_path_source):
self.fs_util.symlink(item_path_source_rel, name,
opts.no_overwrite_mode)
else:
self.fs_util.symlink(item_path_source, name,
opts.no_overwrite_mode)
def process(
self, conf_tree, item, orig_keys=None, orig_value=None, **kwargs
):
"""Install files according to [file:*] in conf_tree.
kwargs["no_overwrite_mode"]: fail if a target file already exists.
"""
# Find all the "file:*" nodes.
nodes = {}
if item == self.SCHEME:
for key, node in list(conf_tree.node.value.items()):
if node.is_ignored() or not key.startswith(self.PREFIX):
continue
nodes[key] = node
else:
node = conf_tree.node.get([item], no_ignore=True)
if node is None:
raise ConfigProcessError(orig_keys, item)
nodes[item] = node
if not nodes:
return
# Create database to store information for incremental updates,
# if it does not already exist.
loc_dao = LocDAO()
loc_dao.create()
cwd = os.getcwd()
file_install_root = conf_tree.node.get_value(
["file-install-root"], os.getenv("ROSE_FILE_INSTALL_ROOT", None)
)
if file_install_root:
file_install_root = env_var_process(file_install_root)
self.manager.fs_util.makedirs(file_install_root)
self.manager.fs_util.chdir(file_install_root)
try:
self._process(conf_tree, nodes, loc_dao, **kwargs)
finally:
if cwd != os.getcwd():
self.manager.fs_util.chdir(cwd)
async def pull(self, loc, conf_tree):
"""Write namelist to loc.cache."""
sections = self.parse(loc, conf_tree)
if loc.name.endswith("(:)"):
sections.sort(key=cmp_to_key(metomi.rose.config.sort_settings))
with open(loc.cache, "wb") as handle:
for section in sections:
section_value = conf_tree.node.get_value([section])
group = RE_NAMELIST_GROUP.match(section).group(1)
nlg = "&" + group + "\n"
for key, node in sorted(section_value.items()):
if node.state:
continue
try:
value = env_var_process(node.value)
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError([section, key], node.value,
exc)
nlg += "%s=%s,\n" % (key, value)
nlg += "/" + "\n"
handle.write(nlg.encode('UTF-8'))
self.manager.handle_event(NamelistEvent(nlg))
def get_rose_vars_from_config_node(config, config_node, environ):
"""Load template variables from a Rose config node.
This uses only the provided config node and environment variables
- there is no system interaction.
Args:
config (dict):
Object which will be populated with the results.
config_node (metomi.rose.config.ConfigNode):
Configuration node representing the Rose suite configuration.
environ (dict):
Dictionary of environment variables
"""
templating = None
# Don't allow multiple templating sections.
templating = identify_templating_section(config_node)
if templating != 'template variables':
config['templating_detected'] = templating.replace(':suite.rc', '')
else:
config['templating_detected'] = templating
# Create env section if it doesn't already exist.
if 'env' not in config_node.value:
config_node.set(['env'])
if templating not in config_node.value:
config_node.set([templating])
# Get Rose Orig host:
rose_orig_host = get_host()
# For each section process variables and add standard variables.
for section in ['env', templating]:
# This loop handles standard variables.
# CYLC_VERSION - If it's in the config, remove it.
# ROSE_VERSION - If it's in the config, replace it.
# ROSE_ORIG_HOST - If it's the config, replace it, unless it has a
# comment marking it as having been saved by ``cylc install``.
# In all cases warn users if the value in their config is not used.
for var_name, replace_with in [('ROSE_ORIG_HOST', rose_orig_host),
('ROSE_VERSION', ROSE_VERSION),
('CYLC_VERSION', SET_BY_CYLC)]:
# Warn if we're we're going to override a variable:
if override_this_variable(config_node, section, var_name):
user_var = config_node[section].value[var_name].value
LOG.warning(
f'[{section}]{var_name}={user_var} from rose-suite.conf '
f'will be ignored: {var_name} will be: {replace_with}')
# Handle replacement of stored variable if appropriate:
if replace_with == SET_BY_CYLC:
config_node[section].unset([var_name])
elif not rose_orig_host_set_by_cylc_install(
config_node, section, var_name):
config_node[section].set([var_name], replace_with)
# Use env_var_process to process variables which may need expanding.
for key, node in config_node.value[section].value.items():
try:
config_node.value[section].value[key].value = env_var_process(
node.value, environ=environ)
if section == 'env':
environ[key] = node.value
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError(['env', key], node.value, exc)
# For each of the template language sections extract items to a simple
# dict to be returned.
if 'env' in config_node.value:
config['env'] = {
item[0][1]: item[1].value
for item in config_node.value['env'].walk()
}
if templating in config_node.value:
config['template_variables'] = {
item[0][1]: item[1].value
for item in config_node.value[templating].walk()
}
elif 'template variables' in config_node.value:
config['template_variables'] = {
item[0][1]: item[1].value
for item in config_node.value['template variables'].walk()
}
# Add the entire config to ROSE_SUITE_VARIABLES to allow for programatic
# access.
if templating is not None:
with patch_jinja2_leading_zeros():
# BACK COMPAT: patch_jinja2_leading_zeros
# back support zero-padded integers for a limited time to help
# users migrate before upgrading cylc-flow to Jinja2>=3.1
parser = Parser()
for key, value in config['template_variables'].items():
# The special variables are already Python variables.
if key not in ['ROSE_ORIG_HOST', 'ROSE_VERSION', 'ROSE_SITE']:
try:
config['template_variables'][key] = (
parser.literal_eval(value))
except Exception:
raise ConfigProcessError(
[templating, key], value,
f'Invalid template variable: {value}'
'\nMust be a valid Python or Jinja2 literal'
' (note strings "must be quoted").') from None
# Add ROSE_SUITE_VARIABLES to config of templating engines in use.
if templating is not None:
config['template_variables']['ROSE_SUITE_VARIABLES'] = config[
'template_variables']
def get_rose_vars_from_config_node(config, config_node, environ):
"""Load template variables from a Rose config node.
This uses only the provided config node and environment variables
- there is no system interaction.
Args:
config (dict):
Object which will be populated with the results.
config_node (metomi.rose.config.ConfigNode):
Configuration node representing the Rose suite configuration.
environ (dict):
Dictionary of environment variables
"""
templating = None
sections = {'jinja2:suite.rc', 'empy:suite.rc', 'template variables'}
# Don't allow multiple templating sections.
defined_sections = sections.intersection(set(config_node.value))
if len(defined_sections) > 1:
raise MultipleTemplatingEnginesError(
"You should not define more than one templating section. "
f"You defined:\n\t{'; '.join(defined_sections)}")
elif len(defined_sections) == 1:
templating, = defined_sections
if templating != 'template variables':
config['templating_detected'] = templating.replace(':suite.rc', '')
else:
config['templating_detected'] = templating
# Create env section if it doesn't already exist.
if 'env' not in config_node.value:
config_node.set(['env'])
# Get Values for standard ROSE variables (ROSE_ORIG_HOST and ROSE_SITE).
rose_orig_host = get_host()
rose_site = ResourceLocator().get_conf().get_value(['site'], '')
# For each section process variables and add standard variables.
for section in ['env', templating]:
if section not in config_node.value:
continue
# Add standard ROSE_VARIABLES
config_node[section].set(['ROSE_SITE'], rose_site)
config_node[section].set(['ROSE_VERSION'], ROSE_VERSION)
config_node[section].set(['ROSE_ORIG_HOST'], rose_orig_host)
# Use env_var_process to process variables which may need expanding.
for key, node in config_node.value[section].value.items():
try:
config_node.value[section].value[key].value = env_var_process(
node.value, environ=environ)
if section == 'env':
environ[key] = node.value
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError(['env', key], node.value, exc)
# For each of the template language sections extract items to a simple
# dict to be returned.
if 'env' in config_node.value:
config['env'] = {
item[0][1]: item[1].value
for item in config_node.value['env'].walk()
}
if templating in config_node.value:
config['template_variables'] = {
item[0][1]: item[1].value
for item in config_node.value[templating].walk()
}
elif 'template variables' in config_node.value:
config['template_variables'] = {
item[0][1]: item[1].value
for item in config_node.value['template variables'].walk()
}
# Add the entire config to ROSE_SUITE_VARIABLES to allow for programatic
# access.
if templating is not None:
parser = Parser()
for key, value in config['template_variables'].items():
# The special variables are already Python variables.
if key not in ['ROSE_ORIG_HOST', 'ROSE_VERSION', 'ROSE_SITE']:
try:
config['template_variables'][key] = (
parser.literal_eval(value))
except Exception:
raise ConfigProcessError(
[templating, key], value,
f'Invalid template variable: {value}'
'\nMust be a valid Python or Jinja2 literal'
' (note strings "must be quoted").') from None
# Add ROSE_SUITE_VARIABLES to config of templating engines in use.
if templating is not None:
config['template_variables']['ROSE_SUITE_VARIABLES'] = config[
'template_variables']
def process(self,
conf_tree,
item,
orig_keys=None,
orig_value=None,
**kwargs):
"""Process [jinja2:*] in "conf_tree.node".
Arguments:
conf_tree:
The relevant metomi.rose.config_tree.ConfigTree object with the
full configuration.
item: The current configuration item to process.
orig_keys:
The keys for locating the originating setting in conf_tree in a
recursive processing. None implies a top level call.
orig_value: The value of orig_keys in conf_tree.
**kwargs:
environ (dict): suite level environment variables.
"""
for s_key, s_node in sorted(conf_tree.node.value.items()):
if (s_node.is_ignored() or not s_key.startswith(self.PREFIX)
or not s_node.value):
continue
target = s_key[len(self.PREFIX):]
source = os.path.join(conf_tree.files[target], target)
if not os.access(source, os.F_OK | os.R_OK):
continue
scheme_ln = self.SCHEME_TEMPL % self.SCHEME
msg_init_ln = self.COMMENT_TEMPL % self.MSG_INIT
msg_done_ln = self.COMMENT_TEMPL % self.MSG_DONE
tmp_file = NamedTemporaryFile()
tmp_file.write(scheme_ln.encode('UTF-8'))
tmp_file.write(msg_init_ln.encode('UTF-8'))
suite_variables = ['{']
for key, node in sorted(s_node.value.items()):
if node.is_ignored():
continue
try:
value = env_var_process(node.value)
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError([s_key, key], node.value, exc)
tmp_file.write(
(self.ASSIGN_TEMPL % (key, value)).encode('UTF-8'))
suite_variables.append(" '%s': %s," % (key, key))
suite_variables.append('}')
suite_variables = self.ASSIGN_TEMPL % ('ROSE_SUITE_VARIABLES',
'\n'.join(suite_variables))
tmp_file.write(suite_variables.encode('UTF-8'))
environ = kwargs.get("environ")
if environ:
tmp_file.write('[cylc]\n'.encode('UTF-8'))
tmp_file.write(' [[environment]]\n'.encode('UTF-8'))
for key, value in sorted(environ.items()):
tmp_file.write(
(' %s=%s\n' % (key, value)).encode('UTF-8'))
tmp_file.write(msg_done_ln.encode('UTF-8'))
line_n = 0
is_in_old_insert = False
for line in open(source):
line_n += 1
if line_n == 1 and line.strip().lower() == scheme_ln.strip():
continue
elif line_n == 2 and line == msg_init_ln:
is_in_old_insert = True
continue
elif is_in_old_insert and line == msg_done_ln:
is_in_old_insert = False
continue
elif is_in_old_insert:
continue
tmp_file.write(line.encode('UTF-8'))
tmp_file.seek(0)
if os.access(target, os.F_OK | os.R_OK):
if filecmp.cmp(target, tmp_file.name): # identical
tmp_file.close()
continue
else:
self.manager.fs_util.delete(target)
# Write content to target
target_file = open(target, "w")
for line in tmp_file:
try:
target_file.write(line)
except TypeError:
target_file.write(line.decode())
event = FileSystemEvent(FileSystemEvent.INSTALL, target)
self.manager.handle_event(event)
tmp_file.close()
def _get_prune_globs(self, app_runner, conf_tree):
"""Return (globs, cycles).
where:
* globs is for matching items to prune.
* cycles is a set of relevant cycles.
"""
globs = []
nodes = conf_tree.node.get_value([self.SECTION])
if nodes is None:
return [], set()
cycle_formats = {}
for key, node in nodes.items():
if node.is_ignored():
continue
if key.startswith("cycle-format{") and key.endswith("}"):
fmt = key[len("cycle-format{"):-1]
try:
cycle_formats[fmt] = env_var_process(node.value)
# Check formats are valid
if self._get_cycling_mode() == "integer":
cycle_formats[fmt] % 0
else:
app_runner.date_time_oper.date_format(
cycle_formats[fmt])
except (UnboundEnvironmentVariableError, ValueError) as exc:
raise ConfigValueError([self.SECTION, key], node.value,
exc)
cycle_set = set()
for key, node in sorted(nodes.items()):
if node.is_ignored():
continue
if key == "prune-datac-at": # backward compat
head = "share/cycle"
elif key == "prune-work-at": # backward compat
head = "work"
elif key.startswith("prune{") and key.endswith("}"):
head = key[len("prune{"):-1].strip() # remove "prune{" and "}"
else:
continue
for cycle, cycle_args in self._get_conf(app_runner,
conf_tree,
key,
max_args=1):
cycle_set.add(cycle)
if cycle_args:
cycle_strs = {"cycle": cycle}
for cycle_key, cycle_format in cycle_formats.items():
if self._get_cycling_mode() == "integer":
cycle_strs[cycle_key] = cycle_format % int(cycle)
else: # date time cycling
cycle_point = app_runner.date_time_oper.date_parse(
cycle)[0]
cycle_strs[
cycle_key] = app_runner.date_time_oper.date_format(
cycle_format, cycle_point)
for tail_glob in shlex.split(cycle_args.pop()):
glob_ = tail_glob % cycle_strs
if glob_ == tail_glob: # no substitution
glob_ = os.path.join(cycle, tail_glob)
globs.append(os.path.join(head, glob_))
else:
globs.append(os.path.join(head, cycle))
return globs, cycle_set
def main():
"""Implement the "rose config" command."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options("default", "env_var_process_mode", "files",
"keys", "meta", "meta_key", "no_ignore",
"no_opts", "print_conf_mode")
opts, args = opt_parser.parse_args()
report = Reporter(opts.verbosity - opts.quietness)
metomi.rose.macro.add_meta_paths()
if opts.meta_key:
opts.meta = True
if opts.files and opts.meta_key:
report(Exception("Cannot specify both a file and meta key."))
sys.exit(1)
config_loader = ConfigLoader()
sources = []
if opts.files:
root_node = ConfigNode()
for fname in opts.files:
if fname == "-":
sources.append(sys.stdin)
else:
if opts.meta:
try:
root_node = config_loader.load(fname)
except ConfigSyntaxError as exc:
report(exc)
sys.exit(1)
rel_path = os.sep.join(fname.split(os.sep)[:-1])
fpath = get_meta_path(root_node, rel_path)
if fpath is None:
report(MetadataNotFoundEvent(fname))
else:
sources.append(fpath)
else:
sources.append(fname)
elif opts.meta:
root_node = ConfigNode()
if opts.meta_key:
root_node.set(["meta"], opts.meta_key)
else:
fname = os.path.join(os.getcwd(), metomi.rose.SUB_CONFIG_NAME)
try:
root_node = config_loader.load(fname)
except ConfigSyntaxError as exc:
report(exc)
sys.exit(1)
fpath = get_meta_path(root_node, meta_key=opts.meta_key)
root_node.unset(["meta"])
if fpath is None:
report(Exception("Metadata not found"))
sys.exit(1)
else:
sources.append(fpath)
else:
root_node = ResourceLocator.default().get_conf()
for source in sources:
try:
if opts.meta or opts.no_opts:
config_loader.load(source, root_node)
else:
config_loader.load_with_opts(source, root_node)
except (ConfigSyntaxError, IOError) as exc:
report(exc)
sys.exit(1)
if source is sys.stdin:
source.close()
if opts.quietness:
sys.exit(root_node.get(args, opts.no_ignore) is None)
if opts.keys_mode:
try:
keys = list(root_node.get(args, opts.no_ignore).value)
except AttributeError:
sys.exit(1)
keys.sort()
for key in keys:
print(key)
sys.exit()
conf_dump = ConfigDumper()
if len(args) == 0:
conf_dump(root_node, concat_mode=opts.print_conf_mode)
sys.exit()
node = root_node.get(args, opts.no_ignore)
if node is not None and isinstance(node.value, dict):
if opts.print_conf_mode:
conf_dump(ConfigNode().set(args, node.value), concat_mode=True)
sys.exit()
keys = list(node.value)
keys.sort()
for key in keys:
node_of_key = node.get([key], opts.no_ignore)
if node_of_key:
value = node_of_key.value
state = node_of_key.state
string = "%s%s=%s" % (state, key, value)
lines = string.splitlines()
print(lines[0])
i_equal = len(state + key) + 1
for line in lines[1:]:
print(" " * i_equal + line)
sys.exit()
if node is None:
if opts.default is None:
sys.exit(1)
value = opts.default
elif opts.env_var_process_mode:
value = env_var_process(node.value)
else:
value = node.value
if opts.print_conf_mode:
conf_dump(ConfigNode().set(args, value), concat_mode=True)
else:
print(value)
sys.exit()
def _load_tasks(self):
"""Populate the list of analysis tasks from the app config."""
# Fill a dictionary of tasks and extract their options and values
# - skipping any which are user/trigger-ignored
_tasks = {}
for keys, node in self.config.walk(no_ignore=True):
task = keys[0]
if task.startswith("ana:"):
# Capture the options only and save them to the tasks dict
task = task.split(":", 1)[1]
if len(keys) == 2:
# The app may define a section containing rose_ana
# config settings; add these to the config dictionary (if
# any) of the names match existing config options from the
# global config it will be overwritten)
if task == "config":
# Process any environment variables first
value = env_var_process(node.value)
self.ana_config[keys[1]] = value
continue
_tasks.setdefault(task, {})
# If the value contains newlines, split it into a list
# and either way remove any quotation marks and process
# any environment variables
value = env_var_process(node.value)
values = value.split("\n")
for ival, value in enumerate(values):
values[ival] = (
re.sub(r"^((?:'|\")*)(.*)(\1)$", r"\2", value))
# If the user passed a blank curled-braces expression
# it should be expanded to contain each of the arguments
# passed to rose_ana
new_values = []
for value in values:
if "{}" in value:
if self.args is not None and len(self.args) > 0:
for arg in self.args:
new_values.append(value.replace("{}", arg))
else:
new_values.append(value)
else:
new_values.append(value)
values = new_values
if len(values) == 1:
values = values[0]
_tasks[task][keys[1]] = values
# Can now populate the output task list with analysis objects
self.analysis_tasks = []
for name in sorted(_tasks.keys()):
options = _tasks[name]
options["full_task_name"] = name
# Create an analysis object for each task, passing through
# all options given to the section in the app, the given name
# starts with the comparison type and then optionally a
# name/description, extract this here
match = re.match(r"(?P<atype>[\w\.]+)(?:\((?P<descr>.*)\)|)", name)
if match:
options["description"] = match.group("descr")
atype = match.group("atype")
# Assuming this analysis task has been loaded by the app, create
# an instance of the task, passing the options to it
if atype in self.methods:
self.analysis_tasks.append(self.methods[atype](self, options))
else:
# If the analysis type isn't matched by one of the loaded
# methods, report the error and return a placeholder
# in its place (so that this tasks' main method can show
# the task as "failed")
msg = "Unrecognised analysis type: {0}"
self.reporter(msg.format(atype), prefix="[FAIL] ")
# Create a simple object to return - when the run_analysis
# method is called by the main loop it will simply raise
# an exception, triggering the "error" trap
class Dummy(AnalysisTask):
def run_analysis(self):
raise ImportError(msg.format(atype))
self.analysis_tasks.append(Dummy(self, options))
def main():
"""Implement the "rose config" command."""
opt_parser = RoseOptionParser(description='''
Parse and print rose configuration files.
With no option and no argument, print the rose site + user configuration.
EXAMPLES
# Print the value of OPTION in SECTION.
rose config SECTION OPTION
# Print the value of OPTION in SECTION in FILE.
rose config --file=FILE SECTION OPTION
# Print the value of OPTION in SECTION if exists, or VALUE otherwise.
rose config --default=VALUE SECTION OPTION
# Print the OPTION=VALUE pairs in SECTION.
rose config SECTION
# Print the value of a top level OPTION.
rose config OPTION
# Print the OPTION keys in SECTION.
rose config --keys SECTION
# Print the SECTION keys.
rose config --keys
# Exit with 0 if OPTION exists in SECTION, or 1 otherwise.
rose config -q SECTION OPTION
# Exit with 0 if SECTION exists, or 1 otherwise.
rose config -q SECTION
# Combine the configurations in FILE1 and FILE2, and dump the result.
rose config --file=FILE1 --file=FILE2
# Print the value of OPTION in SECTION of the metadata associated with
# the specified config FILE
rose config --file=FILE --meta SECTION OPTION
# Print the value of a specified metadata KEY
rose config --meta-key=KEY
''',
epilog='''
ENVIRONMENT VARIABLES
optional ROSE_META_PATH
Prepend `$ROSE_META_PATH` to the metadata search path.
''')
opt_parser.add_my_options(
"default",
"env_var_process_mode",
"files",
"keys",
"meta",
"meta_key",
"no_ignore",
"no_opts",
"print_conf_mode",
)
# the quietness argument is non-standard for this command
opt_parser.modify_option(
'quietness',
help=("Exit with 0 if the specified `SECTION` and/or `OPTION` exist in"
" the configuration, or 1 otherwise."),
)
opts, args = opt_parser.parse_args()
report = Reporter(opts.verbosity - opts.quietness)
metomi.rose.macro.add_meta_paths()
if opts.meta_key:
opts.meta = True
if opts.files and opts.meta_key:
report(Exception("Cannot specify both a file and meta key."))
sys.exit(1)
config_loader = ConfigLoader()
sources = []
if opts.files:
root_node = ConfigNode()
for fname in opts.files:
if fname == "-":
sources.append(sys.stdin)
else:
if opts.meta:
try:
root_node = config_loader.load(fname)
except ConfigSyntaxError as exc:
report(exc)
sys.exit(1)
rel_path = os.sep.join(fname.split(os.sep)[:-1])
fpath = get_meta_path(root_node, rel_path)
if fpath is None:
report(MetadataNotFoundEvent(fname))
else:
sources.append(fpath)
else:
sources.append(fname)
elif opts.meta:
root_node = ConfigNode()
if opts.meta_key:
root_node.set(["meta"], opts.meta_key)
else:
fname = os.path.join(os.getcwd(), metomi.rose.SUB_CONFIG_NAME)
try:
root_node = config_loader.load(fname)
except ConfigSyntaxError as exc:
report(exc)
sys.exit(1)
fpath = get_meta_path(root_node, meta_key=opts.meta_key)
root_node.unset(["meta"])
if fpath is None:
report(Exception("Metadata not found"))
sys.exit(1)
else:
sources.append(fpath)
else:
root_node = ResourceLocator.default().get_conf()
for source in sources:
try:
if opts.meta or opts.no_opts:
config_loader.load(source, root_node)
else:
config_loader.load_with_opts(source, root_node)
except (ConfigSyntaxError, IOError) as exc:
report(exc)
sys.exit(1)
if source is sys.stdin:
source.close()
if opts.quietness:
sys.exit(root_node.get(args, opts.no_ignore) is None)
if opts.keys_mode:
try:
keys = list(root_node.get(args, opts.no_ignore).value)
except AttributeError:
sys.exit(1)
keys.sort()
for key in keys:
print(key)
sys.exit()
conf_dump = ConfigDumper()
if len(args) == 0:
conf_dump(root_node, concat_mode=opts.print_conf_mode)
sys.exit()
node = root_node.get(args, opts.no_ignore)
if node is not None and isinstance(node.value, dict):
if opts.print_conf_mode:
conf_dump(ConfigNode().set(args, node.value), concat_mode=True)
sys.exit()
keys = list(node.value)
keys.sort()
for key in keys:
node_of_key = node.get([key], opts.no_ignore)
if node_of_key:
value = node_of_key.value
state = node_of_key.state
string = "%s%s=%s" % (state, key, value)
lines = string.splitlines()
print(lines[0])
i_equal = len(state + key) + 1
for line in lines[1:]:
print(" " * i_equal + line)
sys.exit()
if node is None:
if opts.default is None:
sys.exit(1)
value = opts.default
elif opts.env_var_process_mode:
value = env_var_process(node.value)
else:
value = node.value
if opts.print_conf_mode:
conf_dump(ConfigNode().set(args, value), concat_mode=True)
else:
print(value)
sys.exit()
def _get_conf(self, app_runner, conf_tree, key, max_args=0):
"""Get a list of cycles from a configuration setting.
key -- An option key in self.SECTION to locate the setting.
max_args -- Maximum number of extra arguments for an item in the list.
The value of the setting is expected to be split by shlex.split into a
list of items. If max_args == 0, an item should be a string
representing a cycle or an cycle offset. If max_args > 0, the cycle
or cycle offset string can, optionally, have arguments. The arguments
are delimited by colons ":".
E.g.:
prune-remote-logs-at=-PT6H -PT12H
prune-server-logs-at=-P7D
prune-datac-at=-PT6H:foo/* -PT12H:'bar/* baz/*' -P1D
prune-work-at=-PT6H:t1*:*.tar -PT12H:t1*: -PT12H:*.gz -P1D
If max_args == 0, return a list of cycles.
If max_args > 0, return a list of (cycle, [arg, ...])
"""
items_str = conf_tree.node.get_value([self.SECTION, key])
if items_str is None:
return []
try:
items_str = env_var_process(items_str)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError([self.SECTION, key], items_str, exc)
items = []
ref_point_str = os.getenv(RoseDateTimeOperator.TASK_CYCLE_TIME_ENV)
try:
ref_point = None
ref_fmt = None
for item_str in shlex.split(items_str):
args = item_str.split(":", max_args)
when = args.pop(0)
cycle = when
if ref_point_str is not None:
if self._get_cycling_mode() == "integer":
# Integer cycling
if "P" in when: # "when" is an offset
cycle = str(
int(ref_point_str) +
int(when.replace("P", "")))
else: # "when" is a cycle point
cycle = str(when)
else:
# Date-time cycling
if ref_fmt is None:
(
ref_point,
ref_fmt,
) = app_runner.date_time_oper.date_parse(
ref_point_str)
try:
time_point = app_runner.date_time_oper.date_parse(
when)[0]
except ValueError:
time_point = app_runner.date_time_oper.date_shift(
ref_point, when)
cycle = app_runner.date_time_oper.date_format(
ref_fmt, time_point)
if max_args:
items.append((cycle, args))
else:
items.append(cycle)
except ValueError as exc:
raise ConfigValueError([self.SECTION, key], items_str, exc)
return items
def _process(self, conf_tree, nodes, loc_dao, **kwargs):
"""Helper for self.process."""
# Ensure that everything is overwritable
# Ensure that container directories exist
for key, node in sorted(nodes.items()):
try:
name = env_var_process(key[len(self.PREFIX) :])
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError([key], key, exc)
if os.path.exists(name) and kwargs.get("no_overwrite_mode"):
raise ConfigProcessError([key], None, FileOverwriteError(name))
self.manager.fs_util.makedirs(self.manager.fs_util.dirname(name))
# Gets a list of sources and targets
sources = {}
targets = {}
for key, node in sorted(nodes.items()):
# N.B. no need to catch UnboundEnvironmentVariableError here
# because any exception should been caught earlier.
name = env_var_process(key[len(self.PREFIX) :])
targets[name] = Loc(name)
targets[name].action_key = Loc.A_INSTALL
targets[name].mode = node.get_value(["mode"])
if targets[name].mode and targets[name].mode not in Loc.MODES:
raise ConfigProcessError([key, "mode"], targets[name].mode)
target_sources = []
for k in ["content", "source"]: # "content" for back compat
source_str = node.get_value([k])
if source_str is None:
continue
try:
source_str = env_var_process(source_str)
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError([key, k], source_str, exc)
source_names = []
for raw_source_glob in shlex.split(source_str):
source_glob = raw_source_glob
if raw_source_glob.startswith(
"("
) and raw_source_glob.endswith(")"):
source_glob = raw_source_glob[1:-1]
names = glob(source_glob)
if names:
source_names += sorted(names)
else:
source_names.append(raw_source_glob)
for raw_source_name in source_names:
source_name = raw_source_name
is_optional = raw_source_name.startswith(
"("
) and raw_source_name.endswith(")")
if is_optional:
source_name = raw_source_name[1:-1]
if source_name.startswith("~"):
source_name = os.path.expanduser(source_name)
if targets[name].mode in [
targets[name].MODE_SYMLINK,
targets[name].MODE_SYMLINK_PLUS,
]:
if targets[name].real_name:
# Symlink mode can only have 1 source
raise ConfigProcessError([key, k], source_str)
targets[name].real_name = source_name
else:
if source_name not in sources:
sources[source_name] = Loc(source_name)
sources[source_name].action_key = Loc.A_SOURCE
sources[source_name].is_optional = is_optional
sources[source_name].used_by_names.append(name)
target_sources.append(sources[source_name])
targets[name].dep_locs = target_sources
if not targets[name].real_name and targets[name].mode in [
targets[name].MODE_SYMLINK,
targets[name].MODE_SYMLINK_PLUS,
]:
raise ConfigProcessError([key, "source"], None)
# Determine the scheme of the location from configuration.
config_schemes_str = conf_tree.node.get_value(["schemes"])
config_schemes = [] # [(pattern, scheme), ...]
if config_schemes_str:
for line in config_schemes_str.splitlines():
pattern, scheme = line.split("=", 1)
pattern = pattern.strip()
scheme = scheme.strip()
config_schemes.append((pattern, scheme))
# Where applicable, determine for each source:
# * Its real name.
# * The checksums of its paths.
# * Whether it can be considered unchanged.
for source in list(sources.values()):
try:
for pattern, scheme in config_schemes:
if fnmatch(source.name, pattern):
source.scheme = scheme
break
self.loc_handlers_manager.parse(source, conf_tree)
except ValueError:
if source.is_optional:
sources.pop(source.name)
for name in source.used_by_names:
targets[name].dep_locs.remove(source)
event = SourceSkipEvent(name, source.name)
self.handle_event(event)
continue
else:
raise ConfigProcessError(
["file:" + source.used_by_names[0], "source"],
source.name,
)
prev_source = loc_dao.select(source.name)
source.is_out_of_date = (
not prev_source
or (not source.key and not source.paths)
or prev_source.scheme != source.scheme
or prev_source.loc_type != source.loc_type
or prev_source.key != source.key
or sorted(prev_source.paths) != sorted(source.paths)
)
# Inspect each target to see if it is out of date:
# * Target does not already exist.
# * Target exists, but does not have a database entry.
# * Target exists, but does not match settings in database.
# * Target exists, but a source cannot be considered unchanged.
for target in list(targets.values()):
if target.real_name:
target.is_out_of_date = not os.path.islink(
target.name
) or target.real_name != os.readlink(target.name)
elif target.mode == target.MODE_MKDIR:
target.is_out_of_date = os.path.islink(
target.name
) or not os.path.isdir(target.name)
else:
if os.path.exists(target.name) and not os.path.islink(
target.name
):
for path, checksum, access_mode in get_checksum(
target.name
):
target.add_path(path, checksum, access_mode)
target.paths.sort()
prev_target = loc_dao.select(target.name)
target.is_out_of_date = (
os.path.islink(target.name)
or not os.path.exists(target.name)
or prev_target is None
or prev_target.mode != target.mode
or len(prev_target.paths) != len(target.paths)
)
if not target.is_out_of_date:
prev_target.paths.sort()
for prev_path, path in zip(
prev_target.paths, target.paths
):
if prev_path != path:
target.is_out_of_date = True
break
# See if any sources have changed names.
if not target.is_out_of_date:
conn = loc_dao.get_conn()
prev_dep_locs = conn.execute(
"""
SELECT *
FROM dep_names
WHERE name=?
ORDER BY ROWID
""",
[target.name],
).fetchall()
prev_dep_locs = [i[1] for i in prev_dep_locs]
prev_dep_locs = [loc_dao.select(i) for i in prev_dep_locs]
if [i.name for i in prev_dep_locs] != [
i.name for i in target.dep_locs
]:
target.is_out_of_date = True
# See if any sources out of date
if not target.is_out_of_date:
for dep_loc in target.dep_locs:
if dep_loc.is_out_of_date:
target.is_out_of_date = True
break
if target.is_out_of_date:
target.paths = None
loc_dao.delete_locs.append(target)
# Set up jobs for rebuilding all out-of-date targets.
jobs = {}
for name, target in sorted(targets.items()):
if not target.is_out_of_date:
self.handle_event(FileUnchangedEvent(target, level=Event.V))
continue
if target.mode in [target.MODE_SYMLINK, target.MODE_SYMLINK_PLUS]:
if target.mode == target.MODE_SYMLINK_PLUS:
try:
os.stat(target.real_name)
except OSError as exc:
raise ConfigProcessError(
[self.PREFIX + target.name, "source"],
target.real_name,
exc,
)
self.manager.fs_util.symlink(target.real_name, target.name)
loc_dao.update_locs.append(target)
elif target.mode == target.MODE_MKDIR:
if os.path.islink(target.name):
self.manager.fs_util.delete(target.name)
self.manager.fs_util.makedirs(target.name)
loc_dao.update_locs.append(target)
target.loc_type = target.TYPE_TREE
target.add_path(target.BLOB, None, None)
elif target.dep_locs:
if os.path.islink(target.name):
self.manager.fs_util.delete(target.name)
jobs[target.name] = JobProxy(target)
for source in target.dep_locs:
if source.name not in jobs:
jobs[source.name] = JobProxy(source)
jobs[source.name].event_level = Event.V
job = jobs[source.name]
jobs[target.name].pending_for[source.name] = job
p_name = target.name
while (
os.path.dirname(p_name)
and os.path.dirname(p_name) != p_name
):
p_name = os.path.dirname(p_name)
if p_name in jobs:
jobs[target.name].pending_for[p_name] = jobs[p_name]
else:
self.manager.fs_util.install(target.name)
target.loc_type = target.TYPE_BLOB
for path, checksum, access_mode in get_checksum(target.name):
target.add_path(path, checksum, access_mode)
loc_dao.update_locs.append(target)
loc_dao.execute_queued_items()
# If relevant, use job runner to get sources and build targets
if jobs:
work_dir = mkdtemp()
try:
nproc_keys = ["rose.config_processors.fileinstall", "nproc"]
nproc_str = conf_tree.node.get_value(nproc_keys)
nproc = None
if nproc_str is not None:
nproc = int(nproc_str)
job_runner = JobRunner(self, nproc)
job_runner(JobManager(jobs), conf_tree, loc_dao, work_dir)
except ValueError as exc:
if exc.args and exc.args[0] in jobs:
job = jobs[exc.args[0]]
if job.context.action_key == Loc.A_SOURCE:
source = job.context
keys = [
self.PREFIX + source.used_by_names[0],
"source",
]
raise ConfigProcessError(keys, source.name)
raise exc
finally:
loc_dao.execute_queued_items()
rmtree(work_dir)
# Target checksum compare and report
for target in list(targets.values()):
if (
not target.is_out_of_date
or target.loc_type == target.TYPE_TREE
):
continue
keys = [self.PREFIX + target.name, "checksum"]
checksum_expected = conf_tree.node.get_value(keys)
if checksum_expected is None:
continue
checksum = target.paths[0].checksum
if checksum_expected:
if len(checksum_expected) != len(checksum):
algorithm = guess_checksum_algorithm(checksum_expected)
if algorithm:
checksum = get_checksum_func(algorithm)(target.name)
if checksum_expected != checksum:
raise ConfigProcessError(
keys,
checksum_expected,
ChecksumError(checksum_expected, checksum),
)
event = ChecksumEvent(target.name, target.paths[0].checksum)
self.handle_event(event)