def process(self,
conf_tree,
item,
orig_keys=None,
orig_value=None,
**kwargs):
"""Export environment variables in an [env] in "conf_tree.node"."""
env_node = conf_tree.node.get([item], no_ignore=True)
if env_node is None:
return
if "UNDEF" in os.environ:
os.environ.pop("UNDEF")
environ = {}
if env_node and not env_node.state:
for key, node in env_node.value.items():
if node.state:
continue
try:
environ[key] = env_var_process(node.value)
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError([item, key], node.value, exc)
environ[key] = os.path.expanduser(environ[key]) # ~ expansion
for key, value in sorted(environ.items()):
env_export(key, value, self.manager.event_handler)
return environ
def process(
self, conf_tree, item, orig_keys=None, orig_value=None, **kwargs
):
"""Install files according to [file:*] in conf_tree.
kwargs["no_overwrite_mode"]: fail if a target file already exists.
"""
# Find all the "file:*" nodes.
nodes = {}
if item == self.SCHEME:
for key, node in list(conf_tree.node.value.items()):
if node.is_ignored() or not key.startswith(self.PREFIX):
continue
nodes[key] = node
else:
node = conf_tree.node.get([item], no_ignore=True)
if node is None:
raise ConfigProcessError(orig_keys, item)
nodes[item] = node
if not nodes:
return
# Create database to store information for incremental updates,
# if it does not already exist.
loc_dao = LocDAO()
loc_dao.create()
cwd = os.getcwd()
file_install_root = conf_tree.node.get_value(
["file-install-root"], os.getenv("ROSE_FILE_INSTALL_ROOT", None)
)
if file_install_root:
file_install_root = env_var_process(file_install_root)
self.manager.fs_util.makedirs(file_install_root)
self.manager.fs_util.chdir(file_install_root)
try:
self._process(conf_tree, nodes, loc_dao, **kwargs)
finally:
if cwd != os.getcwd():
self.manager.fs_util.chdir(cwd)
async def pull(self, loc, conf_tree):
"""Write namelist to loc.cache."""
sections = self.parse(loc, conf_tree)
if loc.name.endswith("(:)"):
sections.sort(key=cmp_to_key(metomi.rose.config.sort_settings))
with open(loc.cache, "wb") as handle:
for section in sections:
section_value = conf_tree.node.get_value([section])
group = RE_NAMELIST_GROUP.match(section).group(1)
nlg = "&" + group + "\n"
for key, node in sorted(section_value.items()):
if node.state:
continue
try:
value = env_var_process(node.value)
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError([section, key], node.value,
exc)
nlg += "%s=%s,\n" % (key, value)
nlg += "/" + "\n"
handle.write(nlg.encode('UTF-8'))
self.manager.handle_event(NamelistEvent(nlg))
def get_rose_vars_from_config_node(config, config_node, environ):
"""Load template variables from a Rose config node.
This uses only the provided config node and environment variables
- there is no system interaction.
Args:
config (dict):
Object which will be populated with the results.
config_node (metomi.rose.config.ConfigNode):
Configuration node representing the Rose suite configuration.
environ (dict):
Dictionary of environment variables
"""
templating = None
# Don't allow multiple templating sections.
templating = identify_templating_section(config_node)
if templating != 'template variables':
config['templating_detected'] = templating.replace(':suite.rc', '')
else:
config['templating_detected'] = templating
# Create env section if it doesn't already exist.
if 'env' not in config_node.value:
config_node.set(['env'])
if templating not in config_node.value:
config_node.set([templating])
# Get Rose Orig host:
rose_orig_host = get_host()
# For each section process variables and add standard variables.
for section in ['env', templating]:
# This loop handles standard variables.
# CYLC_VERSION - If it's in the config, remove it.
# ROSE_VERSION - If it's in the config, replace it.
# ROSE_ORIG_HOST - If it's the config, replace it, unless it has a
# comment marking it as having been saved by ``cylc install``.
# In all cases warn users if the value in their config is not used.
for var_name, replace_with in [('ROSE_ORIG_HOST', rose_orig_host),
('ROSE_VERSION', ROSE_VERSION),
('CYLC_VERSION', SET_BY_CYLC)]:
# Warn if we're we're going to override a variable:
if override_this_variable(config_node, section, var_name):
user_var = config_node[section].value[var_name].value
LOG.warning(
f'[{section}]{var_name}={user_var} from rose-suite.conf '
f'will be ignored: {var_name} will be: {replace_with}')
# Handle replacement of stored variable if appropriate:
if replace_with == SET_BY_CYLC:
config_node[section].unset([var_name])
elif not rose_orig_host_set_by_cylc_install(
config_node, section, var_name):
config_node[section].set([var_name], replace_with)
# Use env_var_process to process variables which may need expanding.
for key, node in config_node.value[section].value.items():
try:
config_node.value[section].value[key].value = env_var_process(
node.value, environ=environ)
if section == 'env':
environ[key] = node.value
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError(['env', key], node.value, exc)
# For each of the template language sections extract items to a simple
# dict to be returned.
if 'env' in config_node.value:
config['env'] = {
item[0][1]: item[1].value
for item in config_node.value['env'].walk()
}
if templating in config_node.value:
config['template_variables'] = {
item[0][1]: item[1].value
for item in config_node.value[templating].walk()
}
elif 'template variables' in config_node.value:
config['template_variables'] = {
item[0][1]: item[1].value
for item in config_node.value['template variables'].walk()
}
# Add the entire config to ROSE_SUITE_VARIABLES to allow for programatic
# access.
if templating is not None:
with patch_jinja2_leading_zeros():
# BACK COMPAT: patch_jinja2_leading_zeros
# back support zero-padded integers for a limited time to help
# users migrate before upgrading cylc-flow to Jinja2>=3.1
parser = Parser()
for key, value in config['template_variables'].items():
# The special variables are already Python variables.
if key not in ['ROSE_ORIG_HOST', 'ROSE_VERSION', 'ROSE_SITE']:
try:
config['template_variables'][key] = (
parser.literal_eval(value))
except Exception:
raise ConfigProcessError(
[templating, key], value,
f'Invalid template variable: {value}'
'\nMust be a valid Python or Jinja2 literal'
' (note strings "must be quoted").') from None
# Add ROSE_SUITE_VARIABLES to config of templating engines in use.
if templating is not None:
config['template_variables']['ROSE_SUITE_VARIABLES'] = config[
'template_variables']
def process(self,
conf_tree,
item,
orig_keys=None,
orig_value=None,
**kwargs):
"""Process [jinja2:*] in "conf_tree.node".
Arguments:
conf_tree:
The relevant metomi.rose.config_tree.ConfigTree object with the
full configuration.
item: The current configuration item to process.
orig_keys:
The keys for locating the originating setting in conf_tree in a
recursive processing. None implies a top level call.
orig_value: The value of orig_keys in conf_tree.
**kwargs:
environ (dict): suite level environment variables.
"""
for s_key, s_node in sorted(conf_tree.node.value.items()):
if (s_node.is_ignored() or not s_key.startswith(self.PREFIX)
or not s_node.value):
continue
target = s_key[len(self.PREFIX):]
source = os.path.join(conf_tree.files[target], target)
if not os.access(source, os.F_OK | os.R_OK):
continue
scheme_ln = self.SCHEME_TEMPL % self.SCHEME
msg_init_ln = self.COMMENT_TEMPL % self.MSG_INIT
msg_done_ln = self.COMMENT_TEMPL % self.MSG_DONE
tmp_file = NamedTemporaryFile()
tmp_file.write(scheme_ln.encode('UTF-8'))
tmp_file.write(msg_init_ln.encode('UTF-8'))
suite_variables = ['{']
for key, node in sorted(s_node.value.items()):
if node.is_ignored():
continue
try:
value = env_var_process(node.value)
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError([s_key, key], node.value, exc)
tmp_file.write(
(self.ASSIGN_TEMPL % (key, value)).encode('UTF-8'))
suite_variables.append(" '%s': %s," % (key, key))
suite_variables.append('}')
suite_variables = self.ASSIGN_TEMPL % ('ROSE_SUITE_VARIABLES',
'\n'.join(suite_variables))
tmp_file.write(suite_variables.encode('UTF-8'))
environ = kwargs.get("environ")
if environ:
tmp_file.write('[cylc]\n'.encode('UTF-8'))
tmp_file.write(' [[environment]]\n'.encode('UTF-8'))
for key, value in sorted(environ.items()):
tmp_file.write(
(' %s=%s\n' % (key, value)).encode('UTF-8'))
tmp_file.write(msg_done_ln.encode('UTF-8'))
line_n = 0
is_in_old_insert = False
for line in open(source):
line_n += 1
if line_n == 1 and line.strip().lower() == scheme_ln.strip():
continue
elif line_n == 2 and line == msg_init_ln:
is_in_old_insert = True
continue
elif is_in_old_insert and line == msg_done_ln:
is_in_old_insert = False
continue
elif is_in_old_insert:
continue
tmp_file.write(line.encode('UTF-8'))
tmp_file.seek(0)
if os.access(target, os.F_OK | os.R_OK):
if filecmp.cmp(target, tmp_file.name): # identical
tmp_file.close()
continue
else:
self.manager.fs_util.delete(target)
# Write content to target
target_file = open(target, "w")
for line in tmp_file:
try:
target_file.write(line)
except TypeError:
target_file.write(line.decode())
event = FileSystemEvent(FileSystemEvent.INSTALL, target)
self.manager.handle_event(event)
tmp_file.close()
def _process(self, conf_tree, nodes, loc_dao, **kwargs):
"""Helper for self.process."""
# Ensure that everything is overwritable
# Ensure that container directories exist
for key, node in sorted(nodes.items()):
try:
name = env_var_process(key[len(self.PREFIX) :])
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError([key], key, exc)
if os.path.exists(name) and kwargs.get("no_overwrite_mode"):
raise ConfigProcessError([key], None, FileOverwriteError(name))
self.manager.fs_util.makedirs(self.manager.fs_util.dirname(name))
# Gets a list of sources and targets
sources = {}
targets = {}
for key, node in sorted(nodes.items()):
# N.B. no need to catch UnboundEnvironmentVariableError here
# because any exception should been caught earlier.
name = env_var_process(key[len(self.PREFIX) :])
targets[name] = Loc(name)
targets[name].action_key = Loc.A_INSTALL
targets[name].mode = node.get_value(["mode"])
if targets[name].mode and targets[name].mode not in Loc.MODES:
raise ConfigProcessError([key, "mode"], targets[name].mode)
target_sources = []
for k in ["content", "source"]: # "content" for back compat
source_str = node.get_value([k])
if source_str is None:
continue
try:
source_str = env_var_process(source_str)
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError([key, k], source_str, exc)
source_names = []
for raw_source_glob in shlex.split(source_str):
source_glob = raw_source_glob
if raw_source_glob.startswith(
"("
) and raw_source_glob.endswith(")"):
source_glob = raw_source_glob[1:-1]
names = glob(source_glob)
if names:
source_names += sorted(names)
else:
source_names.append(raw_source_glob)
for raw_source_name in source_names:
source_name = raw_source_name
is_optional = raw_source_name.startswith(
"("
) and raw_source_name.endswith(")")
if is_optional:
source_name = raw_source_name[1:-1]
if source_name.startswith("~"):
source_name = os.path.expanduser(source_name)
if targets[name].mode in [
targets[name].MODE_SYMLINK,
targets[name].MODE_SYMLINK_PLUS,
]:
if targets[name].real_name:
# Symlink mode can only have 1 source
raise ConfigProcessError([key, k], source_str)
targets[name].real_name = source_name
else:
if source_name not in sources:
sources[source_name] = Loc(source_name)
sources[source_name].action_key = Loc.A_SOURCE
sources[source_name].is_optional = is_optional
sources[source_name].used_by_names.append(name)
target_sources.append(sources[source_name])
targets[name].dep_locs = target_sources
if not targets[name].real_name and targets[name].mode in [
targets[name].MODE_SYMLINK,
targets[name].MODE_SYMLINK_PLUS,
]:
raise ConfigProcessError([key, "source"], None)
# Determine the scheme of the location from configuration.
config_schemes_str = conf_tree.node.get_value(["schemes"])
config_schemes = [] # [(pattern, scheme), ...]
if config_schemes_str:
for line in config_schemes_str.splitlines():
pattern, scheme = line.split("=", 1)
pattern = pattern.strip()
scheme = scheme.strip()
config_schemes.append((pattern, scheme))
# Where applicable, determine for each source:
# * Its real name.
# * The checksums of its paths.
# * Whether it can be considered unchanged.
for source in list(sources.values()):
try:
for pattern, scheme in config_schemes:
if fnmatch(source.name, pattern):
source.scheme = scheme
break
self.loc_handlers_manager.parse(source, conf_tree)
except ValueError:
if source.is_optional:
sources.pop(source.name)
for name in source.used_by_names:
targets[name].dep_locs.remove(source)
event = SourceSkipEvent(name, source.name)
self.handle_event(event)
continue
else:
raise ConfigProcessError(
["file:" + source.used_by_names[0], "source"],
source.name,
)
prev_source = loc_dao.select(source.name)
source.is_out_of_date = (
not prev_source
or (not source.key and not source.paths)
or prev_source.scheme != source.scheme
or prev_source.loc_type != source.loc_type
or prev_source.key != source.key
or sorted(prev_source.paths) != sorted(source.paths)
)
# Inspect each target to see if it is out of date:
# * Target does not already exist.
# * Target exists, but does not have a database entry.
# * Target exists, but does not match settings in database.
# * Target exists, but a source cannot be considered unchanged.
for target in list(targets.values()):
if target.real_name:
target.is_out_of_date = not os.path.islink(
target.name
) or target.real_name != os.readlink(target.name)
elif target.mode == target.MODE_MKDIR:
target.is_out_of_date = os.path.islink(
target.name
) or not os.path.isdir(target.name)
else:
if os.path.exists(target.name) and not os.path.islink(
target.name
):
for path, checksum, access_mode in get_checksum(
target.name
):
target.add_path(path, checksum, access_mode)
target.paths.sort()
prev_target = loc_dao.select(target.name)
target.is_out_of_date = (
os.path.islink(target.name)
or not os.path.exists(target.name)
or prev_target is None
or prev_target.mode != target.mode
or len(prev_target.paths) != len(target.paths)
)
if not target.is_out_of_date:
prev_target.paths.sort()
for prev_path, path in zip(
prev_target.paths, target.paths
):
if prev_path != path:
target.is_out_of_date = True
break
# See if any sources have changed names.
if not target.is_out_of_date:
conn = loc_dao.get_conn()
prev_dep_locs = conn.execute(
"""
SELECT *
FROM dep_names
WHERE name=?
ORDER BY ROWID
""",
[target.name],
).fetchall()
prev_dep_locs = [i[1] for i in prev_dep_locs]
prev_dep_locs = [loc_dao.select(i) for i in prev_dep_locs]
if [i.name for i in prev_dep_locs] != [
i.name for i in target.dep_locs
]:
target.is_out_of_date = True
# See if any sources out of date
if not target.is_out_of_date:
for dep_loc in target.dep_locs:
if dep_loc.is_out_of_date:
target.is_out_of_date = True
break
if target.is_out_of_date:
target.paths = None
loc_dao.delete_locs.append(target)
# Set up jobs for rebuilding all out-of-date targets.
jobs = {}
for name, target in sorted(targets.items()):
if not target.is_out_of_date:
self.handle_event(FileUnchangedEvent(target, level=Event.V))
continue
if target.mode in [target.MODE_SYMLINK, target.MODE_SYMLINK_PLUS]:
if target.mode == target.MODE_SYMLINK_PLUS:
try:
os.stat(target.real_name)
except OSError as exc:
raise ConfigProcessError(
[self.PREFIX + target.name, "source"],
target.real_name,
exc,
)
self.manager.fs_util.symlink(target.real_name, target.name)
loc_dao.update_locs.append(target)
elif target.mode == target.MODE_MKDIR:
if os.path.islink(target.name):
self.manager.fs_util.delete(target.name)
self.manager.fs_util.makedirs(target.name)
loc_dao.update_locs.append(target)
target.loc_type = target.TYPE_TREE
target.add_path(target.BLOB, None, None)
elif target.dep_locs:
if os.path.islink(target.name):
self.manager.fs_util.delete(target.name)
jobs[target.name] = JobProxy(target)
for source in target.dep_locs:
if source.name not in jobs:
jobs[source.name] = JobProxy(source)
jobs[source.name].event_level = Event.V
job = jobs[source.name]
jobs[target.name].pending_for[source.name] = job
p_name = target.name
while (
os.path.dirname(p_name)
and os.path.dirname(p_name) != p_name
):
p_name = os.path.dirname(p_name)
if p_name in jobs:
jobs[target.name].pending_for[p_name] = jobs[p_name]
else:
self.manager.fs_util.install(target.name)
target.loc_type = target.TYPE_BLOB
for path, checksum, access_mode in get_checksum(target.name):
target.add_path(path, checksum, access_mode)
loc_dao.update_locs.append(target)
loc_dao.execute_queued_items()
# If relevant, use job runner to get sources and build targets
if jobs:
work_dir = mkdtemp()
try:
nproc_keys = ["rose.config_processors.fileinstall", "nproc"]
nproc_str = conf_tree.node.get_value(nproc_keys)
nproc = None
if nproc_str is not None:
nproc = int(nproc_str)
job_runner = JobRunner(self, nproc)
job_runner(JobManager(jobs), conf_tree, loc_dao, work_dir)
except ValueError as exc:
if exc.args and exc.args[0] in jobs:
job = jobs[exc.args[0]]
if job.context.action_key == Loc.A_SOURCE:
source = job.context
keys = [
self.PREFIX + source.used_by_names[0],
"source",
]
raise ConfigProcessError(keys, source.name)
raise exc
finally:
loc_dao.execute_queued_items()
rmtree(work_dir)
# Target checksum compare and report
for target in list(targets.values()):
if (
not target.is_out_of_date
or target.loc_type == target.TYPE_TREE
):
continue
keys = [self.PREFIX + target.name, "checksum"]
checksum_expected = conf_tree.node.get_value(keys)
if checksum_expected is None:
continue
checksum = target.paths[0].checksum
if checksum_expected:
if len(checksum_expected) != len(checksum):
algorithm = guess_checksum_algorithm(checksum_expected)
if algorithm:
checksum = get_checksum_func(algorithm)(target.name)
if checksum_expected != checksum:
raise ConfigProcessError(
keys,
checksum_expected,
ChecksumError(checksum_expected, checksum),
)
event = ChecksumEvent(target.name, target.paths[0].checksum)
self.handle_event(event)
def get_rose_vars_from_config_node(config, config_node, environ):
"""Load template variables from a Rose config node.
This uses only the provided config node and environment variables
- there is no system interaction.
Args:
config (dict):
Object which will be populated with the results.
config_node (metomi.rose.config.ConfigNode):
Configuration node representing the Rose suite configuration.
environ (dict):
Dictionary of environment variables
"""
templating = None
sections = {'jinja2:suite.rc', 'empy:suite.rc', 'template variables'}
# Don't allow multiple templating sections.
defined_sections = sections.intersection(set(config_node.value))
if len(defined_sections) > 1:
raise MultipleTemplatingEnginesError(
"You should not define more than one templating section. "
f"You defined:\n\t{'; '.join(defined_sections)}")
elif len(defined_sections) == 1:
templating, = defined_sections
if templating != 'template variables':
config['templating_detected'] = templating.replace(':suite.rc', '')
else:
config['templating_detected'] = templating
# Create env section if it doesn't already exist.
if 'env' not in config_node.value:
config_node.set(['env'])
# Get Values for standard ROSE variables (ROSE_ORIG_HOST and ROSE_SITE).
rose_orig_host = get_host()
rose_site = ResourceLocator().get_conf().get_value(['site'], '')
# For each section process variables and add standard variables.
for section in ['env', templating]:
if section not in config_node.value:
continue
# Add standard ROSE_VARIABLES
config_node[section].set(['ROSE_SITE'], rose_site)
config_node[section].set(['ROSE_VERSION'], ROSE_VERSION)
config_node[section].set(['ROSE_ORIG_HOST'], rose_orig_host)
# Use env_var_process to process variables which may need expanding.
for key, node in config_node.value[section].value.items():
try:
config_node.value[section].value[key].value = env_var_process(
node.value, environ=environ)
if section == 'env':
environ[key] = node.value
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError(['env', key], node.value, exc)
# For each of the template language sections extract items to a simple
# dict to be returned.
if 'env' in config_node.value:
config['env'] = {
item[0][1]: item[1].value
for item in config_node.value['env'].walk()
}
if templating in config_node.value:
config['template_variables'] = {
item[0][1]: item[1].value
for item in config_node.value[templating].walk()
}
elif 'template variables' in config_node.value:
config['template_variables'] = {
item[0][1]: item[1].value
for item in config_node.value['template variables'].walk()
}
# Add the entire config to ROSE_SUITE_VARIABLES to allow for programatic
# access.
if templating is not None:
parser = Parser()
for key, value in config['template_variables'].items():
# The special variables are already Python variables.
if key not in ['ROSE_ORIG_HOST', 'ROSE_VERSION', 'ROSE_SITE']:
try:
config['template_variables'][key] = (
parser.literal_eval(value))
except Exception:
raise ConfigProcessError(
[templating, key], value,
f'Invalid template variable: {value}'
'\nMust be a valid Python or Jinja2 literal'
' (note strings "must be quoted").') from None
# Add ROSE_SUITE_VARIABLES to config of templating engines in use.
if templating is not None:
config['template_variables']['ROSE_SUITE_VARIABLES'] = config[
'template_variables']