def process(self,
conf_tree,
item,
orig_keys=None,
orig_value=None,
**kwargs):
"""Export environment variables in an [env] in "conf_tree.node"."""
env_node = conf_tree.node.get([item], no_ignore=True)
if env_node is None:
return
if os.environ.has_key("UNDEF"):
os.environ.pop("UNDEF")
environ = {}
if env_node and not env_node.state:
for key, node in env_node.value.iteritems():
if node.state:
continue
try:
environ[key] = env_var_process(node.value)
except UnboundEnvironmentVariableError as e:
raise ConfigProcessError([item, key], node.value, e)
environ[key] = os.path.expanduser(environ[key]) # ~ expansion
for key, value in sorted(environ.items()):
env_export(key, value, self.manager.event_handler)
return environ
def process(self, conf_tree, item, orig_keys=None, orig_value=None, **_):
"""Process [jinja2:*] in "conf_tree.node"."""
for s_key, s_node in sorted(conf_tree.node.value.items()):
if (s_node.is_ignored() or not s_key.startswith(self.PREFIX)
or not s_node.value):
continue
target = s_key[len(self.PREFIX):]
source = os.path.join(conf_tree.files[target], target)
if not os.access(source, os.F_OK | os.R_OK):
continue
scheme_ln = self.SCHEME_TEMPL % self.SCHEME
msg_init_ln = self.COMMENT_TEMPL % self.MSG_INIT
msg_done_ln = self.COMMENT_TEMPL % self.MSG_DONE
tmp_file = NamedTemporaryFile()
tmp_file.write(scheme_ln)
tmp_file.write(msg_init_ln)
for key, node in sorted(s_node.value.items()):
if node.is_ignored():
continue
try:
value = env_var_process(node.value)
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError([s_key, key], node.value, exc)
tmp_file.write(self.ASSIGN_TEMPL % (key, value))
tmp_file.write(msg_done_ln)
line_n = 0
is_in_old_insert = False
for line in open(source):
line_n += 1
if line_n == 1 and line.strip().lower() == scheme_ln.strip():
continue
elif line_n == 2 and line == msg_init_ln:
is_in_old_insert = True
continue
elif is_in_old_insert and line == msg_done_ln:
is_in_old_insert = False
continue
elif is_in_old_insert:
continue
tmp_file.write(line)
tmp_file.seek(0)
if os.access(target, os.F_OK | os.R_OK):
if filecmp.cmp(target, tmp_file.name): # identical
tmp_file.close()
continue
else:
self.manager.fs_util.delete(target)
# Write content to target
target_file = open(target, "w")
for line in tmp_file:
target_file.write(line)
event = FileSystemEvent(FileSystemEvent.INSTALL, target)
self.manager.handle_event(event)
tmp_file.close()
def process(self,
conf_tree,
item,
orig_keys=None,
orig_value=None,
**kwargs):
"""Install files according to [file:*] in conf_tree.
kwargs["no_overwrite_mode"]: fail if a target file already exists.
"""
# Find all the "file:*" nodes.
nodes = {}
if item == self.SCHEME:
for key, node in conf_tree.node.value.items():
if node.is_ignored() or not key.startswith(self.PREFIX):
continue
nodes[key] = node
else:
node = conf_tree.node.get([item], no_ignore=True)
if node is None:
raise ConfigProcessError(orig_keys, item)
nodes[item] = node
if not nodes:
return
# Create database to store information for incremental updates,
# if it does not already exist.
loc_dao = LocDAO()
loc_dao.create()
cwd = os.getcwd()
file_install_root = conf_tree.node.get_value(
["file-install-root"], os.getenv("ROSE_FILE_INSTALL_ROOT", None))
if file_install_root:
file_install_root = env_var_process(file_install_root)
self.manager.fs_util.makedirs(file_install_root)
self.manager.fs_util.chdir(file_install_root)
try:
self._process(conf_tree, nodes, loc_dao, **kwargs)
finally:
if cwd != os.getcwd():
self.manager.fs_util.chdir(cwd)
def pull(self, loc, conf_tree):
"""Write namelist to loc.cache."""
sections = self.parse(loc, conf_tree)
if loc.name.endswith("(:)"):
sections.sort(rose.config.sort_settings)
handle = open(loc.cache, "wb")
for section in sections:
section_value = conf_tree.node.get_value([section])
group = RE_NAMELIST_GROUP.match(section).group(1)
nlg = "&" + group + "\n"
for key, node in sorted(section_value.items()):
if node.state:
continue
try:
value = env_var_process(node.value)
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError([section, key], node.value, exc)
nlg += "%s=%s,\n" % (key, value)
nlg += "/" + "\n"
handle.write(nlg)
self.manager.handle_event(NamelistEvent(nlg))
handle.close()
def _process(self, conf_tree, nodes, loc_dao, **kwargs):
"""Helper for self.process."""
# Ensure that everything is overwritable
# Ensure that container directories exist
for key, node in sorted(nodes.items()):
try:
name = env_var_process(key[len(self.PREFIX):])
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError([key], key, exc)
if os.path.exists(name) and kwargs.get("no_overwrite_mode"):
exc = FileOverwriteError(name)
raise ConfigProcessError([key], None, exc)
self.manager.fs_util.makedirs(self.manager.fs_util.dirname(name))
# Gets a list of sources and targets
sources = {}
targets = {}
for key, node in sorted(nodes.items()):
# N.B. no need to catch UnboundEnvironmentVariableError here
# because any exception should been caught earlier.
name = env_var_process(key[len(self.PREFIX):])
targets[name] = Loc(name)
targets[name].action_key = Loc.A_INSTALL
targets[name].mode = node.get_value(["mode"])
if targets[name].mode and targets[name].mode not in Loc.MODES:
raise ConfigProcessError([key, "mode"], targets[name].mode)
target_sources = []
for k in ["content", "source"]:
source_str = node.get_value([k])
if source_str is None:
continue
try:
source_str = env_var_process(source_str)
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError([key, k], source_str, exc)
source_names = []
for raw_source_glob in shlex.split(source_str):
source_glob = raw_source_glob
if (raw_source_glob.startswith("(")
and raw_source_glob.endswith(")")):
source_glob = raw_source_glob[1:-1]
names = glob(source_glob)
if names:
source_names += sorted(names)
else:
source_names.append(raw_source_glob)
for raw_source_name in source_names:
source_name = raw_source_name
is_optional = (raw_source_name.startswith("(")
and raw_source_name.endswith(")"))
if is_optional:
source_name = raw_source_name[1:-1]
if source_name.startswith("~"):
source_name = os.path.expanduser(source_name)
if targets[name].mode == targets[name].MODE_SYMLINK:
if targets[name].real_name:
# Symlink mode can only have 1 source
raise ConfigProcessError([key, k], source_str)
targets[name].real_name = source_name
else:
if source_name not in sources:
sources[source_name] = Loc(source_name)
sources[source_name].action_key = Loc.A_SOURCE
sources[source_name].is_optional = is_optional
sources[source_name].used_by_names.append(name)
target_sources.append(sources[source_name])
targets[name].dep_locs = target_sources
if (targets[name].mode == targets[name].MODE_SYMLINK
and not targets[name].real_name):
raise ConfigProcessError([key, "source"], None)
# Determine the scheme of the location from configuration.
config_schemes_str = conf_tree.node.get_value(["schemes"])
config_schemes = [] # [(pattern, scheme), ...]
if config_schemes_str:
for line in config_schemes_str.splitlines():
pattern, scheme = line.split("=", 1)
pattern = pattern.strip()
scheme = scheme.strip()
config_schemes.append((pattern, scheme))
# Where applicable, determine for each source:
# * Its real name.
# * The checksums of its paths.
# * Whether it can be considered unchanged.
for source in sources.values():
try:
for pattern, scheme in config_schemes:
if fnmatch(source.name, pattern):
source.scheme = scheme
break
self.loc_handlers_manager.parse(source, conf_tree)
except ValueError as exc:
if source.is_optional:
sources.pop(source.name)
for name in source.used_by_names:
targets[name].dep_locs.remove(source)
event = SourceSkipEvent(name, source.name)
self.handle_event(event)
continue
else:
raise ConfigProcessError(
["file:" + source.used_by_names[0], "source"],
source.name)
prev_source = loc_dao.select(source.name)
source.is_out_of_date = (
not prev_source or (not source.key and not source.paths)
or prev_source.scheme != source.scheme
or prev_source.loc_type != source.loc_type
or prev_source.key != source.key
or sorted(prev_source.paths) != sorted(source.paths))
# Inspect each target to see if it is out of date:
# * Target does not already exist.
# * Target exists, but does not have a database entry.
# * Target exists, but does not match settings in database.
# * Target exists, but a source cannot be considered unchanged.
for target in targets.values():
if target.real_name:
target.is_out_of_date = (
not os.path.islink(target.name)
or target.real_name != os.readlink(target.name))
elif target.mode == target.MODE_MKDIR:
target.is_out_of_date = (os.path.islink(target.name)
or not os.path.isdir(target.name))
else:
# See if target is modified compared with previous record
if (os.path.exists(target.name)
and not os.path.islink(target.name)):
for path, checksum, access_mode in get_checksum(
target.name):
target.add_path(path, checksum, access_mode)
target.paths.sort()
prev_target = loc_dao.select(target.name)
target.is_out_of_date = (
os.path.islink(target.name)
or not os.path.exists(target.name) or prev_target is None
or prev_target.mode != target.mode
or len(prev_target.paths) != len(target.paths))
if not target.is_out_of_date:
for prev_path, path in zip(prev_target.paths,
target.paths):
if prev_path != path:
target.is_out_of_date = True
break
# See if any sources out of date
if not target.is_out_of_date:
for dep_loc in target.dep_locs:
if dep_loc.is_out_of_date:
target.is_out_of_date = True
break
if target.is_out_of_date:
target.paths = None
loc_dao.delete(target)
# Set up jobs for rebuilding all out-of-date targets.
jobs = {}
for name, target in sorted(targets.items()):
if not target.is_out_of_date:
self.handle_event(FileUnchangedEvent(target, level=Event.V))
continue
if target.mode == target.MODE_SYMLINK:
self.manager.fs_util.symlink(target.real_name, target.name)
loc_dao.update(target)
elif target.mode == target.MODE_MKDIR:
if os.path.islink(target.name):
self.manager.fs_util.delete(target.name)
self.manager.fs_util.makedirs(target.name)
loc_dao.update(target)
target.loc_type = target.TYPE_TREE
target.add_path(target.BLOB, None, None)
elif target.dep_locs:
if os.path.islink(target.name):
self.manager.fs_util.delete(target.name)
jobs[target.name] = JobProxy(target)
for source in target.dep_locs:
if source.name not in jobs:
jobs[source.name] = JobProxy(source)
jobs[source.name].event_level = Event.V
job = jobs[source.name]
jobs[target.name].pending_for[source.name] = job
p_name = target.name
while (os.path.dirname(p_name)
and os.path.dirname(p_name) != p_name):
p_name = os.path.dirname(p_name)
if p_name in jobs:
jobs[target.name].pending_for[p_name] = jobs[p_name]
else:
self.manager.fs_util.install(target.name)
target.loc_type = target.TYPE_BLOB
for path, checksum, access_mode in get_checksum(target.name):
target.add_path(path, checksum, access_mode)
loc_dao.update(target)
if jobs:
work_dir = mkdtemp()
try:
nproc_keys = ["rose.config_processors.fileinstall", "nproc"]
nproc_str = conf_tree.node.get_value(nproc_keys)
nproc = None
if nproc_str is not None:
nproc = int(nproc_str)
job_runner = JobRunner(self, nproc)
job_runner(JobManager(jobs), conf_tree, loc_dao, work_dir)
except ValueError as exc:
if exc.args and exc.args[0] in jobs:
job = jobs[exc.args[0]]
if job.context.action_key == Loc.A_SOURCE:
source = job.context
keys = [
self.PREFIX + source.used_by_names[0], "source"
]
raise ConfigProcessError(keys, source.name)
raise exc
finally:
rmtree(work_dir)
# Target checksum compare and report
for target in targets.values():
if (not target.is_out_of_date
or target.loc_type == target.TYPE_TREE):
continue
keys = [self.PREFIX + target.name, "checksum"]
checksum_expected = conf_tree.node.get_value(keys)
if checksum_expected is None:
continue
checksum = target.paths[0].checksum
if checksum_expected:
if len(checksum_expected) != len(checksum):
algorithm = guess_checksum_algorithm(checksum_expected)
if algorithm:
checksum = get_checksum_func(algorithm)(target.name)
if checksum_expected != checksum:
exc = ChecksumError(checksum_expected, checksum)
raise ConfigProcessError(keys, checksum_expected, exc)
event = ChecksumEvent(target.name, target.paths[0].checksum)
self.handle_event(event)
def process(self,
conf_tree,
item,
orig_keys=None,
orig_value=None,
**kwargs):
"""Process [jinja2:*] in "conf_tree.node".
Arguments:
conf_tree:
The relevant rose.config_tree.ConfigTree object with the full
configuration.
item: The current configuration item to process.
orig_keys:
The keys for locating the originating setting in conf_tree in a
recursive processing. None implies a top level call.
orig_value: The value of orig_keys in conf_tree.
**kwargs:
environ (dict): suite level environment variables.
"""
for s_key, s_node in sorted(conf_tree.node.value.items()):
if (s_node.is_ignored() or not s_key.startswith(self.PREFIX)
or not s_node.value):
continue
target = s_key[len(self.PREFIX):]
source = os.path.join(conf_tree.files[target], target)
if not os.access(source, os.F_OK | os.R_OK):
continue
scheme_ln = self.SCHEME_TEMPL % self.SCHEME
msg_init_ln = self.COMMENT_TEMPL % self.MSG_INIT
msg_done_ln = self.COMMENT_TEMPL % self.MSG_DONE
tmp_file = NamedTemporaryFile()
tmp_file.write(scheme_ln)
tmp_file.write(msg_init_ln)
for key, node in sorted(s_node.value.items()):
if node.is_ignored():
continue
try:
value = env_var_process(node.value)
except UnboundEnvironmentVariableError as exc:
raise ConfigProcessError([s_key, key], node.value, exc)
tmp_file.write(self.ASSIGN_TEMPL % (key, value))
environ = kwargs.get("environ")
if environ:
tmp_file.write('[cylc]\n')
tmp_file.write(' [[environment]]\n')
for key, value in sorted(environ.items()):
tmp_file.write(' %s=%s\n' % (key, value))
tmp_file.write(msg_done_ln)
line_n = 0
is_in_old_insert = False
for line in open(source):
line_n += 1
if line_n == 1 and line.strip().lower() == scheme_ln.strip():
continue
elif line_n == 2 and line == msg_init_ln:
is_in_old_insert = True
continue
elif is_in_old_insert and line == msg_done_ln:
is_in_old_insert = False
continue
elif is_in_old_insert:
continue
tmp_file.write(line)
tmp_file.seek(0)
if os.access(target, os.F_OK | os.R_OK):
if filecmp.cmp(target, tmp_file.name): # identical
tmp_file.close()
continue
else:
self.manager.fs_util.delete(target)
# Write content to target
target_file = open(target, "w")
for line in tmp_file:
target_file.write(line)
event = FileSystemEvent(FileSystemEvent.INSTALL, target)
self.manager.handle_event(event)
tmp_file.close()
