def _run(self, dao, app_runner, config):
"""Transform and archive suite files.
This application is designed to work under "rose task-run" in a suite.
"""
compress_manager = SchemeHandlersManager(
[os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))],
"rose.apps.rose_arch_compressions",
["compress_sources"],
None, app_runner)
# Set up the targets
s_key_tails = set()
targets = []
for t_key, t_node in sorted(config.value.items()):
if t_node.is_ignored() or ":" not in t_key:
continue
s_key_head, s_key_tail = t_key.split(":", 1)
if s_key_head != self.SECTION or not s_key_tail:
continue
# Determine target path.
s_key_tail = t_key.split(":", 1)[1]
try:
s_key_tail = env_var_process(s_key_tail)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError([t_key, ""], "", exc)
# If parenthesised target is optional.
is_compulsory_target = True
if s_key_tail.startswith("(") and s_key_tail.endswith(")"):
s_key_tail = s_key_tail[1:-1]
is_compulsory_target = False
# Don't permit duplicate targets.
if s_key_tail in s_key_tails:
raise RoseArchDuplicateError([t_key], '', s_key_tail)
else:
s_key_tails.add(s_key_tail)
target = self._run_target_setup(
app_runner, compress_manager, config, t_key, s_key_tail,
t_node, is_compulsory_target)
old_target = dao.select(target.name)
if old_target is None or old_target != target:
dao.delete(target)
else:
target.status = target.ST_OLD
targets.append(target)
targets.sort(key=lambda target: target.name)
# Delete from database items that are no longer relevant
dao.delete_all(filter_targets=targets)
# Update the targets
for target in targets:
self._run_target_update(dao, app_runner, compress_manager, target)
return [target.status for target in targets].count(
RoseArchTarget.ST_BAD)
def _conf_value(conf_tree, keys, default=None):
"""Return conf setting value, with env var processed."""
value = conf_tree.node.get_value(keys, default)
if value is None:
return
try:
return env_var_process(value)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError(keys, value, exc)
def _get_conf(self, r_node, t_node, key, compulsory=False, default=None):
"""Return the value of a configuration."""
value = t_node.get_value([key],
r_node.get_value([self.SECTION, key],
default=default))
if compulsory and not value:
raise CompulsoryConfigValueError([key], None, KeyError(key))
if value:
try:
value = env_var_process(value)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError([key], value, exc)
return value
def _get_prune_globs(self, app_runner, conf_tree):
"""Return (globs, cycles).
where:
* globs is for matching items to prune.
* cycles is a set of relevant cycles.
"""
globs = []
nodes = conf_tree.node.get_value([self.SECTION])
if nodes is None:
return [], set()
cycle_formats = {}
for key, node in nodes.items():
if node.is_ignored():
continue
if key.startswith("cycle-format{") and key.endswith("}"):
fmt = key[len("cycle-format{"):-1]
try:
cycle_formats[fmt] = env_var_process(node.value)
# Check formats are valid
if self._get_cycling_mode() == "integer":
cycle_formats[fmt] % 0
else:
app_runner.date_time_oper.date_format(
cycle_formats[fmt])
except (UnboundEnvironmentVariableError, ValueError) as exc:
raise ConfigValueError([self.SECTION, key], node.value,
exc)
cycle_set = set()
for key, node in sorted(nodes.items()):
if node.is_ignored():
continue
if key == "prune-datac-at": # backward compat
head = "share/cycle"
elif key == "prune-work-at": # backward compat
head = "work"
elif key.startswith("prune{") and key.endswith("}"):
head = key[len("prune{"):-1].strip() # remove "prune{" and "}"
else:
continue
for cycle, cycle_args in self._get_conf(app_runner,
conf_tree,
key,
max_args=1):
cycle_set.add(cycle)
if cycle_args:
cycle_strs = {"cycle": cycle}
for cycle_key, cycle_format in cycle_formats.items():
if self._get_cycling_mode() == "integer":
cycle_strs[cycle_key] = cycle_format % int(cycle)
else: # date time cycling
cycle_point = app_runner.date_time_oper.date_parse(
cycle)[0]
cycle_strs[
cycle_key] = app_runner.date_time_oper.date_format(
cycle_format, cycle_point)
for tail_glob in shlex.split(cycle_args.pop()):
glob_ = tail_glob % cycle_strs
if glob_ == tail_glob: # no substitution
glob_ = os.path.join(cycle, tail_glob)
globs.append(os.path.join(head, glob_))
else:
globs.append(os.path.join(head, cycle))
return globs, cycle_set
def _get_conf(self, app_runner, conf_tree, key, max_args=0):
"""Get a list of cycles from a configuration setting.
key -- An option key in self.SECTION to locate the setting.
max_args -- Maximum number of extra arguments for an item in the list.
The value of the setting is expected to be split by shlex.split into a
list of items. If max_args == 0, an item should be a string
representing a cycle or an cycle offset. If max_args > 0, the cycle
or cycle offset string can, optionally, have arguments. The arguments
are delimited by colons ":".
E.g.:
prune-remote-logs-at=-PT6H -PT12H
prune-server-logs-at=-P7D
prune-datac-at=-PT6H:foo/* -PT12H:'bar/* baz/*' -P1D
prune-work-at=-PT6H:t1*:*.tar -PT12H:t1*: -PT12H:*.gz -P1D
If max_args == 0, return a list of cycles.
If max_args > 0, return a list of (cycle, [arg, ...])
"""
items_str = conf_tree.node.get_value([self.SECTION, key])
if items_str is None:
return []
try:
items_str = env_var_process(items_str)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError([self.SECTION, key], items_str, exc)
items = []
ref_point_str = os.getenv(RoseDateTimeOperator.TASK_CYCLE_TIME_ENV)
try:
ref_point = None
ref_fmt = None
for item_str in shlex.split(items_str):
args = item_str.split(":", max_args)
when = args.pop(0)
cycle = when
if ref_point_str is not None:
if self._get_cycling_mode() == "integer":
# Integer cycling
if "P" in when: # "when" is an offset
cycle = str(
int(ref_point_str) +
int(when.replace("P", "")))
else: # "when" is a cycle point
cycle = str(when)
else:
# Date-time cycling
if ref_fmt is None:
(
ref_point,
ref_fmt,
) = app_runner.date_time_oper.date_parse(
ref_point_str)
try:
time_point = app_runner.date_time_oper.date_parse(
when)[0]
except ValueError:
time_point = app_runner.date_time_oper.date_shift(
ref_point, when)
cycle = app_runner.date_time_oper.date_format(
ref_fmt, time_point)
if max_args:
items.append((cycle, args))
else:
items.append(cycle)
except ValueError as exc:
raise ConfigValueError([self.SECTION, key], items_str, exc)
return items
def run(self, app_runner, conf_tree, opts, args, uuid, work_files):
""" Run multiple instances of a command using sets of specified args"""
# Counts for reporting purposes
run_ok = 0
run_fail = 0
run_skip = 0
notrun = 0
# Allow naming of individual calls
self.invocation_names = conf_tree.node.get_value([self.BUNCH_SECTION,
"names"])
if self.invocation_names:
self.invocation_names = shlex.split(
metomi.rose.env.env_var_process(self.invocation_names))
if len(set(self.invocation_names)) != len(self.invocation_names):
raise ConfigValueError([self.BUNCH_SECTION, "names"],
self.invocation_names,
"names must be unique")
self.fail_mode = metomi.rose.env.env_var_process(
conf_tree.node.get_value(
[self.BUNCH_SECTION, "fail-mode"], self.TYPE_CONTINUE_ON_FAIL))
if self.fail_mode not in self.FAIL_MODE_TYPES:
raise ConfigValueError([self.BUNCH_SECTION, "fail-mode"],
self.fail_mode,
"not a valid setting")
self.incremental = conf_tree.node.get_value([self.BUNCH_SECTION,
"incremental"],
"true")
if self.incremental:
self.incremental = metomi.rose.env.env_var_process(
self.incremental)
self.isformatted = True
self.command = metomi.rose.env.env_var_process(
conf_tree.node.get_value([self.BUNCH_SECTION, "command-format"]))
if not self.command:
self.isformatted = False
self.command = app_runner.get_command(conf_tree, opts, args)
if not self.command:
raise CommandNotDefinedError()
# Set up command-instances if needed
instances = conf_tree.node.get_value([self.BUNCH_SECTION,
"command-instances"])
if instances:
try:
instances = range(
int(metomi.rose.env.env_var_process(instances)))
except ValueError:
raise ConfigValueError([self.BUNCH_SECTION,
"command-instances"],
instances,
"not an integer value")
# Argument lists
multi_args = conf_tree.node.get_value([self.ARGS_SECTION], {})
bunch_args_names = []
bunch_args_values = []
for key, val in multi_args.items():
bunch_args_names.append(key)
bunch_args_values.append(
shlex.split(metomi.rose.env.env_var_process(val.value)))
# Update the argument values based on the argument-mode
argument_mode = conf_tree.node.get_value([self.BUNCH_SECTION,
"argument-mode"],
self.DEFAULT_ARGUMENT_MODE)
if argument_mode == self.DEFAULT_ARGUMENT_MODE:
pass
elif argument_mode in self.ACCEPTED_ARGUMENT_MODES:
# The behaviour of of izip and izip_longest are special cases
# because:
# * izip was deprecated in Python3 use zip
# * itertools.izip_longest was renamed and requires the fillvalue
# kwarg
if argument_mode in ['zip', 'izip']:
_permutations = zip(*bunch_args_values)
elif argument_mode in ['zip_longest', 'izip_longest']:
_permutations = itertools.zip_longest(*bunch_args_values,
fillvalue="")
else:
iteration_cmd = getattr(itertools, argument_mode)
_permutations = iteration_cmd(*bunch_args_values)
# Reconstruct the bunch_args_values
_permutations = list(_permutations)
for index, _ in enumerate(bunch_args_values):
bunch_args_values[index] = [v[index] for v in _permutations]
else:
raise ConfigValueError([self.BUNCH_SECTION,
"argument-mode"],
argument_mode,
"must be one of %s" %
self.ACCEPTED_ARGUMENT_MODES)
# Validate runlists
if not self.invocation_names:
if instances:
arglength = len(instances)
else:
arglength = len(bunch_args_values[0])
self.invocation_names = list(range(0, arglength))
else:
arglength = len(self.invocation_names)
for item, vals in zip(bunch_args_names, bunch_args_values):
if len(vals) != arglength:
raise ConfigValueError([self.ARGS_SECTION, item],
conf_tree.node.get_value(
[self.ARGS_SECTION, item]),
"inconsistent arg lengths")
if conf_tree.node.get_value([self.ARGS_SECTION, "command-instances"]):
raise ConfigValueError([self.ARGS_SECTION, "command-instances"],
conf_tree.node.get_value(
[self.ARGS_SECTION, "command-instances"]),
"reserved keyword")
if conf_tree.node.get_value([self.ARGS_SECTION, "COMMAND_INSTANCES"]):
raise ConfigValueError([self.ARGS_SECTION, "COMMAND_INSTANCES"],
conf_tree.node.get_value(
[self.ARGS_SECTION, "COMMAND_INSTANCES"]),
"reserved keyword")
if instances and arglength != len(instances):
raise ConfigValueError([self.BUNCH_SECTION, "command-instances"],
instances, "inconsistent arg lengths")
# Set max number of processes to run at once
max_procs = conf_tree.node.get_value([self.BUNCH_SECTION, "pool-size"])
if max_procs:
max_procs = int(metomi.rose.env.env_var_process(max_procs))
else:
max_procs = arglength
if self.incremental == "true":
self.dao = RoseBunchDAO(conf_tree)
else:
self.dao = None
commands = {}
for vals in zip(range(arglength), self.invocation_names,
*bunch_args_values):
index, name, bunch_args_vals = vals[0], vals[1], vals[2:]
argsdict = dict(zip(bunch_args_names, bunch_args_vals))
if instances:
if self.isformatted:
argsdict["command-instances"] = instances[index]
else:
argsdict["COMMAND_INSTANCES"] = str(instances[index])
commands[name] = RoseBunchCmd(name, self.command, argsdict,
self.isformatted)
procs = {}
if 'ROSE_TASK_LOG_DIR' in os.environ:
log_format = os.path.join(os.environ['ROSE_TASK_LOG_DIR'], "%s")
else:
log_format = os.path.join(os.getcwd(), "%s")
failed = {}
abort = False
while procs or (commands and not abort):
for key, proc in list(procs.items()):
if proc.poll() is not None:
procs.pop(key)
if proc.returncode:
failed[key] = proc.returncode
run_fail += 1
app_runner.handle_event(RosePopenError(str(key),
proc.returncode,
None, None))
if self.dao:
self.dao.update_command_state(key, self.dao.S_FAIL)
if self.fail_mode == self.TYPE_ABORT_ON_FAIL:
abort = True
app_runner.handle_event(AbortEvent())
else:
run_ok += 1
app_runner.handle_event(SucceededEvent(key),
prefix=self.PREFIX_OK)
if self.dao:
self.dao.update_command_state(key, self.dao.S_PASS)
while len(procs) < max_procs and commands and not abort:
key = self.invocation_names[0]
command = commands.pop(key)
self.invocation_names.pop(0)
cmd = command.get_command()
cmd_stdout = log_format % command.get_out_file()
cmd_stderr = log_format % command.get_err_file()
prefix = command.get_log_prefix()
bunch_environ = os.environ
if not command.isformatted:
bunch_environ.update(command.argsdict)
bunch_environ['ROSE_BUNCH_LOG_PREFIX'] = prefix
if self.dao:
if self.dao.check_has_succeeded(key):
run_skip += 1
app_runner.handle_event(PreviousSuccessEvent(key),
prefix=self.PREFIX_PASS)
continue
else:
self.dao.add_command(key)
app_runner.handle_event(LaunchEvent(key, cmd))
procs[key] = app_runner.popen.run_bg(
cmd,
shell=True,
stdout=open(cmd_stdout, 'w'),
stderr=open(cmd_stderr, 'w'),
env=bunch_environ)
sleep(self.SLEEP_DURATION)
if abort and commands:
for key in self.invocation_names:
notrun += 1
cmd = commands.pop(key).get_command()
app_runner.handle_event(NotRunEvent(key, cmd),
prefix=self.PREFIX_NOTRUN)
if self.dao:
self.dao.close()
# Report summary data in job.out file
app_runner.handle_event(SummaryEvent(
run_ok, run_fail, run_skip, notrun))
if failed:
return 1
else:
return 0
def _run_target_setup(self,
app_runner,
compress_manager,
config,
t_key,
s_key_tail,
t_node,
is_compulsory_target=True):
"""Helper for _run. Set up a target."""
target_prefix = self._get_conf(config,
t_node,
"target-prefix",
default="")
target = RoseArchTarget(target_prefix + s_key_tail)
target.command_format = self._get_conf(config,
t_node,
"command-format",
compulsory=True)
try:
target.command_format % {"sources": "", "target": ""}
except KeyError as exc:
target.status = target.ST_BAD
app_runner.handle_event(
RoseArchValueError(target.name, "command-format",
target.command_format,
type(exc).__name__, exc))
target.source_edit_format = self._get_conf(config,
t_node,
"source-edit-format",
default="")
try:
target.source_edit_format % {"in": "", "out": ""}
except KeyError as exc:
target.status = target.ST_BAD
app_runner.handle_event(
RoseArchValueError(target.name, "source-edit-format",
target.source_edit_format,
type(exc).__name__, exc))
update_check_str = self._get_conf(config, t_node, "update-check")
try:
checksum_func = get_checksum_func(update_check_str)
except ValueError as exc:
raise RoseArchValueError(target.name, "update-check",
update_check_str,
type(exc).__name__, exc)
source_prefix = self._get_conf(config,
t_node,
"source-prefix",
default="")
for source_glob in shlex.split(
self._get_conf(config, t_node, "source", compulsory=True)):
is_compulsory_source = is_compulsory_target
if source_glob.startswith("(") and source_glob.endswith(")"):
source_glob = source_glob[1:-1]
is_compulsory_source = False
paths = glob(source_prefix + source_glob)
if not paths:
exc = OSError(errno.ENOENT, os.strerror(errno.ENOENT),
source_prefix + source_glob)
app_runner.handle_event(
ConfigValueError([t_key, "source"], source_glob, exc))
if is_compulsory_source:
target.status = target.ST_BAD
continue
for path in paths:
# N.B. source_prefix may not be a directory
name = path[len(source_prefix):]
for path_, checksum, _ in get_checksum(path, checksum_func):
if checksum is None: # is directory
continue
if path_:
target.sources[checksum] = RoseArchSource(
checksum, os.path.join(name, path_),
os.path.join(path, path_))
else: # path is a file
target.sources[checksum] = RoseArchSource(
checksum, name, path)
if not target.sources:
if is_compulsory_target:
target.status = target.ST_BAD
else:
target.status = target.ST_NULL
target.compress_scheme = self._get_conf(config, t_node, "compress")
if not target.compress_scheme:
target_base = target.name
if "/" in target.name:
target_base = target.name.rsplit("/", 1)[1]
if "." in target_base:
tail = target_base.split(".", 1)[1]
if compress_manager.get_handler(tail):
target.compress_scheme = tail
elif compress_manager.get_handler(target.compress_scheme) is None:
app_runner.handle_event(
ConfigValueError([t_key, "compress"], target.compress_scheme,
KeyError(target.compress_scheme)))
target.status = target.ST_BAD
rename_format = self._get_conf(config, t_node, "rename-format")
if rename_format:
rename_parser_str = self._get_conf(config, t_node, "rename-parser")
if rename_parser_str:
try:
rename_parser = re.compile(rename_parser_str)
except re.error as exc:
raise RoseArchValueError(target.name, "rename-parser",
rename_parser_str,
type(exc).__name__, exc)
else:
rename_parser = None
for source in target.sources.values():
dict_ = {
"cycle": os.getenv("ROSE_TASK_CYCLE_TIME"),
"name": source.name
}
if rename_parser:
match = rename_parser.match(source.name)
if match:
dict_.update(match.groupdict())
try:
source.name = rename_format % dict_
except (KeyError, ValueError) as exc:
raise RoseArchValueError(target.name, "rename-format",
rename_format,
type(exc).__name__, exc)
return target