def _run(self, dao, app_runner, config):
"""Transform and archive suite files.
This application is designed to work under "rose task-run" in a suite.
"""
compress_manager = SchemeHandlersManager(
[os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))],
"rose.apps.rose_arch_compressions", ["compress_sources"], None,
app_runner)
# Set up the targets
s_key_tails = set()
targets = []
for t_key, t_node in sorted(config.value.items()):
if t_node.is_ignored() or ":" not in t_key:
continue
s_key_head, s_key_tail = t_key.split(":", 1)
if s_key_head != self.SECTION or not s_key_tail:
continue
# Determine target path.
s_key_tail = t_key.split(":", 1)[1]
try:
s_key_tail = env_var_process(s_key_tail)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError([t_key, ""], "", exc)
# If parenthesised target is optional.
is_compulsory_target = True
if s_key_tail.startswith("(") and s_key_tail.endswith(")"):
s_key_tail = s_key_tail[1:-1]
is_compulsory_target = False
# Don't permit duplicate targets.
if s_key_tail in s_key_tails:
raise RoseArchDuplicateError([t_key], '', s_key_tail)
else:
s_key_tails.add(s_key_tail)
target = self._run_target_setup(app_runner, compress_manager,
config, t_key, s_key_tail, t_node,
is_compulsory_target)
old_target = dao.select(target.name)
if old_target is None or old_target != target:
dao.delete(target)
else:
target.status = target.ST_OLD
targets.append(target)
targets.sort(key=lambda target: target.name)
# Delete from database items that are no longer relevant
dao.delete_all(filter_targets=targets)
# Update the targets
for target in targets:
self._run_target_update(dao, app_runner, compress_manager, target)
return [target.status
for target in targets].count(RoseArchTarget.ST_BAD)
def _get_conf(self, conf_tree, key, max_args=0):
"""Get a list of cycles from a configuration setting.
key -- An option key in self.SECTION to locate the setting.
max_args -- Maximum number of extra arguments for an item in the list.
The value of the setting is expected to be split by shlex.split into a
list of items. If max_args == 0, an item should be a string
representing a cycle or an cycle offset. If max_args > 0, the cycle
or cycle offset string can, optionally, have arguments. The arguments
are delimited by colons ":".
E.g.:
prune-remote-logs-at=-6h -12h
prune-server-logs-at=-7d
prune-datac-at=-6h:foo/* -12h:'bar/* baz/*' -1d
prune-work-at=-6h:t1*:*.tar -12h:t1*: -12h:*.gz -1d
If max_args == 0, return a list of cycles.
If max_args > 0, return a list of (cycle, [arg, ...])
"""
items_str = conf_tree.node.get_value([self.SECTION, key])
if items_str is None:
return []
try:
items_str = env_var_process(items_str)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError([self.SECTION, key], items_str, exc)
items = []
ref_time_point = os.getenv(
RoseDateTimeOperator.TASK_CYCLE_TIME_MODE_ENV)
date_time_oper = RoseDateTimeOperator(ref_time_point=ref_time_point)
for item_str in shlex.split(items_str):
args = item_str.split(":", max_args)
offset = args.pop(0)
cycle = offset
if ref_time_point:
if os.getenv("ROSE_CYCLING_MODE") == "integer":
try:
cycle = str(int(ref_time_point) +
int(offset.replace("P","")))
except ValueError:
pass
else:
try:
time_point, parse_format = date_time_oper.date_parse()
time_point = date_time_oper.date_shift(time_point, offset)
cycle = date_time_oper.date_format(
parse_format,
time_point)
except ValueError:
pass
if max_args:
items.append((cycle, args))
else:
items.append(cycle)
return items
def _conf_value(conf_tree, keys, default=None):
"""Return conf setting value, with env var processed."""
value = conf_tree.node.get_value(keys, default)
if value is None:
return
try:
return env_var_process(value)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError(keys, value, exc)
def _get_prune_globs(self, app_runner, conf_tree):
"""Return prune globs."""
globs = []
nodes = conf_tree.node.get_value([self.SECTION])
if nodes is None:
return []
cycle_formats = {}
for key, node in nodes.items():
if node.is_ignored():
continue
if key.startswith("cycle-format{") and key.endswith("}"):
fmt = key[len("cycle-format{"):-1]
try:
cycle_formats[fmt] = env_var_process(node.value)
# Check formats are valid
if self._get_cycling_mode() == "integer":
cycle_formats[fmt] % 0
else:
app_runner.date_time_oper.date_format(
cycle_formats[fmt])
except (UnboundEnvironmentVariableError, ValueError) as exc:
raise ConfigValueError([self.SECTION, key], node.value,
exc)
for key, node in sorted(nodes.items()):
if node.is_ignored():
continue
if key == "prune-datac-at": # backward compat
head = "share/cycle"
elif key == "prune-work-at": # backward compat
head = "work"
elif key.startswith("prune{") and key.endswith("}"):
head = key[len("prune{"):-1].strip() # remove "prune{" and "}"
else:
continue
for cycle, cycle_args in self._get_conf(app_runner,
conf_tree,
key,
max_args=1):
if cycle_args:
cycle_strs = {"cycle": cycle}
for key, cycle_format in cycle_formats.items():
if self._get_cycling_mode() == "integer":
cycle_strs[key] = cycle_format % int(cycle)
else: # date time cycling
cycle_point = (
app_runner.date_time_oper.date_parse(cycle)[0])
cycle_strs[key] = (
app_runner.date_time_oper.date_format(
cycle_format, cycle_point))
for tail_glob in shlex.split(cycle_args.pop()):
glob_ = tail_glob % cycle_strs
if glob_ == tail_glob: # no substitution
glob_ = os.path.join(cycle, tail_glob)
globs.append(os.path.join(head, glob_))
else:
globs.append(os.path.join(head, cycle))
return globs
def _get_conf(self, r_node, t_node, key, compulsory=False, default=None):
"""Return the value of a configuration."""
value = t_node.get_value([key],
r_node.get_value([self.SECTION, key],
default=default))
if compulsory and not value:
raise CompulsoryConfigValueError([key], None, KeyError(key))
if value:
try:
value = env_var_process(value)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError([key], value, exc)
return value
def run(self, app_runner, conf_tree, opts, args, uuid, work_files):
""" Run multiple instaces of a command using sets of specified args"""
# Counts for reporting purposes
run_ok = 0
run_fail = 0
run_skip = 0
notrun = 0
# Allow naming of individual calls
self.invocation_names = conf_tree.node.get_value(
[self.BUNCH_SECTION, "names"])
if self.invocation_names:
self.invocation_names = shlex.split(
rose.env.env_var_process(self.invocation_names))
if len(set(self.invocation_names)) != len(self.invocation_names):
raise ConfigValueError([self.BUNCH_SECTION, "names"],
self.invocation_names,
"names must be unique")
self.fail_mode = rose.env.env_var_process(
conf_tree.node.get_value([self.BUNCH_SECTION, "fail-mode"],
self.TYPE_CONTINUE_ON_FAIL))
if self.fail_mode not in self.FAIL_MODE_TYPES:
raise ConfigValueError([self.BUNCH_SECTION, "fail-mode"],
fail_mode, "not a valid setting")
self.incremental = conf_tree.node.get_value(
[self.BUNCH_SECTION, "incremental"], "true")
if self.incremental:
self.incremental = rose.env.env_var_process(self.incremental)
multi_args = conf_tree.node.get_value([self.ARGS_SECTION], {})
for key, val in multi_args.items():
multi_args[key].value = rose.env.env_var_process(val.value)
self.command_format = rose.env.env_var_process(
conf_tree.node.get_value([self.BUNCH_SECTION, "command-format"]))
if not self.command_format:
raise CompulsoryConfigValueError(
[self.BUNCH_SECTION, "command-format"], None,
KeyError("command-format"))
# Set up command-instances if needed
instances = conf_tree.node.get_value(
[self.BUNCH_SECTION, "command-instances"])
if instances:
try:
instances = range(int(rose.env.env_var_process(instances)))
except ValueError:
raise ConfigValueError(
[self.BUNCH_SECTION, "command-instances"], instances,
"not an integer value")
# Validate runlists
if not self.invocation_names:
if instances:
arglength = len(instances)
else:
item, val = sorted(multi_args.items())[0]
arglength = len(shlex.split(val.value))
self.invocation_names = range(0, arglength)
else:
arglength = len(self.invocation_names)
for item, val in sorted(multi_args.items()):
if len(shlex.split(val.value)) != arglength:
raise ConfigValueError([self.ARGS_SECTION, item],
conf_tree.node.get_value(
[self.ARGS_SECTION, item]),
"inconsistent arg lengths")
if conf_tree.node.get_value([self.ARGS_SECTION, "command-instances"]):
raise ConfigValueError([self.ARGS_SECTION, "command-instances"],
conf_tree.node.get_value([
self.ARGS_SECTION, "command-instances"
]), "reserved keyword")
if instances and arglength != len(instances):
raise ConfigValueError([self.BUNCH_SECTION, "command-instances"],
instances, "inconsistent arg lengths")
# Set max number of processes to run at once
max_procs = conf_tree.node.get_value([self.BUNCH_SECTION, "pool-size"])
if max_procs:
self.MAX_PROCS = int(rose.env.env_var_process(max_procs))
else:
self.MAX_PROCS = arglength
if self.incremental == "true":
self.dao = RoseBunchDAO(conf_tree)
else:
self.dao = None
commands = {}
for index, name in enumerate(self.invocation_names):
invocation = RoseBunchCmd(name, self.command_format, index)
for key, vals in sorted(multi_args.items()):
invocation.argsdict[key] = shlex.split(vals.value)[index]
if instances:
invocation.argsdict["command-instances"] = instances[index]
commands[name] = invocation
procs = {}
if 'ROSE_TASK_LOG_DIR' in os.environ:
log_format = os.path.join(os.environ['ROSE_TASK_LOG_DIR'], "%s")
else:
log_format = os.path.join(os.getcwd(), "%s")
failed = {}
abort = False
while procs or (commands and not abort):
for key, proc in procs.items():
if proc.poll() is not None:
procs.pop(key)
if proc.returncode:
failed[key] = proc.returncode
run_fail += 1
app_runner.handle_event(
RosePopenError(str(key), proc.returncode, None,
None))
if self.dao:
self.dao.update_command_state(key, self.dao.S_FAIL)
if self.fail_mode == self.TYPE_ABORT_ON_FAIL:
abort = True
app_runner.handle_event(AbortEvent())
else:
run_ok += 1
app_runner.handle_event(SucceededEvent(key),
prefix=self.PREFIX_OK)
if self.dao:
self.dao.update_command_state(key, self.dao.S_PASS)
while len(procs) < self.MAX_PROCS and commands and not abort:
key = self.invocation_names[0]
command = commands.pop(key)
self.invocation_names.pop(0)
cmd = command.get_command()
cmd_stdout = log_format % command.get_out_file()
cmd_stderr = log_format % command.get_err_file()
prefix = command.get_log_prefix()
bunch_environ = os.environ
bunch_environ['ROSE_BUNCH_LOG_PREFIX'] = prefix
if self.dao:
if self.dao.check_has_succeeded(key):
run_skip += 1
app_runner.handle_event(PreviousSuccessEvent(key),
prefix=self.PREFIX_PASS)
continue
else:
self.dao.add_command(key)
app_runner.handle_event(LaunchEvent(key, cmd))
procs[key] = \
app_runner.popen.run_bg(cmd,
shell=True,
stdout=open(cmd_stdout, 'w'),
stderr=open(cmd_stderr, 'w'),
env=bunch_environ)
sleep(self.SLEEP_DURATION)
if abort and commands:
for key in self.invocation_names:
notrun += 1
cmd = commands.pop(key).get_command()
app_runner.handle_event(NotRunEvent(key, cmd),
prefix=self.PREFIX_NOTRUN)
if self.dao:
self.dao.close()
# Report summary data in job.out file
app_runner.handle_event(
SummaryEvent(run_ok, run_fail, run_skip, notrun))
if failed:
return 1
else:
return 0
def _run_target_setup(self, app_runner, compress_manager, config, t_key,
t_node):
"""Helper for _run. Set up a target."""
target_prefix = self._get_conf(config,
t_node,
"target-prefix",
default="")
s_key_tail = t_key.split(":", 1)[1]
try:
s_key_tail = env_var_process(s_key_tail)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError([t_key, ""], "", exc)
is_compulsory_target = True
if s_key_tail.startswith("(") and s_key_tail.endswith(")"):
s_key_tail = s_key_tail[1:-1]
is_compulsory_target = False
target = RoseArchTarget(target_prefix + s_key_tail)
target.command_format = self._get_conf(config,
t_node,
"command-format",
compulsory=True)
try:
target.command_format % {"sources": "", "target": ""}
except KeyError as exc:
target.status = target.ST_BAD
app_runner.handle_event(
RoseArchValueError(target.name, "command-format",
target.command_format,
type(exc).__name__, exc))
target.source_edit_format = self._get_conf(config,
t_node,
"source-edit-format",
default="")
try:
target.source_edit_format % {"in": "", "out": ""}
except KeyError as exc:
target.status = target.ST_BAD
app_runner.handle_event(
RoseArchValueError(target.name, "source-edit-format",
target.source_edit_format,
type(exc).__name__, exc))
update_check_str = self._get_conf(config, t_node, "update-check")
try:
checksum_func = get_checksum_func(update_check_str)
except ValueError as exc:
raise RoseArchValueError(target.name, "update-check",
update_check_str,
type(exc).__name__, exc)
source_prefix = self._get_conf(config,
t_node,
"source-prefix",
default="")
for source_glob in shlex.split(
self._get_conf(config, t_node, "source", compulsory=True)):
is_compulsory_source = is_compulsory_target
if source_glob.startswith("(") and source_glob.endswith(")"):
source_glob = source_glob[1:-1]
is_compulsory_source = False
paths = glob(source_prefix + source_glob)
if not paths:
exc = OSError(errno.ENOENT, os.strerror(errno.ENOENT),
source_prefix + source_glob)
app_runner.handle_event(
ConfigValueError([t_key, "source"], source_glob, exc))
if is_compulsory_source:
target.status = target.ST_BAD
continue
for path in paths:
# N.B. source_prefix may not be a directory
name = path[len(source_prefix):]
for path_, checksum, _ in get_checksum(path, checksum_func):
if checksum is None: # is directory
continue
if path_:
target.sources[checksum] = RoseArchSource(
checksum, os.path.join(name, path_),
os.path.join(path, path_))
else: # path is a file
target.sources[checksum] = RoseArchSource(
checksum, name, path)
if not target.sources:
if is_compulsory_target:
target.status = target.ST_BAD
else:
target.status = target.ST_NULL
target.compress_scheme = self._get_conf(config, t_node, "compress")
if not target.compress_scheme:
target_base = target.name
if "/" in target.name:
target_base = target.name.rsplit("/", 1)[1]
if "." in target_base:
tail = target_base.split(".", 1)[1]
if compress_manager.get_handler(tail):
target.compress_scheme = tail
elif compress_manager.get_handler(target.compress_scheme) is None:
app_runner.handle_event(
ConfigValueError([t_key, "compress"], target.compress_scheme,
KeyError(target.compress_scheme)))
target.status = target.ST_BAD
rename_format = self._get_conf(config, t_node, "rename-format")
if rename_format:
rename_parser_str = self._get_conf(config, t_node, "rename-parser")
if rename_parser_str:
try:
rename_parser = re.compile(rename_parser_str)
except re.error as exc:
raise RoseArchValueError(target.name, "rename-parser",
rename_parser_str,
type(exc).__name__, exc)
else:
rename_parser = None
for source in target.sources.values():
dict_ = {
"cycle": os.getenv("ROSE_TASK_CYCLE_TIME"),
"name": source.name
}
if rename_parser:
match = rename_parser.match(source.name)
if match:
dict_.update(match.groupdict())
try:
source.name = rename_format % dict_
except (KeyError, ValueError) as exc:
raise RoseArchValueError(target.name, "rename-format",
rename_format,
type(exc).__name__, exc)
return target
def _run(self, dao, app_runner, config):
"""Transform and archive suite files.
This application is designed to work under "rose task-run" in a suite.
"""
path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
compress_manager = SchemeHandlersManager(
[path], "rose.apps.rose_arch_compressions", ["compress_sources"],
None, app_runner)
# Set up the targets
cycle = os.getenv("ROSE_TASK_CYCLE_TIME")
targets = []
for t_key, t_node in sorted(config.value.items()):
if t_node.is_ignored() or ":" not in t_key:
continue
s_key_head, s_key_tail = t_key.split(":", 1)
if s_key_head != self.SECTION or not s_key_tail:
continue
target_prefix = self._get_conf(config,
t_node,
"target-prefix",
default="")
try:
s_key_tail = env_var_process(s_key_tail)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError([t_key, ""], "", exc)
target_name = target_prefix + s_key_tail
target = RoseArchTarget(target_name)
target.command_format = self._get_conf(config,
t_node,
"command-format",
compulsory=True)
try:
target.command_format % {"sources": "", "target": ""}
except KeyError as exc:
target.status = target.ST_BAD
app_runner.handle_event(
RoseArchValueError(target.name, "command-format",
target.command_format,
type(exc).__name__, exc))
source_str = self._get_conf(config,
t_node,
"source",
compulsory=True)
source_prefix = self._get_conf(config,
t_node,
"source-prefix",
default="")
target.source_edit_format = self._get_conf(config,
t_node,
"source-edit-format",
default="")
try:
target.source_edit_format % {"in": "", "out": ""}
except KeyError as exc:
target.status = target.ST_BAD
app_runner.handle_event(
RoseArchValueError(target.name, "source-edit-format",
target.source_edit_format,
type(exc).__name__, exc))
update_check_str = self._get_conf(config,
t_node,
"update-check",
default="md5sum")
try:
checksum_func = get_checksum_func(update_check_str)
except KeyError as exc:
raise RoseArchValueError(target.name, "update-check",
update_check_str,
type(exc).__name__, exc)
for source_glob in shlex.split(source_str):
paths = glob(source_prefix + source_glob)
if not paths:
exc = OSError(errno.ENOENT, os.strerror(errno.ENOENT),
source_glob)
app_runner.handle_event(
ConfigValueError([t_key, "source"], source_glob, exc))
target.status = target.ST_BAD
continue
for path in paths:
# N.B. source_prefix may not be a directory
name = path[len(source_prefix):]
for path_, checksum, _ in get_checksum(
path, checksum_func):
if checksum is None: # is directory
continue
if path_:
target.sources[checksum] = RoseArchSource(
checksum, os.path.join(name, path_),
os.path.join(path, path_))
else: # path is a file
target.sources[checksum] = RoseArchSource(
checksum, name, path)
target.compress_scheme = self._get_conf(config, t_node, "compress")
if target.compress_scheme:
if (compress_manager.get_handler(target.compress_scheme) is
None):
app_runner.handle_event(
ConfigValueError([t_key, "compress"],
target.compress_scheme,
KeyError(target.compress_scheme)))
target.status = target.ST_BAD
else:
target_base = target.name
if "/" in target.name:
target_base = target.name.rsplit("/", 1)[1]
if "." in target_base:
tail = target_base.split(".", 1)[1]
if compress_manager.get_handler(tail):
target.compress_scheme = tail
rename_format = self._get_conf(config, t_node, "rename-format")
if rename_format:
rename_parser_str = self._get_conf(config, t_node,
"rename-parser")
if rename_parser_str:
try:
rename_parser = re.compile(rename_parser_str)
except re.error as exc:
raise RoseArchValueError(target.name, "rename-parser",
rename_parser_str,
type(exc).__name__, exc)
else:
rename_parser = None
for source in target.sources.values():
dict_ = {"cycle": cycle, "name": source.name}
if rename_parser:
match = rename_parser.match(source.name)
if match:
dict_.update(match.groupdict())
try:
source.name = rename_format % dict_
except (KeyError, ValueError) as exc:
raise RoseArchValueError(target.name, "rename-format",
rename_format,
type(exc).__name__, exc)
old_target = dao.select(target.name)
if old_target is None or old_target != target:
dao.delete(target)
else:
target.status = target.ST_OLD
targets.append(target)
# Delete from database items that are no longer relevant
dao.delete_all(filter_targets=targets)
# Update the targets
for target in targets:
if target.status == target.ST_OLD:
app_runner.handle_event(RoseArchEvent(target))
continue
target.command_rc = 1
dao.insert(target)
if target.status == target.ST_BAD:
app_runner.handle_event(RoseArchEvent(target))
continue
work_dir = mkdtemp()
t_init = time()
t_tran, t_arch = t_init, t_init
ret_code = None
try:
# Rename/edit sources
target.status = target.ST_BAD
rename_required = False
for source in target.sources.values():
if source.name != source.orig_name:
rename_required = True
break
if rename_required or target.source_edit_format:
for source in target.sources.values():
source.path = os.path.join(work_dir, source.name)
source_path_d = os.path.dirname(source.path)
app_runner.fs_util.makedirs(source_path_d)
if target.source_edit_format:
fmt_args = {
"in": source.orig_path,
"out": source.path
}
command = target.source_edit_format % fmt_args
app_runner.popen.run_ok(command, shell=True)
else:
app_runner.fs_util.symlink(source.orig_path,
source.path)
# Compress sources
if target.compress_scheme:
handler = compress_manager.get_handler(
target.compress_scheme)
handler.compress_sources(target, work_dir)
t_tran = time()
# Run archive command
sources = []
if target.work_source_path:
sources = [target.work_source_path]
else:
for source in target.sources.values():
sources.append(source.path)
sources_str = app_runner.popen.list_to_shell_str(sources)
target_str = app_runner.popen.list_to_shell_str([target.name])
command = target.command_format % {
"sources": sources_str,
"target": target_str
}
ret_code, out, err = app_runner.popen.run(command, shell=True)
t_arch = time()
if ret_code:
app_runner.handle_event(
RosePopenError([command], ret_code, out, err))
else:
target.status = target.ST_NEW
app_runner.handle_event(err, kind=Event.KIND_ERR)
app_runner.handle_event(out)
target.command_rc = ret_code
dao.update_command_rc(target)
finally:
app_runner.fs_util.delete(work_dir)
app_runner.handle_event(
RoseArchEvent(target, [t_init, t_tran, t_arch], ret_code))
return [target.status
for target in targets].count(RoseArchTarget.ST_BAD)
def _get_conf(self, app_runner, conf_tree, key, max_args=0):
"""Get a list of cycles from a configuration setting.
key -- An option key in self.SECTION to locate the setting.
max_args -- Maximum number of extra arguments for an item in the list.
The value of the setting is expected to be split by shlex.split into a
list of items. If max_args == 0, an item should be a string
representing a cycle or an cycle offset. If max_args > 0, the cycle
or cycle offset string can, optionally, have arguments. The arguments
are delimited by colons ":".
E.g.:
prune-remote-logs-at=-PT6H -PT12H
prune-server-logs-at=-P7D
prune-datac-at=-PT6H:foo/* -PT12H:'bar/* baz/*' -P1D
prune-work-at=-PT6H:t1*:*.tar -PT12H:t1*: -PT12H:*.gz -P1D
If max_args == 0, return a list of cycles.
If max_args > 0, return a list of (cycle, [arg, ...])
"""
items_str = conf_tree.node.get_value([self.SECTION, key])
if items_str is None:
return []
try:
items_str = env_var_process(items_str)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError([self.SECTION, key], items_str, exc)
items = []
ref_point_str = os.getenv(RoseDateTimeOperator.TASK_CYCLE_TIME_ENV)
try:
ref_point = None
ref_fmt = None
for item_str in shlex.split(items_str):
args = item_str.split(":", max_args)
when = args.pop(0)
cycle = when
if ref_point_str is not None:
if self._get_cycling_mode() == "integer":
# Integer cycling
if "P" in when: # "when" is an offset
cycle = str(
int(ref_point_str) +
int(when.replace("P", "")))
else: # "when" is a cycle point
cycle = str(when)
else:
# Date-time cycling
if ref_fmt is None:
ref_point, ref_fmt = (app_runner.date_time_oper.
date_parse(ref_point_str))
try:
time_point = app_runner.date_time_oper.date_parse(
when)[0]
except ValueError:
time_point = app_runner.date_time_oper.date_shift(
ref_point, when)
cycle = app_runner.date_time_oper.date_format(
ref_fmt, time_point)
if max_args:
items.append((cycle, args))
else:
items.append(cycle)
except ValueError as exc:
raise ConfigValueError([self.SECTION, key], items_str, exc)
return items
def run(self, app_runner, conf_tree, opts, args, uuid, work_files):
""" Run multiple instances of a command using sets of specified args"""
# Counts for reporting purposes
run_ok = 0
run_fail = 0
run_skip = 0
notrun = 0
# Allow naming of individual calls
self.invocation_names = conf_tree.node.get_value(
[self.BUNCH_SECTION, "names"])
if self.invocation_names:
self.invocation_names = shlex.split(
rose.env.env_var_process(self.invocation_names))
if len(set(self.invocation_names)) != len(self.invocation_names):
raise ConfigValueError([self.BUNCH_SECTION, "names"],
self.invocation_names,
"names must be unique")
self.fail_mode = rose.env.env_var_process(
conf_tree.node.get_value([self.BUNCH_SECTION, "fail-mode"],
self.TYPE_CONTINUE_ON_FAIL))
if self.fail_mode not in self.FAIL_MODE_TYPES:
raise ConfigValueError([self.BUNCH_SECTION, "fail-mode"],
self.fail_mode, "not a valid setting")
self.incremental = conf_tree.node.get_value(
[self.BUNCH_SECTION, "incremental"], "true")
if self.incremental:
self.incremental = rose.env.env_var_process(self.incremental)
self.isformatted = True
self.command = rose.env.env_var_process(
conf_tree.node.get_value([self.BUNCH_SECTION, "command-format"]))
if not self.command:
self.isformatted = False
self.command = app_runner.get_command(conf_tree, opts, args)
if not self.command:
raise CommandNotDefinedError()
# Set up command-instances if needed
instances = conf_tree.node.get_value(
[self.BUNCH_SECTION, "command-instances"])
if instances:
try:
instances = range(int(rose.env.env_var_process(instances)))
except ValueError:
raise ConfigValueError(
[self.BUNCH_SECTION, "command-instances"], instances,
"not an integer value")
# Argument lists
multi_args = conf_tree.node.get_value([self.ARGS_SECTION], {})
bunch_args_names = []
bunch_args_values = []
for key, val in multi_args.items():
bunch_args_names.append(key)
bunch_args_values.append(
shlex.split(rose.env.env_var_process(val.value)))
# Update the argument values based on the argument-mode
argument_mode = conf_tree.node.get_value(
[self.BUNCH_SECTION, "argument-mode"], self.DEFAULT_ARGUMENT_MODE)
if argument_mode == self.DEFAULT_ARGUMENT_MODE:
pass
elif argument_mode in self.ACCEPTED_ARGUMENT_MODES:
_itertools_cmd = getattr(itertools, argument_mode)
if argument_mode == "izip_longest":
_permutations = _itertools_cmd(*bunch_args_values,
fillvalue="")
else:
_permutations = _itertools_cmd(*bunch_args_values)
# Reconstruct the bunch_args_values
_permutations = list(_permutations)
for index, _ in enumerate(bunch_args_values):
bunch_args_values[index] = [v[index] for v in _permutations]
else:
raise ConfigValueError([self.BUNCH_SECTION, "argument-mode"],
argument_mode, "must be one of %s" %
self.ACCEPTED_ARGUMENT_MODES)
# Validate runlists
if not self.invocation_names:
if instances:
arglength = len(instances)
else:
arglength = len(bunch_args_values[0])
self.invocation_names = range(0, arglength)
else:
arglength = len(self.invocation_names)
for item, vals in zip(bunch_args_names, bunch_args_values):
if len(vals) != arglength:
raise ConfigValueError([self.ARGS_SECTION, item],
conf_tree.node.get_value(
[self.ARGS_SECTION, item]),
"inconsistent arg lengths")
if conf_tree.node.get_value([self.ARGS_SECTION, "command-instances"]):
raise ConfigValueError([self.ARGS_SECTION, "command-instances"],
conf_tree.node.get_value([
self.ARGS_SECTION, "command-instances"
]), "reserved keyword")
if conf_tree.node.get_value([self.ARGS_SECTION, "COMMAND_INSTANCES"]):
raise ConfigValueError([self.ARGS_SECTION, "COMMAND_INSTANCES"],
conf_tree.node.get_value([
self.ARGS_SECTION, "COMMAND_INSTANCES"
]), "reserved keyword")
if instances and arglength != len(instances):
raise ConfigValueError([self.BUNCH_SECTION, "command-instances"],
instances, "inconsistent arg lengths")
# Set max number of processes to run at once
max_procs = conf_tree.node.get_value([self.BUNCH_SECTION, "pool-size"])
if max_procs:
max_procs = int(rose.env.env_var_process(max_procs))
else:
max_procs = arglength
if self.incremental == "true":
self.dao = RoseBunchDAO(conf_tree)
else:
self.dao = None
commands = {}
for vals in zip(range(arglength), self.invocation_names,
*bunch_args_values):
index, name, bunch_args_vals = vals[0], vals[1], vals[2:]
argsdict = dict(zip(bunch_args_names, bunch_args_vals))
if instances:
if self.isformatted:
argsdict["command-instances"] = instances[index]
else:
argsdict["COMMAND_INSTANCES"] = str(instances[index])
commands[name] = RoseBunchCmd(name, self.command, argsdict,
self.isformatted)
procs = {}
if 'ROSE_TASK_LOG_DIR' in os.environ:
log_format = os.path.join(os.environ['ROSE_TASK_LOG_DIR'], "%s")
else:
log_format = os.path.join(os.getcwd(), "%s")
failed = {}
abort = False
while procs or (commands and not abort):
for key, proc in procs.items():
if proc.poll() is not None:
procs.pop(key)
if proc.returncode:
failed[key] = proc.returncode
run_fail += 1
app_runner.handle_event(
RosePopenError(str(key), proc.returncode, None,
None))
if self.dao:
self.dao.update_command_state(key, self.dao.S_FAIL)
if self.fail_mode == self.TYPE_ABORT_ON_FAIL:
abort = True
app_runner.handle_event(AbortEvent())
else:
run_ok += 1
app_runner.handle_event(SucceededEvent(key),
prefix=self.PREFIX_OK)
if self.dao:
self.dao.update_command_state(key, self.dao.S_PASS)
while len(procs) < max_procs and commands and not abort:
key = self.invocation_names[0]
command = commands.pop(key)
self.invocation_names.pop(0)
cmd = command.get_command()
cmd_stdout = log_format % command.get_out_file()
cmd_stderr = log_format % command.get_err_file()
prefix = command.get_log_prefix()
bunch_environ = os.environ
if not command.isformatted:
bunch_environ.update(command.argsdict)
bunch_environ['ROSE_BUNCH_LOG_PREFIX'] = prefix
if self.dao:
if self.dao.check_has_succeeded(key):
run_skip += 1
app_runner.handle_event(PreviousSuccessEvent(key),
prefix=self.PREFIX_PASS)
continue
else:
self.dao.add_command(key)
app_runner.handle_event(LaunchEvent(key, cmd))
procs[key] = app_runner.popen.run_bg(
cmd,
shell=True,
stdout=open(cmd_stdout, 'w'),
stderr=open(cmd_stderr, 'w'),
env=bunch_environ)
sleep(self.SLEEP_DURATION)
if abort and commands:
for key in self.invocation_names:
notrun += 1
cmd = commands.pop(key).get_command()
app_runner.handle_event(NotRunEvent(key, cmd),
prefix=self.PREFIX_NOTRUN)
if self.dao:
self.dao.close()
# Report summary data in job.out file
app_runner.handle_event(
SummaryEvent(run_ok, run_fail, run_skip, notrun))
if failed:
return 1
else:
return 0
