def run_impl(self, opts, args, uuid, work_files):
"""Run application configuration as a suite task."""
# "rose task-env"
t_prop = self.suite_engine_proc.get_task_props(
cycle=opts.cycle,
cycle_offsets=opts.cycle_offsets,
prefix_delim=opts.prefix_delim,
suffix_delim=opts.suffix_delim)
is_changed = False
for key, value in t_prop:
if os.getenv(key) != value:
env_export(key, value, self.event_handler)
is_changed = True
path_globs = opts.path_globs
if path_globs is None:
path_globs = []
prepend_paths_map = get_prepend_paths(self.event_handler,
t_prop.suite_dir,
path_globs,
full_mode=is_changed)
for key, prepend_paths in prepend_paths_map.items():
orig_paths = []
orig_v = os.getenv(key, "")
if orig_v:
orig_paths = orig_v.split(os.pathsep)
value = os.pathsep.join(prepend_paths + orig_paths)
env_export(key, value, self.event_handler)
# Name association with builtin applications
builtin_app = None
if opts.app_mode is None:
builtin_apps_manager = self.app_runner.builtins_manager
builtin_app = builtin_apps_manager.guess_handler(t_prop.task_name)
if builtin_app is not None:
opts.app_mode = builtin_app.SCHEME
# Determine what app config to use
if not opts.conf_dir:
for app_key in [opts.app_key, os.getenv("ROSE_TASK_APP")]:
if app_key is not None:
conf_dir = os.path.join(t_prop.suite_dir, "app", app_key)
if not os.path.isdir(conf_dir):
raise TaskAppNotFoundError(t_prop.task_name, app_key)
break
else:
app_key = t_prop.task_name
conf_dir = os.path.join(t_prop.suite_dir, "app",
t_prop.task_name)
if (not os.path.isdir(conf_dir) and builtin_app is not None
and builtin_app.get_app_key(t_prop.task_name)):
# A builtin application may select a different app_key
# based on the task name.
app_key = builtin_app.get_app_key(t_prop.task_name)
conf_dir = os.path.join(t_prop.suite_dir, "app", app_key)
if not os.path.isdir(conf_dir):
raise TaskAppNotFoundError(t_prop.task_name, app_key)
opts.conf_dir = conf_dir
return self.app_runner(opts, args)
def process(self,
conf_tree,
item,
orig_keys=None,
orig_value=None,
**kwargs):
"""Export environment variables in an [env] in "conf_tree.node"."""
env_node = conf_tree.node.get([item], no_ignore=True)
if env_node is None:
return
if os.environ.has_key("UNDEF"):
os.environ.pop("UNDEF")
environ = {}
if env_node and not env_node.state:
for key, node in env_node.value.iteritems():
if node.state:
continue
try:
environ[key] = env_var_process(node.value)
except UnboundEnvironmentVariableError as e:
raise ConfigProcessError([item, key], node.value, e)
environ[key] = os.path.expanduser(environ[key]) # ~ expansion
for key, value in sorted(environ.items()):
env_export(key, value, self.manager.event_handler)
return environ
def run_impl(self, opts, args, uuid, work_files):
# "rose task-env"
t = self.suite_engine_proc.get_task_props(
cycle=opts.cycle,
cycle_offsets=opts.cycle_offsets,
prefix_delim=opts.prefix_delim,
suffix_delim=opts.suffix_delim)
is_changed = False
for k, v in t:
if os.getenv(k) != v:
env_export(k, v, self.event_handler)
is_changed = True
path_globs = opts.path_globs
if path_globs is None:
path_globs = []
prepend_paths_map = get_prepend_paths(self.event_handler,
t.suite_dir,
path_globs,
full_mode=is_changed)
for k, prepend_paths in prepend_paths_map.items():
orig_paths = []
orig_v = os.getenv(k, "")
if orig_v:
orig_paths = orig_v.split(os.pathsep)
v = os.pathsep.join(prepend_paths + orig_paths)
env_export(k, v, self.event_handler)
# Name association with builtin applications
builtin_app = None
if opts.app_mode is None:
builtin_apps_manager = self.app_runner.builtins_manager
builtin_app = builtin_apps_manager.guess_handler(t.task_name)
if builtin_app is not None:
opts.app_mode = builtin_app.SCHEME
# Determine what app config to use
if not opts.conf_dir:
for app_key in [opts.app_key, os.getenv("ROSE_TASK_APP")]:
if app_key is not None:
conf_dir = os.path.join(t.suite_dir, "app", app_key)
if not os.path.isdir(conf_dir):
raise TaskAppNotFoundError(t.task_name, app_key)
break
else:
app_key = t.task_name
conf_dir = os.path.join(t.suite_dir, "app", t.task_name)
if (not os.path.isdir(conf_dir) and
builtin_app is not None and
builtin_app.get_app_key(t.task_name)):
# A builtin application may select a different app_key
# based on the task name.
app_key = builtin_app.get_app_key(t.task_name)
conf_dir = os.path.join(t.suite_dir, "app", app_key)
if not os.path.isdir(conf_dir):
raise TaskAppNotFoundError(t.task_name, app_key)
opts.conf_dir = conf_dir
return self.app_runner(opts, args)
def _run_orig(self, app_runner, conf_tree, opts, args, uuid, task, orig_cont_map):
"""Run "fcm make" in original location."""
# Determine the destination
dest_orig_str = _conf_value(conf_tree, ["dest-orig"])
if dest_orig_str is None and _conf_value(conf_tree, ["use-pwd"]) not in ["True", "true"]:
dest_orig_str = os.path.join("share", task.task_name)
dest_orig = dest_orig_str
if dest_orig is not None and not os.path.isabs(dest_orig):
dest_orig = os.path.join(task.suite_dir, dest_orig)
dests = [dest_orig]
# Determine if mirror is necessary or not
# Determine the name of the continuation task
task_name_cont = task.task_name.replace(orig_cont_map[ORIG], orig_cont_map[CONT])
auth = app_runner.suite_engine_proc.get_task_auth(task.suite_name, task_name_cont)
if auth is not None:
dest_cont = _conf_value(conf_tree, ["dest-cont"])
if dest_cont is None:
if dest_orig_str is not None:
dest_cont = dest_orig_str
elif dest_orig:
dest_cont = os.path.join("share", task.task_name)
else:
dest_cont = os.path.join("work", task.task_cycle_time, task_name_cont)
if not os.path.isabs(dest_cont):
dest_cont = os.path.join(task.suite_dir_rel, dest_cont)
dests.append(auth + ":" + dest_cont)
# Environment variables for backward compat. "fcm make"
# supports arguments as extra configurations since version
# 2014-03.
for name in ["ROSE_TASK_MIRROR_TARGET", "MIRROR_TARGET"]:
env_export(name, dests[CONT], app_runner.event_handler)
# "mirror" for backward compat. Use can specify a null string as
# value to switch off the mirror target configuration.
mirror_step = _conf_value(conf_tree, ["mirror-step"], "mirror")
if mirror_step:
args.append("%s.target=%s" % (mirror_step, dests[CONT]))
# "mirror.prop{config-file.name}" requires fcm-2015.05+
make_name_cont = _conf_value(
conf_tree, ["make-name-cont"], orig_cont_map[CONT].replace(orig_cont_map[ORIG], "")
)
if make_name_cont:
args.append("%s.prop{config-file.name}=%s" % (mirror_step, make_name_cont))
# Launch "fcm make"
self._invoke_fcm_make(
app_runner,
conf_tree,
opts,
args,
uuid,
task,
dests,
_conf_value(conf_tree, ["fast-dest-root-orig"]),
_conf_value(conf_tree, ["make-name-orig"]),
)
def _run_orig(self, app_runner, conf_tree, opts, args, uuid, task,
orig_cont_map):
"""Run "fcm make" in original location."""
# Determine the destination
dest_orig_str = _conf_value(conf_tree, ["dest-orig"])
if (dest_orig_str is None and _conf_value(conf_tree, ["use-pwd"])
not in ["True", "true"]):
dest_orig_str = os.path.join("share", task.task_name)
dest_orig = dest_orig_str
if dest_orig is not None and not os.path.isabs(dest_orig):
dest_orig = os.path.join(task.suite_dir, dest_orig)
dests = [dest_orig]
# Determine if mirror is necessary or not
# Determine the name of the continuation task
task_name_cont = task.task_name.replace(orig_cont_map[ORIG],
orig_cont_map[CONT])
auth = app_runner.suite_engine_proc.get_task_auth(
task.suite_name, task_name_cont)
if auth is not None:
dest_cont = _conf_value(conf_tree, ["dest-cont"])
if dest_cont is None:
if dest_orig_str is not None:
dest_cont = dest_orig_str
elif dest_orig:
dest_cont = os.path.join("share", task.task_name)
else:
dest_cont = os.path.join("work", task.task_cycle_time,
task_name_cont)
if not os.path.isabs(dest_cont):
dest_cont = os.path.join(task.suite_dir_rel, dest_cont)
dests.append(auth + ":" + dest_cont)
# Environment variables for backward compat. "fcm make"
# supports arguments as extra configurations since version
# 2014-03.
for name in ["ROSE_TASK_MIRROR_TARGET", "MIRROR_TARGET"]:
env_export(name, dests[CONT], app_runner.event_handler)
# "mirror" for backward compat. Use can specify a null string as
# value to switch off the mirror target configuration.
mirror_step = _conf_value(conf_tree, ["mirror-step"], "mirror")
if mirror_step:
args.append("%s.target=%s" % (mirror_step, dests[CONT]))
# "mirror.prop{config-file.name}" requires fcm-2015.05+
make_name_cont = _conf_value(
conf_tree, ["make-name-cont"],
orig_cont_map[CONT].replace(orig_cont_map[ORIG], ""))
if make_name_cont:
args.append("%s.prop{config-file.name}=%s" %
(mirror_step, make_name_cont))
# Launch "fcm make"
self._invoke_fcm_make(app_runner, conf_tree, opts, args, uuid, task,
dests,
_conf_value(conf_tree, ["fast-dest-root-orig"]),
_conf_value(conf_tree, ["make-name-orig"]))
def run(self, app_runner, conf_tree, opts, args, uuid, work_files):
"""Run "fcm make".
This application will only work under "rose task-run".
"""
t = app_runner.suite_engine_proc.get_task_props()
if t.task_name.startswith(self.SCHEME2):
self._run2(app_runner, conf_tree, opts, args, uuid, work_files, t)
return
task2_name = self.SCHEME2 + t.task_name.replace(self.SCHEME, "")
use_pwd = conf_tree.node.get_value(["use-pwd"]) in ["True", "true"]
auth = app_runner.suite_engine_proc.get_task_auth(
t.suite_name, task2_name)
if auth is not None:
target = auth + ":"
if use_pwd:
target += os.path.join(t.suite_dir_rel, "work", t.task_id)
else:
target += os.path.join(t.suite_dir_rel, "share", t.task_name)
env_export("ROSE_TASK_MIRROR_TARGET", target,
app_runner.event_handler)
# N.B. MIRROR_TARGET deprecated
env_export("MIRROR_TARGET", target, app_runner.event_handler)
cmd = ["fcm", "make"]
for cfg in [
"fcm-make.cfg",
os.path.join(t.suite_dir, "etc", t.task_name + ".cfg")
]:
if os.access(cfg, os.F_OK | os.R_OK):
if cfg != "fcm-make.cfg" or not use_pwd:
cmd += ["-f", os.path.abspath(cfg)]
break
if not use_pwd:
cmd += ["-C", os.path.join(t.suite_dir, "share", t.task_name)]
cmd_opt_jobs = conf_tree.node.get_value(["opt.jobs"],
os.getenv(
"ROSE_TASK_N_JOBS",
self.OPT_JOBS))
cmd += ["-j", cmd_opt_jobs]
cmd_args = conf_tree.node.get_value(["args"],
os.getenv("ROSE_TASK_OPTIONS"))
if cmd_args:
cmd += shlex.split(cmd_args)
if args:
cmd += args
app_runner.popen(*cmd, stdout=sys.stdout, stderr=sys.stderr)
def run(self, app_runner, conf_tree, opts, args, uuid, work_files):
"""Run "fcm make".
This application will only work under "rose task-run".
"""
t = app_runner.suite_engine_proc.get_task_props()
if t.task_name.startswith(self.SCHEME2):
self._run2(app_runner, conf_tree, opts, args, uuid, work_files, t)
return
task2_name = self.SCHEME2 + t.task_name.replace(self.SCHEME, "")
use_pwd = conf_tree.node.get_value(["use-pwd"]) in ["True", "true"]
auth = app_runner.suite_engine_proc.get_task_auth(
t.suite_name, task2_name)
if auth is not None:
target = auth + ":"
if use_pwd:
target += os.path.join(t.suite_dir_rel, "work", t.task_id)
else:
target += os.path.join(t.suite_dir_rel, "share", t.task_name)
env_export("ROSE_TASK_MIRROR_TARGET", target,
app_runner.event_handler)
# N.B. MIRROR_TARGET deprecated
env_export("MIRROR_TARGET", target, app_runner.event_handler)
cmd = ["fcm", "make"]
for cfg in ["fcm-make.cfg",
os.path.join(t.suite_dir, "etc", t.task_name + ".cfg")]:
if os.access(cfg, os.F_OK | os.R_OK):
if cfg != "fcm-make.cfg" or not use_pwd:
cmd += ["-f", os.path.abspath(cfg)]
break
if not use_pwd:
cmd += ["-C", os.path.join(t.suite_dir, "share", t.task_name)]
cmd_opt_jobs = conf_tree.node.get_value(
["opt.jobs"], os.getenv("ROSE_TASK_N_JOBS", self.OPT_JOBS))
cmd += ["-j", cmd_opt_jobs]
cmd_args = conf_tree.node.get_value(["args"],
os.getenv("ROSE_TASK_OPTIONS"))
if cmd_args:
cmd += shlex.split(cmd_args)
if args:
cmd += args
app_runner.popen(*cmd, stdout=sys.stdout, stderr=sys.stderr)
def run_impl_main(self, config, opts, args, uuid, work_files):
t = self.suite_engine_proc.get_task_props()
task2_name = "fcm_make2" + t.task_name.replace("fcm_make", "")
auth = self.suite_engine_proc.get_remote_auth(t.suite_name, task2_name)
if auth is not None:
target = "@".join(auth)
target += ":" + os.path.join(t.suite_dir_rel, "share", t.task_name)
# FIXME: name-space for environment variable?
env_export("MIRROR_TARGET", target, self.event_handler)
cfg = os.path.join(t.suite_dir, "etc", t.task_name + ".cfg")
dir = os.path.join(t.suite_dir, "share", t.task_name)
n_jobs = os.getenv("ROSE_TASK_N_JOBS", "4")
cmd = "fcm make -f %s -C %s -j %s" % (cfg, dir, n_jobs)
if os.getenv("ROSE_TASK_OPTIONS"):
cmd += " " + os.getenv("ROSE_TASK_OPTIONS")
if args:
cmd += " " + self.popen.list_to_shell_str(args)
if os.getenv("ROSE_TASK_PRE_SCRIPT"):
cmd = ". " + os.getenv("ROSE_TASK_PRE_SCRIPT") + " && " + cmd
self.popen(cmd, shell=True, stdout=sys.stdout, stderr=sys.stderr)
def process(self, config, item, orig_keys=None, orig_value=None, **kwargs):
"""Export environment variables in an env section in "config"."""
if config.get([item], no_ignore=True) is None:
return
if os.environ.has_key("UNDEF"):
os.environ.pop("UNDEF")
environ = {}
env_node = config.get([item], no_ignore=True)
if env_node and not env_node.state:
for key, node in env_node.value.iteritems():
if node.state:
continue
try:
environ[key] = env_var_process(node.value)
except UnboundEnvironmentVariableError as e:
raise ConfigProcessError([item, key], node.value, e)
environ[key] = os.path.expanduser(environ[key]) # ~ expansion
for key, value in sorted(environ.items()):
env_export(key, value, self.manager.event_handler)
return environ
def run_impl(self, opts, args, uuid, work_files):
t = self.suite_engine_proc.get_task_props(
cycle=opts.cycle,
cycle_offsets=opts.cycle_offsets,
prefix_delim=opts.prefix_delim,
suffix_delim=opts.suffix_delim)
# Prepend PATH-like variable, site/user configuration
conf = rose.config.default_node()
my_conf = conf.get(["rose-task-run"], no_ignore=True)
for key, node in sorted(my_conf.value.items()):
if node.is_ignored() or not key.startswith("path-prepend"):
continue
env_key = "PATH"
if key != "path-prepend":
env_key = key[len("path-prepend."):]
values = []
for v in node.value.split():
if os.path.exists(v):
values.append(v)
if os.getenv(env_key):
values.append(os.getenv(env_key))
if values:
env_export(env_key, os.pathsep.join(values), self.event_handler)
# Prepend PATH with paths determined by default or specified globs
paths = []
path_globs = list(self.PATH_GLOBS)
if opts.path_globs:
path_globs.extend(opts.path_globs)
for path_glob in path_globs:
if path_glob:
if not os.path.isabs(path_glob):
path_glob = os.path.join(t.suite_dir, path_glob)
for path in glob(path_glob):
paths.append(path)
else:
paths = [] # empty value resets
if paths:
if os.getenv("PATH"):
paths.append(os.getenv("PATH"))
env_export("PATH", os.pathsep.join(paths), self.event_handler)
# Add ROSE_* environment variables
for name, value in t:
if os.getenv(name) != value:
env_export(name, value, self.event_handler)
# Determine what app config to use
if not opts.conf_dir:
app_key = t.task_name
if opts.app_key:
app_key = opts.app_key
elif os.getenv("ROSE_TASK_APP"):
app_key = os.getenv("ROSE_TASK_APP")
opts.conf_dir = os.path.join(t.suite_dir, "app", app_key)
# Run a task util or an app
util = None
if opts.util_key:
util = self._get_task_util(opts.util_key)
elif os.getenv("ROSE_TASK_UTIL"):
util = self._get_task_util(os.getenv("ROSE_TASK_UTIL"))
if opts.auto_util_mode and util is None:
for key, u in reversed(sorted(self._get_task_utils().items())):
if u.can_handle(t.task_name):
util = u
break
if util is None:
return self._run_app(opts, args)
else:
return util(opts, args)