def main():
"""Implement the "rose config" command."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options("default", "env_var_process_mode", "files",
"keys", "meta", "meta_key", "no_ignore",
"no_opts", "print_conf_mode")
opts, args = opt_parser.parse_args()
report = Reporter(opts.verbosity - opts.quietness)
rose.macro.add_meta_paths()
if opts.meta_key:
opts.meta = True
if opts.files and opts.meta_key:
report(Exception("Cannot specify both a file and meta key."))
sys.exit(1)
config_loader = ConfigLoader()
sources = []
if opts.files:
root_node = ConfigNode()
for fname in opts.files:
if fname == "-":
sources.append(sys.stdin)
else:
if opts.meta:
try:
root_node = config_loader.load(fname)
except ConfigSyntaxError as exc:
report(exc)
sys.exit(1)
rel_path = os.sep.join(fname.split(os.sep)[:-1])
fpath = get_meta_path(root_node, rel_path)
if fpath is None:
report(MetadataNotFoundEvent(fname))
else:
sources.append(fpath)
else:
sources.append(fname)
elif opts.meta:
root_node = ConfigNode()
if opts.meta_key:
root_node.set(["meta"], opts.meta_key)
else:
fname = os.path.join(os.getcwd(), rose.SUB_CONFIG_NAME)
try:
root_node = config_loader.load(fname)
except ConfigSyntaxError as exc:
report(exc)
sys.exit(1)
fpath = get_meta_path(root_node, meta_key=opts.meta_key)
root_node.unset(["meta"])
if fpath is None:
report(Exception("Metadata not found"))
sys.exit(1)
else:
sources.append(fpath)
else:
root_node = ResourceLocator.default().get_conf()
for source in sources:
try:
if opts.meta or opts.no_opts:
config_loader.load(source, root_node)
else:
config_loader.load_with_opts(source, root_node)
except (ConfigSyntaxError, IOError) as exc:
report(exc)
sys.exit(1)
if source is sys.stdin:
source.close()
if opts.quietness:
sys.exit(root_node.get(args, opts.no_ignore) is None)
if opts.keys_mode:
try:
keys = root_node.get(args, opts.no_ignore).value.keys()
except AttributeError:
sys.exit(1)
keys.sort()
for key in keys:
print key
sys.exit()
conf_dump = ConfigDumper()
if len(args) == 0:
conf_dump(root_node, concat_mode=opts.print_conf_mode)
sys.exit()
node = root_node.get(args, opts.no_ignore)
if node is not None and isinstance(node.value, dict):
if opts.print_conf_mode:
conf_dump(ConfigNode().set(args, node.value), concat_mode=True)
sys.exit()
keys = node.value.keys()
keys.sort()
for key in keys:
node_of_key = node.get([key], opts.no_ignore)
if node_of_key:
value = node_of_key.value
state = node_of_key.state
string = "%s%s=%s" % (state, key, value)
lines = string.splitlines()
print lines[0]
i_equal = len(state + key) + 1
for line in lines[1:]:
print " " * i_equal + line
sys.exit()
if node is None:
if opts.default is None:
sys.exit(1)
value = opts.default
elif opts.env_var_process_mode:
value = env_var_process(node.value)
else:
value = node.value
if opts.print_conf_mode:
conf_dump(ConfigNode().set(args, value), concat_mode=True)
else:
print value
sys.exit()
def run_impl(self, opts, args, uuid, work_files):
# Log file, temporary
if hasattr(self.event_handler, "contexts"):
t_file = TemporaryFile()
log_context = ReporterContext(None, self.event_handler.VV, t_file)
self.event_handler.contexts[uuid] = log_context
# Check suite engine specific compatibility
self.suite_engine_proc.check_global_conf_compat()
# Suite name from the current working directory
if opts.conf_dir:
self.fs_util.chdir(opts.conf_dir)
opts.conf_dir = os.getcwd()
if opts.defines_suite:
suite_section = "jinja2:" + self.suite_engine_proc.SUITE_CONF
if not opts.defines:
opts.defines = []
for define in opts.defines_suite:
opts.defines.append("[" + suite_section + "]" + define)
# --remote=KEY=VALUE,...
if opts.remote:
# opts.name always set for remote.
return self._run_remote(opts, opts.name)
conf_tree = self.config_load(opts)
self.fs_util.chdir(conf_tree.conf_dirs[0])
suite_name = opts.name
if not opts.name:
suite_name = os.path.basename(os.getcwd())
# Automatic Rose constants
# ROSE_ORIG_HOST: originating host
# ROSE_VERSION: Rose version (not retained in run_mode=="reload")
# Suite engine version
jinja2_section = "jinja2:" + self.suite_engine_proc.SUITE_CONF
my_rose_version = ResourceLocator.default().get_version()
suite_engine_key = self.suite_engine_proc.get_version_env_name()
if opts.run_mode == "reload":
prev_config_path = self.suite_engine_proc.get_suite_dir(
suite_name, "log", "rose-suite-run.conf")
prev_config = ConfigLoader()(prev_config_path)
suite_engine_version = prev_config.get_value(
["env", suite_engine_key])
else:
suite_engine_version = self.suite_engine_proc.get_version()
auto_items = {"ROSE_ORIG_HOST": socket.gethostname(),
"ROSE_VERSION": ResourceLocator.default().get_version(),
suite_engine_key: suite_engine_version}
for key, val in auto_items.items():
requested_value = conf_tree.node.get_value(["env", key])
if requested_value:
if key == "ROSE_VERSION" and val != requested_value:
exc = VersionMismatchError(requested_value, val)
raise ConfigValueError(["env", key], requested_value, exc)
val = requested_value
else:
conf_tree.node.set(["env", key], val,
state=conf_tree.node.STATE_NORMAL)
conf_tree.node.set([jinja2_section, key], '"' + val + '"')
# See if suite is running or not
hosts = []
if opts.host:
hosts.append(opts.host)
conf = ResourceLocator.default().get_conf()
known_hosts = self.host_selector.expand(
conf.get_value(["rose-suite-run", "hosts"], "").split() +
conf.get_value(["rose-suite-run", "scan-hosts"], "").split() +
["localhost"])[0]
known_hosts = list(set(known_hosts))
for known_host in known_hosts:
if known_host not in hosts:
hosts.append(known_host)
if opts.run_mode == "reload":
suite_run_hosts = self.suite_engine_proc.ping(suite_name, hosts)
if not suite_run_hosts:
raise NotRunningError(suite_name)
hosts = suite_run_hosts
else:
self.suite_engine_proc.check_suite_not_running(suite_name, hosts)
# Install the suite to its run location
suite_dir_rel = self._suite_dir_rel(suite_name)
suite_dir = os.path.join(os.path.expanduser("~"), suite_dir_rel)
suite_conf_dir = os.getcwd()
locs_conf = ConfigNode()
if opts.new_mode:
if os.getcwd() == suite_dir:
raise NewModeError("PWD", os.getcwd())
elif opts.run_mode in ["reload", "restart"]:
raise NewModeError("--run", opts.run_mode)
self.suite_run_cleaner.clean(suite_name)
if os.getcwd() != suite_dir:
if opts.run_mode == "run":
self._run_init_dir(opts, suite_name, conf_tree,
locs_conf=locs_conf)
os.chdir(suite_dir)
# Housekeep log files
if not opts.install_only_mode and not opts.local_install_only_mode:
self._run_init_dir_log(opts)
self.fs_util.makedirs("log/suite")
# Rose configuration and version logs
self.fs_util.makedirs("log/rose-conf")
run_mode = opts.run_mode
if run_mode not in ["reload", "restart", "run"]:
run_mode = "run"
mode = run_mode
if opts.install_only_mode:
mode = "install-only"
elif opts.local_install_only_mode:
mode = "local-install-only"
prefix = "rose-conf/%s-%s" % (strftime("%Y%m%dT%H%M%S"), mode)
# Dump the actual configuration as rose-suite-run.conf
ConfigDumper()(conf_tree.node, "log/" + prefix + ".conf")
# Install version information file
write_source_vc_info(
suite_conf_dir, "log/" + prefix + ".version", self.popen)
# If run through rose-stem, install version information files for
# each source tree if they're a working copy
if hasattr(opts, 'source') and hasattr(opts, 'project'):
for i, url in enumerate(opts.source):
if os.path.isdir(url):
write_source_vc_info(
url, "log/" + opts.project[i] + "-" + str(i) +
".version", self.popen)
for ext in [".conf", ".version"]:
self.fs_util.symlink(prefix + ext, "log/rose-suite-run" + ext)
# Move temporary log to permanent log
if hasattr(self.event_handler, "contexts"):
log_file_path = os.path.abspath(
os.path.join("log", "rose-suite-run.log"))
log_file = open(log_file_path, "ab")
temp_log_file = self.event_handler.contexts[uuid].handle
temp_log_file.seek(0)
log_file.write(temp_log_file.read())
self.event_handler.contexts[uuid].handle = log_file
temp_log_file.close()
# Create the suite log view
self.suite_engine_proc.job_logs_db_create(suite_name, close=True)
# Install share/work directories (local)
for name in ["share", "work"]:
self._run_init_dir_work(opts, suite_name, name, conf_tree,
locs_conf=locs_conf)
# Process Environment Variables
environ = self.config_pm(conf_tree, "env")
# Process Files
cwd = os.getcwd()
for rel_path, conf_dir in conf_tree.files.items():
if (conf_dir == cwd or
self.REC_DONT_SYNC.match(rel_path) or
conf_tree.node.get(["jinja2:" + rel_path]) is not None):
continue
# No sub-directories, very slow otherwise
if os.sep in rel_path:
rel_path = rel_path.split(os.sep, 1)[0]
target_key = self.config_pm.get_handler("file").PREFIX + rel_path
target_node = conf_tree.node.get([target_key])
if target_node is None:
conf_tree.node.set([target_key])
target_node = conf_tree.node.get([target_key])
elif target_node.is_ignored():
continue
source_node = target_node.get("source")
if source_node is None:
target_node.set(["source"], os.path.join(conf_dir, rel_path))
elif source_node.is_ignored():
continue
self.config_pm(conf_tree, "file",
no_overwrite_mode=opts.no_overwrite_mode)
# Process Jinja2 configuration
self.config_pm(conf_tree, "jinja2")
# Ask suite engine to parse suite configuration
# and determine if it is up to date (unchanged)
suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf(
suite_name,
opts.strict_mode,
opts.debug_mode)
if opts.local_install_only_mode:
return
# Install suite files to each remote [user@]host
for name in ["", "log/", "share/", "work/"]:
uuid_file = os.path.abspath(name + uuid)
open(uuid_file, "w").close()
work_files.append(uuid_file)
# Install items to user@host
auths = self.suite_engine_proc.get_tasks_auths(suite_name)
queue = [] # [[pipe, command, "ssh"|"rsync", auth], ...]
for auth in sorted(auths):
host = auth
if "@" in auth:
host = auth.split("@", 1)[1]
command = self.popen.get_cmd("ssh", auth, "bash", "--login", "-c")
rose_bin = "rose"
for name in [host, "*"]:
rose_home_node = conf.get(["rose-home-at", name],
no_ignore=True)
if rose_home_node is not None:
rose_bin = "%s/bin/rose" % (rose_home_node.value)
break
# Build remote "rose suite-run" command
rose_sr = "ROSE_VERSION=%s %s" % (my_rose_version, rose_bin)
rose_sr += " suite-run -v -v --name=%s" % suite_name
for key in ["new", "debug", "install-only"]:
attr = key.replace("-", "_") + "_mode"
if getattr(opts, attr, None) is not None:
rose_sr += " --" + key
if opts.log_keep:
rose_sr += " --log-keep=" + opts.log_keep
if opts.log_name:
rose_sr += " --log-name=" + opts.log_name
if not opts.log_archive_mode:
rose_sr += " --no-log-archive"
rose_sr += " --run=" + opts.run_mode
host_confs = ["root-dir", "root-dir-share", "root-dir-work"]
rose_sr += " --remote=uuid=" + uuid
locs_conf.set([auth])
for key in host_confs:
value = self._run_conf(key, host=host, conf_tree=conf_tree)
if value is not None:
val = self.popen.list_to_shell_str([str(value)])
rose_sr += "," + key + "=" + val
locs_conf.set([auth, key], value)
command += ["'" + rose_sr + "'"]
pipe = self.popen.run_bg(*command)
queue.append([pipe, command, "ssh", auth])
while queue:
sleep(self.SLEEP_PIPE)
pipe, command, command_name, auth = queue.pop(0)
if pipe.poll() is None:
queue.append([pipe, command, command_name, auth]) # put it back
continue
ret_code = pipe.wait()
out, err = pipe.communicate()
if ret_code:
raise RosePopenError(command, ret_code, out, err)
if command_name == "rsync":
self.handle_event(out, level=Event.VV)
continue
else:
self.handle_event(out, level=Event.VV, prefix="[%s] " % auth)
for line in out.split("\n"):
if "/" + uuid == line.strip():
locs_conf.unset([auth])
break
else:
filters = {"excludes": [], "includes": []}
for name in ["", "log/", "share/", "work/"]:
filters["excludes"].append(name + uuid)
target = auth + ":" + suite_dir_rel
cmd = self._get_cmd_rsync(target, **filters)
queue.append([self.popen.run_bg(*cmd), cmd, "rsync", auth])
# Install ends
ConfigDumper()(locs_conf, os.path.join("log", "rose-suite-run.locs"))
if opts.install_only_mode:
return
elif opts.run_mode == "reload" and suite_conf_unchanged:
conf_name = self.suite_engine_proc.SUITE_CONF
self.handle_event(SkipReloadEvent(suite_name, conf_name))
return
# Start the suite
self.fs_util.chdir("log")
ret = 0
host = hosts[0]
# FIXME: should sync files to suite host?
if opts.host:
hosts = [host]
# For run and restart, get host for running the suite
if opts.run_mode != "reload" and not opts.host:
hosts = []
val = conf.get_value(["rose-suite-run", "hosts"], "localhost")
known_hosts = self.host_selector.expand(val.split())[0]
for known_host in known_hosts:
if known_host not in hosts:
hosts.append(known_host)
if hosts == ["localhost"]:
host = hosts[0]
else:
host = self.host_selector(hosts)[0][0]
self.handle_event(SuiteHostSelectEvent(suite_name, run_mode, host))
# FIXME: values in environ were expanded in the localhost
self.suite_engine_proc.run(
suite_name, host, environ, opts.run_mode, args)
open("rose-suite-run.host", "w").write(host + "\n")
# Disconnect log file handle, so monitoring tool command will no longer
# be associated with the log file.
self.event_handler.contexts[uuid].handle.close()
self.event_handler.contexts.pop(uuid)
# Launch the monitoring tool
# Note: maybe use os.ttyname(sys.stdout.fileno())?
if os.getenv("DISPLAY") and host and opts.gcontrol_mode:
self.suite_engine_proc.gcontrol(suite_name, host)
return ret
def main():
"""Implement the "rose config" command."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options("default", "env_var_process_mode", "files",
"keys", "meta", "meta_key", "no_ignore",
"no_opts", "print_conf_mode")
opts, args = opt_parser.parse_args()
report = Reporter(opts.verbosity - opts.quietness)
rose.macro.add_site_meta_paths()
rose.macro.add_env_meta_paths()
if opts.meta_key:
opts.meta = True
if opts.files and opts.meta_key:
report(Exception("Cannot specify both a file and meta key."))
sys.exit(1)
config_loader = ConfigLoader()
sources = []
if opts.files:
root_node = ConfigNode()
for fname in opts.files:
if fname == "-":
sources.append(sys.stdin)
else:
if opts.meta:
try:
root_node = config_loader.load(fname)
except ConfigSyntaxError as e:
report(e)
sys.exit(1)
rel_path = os.sep.join(fname.split(os.sep)[:-1])
fpath = get_meta_path(root_node, rel_path)
if fpath is None:
report(MetadataNotFoundEvent(fname))
else:
sources.append(fpath)
else:
sources.append(fname)
elif opts.meta:
root_node = ConfigNode()
if opts.meta_key:
root_node.set(["meta"], opts.meta_key)
else:
fname = os.path.join(
os.getcwd(), rose.SUB_CONFIG_NAME)
try:
root_node = config_loader.load(fname)
except ConfigSyntaxError as e:
report(e)
sys.exit(1)
fpath = get_meta_path(root_node, meta_key=opts.meta_key)
root_node.unset(["meta"])
if fpath is None:
report(Exception("Metadata not found"))
sys.exit(1)
else:
sources.append(fpath)
else:
root_node = ResourceLocator.default().get_conf()
for source in sources:
try:
if opts.meta or opts.no_opts:
config_loader.load(source, root_node)
else:
config_loader.load_with_opts(source, root_node)
except (ConfigSyntaxError, IOError) as exc:
report(exc)
sys.exit(1)
if source is sys.stdin:
source.close()
if opts.quietness:
sys.exit(root_node.get(args, opts.no_ignore) is None)
if opts.keys_mode:
try:
keys = root_node.get(args, opts.no_ignore).value.keys()
except:
sys.exit(1)
keys.sort()
for key in keys:
print key
sys.exit()
conf_dump = ConfigDumper()
if len(args) == 0:
conf_dump(root_node, concat_mode=opts.print_conf_mode)
sys.exit()
node = root_node.get(args, opts.no_ignore)
if node is not None and isinstance(node.value, dict):
if opts.print_conf_mode:
conf_dump(ConfigNode().set(args, node.value), concat_mode=True)
sys.exit()
keys = node.value.keys()
keys.sort()
for key in keys:
node_of_key = node.get([key], opts.no_ignore)
if node_of_key:
value = node_of_key.value
state = node_of_key.state
string = "%s%s=%s" % (state, key, value)
lines = string.splitlines()
print lines[0]
i_equal = len(state + key) + 1
for line in lines[1:]:
print " " * i_equal + line
sys.exit()
if node is None:
if opts.default is None:
sys.exit(1)
value = opts.default
elif opts.env_var_process_mode:
value = env_var_process(node.value)
else:
value = node.value
if opts.print_conf_mode:
conf_dump(ConfigNode().set(args, value), concat_mode=True)
else:
print value
sys.exit()
def run_impl(self, opts, args, uuid, work_files):
# Log file, temporary
if hasattr(self.event_handler, "contexts"):
t_file = TemporaryFile()
log_context = ReporterContext(None, self.event_handler.VV, t_file)
self.event_handler.contexts[uuid] = log_context
# Check suite engine specific compatibility
self.suite_engine_proc.check_global_conf_compat()
# Suite name from the current working directory
if opts.conf_dir:
self.fs_util.chdir(opts.conf_dir)
opts.conf_dir = os.getcwd()
if opts.defines_suite:
suite_section = "jinja2:" + self.suite_engine_proc.SUITE_CONF
if not opts.defines:
opts.defines = []
for define in opts.defines_suite:
opts.defines.append("[" + suite_section + "]" + define)
# --remote=KEY=VALUE,...
if opts.remote:
# opts.name always set for remote.
return self._run_remote(opts, opts.name)
conf_tree = self.config_load(opts)
self.fs_util.chdir(conf_tree.conf_dirs[0])
suite_name = opts.name
if not opts.name:
suite_name = os.path.basename(os.getcwd())
# Automatic Rose constants
# ROSE_ORIG_HOST: originating host
# ROSE_VERSION: Rose version (not retained in run_mode=="reload")
# Suite engine version
jinja2_section = "jinja2:" + self.suite_engine_proc.SUITE_CONF
my_rose_version = ResourceLocator.default().get_version()
suite_engine_key = self.suite_engine_proc.get_version_env_name()
if opts.run_mode in ["reload", "restart"]:
prev_config_path = self.suite_engine_proc.get_suite_dir(
suite_name, "log", "rose-suite-run.conf")
prev_config = ConfigLoader()(prev_config_path)
suite_engine_version = prev_config.get_value(
["env", suite_engine_key])
else:
suite_engine_version = self.suite_engine_proc.get_version()
auto_items = {
"ROSE_ORIG_HOST": self.host_selector.get_local_host(),
"ROSE_VERSION": ResourceLocator.default().get_version(),
suite_engine_key: suite_engine_version
}
for key, val in auto_items.items():
requested_value = conf_tree.node.get_value(["env", key])
if requested_value:
if key == "ROSE_VERSION" and val != requested_value:
exc = VersionMismatchError(requested_value, val)
raise ConfigValueError(["env", key], requested_value, exc)
val = requested_value
else:
conf_tree.node.set(["env", key],
val,
state=conf_tree.node.STATE_NORMAL)
conf_tree.node.set([jinja2_section, key], '"' + val + '"')
# See if suite is running or not
hosts = []
if opts.host:
hosts.append(opts.host)
if opts.run_mode == "reload":
suite_run_hosts = self.suite_engine_proc.get_suite_run_hosts(
None, suite_name, hosts)
if not suite_run_hosts:
raise SuiteNotRunningError(suite_name)
hosts = suite_run_hosts
else:
self.suite_engine_proc.check_suite_not_running(suite_name, hosts)
# Install the suite to its run location
suite_dir_rel = self._suite_dir_rel(suite_name)
suite_dir = os.path.join(os.path.expanduser("~"), suite_dir_rel)
suite_conf_dir = os.getcwd()
locs_conf = ConfigNode()
if opts.new_mode:
if os.getcwd() == suite_dir:
raise NewModeError("PWD", os.getcwd())
elif opts.run_mode in ["reload", "restart"]:
raise NewModeError("--run", opts.run_mode)
self.suite_run_cleaner.clean(suite_name)
if os.getcwd() != suite_dir:
if opts.run_mode == "run":
self._run_init_dir(opts,
suite_name,
conf_tree,
locs_conf=locs_conf)
os.chdir(suite_dir)
# Housekeep log files
if not opts.install_only_mode and not opts.local_install_only_mode:
self._run_init_dir_log(opts)
self.fs_util.makedirs("log/suite")
# Rose configuration and version logs
self.fs_util.makedirs("log/rose-conf")
run_mode = opts.run_mode
if run_mode not in ["reload", "restart", "run"]:
run_mode = "run"
mode = run_mode
if opts.install_only_mode:
mode = "install-only"
elif opts.local_install_only_mode:
mode = "local-install-only"
prefix = "rose-conf/%s-%s" % (strftime("%Y%m%dT%H%M%S"), mode)
# Dump the actual configuration as rose-suite-run.conf
ConfigDumper()(conf_tree.node, "log/" + prefix + ".conf")
# Install version information file
write_source_vc_info(suite_conf_dir, "log/" + prefix + ".version",
self.popen)
# If run through rose-stem, install version information files for
# each source tree if they're a working copy
if hasattr(opts, 'source') and hasattr(opts, 'project'):
for i, url in enumerate(opts.source):
if os.path.isdir(url):
write_source_vc_info(
url,
"log/" + opts.project[i] + "-" + str(i) + ".version",
self.popen)
for ext in [".conf", ".version"]:
self.fs_util.symlink(prefix + ext, "log/rose-suite-run" + ext)
# Move temporary log to permanent log
if hasattr(self.event_handler, "contexts"):
log_file_path = os.path.abspath(
os.path.join("log", "rose-suite-run.log"))
log_file = open(log_file_path, "ab")
temp_log_file = self.event_handler.contexts[uuid].handle
temp_log_file.seek(0)
log_file.write(temp_log_file.read())
self.event_handler.contexts[uuid].handle = log_file
temp_log_file.close()
# Install share/work directories (local)
for name in ["share", "share/cycle", "work"]:
self._run_init_dir_work(opts,
suite_name,
name,
conf_tree,
locs_conf=locs_conf)
# Process Environment Variables
environ = self.config_pm(conf_tree, "env")
# Process Files
cwd = os.getcwd()
for rel_path, conf_dir in conf_tree.files.items():
if (conf_dir == cwd or any([
fnmatchcase(os.sep + rel_path, exclude)
for exclude in self.SYNC_EXCLUDES
]) or conf_tree.node.get(["jinja2:" + rel_path]) is not None):
continue
# No sub-directories, very slow otherwise
if os.sep in rel_path:
rel_path = rel_path.split(os.sep, 1)[0]
target_key = self.config_pm.get_handler("file").PREFIX + rel_path
target_node = conf_tree.node.get([target_key])
if target_node is None:
conf_tree.node.set([target_key])
target_node = conf_tree.node.get([target_key])
elif target_node.is_ignored():
continue
source_node = target_node.get("source")
if source_node is None:
target_node.set(["source"], os.path.join(conf_dir, rel_path))
elif source_node.is_ignored():
continue
self.config_pm(conf_tree,
"file",
no_overwrite_mode=opts.no_overwrite_mode)
# Process Jinja2 configuration
self.config_pm(conf_tree, "jinja2")
# Ask suite engine to parse suite configuration
# and determine if it is up to date (unchanged)
suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf(
suite_name, opts.run_mode, opts.strict_mode, opts.debug_mode)
if opts.local_install_only_mode:
return
# Install suite files to each remote [user@]host
for name in ["", "log/", "share/", "share/cycle/", "work/"]:
uuid_file = os.path.abspath(name + uuid)
open(uuid_file, "w").close()
work_files.append(uuid_file)
# Install items to user@host
conf = ResourceLocator.default().get_conf()
auths = self.suite_engine_proc.get_tasks_auths(suite_name)
queue = [] # [[pipe, command, "ssh"|"rsync", auth], ...]
for auth in sorted(auths):
host = auth
if "@" in auth:
host = auth.split("@", 1)[1]
command = self.popen.get_cmd("ssh", auth, "bash", "--login", "-c")
rose_bin = "rose"
for name in [host, "*"]:
rose_home_node = conf.get(["rose-home-at", name],
no_ignore=True)
if rose_home_node is not None:
rose_bin = "%s/bin/rose" % (rose_home_node.value)
break
# Build remote "rose suite-run" command
rose_sr = "ROSE_VERSION=%s %s" % (my_rose_version, rose_bin)
rose_sr += " suite-run -v -v --name=%s" % suite_name
for key in ["new", "debug", "install-only"]:
attr = key.replace("-", "_") + "_mode"
if getattr(opts, attr, None) is not None:
rose_sr += " --" + key
if opts.log_keep:
rose_sr += " --log-keep=" + opts.log_keep
if opts.log_name:
rose_sr += " --log-name=" + opts.log_name
if not opts.log_archive_mode:
rose_sr += " --no-log-archive"
rose_sr += " --run=" + opts.run_mode
host_confs = [
"root-dir", "root-dir{share}", "root-dir{share/cycle}",
"root-dir{work}"
]
rose_sr += " --remote=uuid=" + uuid
locs_conf.set([auth])
for key in host_confs:
value = self._run_conf(key, host=host, conf_tree=conf_tree)
if value is not None:
val = self.popen.list_to_shell_str([str(value)])
rose_sr += "," + key + "=" + val
locs_conf.set([auth, key], value)
command += ["'" + rose_sr + "'"]
pipe = self.popen.run_bg(*command)
queue.append([pipe, command, "ssh", auth])
while queue:
sleep(self.SLEEP_PIPE)
pipe, command, command_name, auth = queue.pop(0)
if pipe.poll() is None:
queue.append([pipe, command, command_name, auth]) # put back
continue
ret_code = pipe.wait()
out, err = pipe.communicate()
if ret_code:
raise RosePopenError(command, ret_code, out, err)
if command_name == "rsync":
self.handle_event(out, level=Event.VV)
continue
else:
self.handle_event(out, level=Event.VV, prefix="[%s] " % auth)
for line in out.split("\n"):
if "/" + uuid == line.strip():
locs_conf.unset([auth])
break
else:
filters = {"excludes": [], "includes": []}
for name in ["", "log/", "share/", "share/cycle/", "work/"]:
filters["excludes"].append(name + uuid)
target = auth + ":" + suite_dir_rel
cmd = self._get_cmd_rsync(target, **filters)
queue.append([self.popen.run_bg(*cmd), cmd, "rsync", auth])
# Install ends
ConfigDumper()(locs_conf, os.path.join("log", "rose-suite-run.locs"))
if opts.install_only_mode:
return
elif opts.run_mode == "reload" and suite_conf_unchanged:
conf_name = self.suite_engine_proc.SUITE_CONF
self.handle_event(SkipReloadEvent(suite_name, conf_name))
return
# Start the suite
self.fs_util.chdir("log")
ret = 0
# FIXME: should sync files to suite host?
if opts.run_mode != "reload":
if opts.host:
hosts = [opts.host]
else:
names = shlex.split(
conf.get_value(["rose-suite-run", "hosts"], ""))
if names:
hosts += self.host_selector.expand(names)[0]
if (hosts and len(hosts) == 1
and self.host_selector.is_local_host(hosts[0])):
host = "localhost"
elif hosts:
host = self.host_selector(hosts)[0][0]
else:
host = "localhost"
self.handle_event(SuiteHostSelectEvent(suite_name, run_mode, host))
# FIXME: values in environ were expanded in the localhost
self.suite_engine_proc.run(suite_name, host, environ, opts.run_mode,
args)
open("rose-suite-run.host", "w").write(host + "\n")
# Disconnect log file handle, so monitoring tool command will no longer
# be associated with the log file.
self.event_handler.contexts[uuid].handle.close()
self.event_handler.contexts.pop(uuid)
# Launch the monitoring tool
# Note: maybe use os.ttyname(sys.stdout.fileno())?
if os.getenv("DISPLAY") and host and opts.gcontrol_mode:
self.suite_engine_proc.gcontrol(suite_name, host)
return ret
def __init__(self):
self.node = ConfigNode()
self.files = {} # {rel path: root, ...}
self.file_locs = {} # {rel path: [root0, ...], ...}
self.conf_dirs = []
def load(self, conf_dir, conf_name, conf_dir_paths=None, opt_keys=None,
conf_node=None, no_ignore=False):
"""Load a (runtime) configuration directory with inheritance.
Return a ConfigTree object that represents the result.
conf_dir -- The path to the configuration directory to load.
conf_name -- The (base) name of the configuration file.
E.g. "rose-suite.conf".
conf_dir_paths -- A list of directories to locate relative paths to
configurations.
opt_keys -- Optional configuration keys.
conf_node -- A rose.config.ConfigNode to extend, or None to use a
fresh one.
no_ignore -- If True, skip loading ignored config settings.
"""
if not conf_dir_paths:
conf_dir_paths = []
conf_dir = self._search(conf_dir, [os.getcwd()] + conf_dir_paths)
nodes = {} # {conf_dir: node, ...}
conf_file_name = os.path.join(conf_dir, conf_name)
used_keys = []
nodes[conf_dir] = self.node_loader.load_with_opts(
conf_file_name, more_keys=opt_keys, used_keys=used_keys)
conf_tree = ConfigTree()
conf_tree.conf_dirs = mro(
conf_dir, self._get_base_names, conf_name, conf_dir_paths,
opt_keys, used_keys, nodes)
if opt_keys:
bad_keys = []
for opt_key in opt_keys:
if opt_key not in used_keys:
bad_keys.append(opt_key)
if bad_keys:
raise BadOptionalConfigurationKeysError(bad_keys)
if conf_node is None:
conf_tree.node = ConfigNode()
else:
conf_tree.node = conf_node
for t_conf_dir in conf_tree.conf_dirs:
node = nodes[t_conf_dir]
for keys, sub_node in node.walk(no_ignore=no_ignore):
if keys == ["", "import"]:
continue
if conf_tree.node.get(keys) is None:
conf_tree.node.set(keys, sub_node.value, sub_node.state,
sub_node.comments)
for dir_path, dir_names, file_names in os.walk(t_conf_dir):
names = [dir_ for dir_ in dir_names if dir_.startswith(".")]
for name in names:
dir_names.remove(name)
for file_name in file_names:
if file_name == conf_name or file_name.startswith("."):
continue
path = os.path.join(dir_path, file_name)
rel_path = os.path.relpath(path, t_conf_dir)
if rel_path not in conf_tree.files:
conf_tree.files[rel_path] = t_conf_dir
if rel_path not in conf_tree.file_locs:
conf_tree.file_locs[rel_path] = []
conf_tree.file_locs[rel_path].append(t_conf_dir)
return conf_tree
def _clean(self, suite_name, only_items=None):
"""Perform the cleaning operations."""
engine = self.suite_engine_proc
suite_dir_rel = engine.get_suite_dir_rel(suite_name)
locs_file_path = engine.get_suite_dir(suite_name, "log",
"rose-suite-run.locs")
locs_conf = ConfigNode().set(["localhost"], {})
try:
ConfigLoader().load(locs_file_path, locs_conf)
except IOError:
pass
items = self.CLEANABLE_PATHS + [""]
if only_items:
items = only_items
items.sort()
uuid_str = str(uuid4())
for auth, node in sorted(locs_conf.value.items(), self._auth_node_cmp):
locs = []
roots = set([""])
for item in items:
if item:
locs.append(os.path.join(suite_dir_rel, item))
else:
locs.append(suite_dir_rel)
if item and os.path.normpath(item) in self.CLEANABLE_PATHS:
item_root = node.get_value(["root-dir{" + item + "}"])
if item_root is None: # backward compat
item_root = node.get_value(["root-dir-" + item])
elif item == "":
item_root = node.get_value(["root-dir"])
else:
continue
if item_root:
loc_rel = suite_dir_rel
if item:
loc_rel = os.path.join(suite_dir_rel, item)
locs.append(os.path.join(item_root, loc_rel))
roots.add(item_root)
locs.reverse()
# Invoke bash as a login shell. The root location of a path may be
# in $DIR syntax, which can only be expanded correctly in a login
# shell. However, profile scripts invoked on login shell may print
# lots of junks. Hence we use a UUID here as a delimiter. Only
# output after the UUID lines are desirable lines.
command = ["bash", "-l", "-O", "extglob", "-c"]
sh_command = "cd; echo '%s'" % (uuid_str, )
if not self.host_selector.is_local_host(auth):
command = engine.popen.get_cmd("ssh", auth) + command
sh_command += "; ls -d -r %(locs)s; rm -fr %(locs)s" % {
"locs": engine.popen.list_to_shell_str(locs)
}
if not only_items:
# Clean empty directories
# Change directory to root level to avoid cleaning them as
# well For cylc suites, e.g. it can clean up to an empty
# "cylc-run/" directory.
for root in roots:
names = []
# Reverse sort to ensure that e.g. "share/cycle/" is
# cleaned before "share/"
for name in sorted(self.CLEANABLE_PATHS, reverse=True):
names.append(os.path.join(suite_dir_rel, name))
if os.sep in suite_dir_rel:
names.append(os.path.dirname(suite_dir_rel))
sh_command += (
"; " + "(cd %(root)s; " +
"rmdir -p %(names)s 2>/dev/null || true)") % {
"root": root,
"names": engine.popen.list_to_shell_str(names),
}
if self.host_selector.is_local_host(auth):
command.append(sh_command)
else:
command.append(quote(sh_command))
is_after_uuid_str = False
for line in engine.popen(*command)[0].splitlines():
if is_after_uuid_str:
engine.handle_event(
FileSystemEvent(FileSystemEvent.DELETE,
auth + ":" + line.strip()))
elif line == uuid_str:
is_after_uuid_str = True
def run_impl(self, opts, args, uuid, work_files):
# Log file, temporary
if hasattr(self.event_handler, "contexts"):
t_file = TemporaryFile()
log_context = ReporterContext(None, self.event_handler.VV, t_file)
self.event_handler.contexts[uuid] = log_context
# Check suite engine specific compatibility
self.suite_engine_proc.check_global_conf_compat()
# Suite name from the current working directory
if opts.conf_dir:
self.fs_util.chdir(opts.conf_dir)
opts.conf_dir = os.getcwd()
# --remote=KEY=VALUE,...
if opts.remote:
# opts.name always set for remote.
return self._run_remote(opts, opts.name)
conf_tree = self.config_load(opts)
self.fs_util.chdir(conf_tree.conf_dirs[0])
suite_name = opts.name
if not opts.name:
suite_name = os.path.basename(os.getcwd())
# Check suite.rc #! line for template scheme
templ_scheme = "jinja2"
if self.suite_engine_proc.SUITE_CONF in conf_tree.files:
suiterc_path = os.path.join(
conf_tree.files[self.suite_engine_proc.SUITE_CONF],
self.suite_engine_proc.SUITE_CONF)
with open(suiterc_path) as fh:
line = fh.readline()
if line.startswith("#!"):
templ_scheme = line[2:].strip().lower()
suite_section = (templ_scheme + ':' +
self.suite_engine_proc.SUITE_CONF)
extra_defines = []
if opts.defines_suite:
for define in opts.defines_suite:
extra_defines.append("[" + suite_section + "]" + define)
# Automatic Rose constants
# ROSE_ORIG_HOST: originating host
# ROSE_VERSION: Rose version (not retained in run_mode=="reload")
# Suite engine version
my_rose_version = ResourceLocator.default().get_version()
suite_engine_key = self.suite_engine_proc.get_version_env_name()
if opts.run_mode in ["reload", "restart"]:
prev_config_path = self.suite_engine_proc.get_suite_dir(
suite_name, "log", "rose-suite-run.conf")
prev_config = ConfigLoader()(prev_config_path)
suite_engine_version = prev_config.get_value(
["env", suite_engine_key])
else:
suite_engine_version = self.suite_engine_proc.get_version()
resloc = ResourceLocator.default()
auto_items = [
(suite_engine_key, suite_engine_version),
("ROSE_ORIG_HOST", self.host_selector.get_local_host()),
("ROSE_SITE", resloc.get_conf().get_value(['site'], '')),
("ROSE_VERSION", resloc.get_version())]
for key, val in auto_items:
requested_value = conf_tree.node.get_value(["env", key])
if requested_value:
if key == "ROSE_VERSION" and val != requested_value:
exc = VersionMismatchError(requested_value, val)
raise ConfigValueError(["env", key], requested_value, exc)
val = requested_value
else:
conf_tree.node.set(["env", key], val,
state=conf_tree.node.STATE_NORMAL)
extra_defines.append('[%s]%s="%s"' % (suite_section, key, val))
# Pass automatic Rose constants as suite defines
self.conf_tree_loader.node_loader.load_defines(extra_defines,
conf_tree.node)
# See if suite is running or not
if opts.run_mode == "reload":
# Check suite is running
self.suite_engine_proc.get_suite_contact(suite_name)
else:
self.suite_engine_proc.check_suite_not_running(suite_name)
# Install the suite to its run location
suite_dir_rel = self._suite_dir_rel(suite_name)
# Unfortunately a large try/finally block to ensure a temporary folder
# created in validate only mode is cleaned up. Exceptions are not
# caught here
try:
# Process Environment Variables
environ = self.config_pm(conf_tree, "env")
if opts.validate_suite_only_mode:
temp_dir = mkdtemp()
suite_dir = os.path.join(temp_dir, suite_dir_rel)
os.makedirs(suite_dir, 0o0700)
else:
suite_dir = os.path.join(
os.path.expanduser("~"), suite_dir_rel)
suite_conf_dir = os.getcwd()
locs_conf = ConfigNode()
if opts.new_mode:
if os.getcwd() == suite_dir:
raise NewModeError("PWD", os.getcwd())
elif opts.run_mode in ["reload", "restart"]:
raise NewModeError("--run", opts.run_mode)
self.suite_run_cleaner.clean(suite_name)
if os.getcwd() != suite_dir:
if opts.run_mode == "run":
self._run_init_dir(opts, suite_name, conf_tree,
locs_conf=locs_conf)
os.chdir(suite_dir)
# Housekeep log files
now_str = None
if not opts.install_only_mode and not opts.local_install_only_mode:
now_str = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
self._run_init_dir_log(opts, now_str)
self.fs_util.makedirs("log/suite")
# Rose configuration and version logs
self.fs_util.makedirs("log/rose-conf")
run_mode = opts.run_mode
if run_mode not in ["reload", "restart", "run"]:
run_mode = "run"
mode = run_mode
if opts.validate_suite_only_mode:
mode = "validate-suite-only"
elif opts.install_only_mode:
mode = "install-only"
elif opts.local_install_only_mode:
mode = "local-install-only"
prefix = "rose-conf/%s-%s" % (strftime("%Y%m%dT%H%M%S"), mode)
# Dump the actual configuration as rose-suite-run.conf
ConfigDumper()(conf_tree.node, "log/" + prefix + ".conf")
# Install version information file
write_source_vc_info(
suite_conf_dir, "log/" + prefix + ".version", self.popen)
# If run through rose-stem, install version information files for
# each source tree if they're a working copy
if hasattr(opts, 'source') and hasattr(opts, 'project'):
for i, url in enumerate(opts.source):
if os.path.isdir(url):
write_source_vc_info(
url, "log/" + opts.project[i] + "-" + str(i) +
".version", self.popen)
for ext in [".conf", ".version"]:
self.fs_util.symlink(prefix + ext, "log/rose-suite-run" + ext)
# Move temporary log to permanent log
if hasattr(self.event_handler, "contexts"):
log_file_path = os.path.abspath(
os.path.join("log", "rose-suite-run.log"))
log_file = open(log_file_path, "ab")
temp_log_file = self.event_handler.contexts[uuid].handle
temp_log_file.seek(0)
log_file.write(temp_log_file.read())
self.event_handler.contexts[uuid].handle = log_file
temp_log_file.close()
# Process Files
cwd = os.getcwd()
for rel_path, conf_dir in conf_tree.files.items():
if (conf_dir == cwd or
any(fnmatchcase(os.sep + rel_path, exclude)
for exclude in self.SYNC_EXCLUDES) or
conf_tree.node.get(
[templ_scheme + ":" + rel_path]) is not None):
continue
# No sub-directories, very slow otherwise
if os.sep in rel_path:
rel_path = rel_path.split(os.sep, 1)[0]
target_key = self.config_pm.get_handler(
"file").PREFIX + rel_path
target_node = conf_tree.node.get([target_key])
if target_node is None:
conf_tree.node.set([target_key])
target_node = conf_tree.node.get([target_key])
elif target_node.is_ignored():
continue
source_node = target_node.get("source")
if source_node is None:
target_node.set(
["source"], os.path.join(
conf_dir, rel_path))
elif source_node.is_ignored():
continue
self.config_pm(conf_tree, "file",
no_overwrite_mode=opts.no_overwrite_mode)
# Process suite configuration template header
# (e.g. Jinja2:suite.rc, EmPy:suite.rc)
self.config_pm(conf_tree, templ_scheme, environ=environ)
# Ask suite engine to parse suite configuration
# and determine if it is up to date (unchanged)
if opts.validate_suite_only_mode:
suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf(
suite_dir, None, opts.strict_mode,
debug_mode=True)
else:
suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf(
suite_name, opts.run_mode, opts.strict_mode,
opts.debug_mode)
finally:
# Ensure the temporary directory created is cleaned up regardless
# of success or failure
if opts.validate_suite_only_mode and os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
# Only validating so finish now
if opts.validate_suite_only_mode:
return
# Install share/work directories (local)
for name in ["share", "share/cycle", "work"]:
self._run_init_dir_work(
opts, suite_name, name, conf_tree, locs_conf=locs_conf)
if opts.local_install_only_mode:
return
# Install suite files to each remote [user@]host
for name in ["", "log/", "share/", "share/cycle/", "work/"]:
uuid_file = os.path.abspath(name + uuid)
open(uuid_file, "w").close()
work_files.append(uuid_file)
# Install items to user@host
auths = self.suite_engine_proc.get_tasks_auths(suite_name)
proc_queue = [] # [[proc, command, "ssh"|"rsync", auth], ...]
for auth in sorted(auths):
host = auth
if "@" in auth:
host = auth.split("@", 1)[1]
# Remote shell
command = self.popen.get_cmd("ssh", "-n", auth)
# Provide ROSE_VERSION and CYLC_VERSION in the environment
shcommand = "env ROSE_VERSION=%s %s=%s" % (
my_rose_version, suite_engine_key, suite_engine_version)
# Use login shell?
no_login_shell = self._run_conf(
"remote-no-login-shell", host=host, conf_tree=conf_tree)
if not no_login_shell or no_login_shell.lower() != "true":
shcommand += r""" bash -l -c '"$0" "$@"'"""
# Path to "rose" command, if applicable
rose_bin = self._run_conf(
"remote-rose-bin", host=host, conf_tree=conf_tree,
default="rose")
# Build remote "rose suite-run" command
shcommand += " %s suite-run -vv -n %s" % (rose_bin, suite_name)
for key in ["new", "debug", "install-only"]:
attr = key.replace("-", "_") + "_mode"
if getattr(opts, attr, None) is not None:
shcommand += " --%s" % key
if opts.log_keep:
shcommand += " --log-keep=%s" % opts.log_keep
if opts.log_name:
shcommand += " --log-name=%s" % opts.log_name
if not opts.log_archive_mode:
shcommand += " --no-log-archive"
shcommand += " --run=%s" % opts.run_mode
# Build --remote= option
shcommand += " --remote=uuid=%s" % uuid
if now_str is not None:
shcommand += ",now-str=%s" % now_str
host_confs = [
"root-dir",
"root-dir{share}",
"root-dir{share/cycle}",
"root-dir{work}"]
locs_conf.set([auth])
for key in host_confs:
value = self._run_conf(key, host=host, conf_tree=conf_tree)
if value is not None:
val = self.popen.list_to_shell_str([str(value)])
shcommand += ",%s=%s" % (key, pipes.quote(val))
locs_conf.set([auth, key], value)
command.append(shcommand)
proc = self.popen.run_bg(*command)
proc_queue.append([proc, command, "ssh", auth])
while proc_queue:
sleep(self.SLEEP_PIPE)
proc, command, command_name, auth = proc_queue.pop(0)
if proc.poll() is None: # put it back in proc_queue
proc_queue.append([proc, command, command_name, auth])
continue
ret_code = proc.wait()
out, err = proc.communicate()
if ret_code:
raise RosePopenError(command, ret_code, out, err)
if command_name == "rsync":
self.handle_event(out, level=Event.VV)
continue
else:
self.handle_event(out, level=Event.VV, prefix="[%s] " % auth)
for line in out.split("\n"):
if "/" + uuid == line.strip():
locs_conf.unset([auth])
break
else:
filters = {"excludes": [], "includes": []}
for name in ["", "log/", "share/", "share/cycle/", "work/"]:
filters["excludes"].append(name + uuid)
target = auth + ":" + suite_dir_rel
cmd = self._get_cmd_rsync(target, **filters)
proc_queue.append(
[self.popen.run_bg(*cmd), cmd, "rsync", auth])
# Install ends
ConfigDumper()(locs_conf, os.path.join("log", "rose-suite-run.locs"))
if opts.install_only_mode:
return
elif opts.run_mode == "reload" and suite_conf_unchanged:
conf_name = self.suite_engine_proc.SUITE_CONF
self.handle_event(SkipReloadEvent(suite_name, conf_name))
return
# Start the suite
self.fs_util.chdir("log")
self.suite_engine_proc.run(suite_name, opts.host, opts.run_mode, args)
# Disconnect log file handle, so monitoring tool command will no longer
# be associated with the log file.
self.event_handler.contexts[uuid].handle.close()
self.event_handler.contexts.pop(uuid)
# Launch the monitoring tool
# Note: maybe use os.ttyname(sys.stdout.fileno())?
if opts.gcontrol_mode and opts.run_mode != "reload":
self.suite_engine_proc.gcontrol(suite_name)
return 0
def _clean(self, suite_name, only_items=None):
"""Perform the cleaning operations."""
engine = self.suite_engine_proc
suite_dir_rel = engine.get_suite_dir_rel(suite_name)
locs_file_path = engine.get_suite_dir(suite_name, "log",
"rose-suite-run.locs")
locs_conf = ConfigNode().set(["localhost"], {})
try:
ConfigLoader().load(locs_file_path, locs_conf)
except IOError:
pass
items = self.CLEANABLE_ROOTS + [""]
if only_items:
items = only_items
items.sort()
uuid_str = str(uuid4())
for auth, node in sorted(locs_conf.value.items(), self._auth_node_cmp):
locs = []
for item in items:
if item:
locs.append(os.path.join(suite_dir_rel, item))
else:
locs.append(suite_dir_rel)
if item and os.path.normpath(item) in self.CLEANABLE_ROOTS:
conf_key = "root-dir-" + item
elif item == "":
conf_key = "root-dir"
else:
continue
item_root = node.get_value([conf_key])
if item_root:
loc_rel = suite_dir_rel
if item:
loc_rel = os.path.join(suite_dir_rel, item)
locs.append(os.path.join(item_root, loc_rel))
if auth == "localhost":
for loc in locs:
loc = os.path.abspath(env_var_process(loc))
for name in sorted(glob(loc)):
engine.fs_util.delete(name)
else:
# Invoke bash as a login shell. The root location of a path may
# be in $DIR syntax, which can only be expanded correctly in a
# login shell. However, profile scripts invoked on login to the
# remote host may print lots of junks. Hence we use a UUID here
# as a delimiter. Only output after the UUID lines are
# desirable lines.
command = engine.popen.get_cmd("ssh", auth, "bash", "-l", "-c")
command += [
"'echo %(uuid)s; ls -d %(locs)s|sort; rm -rf %(locs)s'" % {
"locs": engine.popen.list_to_shell_str(locs),
"uuid": uuid_str,
},
]
is_after_uuid_str = False
for line in engine.popen(*command)[0].splitlines():
if is_after_uuid_str:
engine.handle_event(
FileSystemEvent(FileSystemEvent.DELETE,
auth + ":" + line.strip()))
elif line == uuid_str:
is_after_uuid_str = True
def run_impl(self, opts, args, uuid, work_files):
# Log file, temporary
if hasattr(self.event_handler, "contexts"):
t_file = TemporaryFile()
log_context = ReporterContext(None, self.event_handler.VV, t_file)
self.event_handler.contexts[uuid] = log_context
# Check suite engine specific compatibility
self.suite_engine_proc.check_global_conf_compat()
# Suite name from the current working directory
if opts.conf_dir:
self.fs_util.chdir(opts.conf_dir)
opts.conf_dir = os.getcwd()
# --remote=KEY=VALUE,...
if opts.remote:
# opts.name always set for remote.
return self._run_remote(opts, opts.name)
conf_tree = self.config_load(opts)
self.fs_util.chdir(conf_tree.conf_dirs[0])
suite_name = opts.name
if not opts.name:
suite_name = os.path.basename(os.getcwd())
# Check suite.rc #! line for template scheme
templ_scheme = "jinja2"
if self.suite_engine_proc.SUITE_CONF in conf_tree.files:
suiterc_path = os.path.join(
conf_tree.files[self.suite_engine_proc.SUITE_CONF],
self.suite_engine_proc.SUITE_CONF)
with open(suiterc_path) as fh:
line = fh.readline()
if line.startswith("#!"):
templ_scheme = line[2:].strip().lower()
suite_section = (templ_scheme + ':' +
self.suite_engine_proc.SUITE_CONF)
extra_defines = []
if opts.defines_suite:
for define in opts.defines_suite:
extra_defines.append("[" + suite_section + "]" + define)
# Automatic Rose constants
# ROSE_ORIG_HOST: originating host
# ROSE_VERSION: Rose version (not retained in run_mode=="reload")
# Suite engine version
my_rose_version = ResourceLocator.default().get_version()
suite_engine_key = self.suite_engine_proc.get_version_env_name()
if opts.run_mode in ["reload", "restart"]:
prev_config_path = self.suite_engine_proc.get_suite_dir(
suite_name, "log", "rose-suite-run.conf")
prev_config = ConfigLoader()(prev_config_path)
suite_engine_version = prev_config.get_value(
["env", suite_engine_key])
else:
suite_engine_version = self.suite_engine_proc.get_version()
resloc = ResourceLocator.default()
auto_items = [(suite_engine_key, suite_engine_version),
("ROSE_ORIG_HOST", self.host_selector.get_local_host()),
("ROSE_SITE", resloc.get_conf().get_value(['site'], '')),
("ROSE_VERSION", resloc.get_version())]
for key, val in auto_items:
requested_value = conf_tree.node.get_value(["env", key])
if requested_value:
if key == "ROSE_VERSION" and val != requested_value:
exc = VersionMismatchError(requested_value, val)
raise ConfigValueError(["env", key], requested_value, exc)
val = requested_value
else:
conf_tree.node.set(["env", key],
val,
state=conf_tree.node.STATE_NORMAL)
extra_defines.append('[%s]%s="%s"' % (suite_section, key, val))
# Pass automatic Rose constants as suite defines
self.conf_tree_loader.node_loader.load_defines(extra_defines,
conf_tree.node)
# See if suite is running or not
if opts.run_mode == "reload":
# Check suite is running
self.suite_engine_proc.get_suite_contact(suite_name)
else:
self.suite_engine_proc.check_suite_not_running(suite_name)
# Install the suite to its run location
suite_dir_rel = self._suite_dir_rel(suite_name)
# Unfortunately a large try/finally block to ensure a temporary folder
# created in validate only mode is cleaned up. Exceptions are not
# caught here
try:
# Process Environment Variables
environ = self.config_pm(conf_tree, "env")
if opts.validate_suite_only_mode:
temp_dir = mkdtemp()
suite_dir = os.path.join(temp_dir, suite_dir_rel)
os.makedirs(suite_dir, 0o0700)
else:
suite_dir = os.path.join(os.path.expanduser("~"),
suite_dir_rel)
suite_conf_dir = os.getcwd()
locs_conf = ConfigNode()
if opts.new_mode:
if os.getcwd() == suite_dir:
raise NewModeError("PWD", os.getcwd())
elif opts.run_mode in ["reload", "restart"]:
raise NewModeError("--run", opts.run_mode)
self.suite_run_cleaner.clean(suite_name)
if os.getcwd() != suite_dir:
if opts.run_mode == "run":
self._run_init_dir(opts,
suite_name,
conf_tree,
locs_conf=locs_conf)
os.chdir(suite_dir)
# Housekeep log files
now_str = None
if not opts.install_only_mode and not opts.local_install_only_mode:
now_str = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
self._run_init_dir_log(opts, now_str)
self.fs_util.makedirs("log/suite")
# Rose configuration and version logs
self.fs_util.makedirs("log/rose-conf")
run_mode = opts.run_mode
if run_mode not in ["reload", "restart", "run"]:
run_mode = "run"
mode = run_mode
if opts.validate_suite_only_mode:
mode = "validate-suite-only"
elif opts.install_only_mode:
mode = "install-only"
elif opts.local_install_only_mode:
mode = "local-install-only"
prefix = "rose-conf/%s-%s" % (strftime("%Y%m%dT%H%M%S"), mode)
# Dump the actual configuration as rose-suite-run.conf
ConfigDumper()(conf_tree.node, "log/" + prefix + ".conf")
# Install version information file
write_source_vc_info(suite_conf_dir, "log/" + prefix + ".version",
self.popen)
# If run through rose-stem, install version information files for
# each source tree if they're a working copy
if hasattr(opts, 'source') and hasattr(opts, 'project'):
for i, url in enumerate(opts.source):
if os.path.isdir(url):
write_source_vc_info(
url, "log/" + opts.project[i] + "-" + str(i) +
".version", self.popen)
for ext in [".conf", ".version"]:
self.fs_util.symlink(prefix + ext, "log/rose-suite-run" + ext)
# Move temporary log to permanent log
if hasattr(self.event_handler, "contexts"):
log_file_path = os.path.abspath(
os.path.join("log", "rose-suite-run.log"))
log_file = open(log_file_path, "ab")
temp_log_file = self.event_handler.contexts[uuid].handle
temp_log_file.seek(0)
log_file.write(temp_log_file.read())
self.event_handler.contexts[uuid].handle = log_file
temp_log_file.close()
# Process Files
cwd = os.getcwd()
for rel_path, conf_dir in conf_tree.files.items():
if (conf_dir == cwd or any(
fnmatchcase(os.sep + rel_path, exclude)
for exclude in self.SYNC_EXCLUDES)
or conf_tree.node.get([templ_scheme + ":" + rel_path
]) is not None):
continue
# No sub-directories, very slow otherwise
if os.sep in rel_path:
rel_path = rel_path.split(os.sep, 1)[0]
target_key = self.config_pm.get_handler(
"file").PREFIX + rel_path
target_node = conf_tree.node.get([target_key])
if target_node is None:
conf_tree.node.set([target_key])
target_node = conf_tree.node.get([target_key])
elif target_node.is_ignored():
continue
source_node = target_node.get("source")
if source_node is None:
target_node.set(["source"],
os.path.join(conf_dir, rel_path))
elif source_node.is_ignored():
continue
self.config_pm(conf_tree,
"file",
no_overwrite_mode=opts.no_overwrite_mode)
# Process suite configuration template header
# (e.g. Jinja2:suite.rc, EmPy:suite.rc)
self.config_pm(conf_tree, templ_scheme, environ=environ)
# Ask suite engine to parse suite configuration
# and determine if it is up to date (unchanged)
if opts.validate_suite_only_mode:
suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf(
suite_dir, None, opts.strict_mode, debug_mode=True)
else:
suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf(
suite_name, opts.run_mode, opts.strict_mode,
opts.debug_mode)
finally:
# Ensure the temporary directory created is cleaned up regardless
# of success or failure
if opts.validate_suite_only_mode and os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
# Only validating so finish now
if opts.validate_suite_only_mode:
return
# Install share/work directories (local)
for name in ["share", "share/cycle", "work"]:
self._run_init_dir_work(opts,
suite_name,
name,
conf_tree,
locs_conf=locs_conf)
if opts.local_install_only_mode:
return
# Install suite files to each remote [user@]host
for name in ["", "log/", "share/", "share/cycle/", "work/"]:
uuid_file = os.path.abspath(name + uuid)
open(uuid_file, "w").close()
work_files.append(uuid_file)
# Install items to user@host
auths = self.suite_engine_proc.get_tasks_auths(suite_name)
proc_queue = [] # [[proc, command, "ssh"|"rsync", auth], ...]
for auth in sorted(auths):
host = auth
if "@" in auth:
host = auth.split("@", 1)[1]
# Remote shell
command = self.popen.get_cmd("ssh", "-n", auth)
# Provide ROSE_VERSION and CYLC_VERSION in the environment
shcommand = "env ROSE_VERSION=%s %s=%s" % (
my_rose_version, suite_engine_key, suite_engine_version)
# Use login shell?
no_login_shell = self._run_conf("remote-no-login-shell",
host=host,
conf_tree=conf_tree)
if not no_login_shell or no_login_shell.lower() != "true":
shcommand += r""" bash -l -c '"$0" "$@"'"""
# Path to "rose" command, if applicable
rose_bin = self._run_conf("remote-rose-bin",
host=host,
conf_tree=conf_tree,
default="rose")
# Build remote "rose suite-run" command
shcommand += " %s suite-run -vv -n %s" % (rose_bin, suite_name)
for key in ["new", "debug", "install-only"]:
attr = key.replace("-", "_") + "_mode"
if getattr(opts, attr, None) is not None:
shcommand += " --%s" % key
if opts.log_keep:
shcommand += " --log-keep=%s" % opts.log_keep
if opts.log_name:
shcommand += " --log-name=%s" % opts.log_name
if not opts.log_archive_mode:
shcommand += " --no-log-archive"
shcommand += " --run=%s" % opts.run_mode
# Build --remote= option
shcommand += " --remote=uuid=%s" % uuid
if now_str is not None:
shcommand += ",now-str=%s" % now_str
host_confs = [
"root-dir", "root-dir{share}", "root-dir{share/cycle}",
"root-dir{work}"
]
locs_conf.set([auth])
for key in host_confs:
value = self._run_conf(key, host=host, conf_tree=conf_tree)
if value is not None:
val = self.popen.list_to_shell_str([str(value)])
shcommand += ",%s=%s" % (key, pipes.quote(val))
locs_conf.set([auth, key], value)
command.append(shcommand)
proc = self.popen.run_bg(*command)
proc_queue.append([proc, command, "ssh", auth])
while proc_queue:
sleep(self.SLEEP_PIPE)
proc, command, command_name, auth = proc_queue.pop(0)
if proc.poll() is None: # put it back in proc_queue
proc_queue.append([proc, command, command_name, auth])
continue
ret_code = proc.wait()
out, err = proc.communicate()
if ret_code:
raise RosePopenError(command, ret_code, out, err)
if command_name == "rsync":
self.handle_event(out, level=Event.VV)
continue
else:
self.handle_event(out, level=Event.VV, prefix="[%s] " % auth)
for line in out.split("\n"):
if "/" + uuid == line.strip():
locs_conf.unset([auth])
break
else:
filters = {"excludes": [], "includes": []}
for name in ["", "log/", "share/", "share/cycle/", "work/"]:
filters["excludes"].append(name + uuid)
target = auth + ":" + suite_dir_rel
cmd = self._get_cmd_rsync(target, **filters)
proc_queue.append(
[self.popen.run_bg(*cmd), cmd, "rsync", auth])
# Install ends
ConfigDumper()(locs_conf, os.path.join("log", "rose-suite-run.locs"))
if opts.install_only_mode:
return
elif opts.run_mode == "reload" and suite_conf_unchanged:
conf_name = self.suite_engine_proc.SUITE_CONF
self.handle_event(SkipReloadEvent(suite_name, conf_name))
return
# Start the suite
self.fs_util.chdir("log")
self.suite_engine_proc.run(suite_name, opts.host, opts.run_mode, args)
# Disconnect log file handle, so monitoring tool command will no longer
# be associated with the log file.
self.event_handler.contexts[uuid].handle.close()
self.event_handler.contexts.pop(uuid)
# Launch the monitoring tool
# Note: maybe use os.ttyname(sys.stdout.fileno())?
if opts.gcontrol_mode and opts.run_mode != "reload":
self.suite_engine_proc.gcontrol(suite_name)
return 0
def _clean(self, suite_name, only_items=None):
"""Perform the cleaning operations."""
engine = self.suite_engine_proc
suite_dir_rel = engine.get_suite_dir_rel(suite_name)
locs_file_path = engine.get_suite_dir(suite_name, "log",
"rose-suite-run.locs")
locs_conf = ConfigNode().set(["localhost"], {})
try:
ConfigLoader().load(locs_file_path, locs_conf)
except IOError:
pass
items = self.CLEANABLE_PATHS + [""]
if only_items:
items = only_items
items.sort()
uuid_str = str(uuid4())
for auth, node in sorted(locs_conf.value.items(), self._auth_node_cmp):
locs = []
roots = set([""])
for item in items:
if item:
locs.append(os.path.join(suite_dir_rel, item))
else:
locs.append(suite_dir_rel)
if item and os.path.normpath(item) in self.CLEANABLE_PATHS:
item_root = node.get_value(["root-dir{" + item + "}"])
if item_root is None: # backward compat
item_root = node.get_value(["root-dir-" + item])
elif item == "":
item_root = node.get_value(["root-dir"])
else:
continue
if item_root:
loc_rel = suite_dir_rel
if item:
loc_rel = os.path.join(suite_dir_rel, item)
locs.append(os.path.join(item_root, loc_rel))
roots.add(item_root)
if self.host_selector.is_local_host(auth):
# Clean relevant items
for loc in locs:
loc = os.path.abspath(env_var_process(loc))
for name in sorted(glob(loc)):
engine.fs_util.delete(name)
# Clean empty directories
# Change directory to root level to avoid cleaning them as well
# For cylc suites, e.g. it can clean up to an empty "cylc-run/"
# directory.
for root in sorted(roots):
cwd = os.getcwd()
if root:
try:
os.chdir(env_var_process(root))
except OSError:
continue
# Reverse sort to ensure that e.g. "share/cycle/" is
# cleaned before "share/"
for name in sorted(self.CLEANABLE_PATHS, reverse=True):
try:
os.removedirs(os.path.join(suite_dir_rel, name))
except OSError:
pass
try:
os.removedirs(suite_dir_rel)
except OSError:
pass
if root:
os.chdir(cwd)
else:
# Invoke bash as a login shell. The root location of a path may
# be in $DIR syntax, which can only be expanded correctly in a
# login shell. However, profile scripts invoked on login to the
# remote host may print lots of junks. Hence we use a UUID here
# as a delimiter. Only output after the UUID lines are
# desirable lines.
command = engine.popen.get_cmd("ssh", auth, "bash", "-l", "-c")
sh_command = (
"echo %(uuid)s; ls -d %(locs)s|sort; rm -fr %(locs)s") % {
"locs": engine.popen.list_to_shell_str(locs),
"uuid": uuid_str,
}
# Clean empty directories
# Change directory to root level to avoid cleaning them as well
# For cylc suites, e.g. it can clean up to an empty "cylc-run/"
# directory.
for root in roots:
names = []
# Reverse sort to ensure that e.g. "share/cycle/" is
# cleaned before "share/"
for name in sorted(self.CLEANABLE_PATHS, reverse=True):
names.append(os.path.join(suite_dir_rel, name))
sh_command += (
"; " +
"(cd %(root)s; rmdir -p %(names)s 2>/dev/null || true)"
) % {
"root": root,
"names": engine.popen.list_to_shell_str(names),
}
command.append(quote(sh_command))
is_after_uuid_str = False
for line in engine.popen(*command)[0].splitlines():
if is_after_uuid_str:
engine.handle_event(
FileSystemEvent(FileSystemEvent.DELETE,
auth + ":" + line.strip()))
elif line == uuid_str:
is_after_uuid_str = True