def __init__(self, *args, **kwargs):
Runner.__init__(self, *args, **kwargs)
self.host_selector = HostSelector(self.event_handler, self.popen)
self.suite_run_cleaner = SuiteRunCleaner(
event_handler=self.event_handler,
host_selector=self.host_selector,
suite_engine_proc=self.suite_engine_proc)
def __init__(self, opts, reporter=None, popen=None, fs_util=None):
self.opts = opts
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
if fs_util is None:
self.fs_util = FileSystemUtil(event_handler=self.reporter)
else:
self.fs_util = fs_util
self.host_selector = HostSelector(event_handler=self.reporter,
popen=self.popen)
def __init__(self, event_handler=None, popen=None, fs_util=None,
host_selector=None, **_):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if host_selector is None:
host_selector = HostSelector(event_handler, popen)
self.host_selector = host_selector
self.date_time_oper = RoseDateTimeOperator()
def __init__(self,
event_handler=None,
host_selector=None,
suite_engine_proc=None):
if event_handler is None:
event_handler = Reporter()
self.event_handler = event_handler
if host_selector is None:
host_selector = HostSelector(event_handler=event_handler)
self.host_selector = host_selector
if suite_engine_proc is None:
suite_engine_proc = SuiteEngineProcessor.get_processor(
event_handler=event_handler)
self.suite_engine_proc = suite_engine_proc
def __init__(self, service_root_mode=False, *args, **kwargs):
self.stopping = False
self.service_root_mode = service_root_mode
self.props = {}
rose_conf = ResourceLocator.default().get_conf()
self.props["title"] = rose_conf.get_value(["rosie-disco", "title"],
self.TITLE)
self.props["host_name"] = rose_conf.get_value(["rosie-disco", "host"])
if self.props["host_name"] is None:
self.props["host_name"] = HostSelector().get_local_host()
if self.props["host_name"] and "." in self.props["host_name"]:
self.props["host_name"] = (self.props["host_name"].split(
".", 1)[0])
self.props["rose_version"] = ResourceLocator.default().get_version()
# Get location of HTML files from package
rosie_lib = os.path.join(
pkg_resources.resource_filename('metomi.rosie', 'lib'), "html",
"template", "rosie-disco")
# Autoescape markup to prevent code injection from user inputs.
self.props["template_env"] = jinja2.Environment(
autoescape=jinja2.select_autoescape(enabled_extensions=("html",
"xml"),
default_for_string=True),
loader=jinja2.FileSystemLoader(rosie_lib))
db_url_map = {}
for key, node in rose_conf.get(["rosie-db"]).value.items():
if key.startswith("db.") and key[3:]:
db_url_map[key[3:]] = node.value
self.db_url_map = db_url_map
# Specify the root URL for the handlers and template.
ROOT = "%s-%s" % (self.NAMESPACE, self.UTIL)
service_root = r"/?"
if self.service_root_mode:
service_root = service_root.replace("?", ROOT + r"/?")
# Set-up the Tornado application request-handling structure.
prefix_handlers = []
class_args = {"props": self.props}
root_class_args = dict(class_args) # mutable so copy for safety
root_class_args.update({"db_url_map": self.db_url_map})
root_handler = (service_root, RosieDiscoServiceRoot, root_class_args)
for key, db_url in self.db_url_map.items():
prefix_class_args = dict(class_args) # mutable so copy for safety
prefix_class_args.update({
"prefix": key,
"db_url": db_url,
"service_root": service_root,
})
handler = (service_root + key + r"/?", RosieDiscoService,
prefix_class_args)
get_handler = (service_root + key + r"/get_(.+)", GetHandler,
prefix_class_args)
hello_handler = (service_root + key + r"/hello/?", HelloHandler,
prefix_class_args)
search_handler = (service_root + key + r"/search", SearchHandler,
prefix_class_args)
query_handler = (service_root + key + r"/query", QueryHandler,
prefix_class_args)
prefix_handlers.extend([
handler, get_handler, hello_handler, search_handler,
query_handler
])
handlers = [root_handler] + prefix_handlers
settings = dict(
autoreload=True,
static_path=ResourceLocator.default().get_util_home(
"lib", "html", "static"),
)
super(RosieDiscoServiceApplication,
self).__init__(handlers, **settings)
class StemRunner(object):
"""Set up options for running a STEM job through Rose."""
def __init__(self, opts, reporter=None, popen=None, fs_util=None):
self.opts = opts
if reporter is None:
self.reporter = Reporter(opts.verbosity - opts.quietness)
else:
self.reporter = reporter
if popen is None:
self.popen = RosePopener(event_handler=self.reporter)
else:
self.popen = popen
if fs_util is None:
self.fs_util = FileSystemUtil(event_handler=self.reporter)
else:
self.fs_util = fs_util
self.host_selector = HostSelector(event_handler=self.reporter,
popen=self.popen)
def _add_define_option(self, var, val):
"""Add a define option passed to the SuiteRunner."""
if self.opts.defines:
self.opts.defines.append(SUITE_RC_PREFIX + var + '=' + val)
else:
self.opts.defines = [SUITE_RC_PREFIX + var + '=' + val]
self.reporter(ConfigVariableSetEvent(var, val))
return
def _get_base_dir(self, item):
"""Given a source tree return the following from 'fcm loc-layout':
* url
* sub_tree
* peg_rev
* root
* project
"""
ret_code, output, stderr = self.popen.run('fcm', 'loc-layout', item)
output = output.decode()
if ret_code != 0:
raise ProjectNotFoundException(item, stderr)
ret = {}
for line in output.splitlines():
if ":" not in line:
continue
key, value = line.split(":", 1)
if key:
if value:
ret[key] = value.strip()
return ret
def _get_project_from_url(self, source_dict):
"""Run 'fcm keyword-print' to work out the project name."""
repo = source_dict['root']
if source_dict['project']:
repo += '/' + source_dict['project']
kpoutput = self.popen.run('fcm', 'kp', source_dict['url'])[1]
project = None
for line in kpoutput.splitlines():
if line.rstrip().endswith(repo.encode('UTF-8')):
kpresult = re.search(r'^location{primary}\[(.*)\]',
line.decode())
if kpresult:
project = kpresult.group(1)
break
return project
def _deduce_mirror(self, source_dict, project):
"""Deduce the mirror location of this source tree."""
# Root location for project
proj_root = source_dict['root'] + '/' + source_dict['project']
# Swap project to mirror
project = re.sub(r'\.x$', r'.xm', project)
mirror_repo = "fcm:" + project
# Generate mirror location
mirror = re.sub(proj_root, mirror_repo, source_dict['url'])
# Remove any sub-tree
mirror = re.sub(source_dict['sub_tree'], r'', mirror)
mirror = re.sub(r'/@', r'@', mirror)
# Add forwards slash after .xm if missing
if '.xm/' not in mirror:
mirror = re.sub(r'\.xm', r'.xm/', mirror)
return mirror
def _ascertain_project(self, item):
"""Set the project name and top-level from 'fcm loc-layout'.
Returns:
* project name
* top-level location of the source tree with revision number
* top-level location of the source tree without revision number
* revision number
"""
project = None
try:
project, item = item.split("=", 1)
except ValueError:
pass
if re.search(r'^\.', item):
item = os.path.abspath(os.path.join(os.getcwd(), item))
if project is not None:
print("[WARN] Forcing project for '{0}' to be '{1}'".format(
item, project))
return project, item, item, '', ''
source_dict = self._get_base_dir(item)
project = self._get_project_from_url(source_dict)
if not project:
raise ProjectNotFoundException(item)
mirror = self._deduce_mirror(source_dict, project)
if 'peg_rev' in source_dict and '@' in item:
revision = '@' + source_dict['peg_rev']
base = re.sub(r'@.*', r'', item)
else:
revision = ''
base = item
# Remove subtree from base and item
if 'sub_tree' in source_dict:
item = re.sub(r'(.*)%s/?$' % (source_dict['sub_tree']),
r'\1',
item,
count=1)
base = re.sub(r'(.*)%s/?$' % (source_dict['sub_tree']),
r'\1',
base,
count=1)
# Remove trailing forwards-slash
item = re.sub(r'/$', r'', item)
base = re.sub(r'/$', r'', base)
# Remove anything after a point
project = re.sub(r'\..*', r'', project)
return project, item, base, revision, mirror
def _generate_name(self):
"""Generate a suite name from the name of the first source tree."""
try:
basedir = self._ascertain_project(os.getcwd())[1]
except ProjectNotFoundException:
if self.opts.conf_dir:
basedir = os.path.abspath(self.opts.conf_dir)
else:
basedir = os.getcwd()
name = os.path.basename(basedir)
self.reporter(NameSetEvent(name))
return name
def _this_suite(self):
"""Find the location of the suite in the first source tree."""
# Get base of first source
basedir = ''
if self.opts.source:
basedir = self.opts.source[0]
else:
basedir = self._ascertain_project(os.getcwd())[1]
suitedir = os.path.join(basedir, DEFAULT_TEST_DIR)
suitefile = os.path.join(suitedir, "rose-suite.conf")
if not os.path.isfile(suitefile):
raise RoseSuiteConfNotFoundException(suitedir)
self._check_suite_version(suitefile)
return suitedir
def _read_auto_opts(self):
"""Read the site metomi.rose.conf file."""
return ResourceLocator.default().get_conf().get_value(
["rose-stem", "automatic-options"])
def _check_suite_version(self, fname):
"""Check the suite is compatible with this version of metomi.rose-stem.
"""
if not os.path.isfile(fname):
raise RoseSuiteConfNotFoundException(os.path.dirname(fname))
config = metomi.rose.config.load(fname)
suite_rose_stem_version = config.get(['ROSE_STEM_VERSION'])
if suite_rose_stem_version:
suite_rose_stem_version = int(suite_rose_stem_version.value)
else:
suite_rose_stem_version = None
if not suite_rose_stem_version == ROSE_STEM_VERSION:
raise RoseStemVersionException(suite_rose_stem_version)
def _prepend_localhost(self, url):
"""Prepend the local hostname to urls which do not point to repository
locations."""
if ':' not in url or url.split(
':', 1)[0] not in ['svn', 'fcm', 'http', 'https', 'svn+ssh']:
url = self.host_selector.get_local_host() + ':' + url
return url
def process(self):
"""Process STEM options into 'rose suite-run' options."""
# Generate options for source trees
repos = {}
repos_with_hosts = {}
if not self.opts.source:
self.opts.source = ['.']
self.opts.project = list()
for i, url in enumerate(self.opts.source):
project, url, base, rev, mirror = self._ascertain_project(url)
self.opts.source[i] = url
self.opts.project.append(project)
# Versions of variables with hostname prepended for working copies
url_host = self._prepend_localhost(url)
base_host = self._prepend_localhost(base)
if project in repos:
repos[project].append(url)
repos_with_hosts[project].append(url_host)
else:
repos[project] = [url]
repos_with_hosts[project] = [url_host]
self._add_define_option('SOURCE_' + project.upper() + '_REV',
'"' + rev + '"')
self._add_define_option('SOURCE_' + project.upper() + '_BASE',
'"' + base + '"')
self._add_define_option(
'HOST_SOURCE_' + project.upper() + '_BASE',
'"' + base_host + '"')
self._add_define_option(
'SOURCE_' + project.upper() + '_MIRROR',
'"' + mirror + '"')
self.reporter(SourceTreeAddedAsBranchEvent(url))
for project, branches in repos.items():
var = 'SOURCE_' + project.upper()
branchstring = RosePopener.list_to_shell_str(branches)
self._add_define_option(var, '"' + branchstring + '"')
for project, branches in repos_with_hosts.items():
var_host = 'HOST_SOURCE_' + project.upper()
branchstring = RosePopener.list_to_shell_str(branches)
self._add_define_option(var_host, '"' + branchstring + '"')
# Generate the variable containing tasks to run
if self.opts.group:
if not self.opts.defines:
self.opts.defines = []
expanded_groups = []
for i in self.opts.group:
expanded_groups.extend(i.split(','))
self.opts.defines.append(SUITE_RC_PREFIX + 'RUN_NAMES=' +
str(expanded_groups))
# Load the config file and return any automatic-options
auto_opts = self._read_auto_opts()
if auto_opts:
automatic_options = auto_opts.split()
for option in automatic_options:
elements = option.split("=")
if len(elements) == 2:
self._add_define_option(elements[0],
'"' + elements[1] + '"')
# Change into the suite directory
if self.opts.conf_dir:
self.reporter(SuiteSelectionEvent(self.opts.conf_dir))
self._check_suite_version(
os.path.join(self.opts.conf_dir, 'rose-suite.conf'))
else:
thissuite = self._this_suite()
self.fs_util.chdir(thissuite)
self.reporter(SuiteSelectionEvent(thissuite))
# Create a default name for the suite; allow override by user
if not self.opts.name:
self.opts.name = self._generate_name()
return self.opts
def run(self, app_runner, conf_tree, opts, args, uuid, work_files):
"""Suite housekeeping application.
This application is designed to work under "rose task-run" in a cycling
suite.
"""
suite_name = os.getenv("ROSE_SUITE_NAME")
if not suite_name:
return
# Tar-gzip job logs on suite host
# Prune job logs on remote hosts and suite host
prune_remote_logs_cycles = self._get_conf(app_runner, conf_tree,
"prune-remote-logs-at")
# Prune job logs on suite host
prune_server_logs_cycles = self._get_conf(app_runner, conf_tree,
"prune-server-logs-at")
archive_logs_cycles = self._get_conf(app_runner, conf_tree,
"archive-logs-at")
if (prune_remote_logs_cycles or prune_server_logs_cycles
or archive_logs_cycles):
tmp_prune_remote_logs_cycles = []
for cycle in prune_remote_logs_cycles:
if cycle not in archive_logs_cycles:
tmp_prune_remote_logs_cycles.append(cycle)
prune_remote_logs_cycles = tmp_prune_remote_logs_cycles
tmp_prune_server_logs_cycles = []
for cycle in prune_server_logs_cycles:
if cycle not in archive_logs_cycles:
tmp_prune_server_logs_cycles.append(cycle)
prune_server_logs_cycles = tmp_prune_server_logs_cycles
if prune_remote_logs_cycles:
app_runner.suite_engine_proc.job_logs_housekeep_remote(
suite_name,
prune_remote_logs_cycles,
prune_remote_mode=True,
)
if prune_server_logs_cycles:
app_runner.suite_engine_proc.job_logs_remove_on_server(
suite_name, prune_server_logs_cycles)
if archive_logs_cycles:
app_runner.suite_engine_proc.job_logs_archive(
suite_name, archive_logs_cycles)
# Prune other directories
globs, cycle_set = self._get_prune_globs(app_runner, conf_tree)
if not globs:
return
suite_engine_proc = app_runner.suite_engine_proc
hosts = suite_engine_proc.get_suite_jobs_auths(
suite_name, [(cycle, None) for cycle in cycle_set])
# A shuffle here should allow the load for doing "rm -rf" to be shared
# between job hosts who share a file system.
shuffle(hosts)
suite_dir_rel = suite_engine_proc.get_suite_dir_rel(suite_name)
form_dict = {"d": suite_dir_rel, "g": " ".join(globs)}
sh_cmd_head = r"set -e; cd %(d)s; " % form_dict
sh_cmd = (r"set +e; ls -d %(g)s; " +
r"set -e; rm -fr %(g)s") % form_dict
cwd = os.getcwd()
host_selector = HostSelector(app_runner.event_handler,
app_runner.popen)
for host in hosts + [host_selector.get_local_host()]:
sdir = None
try:
if host_selector.is_local_host(host):
sdir = suite_engine_proc.get_suite_dir(suite_name)
app_runner.fs_util.chdir(sdir)
out = app_runner.popen.run_ok("bash", "-O", "extglob",
"-c", sh_cmd)[0]
else:
cmd = app_runner.popen.get_cmd(
"ssh",
host,
"bash -O extglob -c '" + sh_cmd_head + sh_cmd + "'",
)
out = app_runner.popen.run_ok(*cmd)[0]
except RosePopenError as exc:
app_runner.handle_event(exc)
else:
if sdir is None:
event = FileSystemEvent(FileSystemEvent.CHDIR,
host + ":" + suite_dir_rel)
app_runner.handle_event(event)
for line in sorted(out.splitlines()):
if not host_selector.is_local_host(host):
line = host + ":" + line
event = FileSystemEvent(FileSystemEvent.DELETE, line)
app_runner.handle_event(event)
finally:
if sdir:
app_runner.fs_util.chdir(cwd)
return
class SuiteRunner(Runner):
"""Invoke a Rose suite."""
SLEEP_PIPE = 0.05
NAME = "suite"
OPTIONS = [
"conf_dir", "defines", "defines_suite", "host", "install_only_mode",
"local_install_only_mode", "log_archive_mode", "log_keep", "log_name",
"name", "new_mode", "no_overwrite_mode", "opt_conf_keys",
"reload_mode", "remote", "restart_mode", "run_mode", "strict_mode",
"validate_suite_only"
]
# Lists of rsync (always) exclude globs
SYNC_EXCLUDES = (
"/.*",
"/cylc-suite.db",
"/log",
"/log.*",
"/state",
"/share",
"/work",
)
def __init__(self, *args, **kwargs):
Runner.__init__(self, *args, **kwargs)
self.host_selector = HostSelector(self.event_handler, self.popen)
self.suite_run_cleaner = SuiteRunCleaner(
event_handler=self.event_handler,
host_selector=self.host_selector,
suite_engine_proc=self.suite_engine_proc)
def run_impl(self, opts, args, uuid, work_files):
# Log file, temporary
if hasattr(self.event_handler, "contexts"):
t_file = TemporaryFile()
log_context = ReporterContext(None, self.event_handler.VV, t_file)
self.event_handler.contexts[uuid] = log_context
# Check suite engine specific compatibility
self.suite_engine_proc.check_global_conf_compat()
# Suite name from the current working directory
if opts.conf_dir:
self.fs_util.chdir(opts.conf_dir)
opts.conf_dir = os.getcwd()
# --remote=KEY=VALUE,...
if opts.remote:
# opts.name always set for remote.
return self._run_remote(opts, opts.name)
conf_tree = self.config_load(opts)
self.fs_util.chdir(conf_tree.conf_dirs[0])
suite_name = opts.name
if not opts.name:
suite_name = os.path.basename(os.getcwd())
# Check suite.rc #! line for template scheme
templ_scheme = "jinja2"
if self.suite_engine_proc.SUITE_CONF in conf_tree.files:
suiterc_path = os.path.join(
conf_tree.files[self.suite_engine_proc.SUITE_CONF],
self.suite_engine_proc.SUITE_CONF)
with open(suiterc_path) as fh:
line = fh.readline()
if line.startswith("#!"):
templ_scheme = line[2:].strip().lower()
suite_section = (templ_scheme + ':' +
self.suite_engine_proc.SUITE_CONF)
extra_defines = []
if opts.defines_suite:
for define in opts.defines_suite:
extra_defines.append("[" + suite_section + "]" + define)
# Automatic Rose constants
# ROSE_ORIG_HOST: originating host
# ROSE_VERSION: Rose version (not retained in run_mode=="reload")
# Suite engine version
my_rose_version = ResourceLocator.default().get_version()
suite_engine_key = self.suite_engine_proc.get_version_env_name()
if opts.run_mode in ["reload", "restart"]:
prev_config_path = self.suite_engine_proc.get_suite_dir(
suite_name, "log", "rose-suite-run.conf")
prev_config = ConfigLoader()(prev_config_path)
suite_engine_version = prev_config.get_value(
["env", suite_engine_key])
else:
suite_engine_version =\
self.suite_engine_proc.get_version().decode()
resloc = ResourceLocator.default()
auto_items = [(suite_engine_key, suite_engine_version),
("ROSE_ORIG_HOST", self.host_selector.get_local_host()),
("ROSE_SITE", resloc.get_conf().get_value(['site'], '')),
("ROSE_VERSION", resloc.get_version())]
for key, val in auto_items:
requested_value = conf_tree.node.get_value(["env", key])
if requested_value:
if key == "ROSE_VERSION" and val != requested_value:
exc = VersionMismatchError(requested_value, val)
raise ConfigValueError(["env", key], requested_value, exc)
val = requested_value
else:
conf_tree.node.set(["env", key],
val,
state=conf_tree.node.STATE_NORMAL)
extra_defines.append('[%s]%s="%s"' % (suite_section, key, val))
# Pass automatic Rose constants as suite defines
self.conf_tree_loader.node_loader.load(extra_defines, conf_tree.node)
# See if suite is running or not
if opts.run_mode == "reload":
# Check suite is running
self.suite_engine_proc.get_suite_contact(suite_name)
else:
self.suite_engine_proc.check_suite_not_running(suite_name)
# Install the suite to its run location
suite_dir_rel = self._suite_dir_rel(suite_name)
# Unfortunately a large try/finally block to ensure a temporary folder
# created in validate only mode is cleaned up. Exceptions are not
# caught here
try:
# Process Environment Variables
environ = self.config_pm(conf_tree, "env")
if opts.validate_suite_only_mode:
temp_dir = mkdtemp()
suite_dir = os.path.join(temp_dir, suite_dir_rel)
os.makedirs(suite_dir, 0o0700)
else:
suite_dir = os.path.join(os.path.expanduser("~"),
suite_dir_rel)
suite_conf_dir = os.getcwd()
locs_conf = ConfigNode()
if opts.new_mode:
if os.getcwd() == suite_dir:
raise NewModeError("PWD", os.getcwd())
elif opts.run_mode in ["reload", "restart"]:
raise NewModeError("--run", opts.run_mode)
self.suite_run_cleaner.clean(suite_name)
if os.getcwd() != suite_dir:
if opts.run_mode == "run":
self._run_init_dir(opts,
suite_name,
conf_tree,
locs_conf=locs_conf)
os.chdir(suite_dir)
# Housekeep log files
now_str = None
if not opts.install_only_mode and not opts.local_install_only_mode:
now_str = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
self._run_init_dir_log(opts, now_str)
self.fs_util.makedirs("log/suite")
# Rose configuration and version logs
self.fs_util.makedirs("log/rose-conf")
run_mode = opts.run_mode
if run_mode not in ["reload", "restart", "run"]:
run_mode = "run"
mode = run_mode
if opts.validate_suite_only_mode:
mode = "validate-suite-only"
elif opts.install_only_mode:
mode = "install-only"
elif opts.local_install_only_mode:
mode = "local-install-only"
prefix = "rose-conf/%s-%s" % (strftime("%Y%m%dT%H%M%S"), mode)
# Dump the actual configuration as rose-suite-run.conf
ConfigDumper()(conf_tree.node, "log/" + prefix + ".conf")
# Install version information file
write_source_vc_info(suite_conf_dir, "log/" + prefix + ".version",
self.popen)
# If run through rose-stem, install version information
# files for each source tree if they're a working copy
if hasattr(opts, 'source') and hasattr(opts, 'project'):
for i, url in enumerate(opts.source):
if os.path.isdir(url):
write_source_vc_info(
url, "log/" + opts.project[i] + "-" + str(i) +
".version", self.popen)
for ext in [".conf", ".version"]:
self.fs_util.symlink(prefix + ext, "log/rose-suite-run" + ext)
# Move temporary log to permanent log
if hasattr(self.event_handler, "contexts"):
log_file_path = os.path.abspath(
os.path.join("log", "rose-suite-run.log"))
log_file = open(log_file_path, "ab")
temp_log_file = self.event_handler.contexts[uuid].handle
temp_log_file.seek(0)
log_file.write(temp_log_file.read())
self.event_handler.contexts[uuid].handle = log_file
temp_log_file.close()
# Process Files
cwd = os.getcwd()
for rel_path, conf_dir in conf_tree.files.items():
if (conf_dir == cwd or any(
fnmatchcase(os.sep + rel_path, exclude)
for exclude in self.SYNC_EXCLUDES)
or conf_tree.node.get([templ_scheme + ":" + rel_path
]) is not None):
continue
# No sub-directories, very slow otherwise
if os.sep in rel_path:
rel_path = rel_path.split(os.sep, 1)[0]
target_key = self.config_pm.get_handler(
"file").PREFIX + rel_path
target_node = conf_tree.node.get([target_key])
if target_node is None:
conf_tree.node.set([target_key])
target_node = conf_tree.node.get([target_key])
elif target_node.is_ignored():
continue
source_node = target_node.get("source")
if source_node is None:
target_node.set(["source"],
os.path.join(conf_dir, rel_path))
elif source_node.is_ignored():
continue
self.config_pm(conf_tree,
"file",
no_overwrite_mode=opts.no_overwrite_mode)
# Process suite configuration template header
# (e.g. Jinja2:suite.rc, EmPy:suite.rc)
self.config_pm(conf_tree, templ_scheme, environ=environ)
# Ask suite engine to parse suite configuration
# and determine if it is up to date (unchanged)
if opts.validate_suite_only_mode:
suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf(
suite_dir, None, opts.strict_mode, debug_mode=True)
else:
suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf(
suite_name, opts.run_mode, opts.strict_mode,
opts.debug_mode)
finally:
# Ensure the temporary directory created is cleaned up regardless
# of success or failure
if opts.validate_suite_only_mode and os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
# Only validating so finish now
if opts.validate_suite_only_mode:
return
# Install share/work directories (local)
for name in ["share", "share/cycle", "work"]:
self._run_init_dir_work(opts,
suite_name,
name,
conf_tree,
locs_conf=locs_conf)
if opts.local_install_only_mode:
return
# Install suite files to each remote [user@]host
for name in ["", "log/", "share/", "share/cycle/", "work/"]:
uuid_file = os.path.abspath(name + uuid)
open(uuid_file, "w").close()
work_files.append(uuid_file)
# Install items to user@host
auths = self.suite_engine_proc.get_tasks_auths(suite_name)
proc_queue = [] # [[proc, command, "ssh"|"rsync", auth], ...]
for auth in sorted(auths):
host = auth
if "@" in auth:
host = auth.split("@", 1)[1]
# Remote shell
command = self.popen.get_cmd("ssh", "-n", auth)
# Provide ROSE_VERSION and CYLC_VERSION in the environment
shcommand = "env ROSE_VERSION=%s %s=%s" % (
my_rose_version, suite_engine_key, suite_engine_version)
# Use login shell?
no_login_shell = self._run_conf("remote-no-login-shell",
host=host,
conf_tree=conf_tree)
if not no_login_shell or no_login_shell.lower() != "true":
shcommand += r""" bash -l -c '"$0" "$@"'"""
# Path to "rose" command, if applicable
rose_bin = self._run_conf("remote-rose-bin",
host=host,
conf_tree=conf_tree,
default="rose")
# Build remote "rose suite-run" command
shcommand += " %s suite-run -vv -n %s" % (rose_bin, suite_name)
for key in ["new", "debug", "install-only"]:
attr = key.replace("-", "_") + "_mode"
if getattr(opts, attr, None) is not None:
shcommand += " --%s" % key
if opts.log_keep:
shcommand += " --log-keep=%s" % opts.log_keep
if opts.log_name:
shcommand += " --log-name=%s" % opts.log_name
if not opts.log_archive_mode:
shcommand += " --no-log-archive"
shcommand += " --run=%s" % opts.run_mode
# Build --remote= option
shcommand += " --remote=uuid=%s" % uuid
if now_str is not None:
shcommand += ",now-str=%s" % now_str
host_confs = [
"root-dir", "root-dir{share}", "root-dir{share/cycle}",
"root-dir{work}"
]
locs_conf.set([auth])
for key in host_confs:
value = self._run_conf(key, host=host, conf_tree=conf_tree)
if value is not None:
val = self.popen.list_to_shell_str([str(value)])
shcommand += ",%s=%s" % (key, pipes.quote(val))
locs_conf.set([auth, key], value)
command.append(shcommand)
proc = self.popen.run_bg(*command)
proc_queue.append([proc, command, "ssh", auth])
while proc_queue:
sleep(self.SLEEP_PIPE)
proc, command, command_name, auth = proc_queue.pop(0)
if proc.poll() is None: # put it back in proc_queue
proc_queue.append([proc, command, command_name, auth])
continue
ret_code = proc.wait()
out, err = proc.communicate()
ret_code, out, err = [
i.decode() if isinstance(i, bytes) else i
for i in [ret_code, out, err]
]
if ret_code:
raise RosePopenError(command, ret_code, out, err)
if command_name == "rsync":
self.handle_event(out, level=Event.VV)
continue
else:
self.handle_event(out, level=Event.VV, prefix="[%s] " % auth)
for line in out.split("\n"):
if "/" + uuid == line.strip():
locs_conf.unset([auth])
break
else:
filters = {"excludes": [], "includes": []}
for name in ["", "log/", "share/", "share/cycle/", "work/"]:
filters["excludes"].append(name + uuid)
target = auth + ":" + suite_dir_rel
cmd = self._get_cmd_rsync(target, **filters)
proc_queue.append(
[self.popen.run_bg(*cmd), cmd, "rsync", auth])
# Install ends
ConfigDumper()(locs_conf, os.path.join("log", "rose-suite-run.locs"))
if opts.install_only_mode:
return
elif opts.run_mode == "reload" and suite_conf_unchanged:
conf_name = self.suite_engine_proc.SUITE_CONF
self.handle_event(SkipReloadEvent(suite_name, conf_name))
return
# Start the suite
self.fs_util.chdir("log")
self.suite_engine_proc.run(suite_name, opts.host, opts.run_mode, args)
# Disconnect log file handle, so monitoring tool command will no longer
# be associated with the log file.
self.event_handler.contexts[uuid].handle.close()
self.event_handler.contexts.pop(uuid)
return 0
@classmethod
def _run_conf(cls,
key,
default=None,
host=None,
conf_tree=None,
r_opts=None):
"""Return the value of a setting given by a key for a given host. If
r_opts is defined, we are already in a remote host, so there is no need
to do a host match. Otherwise, the setting may be found in the run time
configuration, or the default (i.e. site/user configuration). The value
of each setting in the configuration would be in a line delimited list
of PATTERN=VALUE pairs.
"""
if r_opts is not None:
return r_opts.get(key, default)
if host is None:
host = "localhost"
for conf, keys in [(conf_tree.node, []),
(ResourceLocator.default().get_conf(),
["rose-suite-run"])]:
if conf is None:
continue
node_value = conf.get_value(keys + [key])
if node_value is None:
continue
for line in node_value.strip().splitlines():
pattern, value = line.strip().split("=", 1)
if (pattern.startswith("jinja2:")
or pattern.startswith("empy:")):
section, name = pattern.rsplit(":", 1)
p_node = conf.get([section, name], no_ignore=True)
# Values in "jinja2:*" and "empy:*" sections are quoted.
pattern = ast.literal_eval(p_node.value)
if fnmatchcase(host, pattern):
return value.strip()
return default
def _run_init_dir(self,
opts,
suite_name,
conf_tree=None,
r_opts=None,
locs_conf=None):
"""Create the suite's directory."""
suite_dir_rel = self._suite_dir_rel(suite_name)
home = os.path.expanduser("~")
suite_dir_root = self._run_conf("root-dir",
conf_tree=conf_tree,
r_opts=r_opts)
if suite_dir_root:
if locs_conf is not None:
locs_conf.set(["localhost", "root-dir"], suite_dir_root)
suite_dir_root = env_var_process(suite_dir_root)
suite_dir_home = os.path.join(home, suite_dir_rel)
if (suite_dir_root and
os.path.realpath(home) != os.path.realpath(suite_dir_root)):
suite_dir_real = os.path.join(suite_dir_root, suite_dir_rel)
self.fs_util.makedirs(suite_dir_real)
self.fs_util.symlink(suite_dir_real, suite_dir_home,
opts.no_overwrite_mode)
else:
self.fs_util.makedirs(suite_dir_home)
def _run_init_dir_log(self, opts, now_str=None):
"""Create the suite's log/ directory. Housekeep, archive old ones."""
# Do nothing in log append mode if log directory already exists
if opts.run_mode in ["reload", "restart"] and os.path.isdir("log"):
return
# Log directory of this run
if now_str is None:
now_str = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
now_log = "log." + now_str
self.fs_util.makedirs(now_log)
self.fs_util.symlink(now_log, "log")
now_log_name = getattr(opts, "log_name", None)
if now_log_name:
self.fs_util.symlink(now_log, "log." + now_log_name)
# Keep log for this run and named logs
logs = set(glob("log.*") + ["log"])
for log in list(logs):
if os.path.islink(log):
logs.remove(log)
log_link = os.readlink(log)
if log_link in logs:
logs.remove(log_link)
# Housekeep old logs, if necessary
log_keep = getattr(opts, "log_keep", None)
if log_keep:
t_threshold = time() - abs(float(log_keep)) * 86400.0
for log in list(logs):
if os.path.isfile(log):
if t_threshold > os.stat(log).st_mtime:
self.fs_util.delete(log)
logs.remove(log)
else:
for root, _, files in os.walk(log):
keep = False
for file_ in files:
path = os.path.join(root, file_)
if (os.path.exists(path)
and os.stat(path).st_mtime >= t_threshold):
keep = True
break
if keep:
break
else:
self.fs_util.delete(log)
logs.remove(log)
# Archive old logs, if necessary
if getattr(opts, "log_archive_mode", True):
for log in list(logs):
if os.path.isfile(log):
continue
log_tar_gz = log + ".tar.gz"
try:
self.popen.run_simple("tar", "-czf", log_tar_gz, log)
except RosePopenError:
try:
self.fs_util.delete(log_tar_gz)
except OSError:
pass
raise
else:
self.handle_event(SuiteLogArchiveEvent(log_tar_gz, log))
self.fs_util.delete(log)
def _run_init_dir_work(self,
opts,
suite_name,
name,
conf_tree=None,
r_opts=None,
locs_conf=None):
"""Create a named suite's directory."""
item_path = os.path.realpath(name)
item_path_source = item_path
key = "root-dir{" + name + "}"
item_root = self._run_conf(key, conf_tree=conf_tree, r_opts=r_opts)
if item_root is None: # backward compat
item_root = self._run_conf("root-dir-" + name,
conf_tree=conf_tree,
r_opts=r_opts)
if item_root:
if locs_conf is not None:
locs_conf.set(["localhost", key], item_root)
item_root = env_var_process(item_root)
suite_dir_rel = self._suite_dir_rel(suite_name)
if os.path.isabs(item_root):
item_path_source = os.path.join(item_root, suite_dir_rel, name)
else:
item_path_source = item_root
item_path_source = os.path.realpath(item_path_source)
if item_path == item_path_source:
if opts.new_mode:
self.fs_util.delete(name)
self.fs_util.makedirs(name)
else:
if opts.new_mode:
self.fs_util.delete(item_path_source)
self.fs_util.makedirs(item_path_source)
if os.sep in name:
dirname_of_name = os.path.dirname(name)
self.fs_util.makedirs(dirname_of_name)
item_path_source_rel = os.path.relpath(
item_path_source, os.path.realpath(dirname_of_name))
else:
item_path_source_rel = os.path.relpath(item_path_source)
if len(item_path_source_rel) < len(item_path_source):
self.fs_util.symlink(item_path_source_rel, name,
opts.no_overwrite_mode)
else:
self.fs_util.symlink(item_path_source, name,
opts.no_overwrite_mode)
def _run_remote(self, opts, suite_name):
"""rose suite-run --remote=KEY=VALUE,..."""
suite_dir_rel = self._suite_dir_rel(suite_name)
r_opts = {}
for item in opts.remote.split(","):
key, val = item.split("=", 1)
r_opts[key] = val
uuid_file = os.path.join(suite_dir_rel, r_opts["uuid"])
if os.path.exists(uuid_file):
self.handle_event("/" + r_opts["uuid"] + "\n", level=0)
elif opts.new_mode:
self.fs_util.delete(suite_dir_rel)
if opts.run_mode == "run" or not os.path.exists(suite_dir_rel):
self._run_init_dir(opts, suite_name, r_opts=r_opts)
os.chdir(suite_dir_rel)
for name in ["share", "share/cycle", "work"]:
uuid_file = os.path.join(name, r_opts["uuid"])
if os.path.exists(uuid_file):
self.handle_event(name + "/" + r_opts["uuid"] + "\n", level=0)
else:
self._run_init_dir_work(opts, suite_name, name, r_opts=r_opts)
if not opts.install_only_mode:
uuid_file = os.path.join("log", r_opts["uuid"])
if os.path.exists(uuid_file):
self.handle_event("log/" + r_opts["uuid"] + "\n", level=0)
else:
self._run_init_dir_log(opts, r_opts.get("now-str"))
self.fs_util.makedirs("log/suite")
def _get_cmd_rsync(self, target, excludes=None, includes=None):
"""rsync relevant suite items to target."""
if excludes is None:
excludes = []
if includes is None:
includes = []
cmd = self.popen.get_cmd("rsync")
for exclude in excludes + list(self.SYNC_EXCLUDES):
cmd.append("--exclude=" + exclude)
for include in includes:
cmd.append("--include=" + include)
cmd.append("./")
cmd.append(target)
return cmd
def _suite_dir_rel(self, suite_name):
"""Return the relative path to the suite running directory."""
return self.suite_engine_proc.get_suite_dir_rel(suite_name)
