def start(is_main=False):
"""Create the server.
If is_main, invoke cherrypy.quickstart.
Otherwise, return a cherrypy.Application instance.
"""
# Environment variables (not normally defined in WSGI mode)
if os.getenv("ROSE_HOME") is None:
path = os.path.abspath(__file__)
while os.path.dirname(path) != path: # not root
if os.path.basename(path) == "lib":
os.environ["ROSE_HOME"] = os.path.dirname(path)
break
path = os.path.dirname(path)
for key, value in [("ROSE_NS", "rosa"), ("ROSE_UTIL", "ws")]:
if os.getenv(key) is None:
os.environ[key] = value
# CherryPy quick server configuration
rose_conf = ResourceLocator.default().get_conf()
if is_main and rose_conf.get_value(["rosie-ws", "log-dir"]) is not None:
log_dir_value = rose_conf.get_value(["rosie-ws", "log-dir"])
log_dir = env_var_process(os.path.expanduser(log_dir_value))
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
log_file = os.path.join(log_dir, "server.log")
log_error_file = os.path.join(log_dir, "server.err.log")
cherrypy.config["log.error_file"] = log_error_file
cherrypy.config["log.access_file"] = log_file
cherrypy.config["request.error_response"] = _handle_error
cherrypy.config["log.screen"] = False
# Configuration for dynamic pages
db_url_map = {}
for key, node in rose_conf.get(["rosie-db"]).value.items():
if key.startswith("db.") and key[3:]:
db_url_map[key[3:]] = node.value
res_loc = ResourceLocator.default()
html_lib = res_loc.get_util_home("lib", "html")
icon_path = res_loc.locate("images/rosie-icon-trim.png")
tmpl_loader = jinja2.FileSystemLoader(os.path.join(html_lib, "rosie-ws"))
root = Root(jinja2.Environment(loader=tmpl_loader), db_url_map)
# Configuration for static pages
config = {"/etc": {
"tools.staticdir.dir": os.path.join(html_lib, "external"),
"tools.staticdir.on": True},
"/favicon.ico": {
"tools.staticfile.on": True,
"tools.staticfile.filename": icon_path}}
if is_main:
port = int(rose_conf.get_value(["rosie-ws", "port"], 8080))
config.update({"global": {"server.socket_host": "0.0.0.0",
"server.socket_port": port}})
# Start server or return WSGI application
if is_main:
return cherrypy.quickstart(root, "/", config=config)
else:
return cherrypy.Application(root, script_name=None, config=config)
def __init__(self, *args, **kwargs):
self.exposed = True
self.props = {}
rose_conf = ResourceLocator.default().get_conf()
self.props["title"] = rose_conf.get_value(
["rosie-disco", "title"], self.TITLE)
self.props["host_name"] = rose_conf.get_value(["rosie-disco", "host"])
if self.props["host_name"] is None:
self.props["host_name"] = HostSelector().get_local_host()
if self.props["host_name"] and "." in self.props["host_name"]:
self.props["host_name"] = (
self.props["host_name"].split(".", 1)[0])
self.props["rose_version"] = ResourceLocator.default().get_version()
self.props["template_env"] = jinja2.Environment(
loader=jinja2.FileSystemLoader(
ResourceLocator.default().get_util_home(
"lib", "html", "template", "rosie-disco")))
db_url_map = {}
for key, node in rose_conf.get(["rosie-db"]).value.items():
if key.startswith("db.") and key[3:]:
db_url_map[key[3:]] = node.value
self.db_url_map = db_url_map
if not self.db_url_map:
self.db_url_map = {}
for key, db_url in self.db_url_map.items():
setattr(self, key, RosieDiscoService(self.props, key, db_url))
def __init__(self, *args, **kwargs):
if hasattr(kwargs, "prog"):
ns, util = kwargs["prog"].split(None, 1)
resource_loc = ResourceLocator(ns=ns, util=util)
else:
resource_loc = ResourceLocator.default()
kwargs["prog"] = resource_loc.get_util_name()
if not hasattr(kwargs, "usage"):
kwargs["usage"] = resource_loc.get_synopsis()
OptionParser.__init__(self, *args, **kwargs)
self.add_my_options("debug_mode", "quietness", "verbosity")
def main():
"""rosa db-create."""
db_conf = ResourceLocator.default().get_conf().get(["rosie-db"])
if db_conf is not None:
opts = RoseOptionParser().parse_args()[0]
reporter = Reporter(opts.verbosity - opts.quietness)
init = RosieDatabaseInitiator(event_handler=reporter)
conf = ResourceLocator.default().get_conf()
for key in db_conf.value:
if key.startswith("db."):
prefix = key.replace("db.", "", 1)
db_url = conf.get_value(["rosie-db", "db." + prefix])
repos_path = conf.get_value(["rosie-db", "repos." + prefix])
init(db_url, repos_path)
def run(
self,
suite_name,
task_id,
hook_event,
hook_message=None,
should_mail=False,
mail_cc_list=None,
should_shutdown=False,
):
"""
Invoke the hook for a suite.
1. For a task hook, if the task runs remotely, retrieve its log from
the remote host.
2. If "should_mail", send an email notification to the current user,
and those in the "mail_cc_list".
3. If "should_shutdown", shut down the suite.
"""
# Retrieve log and generate code view
task_ids = []
if task_id:
task_ids = [task_id]
self.suite_engine_proc.job_logs_pull_remote(suite_name, task_ids)
# Send email notification if required
if should_mail:
text = ""
if task_id:
text += "Task: %s\n" % task_id
if hook_message:
text += "Message: %s\n" % hook_message
url = self.suite_engine_proc.get_suite_log_url(None, suite_name)
text += "See: %s\n" % (url)
user = pwd.getpwuid(os.getuid()).pw_name
conf = ResourceLocator.default().get_conf()
host = conf.get_value(["rose-suite-hook", "email-host"], default="localhost")
msg = MIMEText(text)
msg["From"] = user + "@" + host
msg["To"] = msg["From"]
if mail_cc_list:
mail_cc_addresses = []
for mail_cc_address in mail_cc_list:
if "@" not in mail_cc_address:
mail_cc_address += "@" + host
mail_cc_addresses.append(mail_cc_address)
msg["Cc"] = ", ".join(mail_cc_addresses)
mail_cc_list = mail_cc_addresses
else:
mail_cc_list = []
msg["Subject"] = "[%s] %s" % (hook_event, suite_name)
smtp_host = conf.get_value(["rose-suite-hook", "smtp-host"], default="localhost")
smtp = SMTP(smtp_host)
smtp.sendmail(msg["From"], [msg["To"]] + mail_cc_list, msg.as_string())
smtp.quit()
# Shut down if required
if should_shutdown:
self.suite_engine_proc.shutdown(suite_name, args=["--kill"])
def __init__(self, prefix, popen=None, prompt_func=None):
self.prefix = prefix
root = self._get_conf_value("ws")
if root is None:
raise UndefinedRosiePrefixWS(self.prefix)
if not root.endswith("/"):
root += "/"
self.root = root
urlparse_res = urlparse(self.root)
self.scheme = urlparse_res[0]
self.host = urlparse_res[1]
self.password_orig = None
self.username_orig = None
self.password = None
self.username = None
if popen is None:
popen = RosePopener()
self.popen = popen
self.prompt_func = prompt_func
res_loc = ResourceLocator.default()
password_stores_str = res_loc.default().get_conf().get_value(
keys=["rosie-id", "prefix-password-store." + self.prefix],
default=self.PASSWORD_STORES_STR)
for password_store_name in shlex.split(password_stores_str):
password_store_cls = self.PASSWORD_STORE_CLASSES.get(
password_store_name)
if password_store_cls is not None and password_store_cls.usable():
self.password_store = password_store_cls()
break
else:
self.password_store = None
self.requests_kwargs = {}
self._init_https_params()
def expand(self, names=None, rank_method=None, thresholds=None):
"""Expand each name in names, and look up rank method for each name.
names, if specified, should be a list of host names or known groups in
the site / user configuration file. Otherwise, the default setting in
the site / user configuration file will be used.
rank_method, if specified, should be the name of a supported ranking
method. If not specified, use the default specified for a host group.
If the default differs in hosts, use "load:15".
"""
conf = ResourceLocator.default().get_conf()
if not names:
node = conf.get(["rose-host-select", "default"],
no_ignore=True)
if node:
names = [node.value]
else:
raise NoHostError()
host_names = []
rank_method_set = set()
thresholds_set = set()
while names:
name = names.pop()
key = "group{" + name + "}"
value = conf.get_value(["rose-host-select", key])
if value is None:
host_names.append(name)
else:
for v in value.split():
names.append(v)
if rank_method is None:
key = "method{" + name + "}"
m = conf.get_value(["rose-host-select", key])
if m is None:
rank_method_set.add(self.RANK_METHOD_DEFAULT)
else:
rank_method_set.add(m)
if thresholds is None:
key = "thresholds{" + name + "}"
t = conf.get_value(["rose-host-select", key])
if t is None:
thresholds_set.add(())
else:
thresholds_set.add(tuple(sorted(shlex.split(t))))
# If default rank method differs in hosts, use load:15.
if rank_method is None:
if len(rank_method_set) == 1:
rank_method = rank_method_set.pop()
else:
rank_method = self.RANK_METHOD_DEFAULT
if thresholds is None:
if len(thresholds_set) == 1:
thresholds = thresholds_set.pop()
return host_names, rank_method, thresholds
def _get_hosts(self, suite_name, host):
if host:
hosts = [host]
else:
conf = ResourceLocator.default().get_conf()
hosts = None
known_hosts = self.host_selector.expand(
conf.get_value(["rose-suite-run", "hosts"], "").split() +
conf.get_value(["rose-suite-run", "scan-hosts"], "").split() +
["localhost"])[0]
known_hosts = list(set(known_hosts))
if known_hosts:
hosts = self.suite_engine_proc.ping(
suite_name,
known_hosts)
if not hosts:
# Try the "rose-suite.host" file in the suite log directory
log = self.suite_engine_proc.get_suite_dir(suite_name, "log")
try:
host_file = os.path.join(log, "rose-suite-run.host")
hosts = [open(host_file).read().strip()]
except IOError:
pass
if not hosts:
hosts = ["localhost"]
return hosts
def __init__(self, *args, **kwargs):
self.exposed = True
self.suite_engine_proc = SuiteEngineProcessor.get_processor()
rose_conf = ResourceLocator.default().get_conf()
self.logo = rose_conf.get_value(["rose-bush", "logo"])
self.title = rose_conf.get_value(["rose-bush", "title"], self.TITLE)
self.host_name = rose_conf.get_value(["rose-bush", "host"])
if self.host_name is None:
self.host_name = HostSelector().get_local_host()
if self.host_name and "." in self.host_name:
self.host_name = self.host_name.split(".", 1)[0]
self.rose_version = ResourceLocator.default().get_version()
template_env = jinja2.Environment(loader=jinja2.FileSystemLoader(
ResourceLocator.default().get_util_home(
"lib", "html", "template", "rose-bush")))
self.template_env = template_env
def get_suite_log_url(self, user_name, suite_name):
"""Return the "rose bush" URL for a user's suite."""
prefix = "~"
if user_name:
prefix += user_name
suite_d = os.path.join(prefix, self.get_suite_dir_rel(suite_name))
suite_d = os.path.expanduser(suite_d)
if not os.path.isdir(suite_d):
raise NoSuiteLogError(user_name, suite_name)
rose_bush_url = None
for f_name in glob(os.path.expanduser("~/.metomi/rose-bush*.status")):
status = {}
for line in open(f_name):
key, value = line.strip().split("=", 1)
status[key] = value
if status.get("host"):
rose_bush_url = "http://" + status["host"]
if status.get("port"):
rose_bush_url += ":" + status["port"]
rose_bush_url += "/"
break
if not rose_bush_url:
conf = ResourceLocator.default().get_conf()
rose_bush_url = conf.get_value(["rose-suite-log", "rose-bush"])
if not rose_bush_url:
return "file://" + suite_d
if not rose_bush_url.endswith("/"):
rose_bush_url += "/"
if not user_name:
user_name = pwd.getpwuid(os.getuid()).pw_name
return rose_bush_url + "/".join(["taskjobs", user_name, suite_name])
def load_override_config(sections, my_globals=None):
if my_globals is None:
my_globals = globals()
for section in sections:
conf = ResourceLocator.default().get_conf().get([section])
if conf is None:
continue
for key, node in conf.value.items():
if node.is_ignored():
continue
try:
cast_value = ast.literal_eval(node.value)
except Exception:
cast_value = node.value
name = key.replace("-", "_").upper()
orig_value = my_globals[name]
if (type(orig_value) is not type(cast_value) and
orig_value is not None):
sys.stderr.write(_OVERRIDE_WARNING_TYPE.format(
section, key, cast_value,
type(orig_value), type(cast_value))
)
continue
if name.startswith("_"):
sys.stderr.write(_OVERRIDE_WARNING_PRIVATE.format(
section, key, name)
)
continue
my_globals[name] = cast_value
def get_cmd(self, key, *args):
"""Return default options and arguments of a known command as a list.
If a setting [external] <key> is defined in the site/user
configuration, use the setting.
Otherwise, if RosePopener.ENVS_OF_CMDS[key] exists, it looks for each
environment variable in the list in RosePopener.ENVS_OF_CMDS[key] in
order. If the environment variable is defined and is not a null string,
use the value of the environment variable.
Otherwise, return RosePopener.CMDS[key]
key: must be a key of RosePopener.CMDS
args: if specified, will be added to the returned list
"""
if key not in self.cmds:
root_node = ResourceLocator.default().get_conf()
node = root_node.get(["external", key], no_ignore=True)
if node is not None:
self.cmds[key] = shlex.split(node.value)
if key not in self.cmds:
for name in self.ENVS_OF_CMDS.get(key, []):
if os.getenv(name): # not None, not null str
self.cmds[key] = shlex.split(os.getenv(name))
break
if key not in self.cmds:
self.cmds[key] = self.CMDS[key]
return self.cmds[key] + list(args)
def _configure(service_cls):
"""Configure cherrypy and return a dict for the specified cherrypy app."""
# Environment variables (not normally defined in WSGI mode)
if not os.getenv("ROSE_HOME"):
path = os.path.abspath(__file__)
while os.path.dirname(path) != path: # not root
if os.path.basename(path) == "lib":
os.environ["ROSE_HOME"] = os.path.dirname(path)
break
path = os.path.dirname(path)
for key, value in (
("ROSE_NS", service_cls.NS), ("ROSE_UTIL", service_cls.UTIL)):
if os.getenv(key) is None:
os.environ[key] = value
# Configuration for HTML library
cherrypy.config["tools.encode.on"] = True
cherrypy.config["tools.encode.encoding"] = "utf-8"
config = {}
static_lib = ResourceLocator.default().get_util_home(
"lib", "html", "static")
for name in os.listdir(static_lib):
path = os.path.join(static_lib, name)
if os.path.isdir(path):
path_key = "tools.staticdir.dir"
bool_key = "tools.staticdir.on"
else:
path_key = "tools.staticfile.filename"
bool_key = "tools.staticfile.on"
config["/" + name] = {path_key: path, bool_key: True}
if name == service_cls.NS + "-favicon.png":
config["/favicon.ico"] = config["/" + name]
return config
def _verify_users(self, status, path, txn_owner, txn_access_list,
bad_changes):
"""Check txn_owner and txn_access_list.
For any invalid users, append to bad_changes and return True.
"""
# The owner and names in access list must be real users
conf = ResourceLocator.default().get_conf()
user_tool_name = conf.get_value(["rosa-svn", "user-tool"])
if not user_tool_name:
return False
user_tool = self.usertools_manager.get_handler(user_tool_name)
txn_users = set([txn_owner] + txn_access_list)
txn_users.discard("*")
bad_users = user_tool.verify_users(txn_users)
for bad_user in bad_users:
if txn_owner == bad_user:
bad_change = BadChange(
status,
path,
BadChange.USER,
"owner=" + bad_user)
bad_changes.append(bad_change)
if bad_user in txn_access_list:
bad_change = BadChange(
status,
path,
BadChange.USER,
"access-list=" + bad_user)
bad_changes.append(bad_change)
return bool(bad_users)
def _search(cls, users, attr_idx):
"""Search LDAP directory for the indexed attr for users.
Attr index can be UID_IDX, CN_IDX or MAIL_IDX.
Return a list containing the results.
"""
conf = ResourceLocator.default().get_conf()
uri = conf.get_value(["rosa-ldap", "uri"])
binddn = conf.get_value(["rosa-ldap", "binddn"])
passwd = ""
passwd_file = conf.get_value(["rosa-ldap", "password-file"],
cls.PASSWD_FILE)
if passwd_file:
passwd = open(os.path.expanduser(passwd_file)).read().strip()
basedn = conf.get_value(["rosa-ldap", "basedn"], "")
filter_str = "(|(uid=" + ")(uid=".join(users) + "))"
filter_more_str = conf.get_value(["rosa-ldap", "filter-more"], "")
if filter_more_str:
filter_str = "(&" + filter_str + filter_more_str + ")"
user_attr_str = conf.get_value(["rosa-ldap", "attrs"], cls.USER_ATTRS)
attr = user_attr_str.split()[attr_idx]
tls_ca_file = conf.get_value(["rosa-ldap", "tls-ca-file"])
if tls_ca_file:
ldap.set_option(ldap.OPT_X_TLS_CACERTFILE, tls_ca_file)
conn = ldap.initialize(uri)
conn.bind_s(binddn, passwd)
results = conn.search_s(basedn, ldap.SCOPE_SUBTREE, filter_str, [attr])
conn.unbind()
return [result[1][attr][0] for result in results]
def _run_conf(
cls, key, default=None, host=None, conf_tree=None, r_opts=None):
"""Return the value of a setting given by a key for a given host. If
r_opts is defined, we are alerady in a remote host, so there is no need
to do a host match. Otherwise, the setting may be found in the run time
configuration, or the default (i.e. site/user configuration). The value
of each setting in the configuration would be in a line delimited list
of PATTERN=VALUE pairs.
"""
if r_opts is not None:
return r_opts.get(key, default)
if host is None:
host = "localhost"
for conf, keys in [
(conf_tree.node, []),
(ResourceLocator.default().get_conf(), ["rose-suite-run"])]:
if conf is None:
continue
node_value = conf.get_value(keys + [key])
if node_value is None:
continue
for line in node_value.strip().splitlines():
pattern, value = line.strip().split("=", 1)
if pattern.startswith("jinja2:"):
section, name = pattern.rsplit(":", 1)
p_node = conf.get([section, name], no_ignore=True)
# Values in "jinja2:*" section are quoted.
pattern = ast.literal_eval(p_node.value)
if fnmatchcase(host, pattern):
return value.strip()
return default
def __init__(self, prefixes=None, prompt_func=None, popen=None,
event_handler=None):
if not event_handler:
event_handler = Reporter()
if not popen:
popen = RosePopener(event_handler=event_handler)
self.event_handler = event_handler
self.popen = popen
self.prompt_func = prompt_func
self.prefixes = []
self.unreachable_prefixes = []
self.auth_managers = {}
conf = ResourceLocator.default().get_conf()
conf_rosie_id = conf.get(["rosie-id"], no_ignore=True)
if conf_rosie_id is None:
raise RosieWSClientConfError()
for key, node in conf_rosie_id.value.items():
if node.is_ignored() or not key.startswith("prefix-ws."):
continue
prefix = key.replace("prefix-ws.", "")
self.auth_managers[prefix] = RosieWSClientAuthManager(
prefix, popen=self.popen, prompt_func=self.prompt_func)
if not prefixes:
prefixes_str = conf_rosie_id.get_value(["prefixes-ws-default"])
if prefixes_str:
prefixes = shlex.split(prefixes_str)
else:
prefixes = sorted(self.auth_managers.keys())
self.set_prefixes(prefixes)
def get_prefix_default(cls):
"""Return the default prefix."""
config = ResourceLocator.default().get_conf()
value = config.get_value(["rosie-id", "prefix-default"])
if not value or not value.strip():
raise SuiteIdPrefixError()
return shlex.split(value)[0]
def get_prefix_default(cls):
"""Return the default prefix."""
config = ResourceLocator.default().get_conf()
value = config.get_value(["rosie-id", "prefix-default"])
if value is None:
raise SuiteIdPrefixError()
return value
def cycles(
self, user, suite, page=1, order=None, per_page=None,
no_fuzzy_time="0", form=None):
"""List cycles of a running or completed suite."""
conf = ResourceLocator.default().get_conf()
per_page_default = int(conf.get_value(
["rose-bush", "cycles-per-page"], self.CYCLES_PER_PAGE))
if not isinstance(per_page, int):
if per_page:
per_page = int(per_page)
else:
per_page = per_page_default
if page and per_page:
page = int(page)
else:
page = 1
data = {
"logo": self.logo,
"title": self.title,
"host": self.host_name,
"user": user,
"suite": suite,
"is_option_on": (
order is not None and order != "time_desc" or
per_page is not None and per_page != per_page_default
),
"order": order,
"rose_version": self.rose_version,
"script": cherrypy.request.script_name,
"method": "cycles",
"no_fuzzy_time": no_fuzzy_time,
"states": {},
"per_page": per_page,
"per_page_default": per_page_default,
"page": page,
"task_status_groups": self.bush_dao.TASK_STATUS_GROUPS,
}
data["entries"], data["of_n_entries"] = (
self.bush_dao.get_suite_cycles_summary(
user, suite, order, per_page, (page - 1) * per_page))
if per_page:
data["n_pages"] = data["of_n_entries"] / per_page
if data["of_n_entries"] % per_page != 0:
data["n_pages"] += 1
else:
data["n_pages"] = 1
data.update(self._get_suite_logs_info(user, suite))
data["states"].update(
self.bush_dao.get_suite_state_summary(user, suite))
data["states"]["last_activity_time"] = (
self.get_last_activity_time(user, suite))
data["time"] = strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
if form == "json":
return simplejson.dumps(data)
try:
return self.template_env.get_template("cycles.html").render(**data)
except jinja2.TemplateError:
traceback.print_exc()
return simplejson.dumps(data)
def __init__(self, *args, **kwargs):
self.exposed = True
self.bush_dao = RoseBushDAO()
rose_conf = ResourceLocator.default().get_conf()
self.logo = rose_conf.get_value(["rose-bush", "logo"])
self.title = rose_conf.get_value(["rose-bush", "title"], self.TITLE)
self.host_name = rose_conf.get_value(["rose-bush", "host"])
if self.host_name is None:
self.host_name = HostSelector().get_local_host()
if self.host_name and "." in self.host_name:
self.host_name = self.host_name.split(".", 1)[0]
self.rose_version = ResourceLocator.default().get_version()
template_env = jinja2.Environment(loader=jinja2.FileSystemLoader(
ResourceLocator.default().get_util_home(
"lib", "html", "template", "rose-bush")))
template_env.filters['urlise'] = self.url2hyperlink
self.template_env = template_env
def __init__(self, template_env):
self.exposed = True
self.suite_engine_proc = SuiteEngineProcessor.get_processor()
self.template_env = template_env
self.host_name = socket.gethostname()
if self.host_name and "." in self.host_name:
self.host_name = self.host_name.split(".", 1)[0]
self.rose_version = ResourceLocator.default().get_version()
def _get_conf_value(self, name, default=None):
"""Return the value of a named conf setting for this prefix."""
conf = ResourceLocator.default().get_conf()
value = conf.get_value(
["rosie-id", "prefix-%s.%s" % (name, self.prefix)],
default=default)
if value:
value = env_var_process(value)
return value
def cycles(
self, user, suite, page=1, order=None, per_page=None, form=None):
"""List cycles of a running or completed suite."""
user_suite_dir = self._get_user_suite_dir(user, suite)
conf = ResourceLocator.default().get_conf()
per_page_default = int(conf.get_value(
["rose-bush", "cycles-per-page"], self.CYCLES_PER_PAGE))
if not isinstance(per_page, int):
if per_page:
per_page = int(per_page)
else:
per_page = per_page_default
if page and per_page:
page = int(page)
else:
page = 1
data = {
"logo": self.logo,
"title": self.title,
"host": self.host_name,
"user": user,
"suite": suite,
"is_option_on": (
order is not None and order != "time_desc" or
per_page is not None and per_page != per_page_default
),
"order": order,
"rose_version": self.rose_version,
"script": cherrypy.request.script_name,
"method": "cycles",
"states": {},
"per_page": per_page,
"per_page_default": per_page_default,
"page": page,
}
data["entries"], data["of_n_entries"] = (
self.suite_engine_proc.get_suite_cycles_summary(
user, suite, order, per_page, (page - 1) * per_page))
if per_page:
data["n_pages"] = data["of_n_entries"] / per_page
if data["of_n_entries"] % per_page != 0:
data["n_pages"] += 1
else:
data["n_pages"] = 1
data.update(self._get_suite_logs_info(user, suite))
data["states"].update(
self.suite_engine_proc.get_suite_state_summary(user, suite))
data["time"] = strftime("%Y-%m-%dT%H:%M:%S+0000", gmtime())
if form == "json":
return simplejson.dumps(data)
try:
template = self.template_env.get_template("cycles.html")
return template.render(**data)
except Exception as exc:
traceback.print_exc(exc)
return simplejson.dumps(data)
def get_prefix_web(cls, prefix=None):
"""Return a url for the prefix repository source url."""
if prefix is None:
prefix = cls.get_prefix_default()
key = "prefix-web." + prefix
config = ResourceLocator.default().get_conf()
value = config.get_value(["rosie-id", key])
if value is None:
raise SuiteIdPrefixError(prefix)
return value.rstrip("/")
def get_local_copy_root(cls):
"""Return the root directory for hosting the local suite copies."""
config = ResourceLocator.default().get_conf()
value = config.get_value(["rosie-id", "local-copy-root"])
if value:
local_copy_root = value
else:
local_copy_root = "$HOME/roses"
local_copy_root = rose.env.env_var_process(local_copy_root)
return local_copy_root
def get_prefix_location(cls, prefix=None):
"""Return the repository location of a given prefix."""
if prefix is None:
prefix = cls.get_prefix_default()
key = "prefix-location." + prefix
config = ResourceLocator.default().get_conf()
value = config.get_value(["rosie-id", key])
if value is None:
raise SuiteIdPrefixError(prefix)
return value.rstrip("/")
def _prompt(self, is_retry=False):
"""Prompt for the username and password, where necessary.
Prompt with zenity or raw_input/getpass.
"""
if (callable(self.prompt_func) and
not hasattr(self.password_store, "prompt_password")):
self.username, self.password = self.prompt_func(
self.username, self.password, is_retry)
return
icon_path = ResourceLocator.default().locate("images/rosie-icon.png")
if is_retry:
username = ""
if self.username:
username = ""
prompt = self.PROMPT_USERNAME % {
"prefix": self.prefix, "root": self.root}
if self.popen.which("zenity") and os.getenv("DISPLAY"):
username = self.popen.run(
"zenity", "--entry",
"--title=Rosie",
"--window-icon=" + icon_path,
"--text=" + prompt)[1].strip()
else:
username = raw_input(prompt)
if not username:
raise KeyboardInterrupt(self.STR_CANCELLED)
if username and username != self.username:
self.username = username
self._load_password()
if self.password:
return
if self.username and self.password is None or is_retry:
prompt = self.PROMPT_PASSWORD % {"prefix": self.prefix,
"root": self.root,
"username": self.username}
if hasattr(self.password_store, "prompt_password"):
password = self.password_store.prompt_password(
prompt, self.scheme, self.host, self.username)
elif self.popen.which("zenity") and os.getenv("DISPLAY"):
password = self.popen.run(
"zenity", "--entry", "--hide-text",
"--title=Rosie",
"--window-icon=" + icon_path,
"--text=" + prompt)[1].strip()
else:
password = getpass(prompt)
if not password:
raise KeyboardInterrupt(self.STR_CANCELLED)
if password and password != self.password:
self.password = password
def initialize(self, props, prefix, db_url, service_root):
self.props = props
self.prefix = prefix
source_option = "prefix-web." + self.prefix
source_url_node = ResourceLocator.default().get_conf().get(
["rosie-id", source_option])
self.source_url = ""
if source_url_node is not None:
self.source_url = source_url_node.value
self.dao = rosie.db.DAO(db_url)
self.service_root = service_root[:-1] # remove the '?' regex aspect
def __init__(self, template_env, prefix, db_url):
self.exposed = True
self.template_env = template_env
self.prefix = prefix
source_option = "prefix-web." + self.prefix
source_url_node = ResourceLocator.default().get_conf().get(
["rosie-id", source_option])
self.source_url = ""
if source_url_node is not None:
self.source_url = source_url_node.value
self.dao = rosie.db.DAO(db_url)
def suites(self,
user,
names=None,
page=1,
order=None,
per_page=None,
no_fuzzy_time="0",
form=None):
"""List (installed) suites of a user.
user -- A string containing a valid user ID
form -- Specify return format. If None, display HTML page. If "json",
return a JSON data structure.
"""
user_suite_dir_root = self._get_user_suite_dir_root(user)
conf = ResourceLocator.default().get_conf()
per_page_default = int(
conf.get_value(["rose-bush", "suites-per-page"],
self.SUITES_PER_PAGE))
if not isinstance(per_page, int):
if per_page:
per_page = int(per_page)
else:
per_page = per_page_default
if page and per_page:
page = int(page)
else:
page = 1
data = {
"logo":
self.logo,
"title":
self.title,
"host":
self.host_name,
"rose_version":
self.rose_version,
"script":
cherrypy.request.script_name,
"method":
"suites",
"no_fuzzy_time":
no_fuzzy_time,
"user":
user,
"is_option_on":
(names and shlex.split(str(names)) != ["*"]
or order is not None and order != "time_desc"
or per_page is not None and per_page != per_page_default),
"names":
names,
"page":
page,
"order":
order,
"per_page":
per_page,
"per_page_default":
per_page_default,
"entries": [],
}
name_globs = ["*"]
if names:
name_globs = shlex.split(str(names))
# Get entries
sub_names = [
".service", "log", "share", "work", self.bush_dao.SUITE_CONF
]
for dirpath, dnames, fnames in os.walk(user_suite_dir_root,
followlinks=True):
if dirpath != user_suite_dir_root and (any(
name in dnames or name in fnames for name in sub_names)):
dnames[:] = []
else:
continue
item = os.path.relpath(dirpath, user_suite_dir_root)
if not any(fnmatch(item, glob_) for glob_ in name_globs):
continue
try:
data["entries"].append({
"name":
item,
"info": {},
"last_activity_time":
(self.get_last_activity_time(user, item))
})
except OSError:
continue
if order == "name_asc":
data["entries"].sort(key=lambda entry: entry["name"])
elif order == "name_desc":
data["entries"].sort(key=lambda entry: entry["name"], reverse=True)
elif order == "time_asc":
data["entries"].sort(self._sort_summary_entries, reverse=True)
else: # order == "time_desc"
data["entries"].sort(self._sort_summary_entries)
data["of_n_entries"] = len(data["entries"])
if per_page:
data["n_pages"] = data["of_n_entries"] / per_page
if data["of_n_entries"] % per_page != 0:
data["n_pages"] += 1
offset = (page - 1) * per_page
data["entries"] = data["entries"][offset:offset + per_page]
else:
data["n_pages"] = 1
# Get suite info for each entry
for entry in data["entries"]:
user_suite_dir = os.path.join(user_suite_dir_root, entry["name"])
rose_suite_info = os.path.join(user_suite_dir, "rose-suite.info")
try:
info_root = rose.config.load(rose_suite_info)
for key, node in info_root.value.items():
if (node.is_ignored() or not isinstance(node.value, str)):
continue
entry["info"][key] = node.value
except (IOError, rose.config.ConfigSyntaxError):
pass
data["time"] = strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
if form == "json":
return json.dumps(data)
template = self.template_env.get_template("suites.html")
return template.render(**data)
def get_file(self, user, suite, path, path_in_tar=None, mode=None):
"""Returns file information / content or a cherrypy response."""
f_name = self._get_user_suite_dir(user, suite, path)
conf = ResourceLocator.default().get_conf()
view_size_max = int(
conf.get_value(["rose-bush", "view-size-max"], self.VIEW_SIZE_MAX))
if path_in_tar:
tar_f = tarfile.open(f_name, "r:gz")
try:
tar_info = tar_f.getmember(path_in_tar)
except KeyError:
raise cherrypy.HTTPError(404)
f_size = tar_info.size
handle = tar_f.extractfile(path_in_tar)
if handle.read(2) == "#!":
mime = self.MIME_TEXT_PLAIN
else:
mime = mimetypes.guess_type(
urllib.pathname2url(path_in_tar))[0]
handle.seek(0)
if (mode == "download" or f_size > view_size_max or mime and
(not mime.startswith("text/") or mime.endswith("html"))):
temp_f = NamedTemporaryFile()
f_bsize = os.fstatvfs(temp_f.fileno()).f_bsize
while True:
bytes_ = handle.read(f_bsize)
if not bytes_:
break
temp_f.write(bytes_)
cherrypy.response.headers["Content-Type"] = mime
try:
return cherrypy.lib.static.serve_file(temp_f.name, mime)
finally:
temp_f.close()
text = handle.read()
else:
f_size = os.stat(f_name).st_size
if open(f_name).read(2) == "#!":
mime = self.MIME_TEXT_PLAIN
else:
mime = mimetypes.guess_type(urllib.pathname2url(f_name))[0]
if not mime:
mime = self.MIME_TEXT_PLAIN
if (mode == "download" or f_size > view_size_max or mime and
(not mime.startswith("text/") or mime.endswith("html"))):
cherrypy.response.headers["Content-Type"] = mime
return cherrypy.lib.static.serve_file(f_name, mime)
text = open(f_name).read()
try:
if mode in [None, "text"]:
text = jinja2.escape(text)
lines = [unicode(line) for line in text.splitlines()]
except UnicodeDecodeError:
if path_in_tar:
handle.seek(0)
# file closed by cherrypy
return cherrypy.lib.static.serve_fileobj(
handle, self.MIME_TEXT_PLAIN)
else:
return cherrypy.lib.static.serve_file(f_name,
self.MIME_TEXT_PLAIN)
else:
if path_in_tar:
handle.close()
name = path
if path_in_tar:
name = "log/" + path_in_tar
job_entry = None
if name.startswith("log/job"):
names = self.bush_dao.parse_job_log_rel_path(name)
if len(names) == 4:
cycle, task, submit_num, _ = names
entries = self.bush_dao.get_suite_job_entries(
user, suite, [cycle], [task], None, None, None, None,
None)[0]
for entry in entries:
if entry["submit_num"] == int(submit_num):
job_entry = entry
break
if fnmatch(os.path.basename(path), "rose*.conf"):
file_content = "rose-conf"
else:
file_content = self.bush_dao.is_conf(path)
return lines, job_entry, file_content, f_name
def suites(self, user, names=None, page=1, order=None, per_page=None,
form=None):
"""List (installed) suites of a user.
user -- A string containing a valid user ID
form -- Specify return format. If None, display HTML page. If "json",
return a JSON data structure.
"""
user_suite_dir_root = self._get_user_suite_dir_root(user)
conf = ResourceLocator.default().get_conf()
per_page_default = int(conf.get_value(
["rose-bush", "suites-per-page"], self.SUITES_PER_PAGE))
if not isinstance(per_page, int):
if per_page:
per_page = int(per_page)
else:
per_page = per_page_default
if page and per_page:
page = int(page)
else:
page = 1
data = {
"logo": self.logo,
"title": self.title,
"host": self.host_name,
"rose_version": self.rose_version,
"script": cherrypy.request.script_name,
"method": "suites",
"user": user,
"is_option_on": (
names and shlex.split(str(names)) != ["*"] or
order is not None and order != "time_desc" or
per_page is not None and per_page != per_page_default
),
"names": names,
"page": page,
"order": order,
"per_page": per_page,
"per_page_default": per_page_default,
"entries": [],
}
name_globs = ["*"]
if names:
name_globs = shlex.split(str(names))
# Get entries
try:
items = os.listdir(user_suite_dir_root)
except OSError:
items = []
for item in items:
if not any([fnmatch(item, glob_) for glob_ in name_globs]):
continue
user_suite_dir = os.path.join(user_suite_dir_root, item)
suite_conf = os.path.join(
user_suite_dir, self.suite_engine_proc.SUITE_CONF)
job_logs_db = os.path.join(
user_suite_dir, self.suite_engine_proc.JOB_LOGS_DB)
if (not os.path.exists(job_logs_db) and
not os.path.exists(suite_conf)):
continue
suite_db = os.path.join(
user_suite_dir, self.suite_engine_proc.SUITE_DB)
try:
last_activity_time = strftime(
"%Y-%m-%dT%H:%M:%S+0000",
gmtime(os.stat(suite_db).st_mtime))
except OSError:
last_activity_time = None
data["entries"].append({
"name": item,
"info": {},
"last_activity_time": last_activity_time})
if order == "name_asc":
data["entries"].sort(key=lambda entry: entry["name"])
elif order == "name_desc":
data["entries"].sort(key=lambda entry: entry["name"], reverse=True)
elif order == "time_asc":
data["entries"].sort(self._sort_summary_entries, reverse=True)
else: # order == "time_desc"
data["entries"].sort(self._sort_summary_entries)
data["of_n_entries"] = len(data["entries"])
if per_page:
data["n_pages"] = data["of_n_entries"] / per_page
if data["of_n_entries"] % per_page != 0:
data["n_pages"] += 1
offset = (page - 1) * per_page
data["entries"] = data["entries"][offset:offset + per_page]
else:
data["n_pages"] = 1
# Get suite info for each entry
for entry in data["entries"]:
user_suite_dir = os.path.join(user_suite_dir_root, entry["name"])
rose_suite_info = os.path.join(user_suite_dir, "rose-suite.info")
try:
info_root = rose.config.load(rose_suite_info)
for key, node in info_root.value.items():
if (node.is_ignored() or
not isinstance(node.value, str)):
continue
entry["info"][key] = node.value
except (IOError, rose.config.ConfigSyntaxError):
pass
data["time"] = strftime("%Y-%m-%dT%H:%M:%S+0000", gmtime())
if form == "json":
return simplejson.dumps(data)
template = self.template_env.get_template("suites.html")
return template.render(**data)
def _read_site_config_and_return_options(self):
"""Read the site rose.conf file."""
return ResourceLocator.default().get_conf().get_value(
["rose-stem", "automatic-options"])
def run_impl(self, opts, args, uuid, work_files):
# Log file, temporary
if hasattr(self.event_handler, "contexts"):
t_file = TemporaryFile()
log_context = ReporterContext(None, self.event_handler.VV, t_file)
self.event_handler.contexts[uuid] = log_context
# Check suite engine specific compatibility
self.suite_engine_proc.check_global_conf_compat()
# Suite name from the current working directory
if opts.conf_dir:
self.fs_util.chdir(opts.conf_dir)
opts.conf_dir = os.getcwd()
# --remote=KEY=VALUE,...
if opts.remote:
# opts.name always set for remote.
return self._run_remote(opts, opts.name)
conf_tree = self.config_load(opts)
self.fs_util.chdir(conf_tree.conf_dirs[0])
suite_name = opts.name
if not opts.name:
suite_name = os.path.basename(os.getcwd())
# Check suite.rc #! line for template scheme
templ_scheme = "jinja2"
if self.suite_engine_proc.SUITE_CONF in conf_tree.files:
suiterc_path = os.path.join(
conf_tree.files[self.suite_engine_proc.SUITE_CONF],
self.suite_engine_proc.SUITE_CONF)
with open(suiterc_path) as fh:
line = fh.readline()
if line.startswith("#!"):
templ_scheme = line[2:].strip().lower()
suite_section = (templ_scheme + ':' +
self.suite_engine_proc.SUITE_CONF)
extra_defines = []
if opts.defines_suite:
for define in opts.defines_suite:
extra_defines.append("[" + suite_section + "]" + define)
# Automatic Rose constants
# ROSE_ORIG_HOST: originating host
# ROSE_VERSION: Rose version (not retained in run_mode=="reload")
# Suite engine version
my_rose_version = ResourceLocator.default().get_version()
suite_engine_key = self.suite_engine_proc.get_version_env_name()
if opts.run_mode in ["reload", "restart"]:
prev_config_path = self.suite_engine_proc.get_suite_dir(
suite_name, "log", "rose-suite-run.conf")
prev_config = ConfigLoader()(prev_config_path)
suite_engine_version = prev_config.get_value(
["env", suite_engine_key])
else:
suite_engine_version = self.suite_engine_proc.get_version()
resloc = ResourceLocator.default()
auto_items = [(suite_engine_key, suite_engine_version),
("ROSE_ORIG_HOST", self.host_selector.get_local_host()),
("ROSE_SITE", resloc.get_conf().get_value(['site'], '')),
("ROSE_VERSION", resloc.get_version())]
for key, val in auto_items:
requested_value = conf_tree.node.get_value(["env", key])
if requested_value:
if key == "ROSE_VERSION" and val != requested_value:
exc = VersionMismatchError(requested_value, val)
raise ConfigValueError(["env", key], requested_value, exc)
val = requested_value
else:
conf_tree.node.set(["env", key],
val,
state=conf_tree.node.STATE_NORMAL)
extra_defines.append('[%s]%s="%s"' % (suite_section, key, val))
# Pass automatic Rose constants as suite defines
self.conf_tree_loader.node_loader.load_defines(extra_defines,
conf_tree.node)
# See if suite is running or not
if opts.run_mode == "reload":
# Check suite is running
self.suite_engine_proc.get_suite_contact(suite_name)
else:
self.suite_engine_proc.check_suite_not_running(suite_name)
# Install the suite to its run location
suite_dir_rel = self._suite_dir_rel(suite_name)
# Unfortunately a large try/finally block to ensure a temporary folder
# created in validate only mode is cleaned up. Exceptions are not
# caught here
try:
# Process Environment Variables
environ = self.config_pm(conf_tree, "env")
if opts.validate_suite_only_mode:
temp_dir = mkdtemp()
suite_dir = os.path.join(temp_dir, suite_dir_rel)
os.makedirs(suite_dir, 0o0700)
else:
suite_dir = os.path.join(os.path.expanduser("~"),
suite_dir_rel)
suite_conf_dir = os.getcwd()
locs_conf = ConfigNode()
if opts.new_mode:
if os.getcwd() == suite_dir:
raise NewModeError("PWD", os.getcwd())
elif opts.run_mode in ["reload", "restart"]:
raise NewModeError("--run", opts.run_mode)
self.suite_run_cleaner.clean(suite_name)
if os.getcwd() != suite_dir:
if opts.run_mode == "run":
self._run_init_dir(opts,
suite_name,
conf_tree,
locs_conf=locs_conf)
os.chdir(suite_dir)
# Housekeep log files
now_str = None
if not opts.install_only_mode and not opts.local_install_only_mode:
now_str = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
self._run_init_dir_log(opts, now_str)
self.fs_util.makedirs("log/suite")
# Rose configuration and version logs
self.fs_util.makedirs("log/rose-conf")
run_mode = opts.run_mode
if run_mode not in ["reload", "restart", "run"]:
run_mode = "run"
mode = run_mode
if opts.validate_suite_only_mode:
mode = "validate-suite-only"
elif opts.install_only_mode:
mode = "install-only"
elif opts.local_install_only_mode:
mode = "local-install-only"
prefix = "rose-conf/%s-%s" % (strftime("%Y%m%dT%H%M%S"), mode)
# Dump the actual configuration as rose-suite-run.conf
ConfigDumper()(conf_tree.node, "log/" + prefix + ".conf")
# Install version information file
write_source_vc_info(suite_conf_dir, "log/" + prefix + ".version",
self.popen)
# If run through rose-stem, install version information files for
# each source tree if they're a working copy
if hasattr(opts, 'source') and hasattr(opts, 'project'):
for i, url in enumerate(opts.source):
if os.path.isdir(url):
write_source_vc_info(
url, "log/" + opts.project[i] + "-" + str(i) +
".version", self.popen)
for ext in [".conf", ".version"]:
self.fs_util.symlink(prefix + ext, "log/rose-suite-run" + ext)
# Move temporary log to permanent log
if hasattr(self.event_handler, "contexts"):
log_file_path = os.path.abspath(
os.path.join("log", "rose-suite-run.log"))
log_file = open(log_file_path, "ab")
temp_log_file = self.event_handler.contexts[uuid].handle
temp_log_file.seek(0)
log_file.write(temp_log_file.read())
self.event_handler.contexts[uuid].handle = log_file
temp_log_file.close()
# Process Files
cwd = os.getcwd()
for rel_path, conf_dir in conf_tree.files.items():
if (conf_dir == cwd or any(
fnmatchcase(os.sep + rel_path, exclude)
for exclude in self.SYNC_EXCLUDES)
or conf_tree.node.get([templ_scheme + ":" + rel_path
]) is not None):
continue
# No sub-directories, very slow otherwise
if os.sep in rel_path:
rel_path = rel_path.split(os.sep, 1)[0]
target_key = self.config_pm.get_handler(
"file").PREFIX + rel_path
target_node = conf_tree.node.get([target_key])
if target_node is None:
conf_tree.node.set([target_key])
target_node = conf_tree.node.get([target_key])
elif target_node.is_ignored():
continue
source_node = target_node.get("source")
if source_node is None:
target_node.set(["source"],
os.path.join(conf_dir, rel_path))
elif source_node.is_ignored():
continue
self.config_pm(conf_tree,
"file",
no_overwrite_mode=opts.no_overwrite_mode)
# Process suite configuration template header
# (e.g. Jinja2:suite.rc, EmPy:suite.rc)
self.config_pm(conf_tree, templ_scheme, environ=environ)
# Ask suite engine to parse suite configuration
# and determine if it is up to date (unchanged)
if opts.validate_suite_only_mode:
suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf(
suite_dir, None, opts.strict_mode, debug_mode=True)
else:
suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf(
suite_name, opts.run_mode, opts.strict_mode,
opts.debug_mode)
finally:
# Ensure the temporary directory created is cleaned up regardless
# of success or failure
if opts.validate_suite_only_mode and os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
# Only validating so finish now
if opts.validate_suite_only_mode:
return
# Install share/work directories (local)
for name in ["share", "share/cycle", "work"]:
self._run_init_dir_work(opts,
suite_name,
name,
conf_tree,
locs_conf=locs_conf)
if opts.local_install_only_mode:
return
# Install suite files to each remote [user@]host
for name in ["", "log/", "share/", "share/cycle/", "work/"]:
uuid_file = os.path.abspath(name + uuid)
open(uuid_file, "w").close()
work_files.append(uuid_file)
# Install items to user@host
auths = self.suite_engine_proc.get_tasks_auths(suite_name)
proc_queue = [] # [[proc, command, "ssh"|"rsync", auth], ...]
for auth in sorted(auths):
host = auth
if "@" in auth:
host = auth.split("@", 1)[1]
# Remote shell
command = self.popen.get_cmd("ssh", "-n", auth)
# Provide ROSE_VERSION and CYLC_VERSION in the environment
shcommand = "env ROSE_VERSION=%s %s=%s" % (
my_rose_version, suite_engine_key, suite_engine_version)
# Use login shell?
no_login_shell = self._run_conf("remote-no-login-shell",
host=host,
conf_tree=conf_tree)
if not no_login_shell or no_login_shell.lower() != "true":
shcommand += r""" bash -l -c '"$0" "$@"'"""
# Path to "rose" command, if applicable
rose_bin = self._run_conf("remote-rose-bin",
host=host,
conf_tree=conf_tree,
default="rose")
# Build remote "rose suite-run" command
shcommand += " %s suite-run -vv -n %s" % (rose_bin, suite_name)
for key in ["new", "debug", "install-only"]:
attr = key.replace("-", "_") + "_mode"
if getattr(opts, attr, None) is not None:
shcommand += " --%s" % key
if opts.log_keep:
shcommand += " --log-keep=%s" % opts.log_keep
if opts.log_name:
shcommand += " --log-name=%s" % opts.log_name
if not opts.log_archive_mode:
shcommand += " --no-log-archive"
shcommand += " --run=%s" % opts.run_mode
# Build --remote= option
shcommand += " --remote=uuid=%s" % uuid
if now_str is not None:
shcommand += ",now-str=%s" % now_str
host_confs = [
"root-dir", "root-dir{share}", "root-dir{share/cycle}",
"root-dir{work}"
]
locs_conf.set([auth])
for key in host_confs:
value = self._run_conf(key, host=host, conf_tree=conf_tree)
if value is not None:
val = self.popen.list_to_shell_str([str(value)])
shcommand += ",%s=%s" % (key, pipes.quote(val))
locs_conf.set([auth, key], value)
command.append(shcommand)
proc = self.popen.run_bg(*command)
proc_queue.append([proc, command, "ssh", auth])
while proc_queue:
sleep(self.SLEEP_PIPE)
proc, command, command_name, auth = proc_queue.pop(0)
if proc.poll() is None: # put it back in proc_queue
proc_queue.append([proc, command, command_name, auth])
continue
ret_code = proc.wait()
out, err = proc.communicate()
if ret_code:
raise RosePopenError(command, ret_code, out, err)
if command_name == "rsync":
self.handle_event(out, level=Event.VV)
continue
else:
self.handle_event(out, level=Event.VV, prefix="[%s] " % auth)
for line in out.split("\n"):
if "/" + uuid == line.strip():
locs_conf.unset([auth])
break
else:
filters = {"excludes": [], "includes": []}
for name in ["", "log/", "share/", "share/cycle/", "work/"]:
filters["excludes"].append(name + uuid)
target = auth + ":" + suite_dir_rel
cmd = self._get_cmd_rsync(target, **filters)
proc_queue.append(
[self.popen.run_bg(*cmd), cmd, "rsync", auth])
# Install ends
ConfigDumper()(locs_conf, os.path.join("log", "rose-suite-run.locs"))
if opts.install_only_mode:
return
elif opts.run_mode == "reload" and suite_conf_unchanged:
conf_name = self.suite_engine_proc.SUITE_CONF
self.handle_event(SkipReloadEvent(suite_name, conf_name))
return
# Start the suite
self.fs_util.chdir("log")
self.suite_engine_proc.run(suite_name, opts.host, opts.run_mode, args)
# Disconnect log file handle, so monitoring tool command will no longer
# be associated with the log file.
self.event_handler.contexts[uuid].handle.close()
self.event_handler.contexts.pop(uuid)
# Launch the monitoring tool
# Note: maybe use os.ttyname(sys.stdout.fileno())?
if opts.gcontrol_mode and opts.run_mode != "reload":
self.suite_engine_proc.gcontrol(suite_name)
return 0
def restart(self,
suite_name=None,
host=None,
gcontrol_mode=None,
args=None):
"""Restart a "cylc" suite."""
# Check suite engine specific compatibility
self.suite_engine_proc.check_global_conf_compat()
if not suite_name:
suite_name = get_suite_name(self.event_handler)
suite_dir = self.suite_engine_proc.get_suite_dir(suite_name)
if not os.path.exists(suite_dir):
raise SuiteNotFoundError(suite_dir)
# Ensure suite is not running
hosts = []
if host:
hosts.append(host)
self.suite_engine_proc.check_suite_not_running(suite_name, hosts)
# Determine suite host to restart suite
if host:
hosts = [host]
else:
hosts = []
val = ResourceLocator.default().get_conf().get_value(
["rose-suite-run", "hosts"], "localhost")
known_hosts = self.host_selector.expand(val.split())[0]
for known_host in known_hosts:
if known_host not in hosts:
hosts.append(known_host)
if hosts == ["localhost"]:
host = hosts[0]
else:
host = self.host_selector(hosts)[0][0]
self.handle_event(SuiteHostSelectEvent(suite_name, "restart", host))
# Suite host environment
run_conf_file_name = self.suite_engine_proc.get_suite_dir(
suite_name, "log", "rose-suite-run.conf")
try:
run_conf = ConfigLoader().load(run_conf_file_name)
except (ConfigSyntaxError, IOError):
environ = None
else:
run_conf_tree = ConfigTree()
run_conf_tree.node = run_conf
environ = self.config_pm(run_conf_tree, "env")
# Restart the suite
self.suite_engine_proc.run(suite_name, host, environ, "restart", args)
# Write suite host name to host file
host_file_name = self.suite_engine_proc.get_suite_dir(
suite_name, "log", "rose-suite-run.host")
open(host_file_name, "w").write(host + "\n")
# Launch the monitoring tool
# Note: maybe use os.ttyname(sys.stdout.fileno())?
if os.getenv("DISPLAY") and host and gcontrol_mode:
self.suite_engine_proc.gcontrol(suite_name, host)
return
def cycles(self,
user,
suite,
page=1,
order=None,
per_page=None,
no_fuzzy_time="0",
form=None):
"""List cycles of a running or completed suite."""
conf = ResourceLocator.default().get_conf()
per_page_default = int(
conf.get_value(["rose-bush", "cycles-per-page"],
self.CYCLES_PER_PAGE))
if not isinstance(per_page, int):
if per_page:
per_page = int(per_page)
else:
per_page = per_page_default
if page and per_page:
page = int(page)
else:
page = 1
data = {
"logo":
self.logo,
"title":
self.title,
"host":
self.host_name,
"user":
user,
"suite":
suite,
"is_option_on":
(order is not None and order != "time_desc"
or per_page is not None and per_page != per_page_default),
"order":
order,
"rose_version":
self.rose_version,
"script":
cherrypy.request.script_name,
"method":
"cycles",
"no_fuzzy_time":
no_fuzzy_time,
"states": {},
"per_page":
per_page,
"per_page_default":
per_page_default,
"page":
page,
"task_status_groups":
self.bush_dao.TASK_STATUS_GROUPS,
}
data["entries"], data["of_n_entries"] = (
self.bush_dao.get_suite_cycles_summary(user, suite, order,
per_page,
(page - 1) * per_page))
if per_page:
data["n_pages"] = data["of_n_entries"] / per_page
if data["of_n_entries"] % per_page != 0:
data["n_pages"] += 1
else:
data["n_pages"] = 1
data.update(self._get_suite_logs_info(user, suite))
data["states"].update(
self.bush_dao.get_suite_state_summary(user, suite))
data["states"]["last_activity_time"] = (self.get_last_activity_time(
user, suite))
data["time"] = strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
if form == "json":
return json.dumps(data)
try:
return self.template_env.get_template("cycles.html").render(**data)
except jinja2.TemplateError:
traceback.print_exc()
return json.dumps(data)
def select(self,
names=None,
rank_method=None,
thresholds=None,
ssh_cmd_timeout=None):
"""Return a list. Element 0 is most desirable.
Each element of the list is a tuple (host, score).
names: a list of known host groups or host names.
rank_method: the ranking method. Can be one of:
load:1, load:5, load:15 (=load =default), fs:FS and
random. The "load" methods determines the load using the
average load as returned by the "uptime" command divided
by the number of CPUs. The "fs" method determines the load
using the usage in the file system specified by FS. The
"mem" method ranks by highest free memory. The "random"
method ranks everything by random.
thresholds: a list of thresholds which each host must not exceed.
Should be in the format rank_method:value, where
rank_method is one of load:1, load:5, load:15 or fs:FS; and
value is number that must be be exceeded.
ssh_cmd_timeout: timeout of SSH commands to hosts. A float in seconds.
"""
host_names, rank_method, thresholds = self.expand(
names, rank_method, thresholds)
# Load scorers, ranking and thresholds
rank_method_arg = None
if rank_method:
if ":" in rank_method:
rank_method, rank_method_arg = rank_method.split(":", 1)
else:
rank_method = self.RANK_METHOD_DEFAULT
rank_conf = ScorerConf(self.get_scorer(rank_method), rank_method_arg)
self.handle_event(RankMethodEvent(rank_method, rank_method_arg))
threshold_confs = []
if thresholds:
for threshold in thresholds:
method = self.RANK_METHOD_DEFAULT
method_arg = None
value = threshold
if ":" in threshold:
head, value = threshold.rsplit(":", 1)
method = head
if ":" in head:
method, method_arg = head.split(":", 1)
try:
float(value)
except ValueError:
raise ValueError(threshold)
scorer = self.get_scorer(method)
if method_arg is None:
method_arg = scorer.ARG
threshold_conf = ScorerConf(self.get_scorer(method),
method_arg, value)
threshold_confs.append(threshold_conf)
if ssh_cmd_timeout is None:
conf = ResourceLocator.default().get_conf()
ssh_cmd_timeout = float(
conf.get_value(["rose-host-select", "timeout"],
self.SSH_CMD_TIMEOUT))
host_name_list = list(host_names)
host_names = []
for host_name in host_name_list:
if self.is_local_host(host_name):
if self.get_local_host() not in host_names:
host_names.append(self.get_local_host())
else:
host_names.append(host_name)
# Random selection with no thresholds. Return the 1st available host.
if rank_conf.method == self.RANK_METHOD_RANDOM and not threshold_confs:
shuffle(host_names)
for host_name in host_names:
if self.is_local_host(host_name):
return [("localhost", 1)]
command = self.popen.get_cmd("ssh", host_name, "true")
proc = self.popen.run_bg(*command, preexec_fn=os.setpgrp)
time0 = time()
while (proc.poll() is None
and time() - time0 <= ssh_cmd_timeout):
sleep(self.SSH_CMD_POLL_DELAY)
if proc.poll() is None:
os.killpg(proc.pid, signal.SIGTERM)
proc.wait()
self.handle_event(TimedOutHostEvent(host_name))
elif proc.wait():
self.handle_event(DeadHostEvent(host_name))
else:
return [(host_name, 1)]
else:
raise NoHostSelectError()
# ssh to each host to return its score(s).
host_proc_dict = {}
for host_name in sorted(host_names):
command = []
if not self.is_local_host(host_name):
command_args = []
command_args.append(host_name)
command = self.popen.get_cmd("ssh", *command_args)
command.append("bash")
stdin = rank_conf.get_command()
for threshold_conf in threshold_confs:
stdin += threshold_conf.get_command()
stdin += "exit\n"
proc = self.popen.run_bg(*command,
stdin=stdin,
preexec_fn=os.setpgrp)
proc.stdin.write(stdin)
proc.stdin.flush()
host_proc_dict[host_name] = proc
# Retrieve score for each host name
host_score_list = []
time0 = time()
while host_proc_dict:
sleep(self.SSH_CMD_POLL_DELAY)
for host_name, proc in host_proc_dict.items():
if proc.poll() is None:
score = None
elif proc.wait():
self.handle_event(DeadHostEvent(host_name))
host_proc_dict.pop(host_name)
else:
out = proc.communicate()[0]
host_proc_dict.pop(host_name)
for threshold_conf in threshold_confs:
try:
is_bad = threshold_conf.check_threshold(out)
score = threshold_conf.command_out_parser(out)
except ValueError:
is_bad = True
score = None
if is_bad:
self.handle_event(
HostThresholdNotMetEvent(
host_name, threshold_conf, score))
break
else:
try:
score = rank_conf.command_out_parser(out)
host_score_list.append((host_name, score))
except ValueError:
score = None
self.handle_event(
HostSelectScoreEvent(host_name, score))
if time() - time0 > ssh_cmd_timeout:
break
# Report timed out hosts
for host_name, proc in sorted(host_proc_dict.items()):
self.handle_event(TimedOutHostEvent(host_name))
os.killpg(proc.pid, signal.SIGTERM)
proc.wait()
if not host_score_list:
raise NoHostSelectError()
host_score_list.sort(lambda a, b: cmp(a[1], b[1]),
reverse=rank_conf.scorer.SIGN < 0)
return host_score_list
def run(self, suite_name, task_id, hook_event, hook_message=None,
should_mail=False, mail_cc_list=None, should_shutdown=False,
should_retrieve_job_logs=False):
"""
Invoke the hook for a suite.
1. For a task hook, if the task runs remotely, retrieve its log from
the remote host.
2. If "should_mail", send an email notification to the current user,
and those in the "mail_cc_list".
3. If "should_shutdown", shut down the suite.
"""
# Retrieve log and populate job logs database
task_ids = []
if task_id and should_retrieve_job_logs:
task_ids = [task_id]
self.suite_engine_proc.job_logs_pull_remote(suite_name, task_ids)
# Send email notification if required
email_exc = None
if should_mail:
text = ""
if task_id:
text += "Task: %s\n" % task_id
if hook_message:
text += "Message: %s\n" % hook_message
url = self.suite_engine_proc.get_suite_log_url(None, suite_name)
text += "See: %s\n" % (url)
user = pwd.getpwuid(os.getuid()).pw_name
conf = ResourceLocator.default().get_conf()
host = conf.get_value(["rose-suite-hook", "email-host"],
default="localhost")
msg = MIMEText(text)
msg["From"] = user + "@" + host
msg["To"] = msg["From"]
if mail_cc_list:
mail_cc_addresses = []
for mail_cc_address in mail_cc_list:
if "@" not in mail_cc_address:
mail_cc_address += "@" + host
mail_cc_addresses.append(mail_cc_address)
msg["Cc"] = ", ".join(mail_cc_addresses)
mail_cc_list = mail_cc_addresses
else:
mail_cc_list = []
msg["Subject"] = "[%s] %s" % (hook_event, suite_name)
smtp_host = conf.get_value(["rose-suite-hook", "smtp-host"],
default="localhost")
try:
smtp = SMTP(smtp_host)
smtp.sendmail(
msg["From"], [msg["To"]] + mail_cc_list, msg.as_string())
smtp.quit()
except (socket.error, SMTPException) as email_exc:
pass
# Shut down if required
if should_shutdown:
self.suite_engine_proc.shutdown(suite_name, args=["--kill"])
if email_exc is not None:
raise
def main():
"""Implement the "rose config" command."""
opt_parser = RoseOptionParser()
opt_parser.add_my_options("default", "env_var_process_mode", "files",
"keys", "meta", "meta_key", "no_ignore",
"no_opts", "print_conf_mode")
opts, args = opt_parser.parse_args()
report = Reporter(opts.verbosity - opts.quietness)
rose.macro.add_meta_paths()
if opts.meta_key:
opts.meta = True
if opts.files and opts.meta_key:
report(Exception("Cannot specify both a file and meta key."))
sys.exit(1)
config_loader = ConfigLoader()
sources = []
if opts.files:
root_node = ConfigNode()
for fname in opts.files:
if fname == "-":
sources.append(sys.stdin)
else:
if opts.meta:
try:
root_node = config_loader.load(fname)
except ConfigSyntaxError as exc:
report(exc)
sys.exit(1)
rel_path = os.sep.join(fname.split(os.sep)[:-1])
fpath = get_meta_path(root_node, rel_path)
if fpath is None:
report(MetadataNotFoundEvent(fname))
else:
sources.append(fpath)
else:
sources.append(fname)
elif opts.meta:
root_node = ConfigNode()
if opts.meta_key:
root_node.set(["meta"], opts.meta_key)
else:
fname = os.path.join(os.getcwd(), rose.SUB_CONFIG_NAME)
try:
root_node = config_loader.load(fname)
except ConfigSyntaxError as exc:
report(exc)
sys.exit(1)
fpath = get_meta_path(root_node, meta_key=opts.meta_key)
root_node.unset(["meta"])
if fpath is None:
report(Exception("Metadata not found"))
sys.exit(1)
else:
sources.append(fpath)
else:
root_node = ResourceLocator.default().get_conf()
for source in sources:
try:
if opts.meta or opts.no_opts:
config_loader.load(source, root_node)
else:
config_loader.load_with_opts(source, root_node)
except (ConfigSyntaxError, IOError) as exc:
report(exc)
sys.exit(1)
if source is sys.stdin:
source.close()
if opts.quietness:
sys.exit(root_node.get(args, opts.no_ignore) is None)
if opts.keys_mode:
try:
keys = root_node.get(args, opts.no_ignore).value.keys()
except AttributeError:
sys.exit(1)
keys.sort()
for key in keys:
print key
sys.exit()
conf_dump = ConfigDumper()
if len(args) == 0:
conf_dump(root_node, concat_mode=opts.print_conf_mode)
sys.exit()
node = root_node.get(args, opts.no_ignore)
if node is not None and isinstance(node.value, dict):
if opts.print_conf_mode:
conf_dump(ConfigNode().set(args, node.value), concat_mode=True)
sys.exit()
keys = node.value.keys()
keys.sort()
for key in keys:
node_of_key = node.get([key], opts.no_ignore)
if node_of_key:
value = node_of_key.value
state = node_of_key.state
string = "%s%s=%s" % (state, key, value)
lines = string.splitlines()
print lines[0]
i_equal = len(state + key) + 1
for line in lines[1:]:
print " " * i_equal + line
sys.exit()
if node is None:
if opts.default is None:
sys.exit(1)
value = opts.default
elif opts.env_var_process_mode:
value = env_var_process(node.value)
else:
value = node.value
if opts.print_conf_mode:
conf_dump(ConfigNode().set(args, value), concat_mode=True)
else:
print value
sys.exit()
def generate_info_config(self, from_id=None, prefix=None, project=None):
"""Generate a rose.config.ConfigNode for a rose-suite.info.
This is suitable for passing into the create method of this
class.
If from_id is defined, copy items from it.
Return the rose.config.ConfigNode instance.
"""
from_project = None
from_title = None
if from_id is not None:
from_info_url = "%s/%s/rose-suite.info@%s" % (
from_id.to_origin(), from_id.branch, from_id.revision)
out_data = self.popen("svn", "cat", from_info_url)[0]
from_config = rose.config.load(StringIO(out_data))
res_loc = ResourceLocator.default()
older_config = None
info_config = rose.config.ConfigNode()
# Determine project if given as a command-line option on create
if from_id is None and project is not None:
info_config.set(["project"], project)
# Set the compulsory fields and use the project and metadata if
# available.
meta_config = load_meta_config(info_config,
config_type=rose.INFO_CONFIG_NAME)
if from_id is None and project is not None:
for node_keys, node in meta_config.walk(no_ignore=True):
if isinstance(node.value, dict):
continue
sect, key = node_keys
value = node.value
sect = sect.translate(None, "=")
if key == "compulsory" and value == "true":
info_config.set([sect], "")
info_config.set(["project"], project)
else:
if from_project is None:
info_config.set(["project"], "")
if from_title is None:
info_config.set(["title"], "")
# Determine prefix
if prefix is None:
if from_id is None:
prefix = SuiteId.get_prefix_default()
else:
prefix = from_id.prefix
# Determine owner:
# 1. From user configuration [rosie-id]prefix-username
# 2. From username of a matching group in [groups] in
# ~/.subversion/servers
# 3. Current user ID
owner = res_loc.get_conf().get_value(
["rosie-id", "prefix-username." + prefix])
if not owner and self.subversion_servers_conf:
servers_conf = rose.config.load(self.subversion_servers_conf)
groups_node = servers_conf.get(["groups"])
if groups_node is not None:
prefix_loc = SuiteId.get_prefix_location(prefix)
prefix_host = urlparse(prefix_loc).hostname
for key, node in groups_node.value.items():
if fnmatch(prefix_host, node.value):
owner = servers_conf.get_value([key, "username"])
break
if not owner:
owner = pwd.getpwuid(os.getuid())[0]
info_config.set(["owner"], owner)
# Copy description
try:
from_id.to_string_with_version()
info_config.set(["description"],
"Copy of %s" % (from_id.to_string_with_version()))
except AttributeError:
pass
# Copy fields provided by the user
try:
from_config.walk(no_ignore=False)
for node_keys, node in from_config.walk(no_ignore=False):
if isinstance(node.value, dict):
continue
sect, key = node_keys
value = node.value
if (key in ["description", "owner", "access-list"]
or (key == "project" and from_project is not None)):
pass
else:
info_config.set([key], value)
except UnboundLocalError:
pass
# Determine access list
access_list_str = res_loc.get_conf().get_value(
["rosie-vc", "access-list-default"])
if access_list_str:
info_config.set(["access-list"], access_list_str)
if from_id is None and project is not None:
for node_keys, node in meta_config.walk(no_ignore=True):
if isinstance(node.value, dict):
continue
sect, key = node_keys
value = node.value
sect = sect.translate(None, "=")
if key == "value-hints" or key == "values":
reminder = ("please remove all commented hints/lines " +
"in the main/top section before saving.")
info_config.set([sect],
rose.variable.array_split(value)[0],
comments=[value, reminder])
if older_config is not None:
for node_keys, node in older_config.walk(no_ignore=True):
if isinstance(node.value, dict):
continue
sect, key = node_keys
value = node.value
info_config.set([key], value)
return info_config
def run_impl(self, opts, args, uuid, work_files):
# Log file, temporary
if hasattr(self.event_handler, "contexts"):
t_file = TemporaryFile()
log_context = ReporterContext(None, self.event_handler.VV, t_file)
self.event_handler.contexts[uuid] = log_context
# Check suite engine specific compatibility
self.suite_engine_proc.check_global_conf_compat()
# Suite name from the current working directory
if opts.conf_dir:
self.fs_util.chdir(opts.conf_dir)
opts.conf_dir = os.getcwd()
if opts.defines_suite:
suite_section = "jinja2:" + self.suite_engine_proc.SUITE_CONF
if not opts.defines:
opts.defines = []
for define in opts.defines_suite:
opts.defines.append("[" + suite_section + "]" + define)
# --remote=KEY=VALUE,...
if opts.remote:
# opts.name always set for remote.
return self._run_remote(opts, opts.name)
conf_tree = self.config_load(opts)
self.fs_util.chdir(conf_tree.conf_dirs[0])
suite_name = opts.name
if not opts.name:
suite_name = os.path.basename(os.getcwd())
# Automatic Rose constants
# ROSE_ORIG_HOST: originating host
# ROSE_VERSION: Rose version (not retained in run_mode=="reload")
# Suite engine version
jinja2_section = "jinja2:" + self.suite_engine_proc.SUITE_CONF
my_rose_version = ResourceLocator.default().get_version()
suite_engine_key = self.suite_engine_proc.get_version_env_name()
if opts.run_mode in ["reload", "restart"]:
prev_config_path = self.suite_engine_proc.get_suite_dir(
suite_name, "log", "rose-suite-run.conf")
prev_config = ConfigLoader()(prev_config_path)
suite_engine_version = prev_config.get_value(
["env", suite_engine_key])
else:
suite_engine_version = self.suite_engine_proc.get_version()
auto_items = {
"ROSE_ORIG_HOST": self.host_selector.get_local_host(),
"ROSE_VERSION": ResourceLocator.default().get_version(),
suite_engine_key: suite_engine_version
}
for key, val in auto_items.items():
requested_value = conf_tree.node.get_value(["env", key])
if requested_value:
if key == "ROSE_VERSION" and val != requested_value:
exc = VersionMismatchError(requested_value, val)
raise ConfigValueError(["env", key], requested_value, exc)
val = requested_value
else:
conf_tree.node.set(["env", key],
val,
state=conf_tree.node.STATE_NORMAL)
conf_tree.node.set([jinja2_section, key], '"' + val + '"')
# See if suite is running or not
hosts = []
if opts.host:
hosts.append(opts.host)
if opts.run_mode == "reload":
suite_run_hosts = self.suite_engine_proc.get_suite_run_hosts(
None, suite_name, hosts)
if not suite_run_hosts:
raise SuiteNotRunningError(suite_name)
hosts = suite_run_hosts
else:
self.suite_engine_proc.check_suite_not_running(suite_name, hosts)
# Install the suite to its run location
suite_dir_rel = self._suite_dir_rel(suite_name)
suite_dir = os.path.join(os.path.expanduser("~"), suite_dir_rel)
suite_conf_dir = os.getcwd()
locs_conf = ConfigNode()
if opts.new_mode:
if os.getcwd() == suite_dir:
raise NewModeError("PWD", os.getcwd())
elif opts.run_mode in ["reload", "restart"]:
raise NewModeError("--run", opts.run_mode)
self.suite_run_cleaner.clean(suite_name)
if os.getcwd() != suite_dir:
if opts.run_mode == "run":
self._run_init_dir(opts,
suite_name,
conf_tree,
locs_conf=locs_conf)
os.chdir(suite_dir)
# Housekeep log files
if not opts.install_only_mode and not opts.local_install_only_mode:
self._run_init_dir_log(opts)
self.fs_util.makedirs("log/suite")
# Rose configuration and version logs
self.fs_util.makedirs("log/rose-conf")
run_mode = opts.run_mode
if run_mode not in ["reload", "restart", "run"]:
run_mode = "run"
mode = run_mode
if opts.install_only_mode:
mode = "install-only"
elif opts.local_install_only_mode:
mode = "local-install-only"
prefix = "rose-conf/%s-%s" % (strftime("%Y%m%dT%H%M%S"), mode)
# Dump the actual configuration as rose-suite-run.conf
ConfigDumper()(conf_tree.node, "log/" + prefix + ".conf")
# Install version information file
write_source_vc_info(suite_conf_dir, "log/" + prefix + ".version",
self.popen)
# If run through rose-stem, install version information files for
# each source tree if they're a working copy
if hasattr(opts, 'source') and hasattr(opts, 'project'):
for i, url in enumerate(opts.source):
if os.path.isdir(url):
write_source_vc_info(
url,
"log/" + opts.project[i] + "-" + str(i) + ".version",
self.popen)
for ext in [".conf", ".version"]:
self.fs_util.symlink(prefix + ext, "log/rose-suite-run" + ext)
# Move temporary log to permanent log
if hasattr(self.event_handler, "contexts"):
log_file_path = os.path.abspath(
os.path.join("log", "rose-suite-run.log"))
log_file = open(log_file_path, "ab")
temp_log_file = self.event_handler.contexts[uuid].handle
temp_log_file.seek(0)
log_file.write(temp_log_file.read())
self.event_handler.contexts[uuid].handle = log_file
temp_log_file.close()
# Install share/work directories (local)
for name in ["share", "share/cycle", "work"]:
self._run_init_dir_work(opts,
suite_name,
name,
conf_tree,
locs_conf=locs_conf)
# Process Environment Variables
environ = self.config_pm(conf_tree, "env")
# Process Files
cwd = os.getcwd()
for rel_path, conf_dir in conf_tree.files.items():
if (conf_dir == cwd or any([
fnmatchcase(os.sep + rel_path, exclude)
for exclude in self.SYNC_EXCLUDES
]) or conf_tree.node.get(["jinja2:" + rel_path]) is not None):
continue
# No sub-directories, very slow otherwise
if os.sep in rel_path:
rel_path = rel_path.split(os.sep, 1)[0]
target_key = self.config_pm.get_handler("file").PREFIX + rel_path
target_node = conf_tree.node.get([target_key])
if target_node is None:
conf_tree.node.set([target_key])
target_node = conf_tree.node.get([target_key])
elif target_node.is_ignored():
continue
source_node = target_node.get("source")
if source_node is None:
target_node.set(["source"], os.path.join(conf_dir, rel_path))
elif source_node.is_ignored():
continue
self.config_pm(conf_tree,
"file",
no_overwrite_mode=opts.no_overwrite_mode)
# Process Jinja2 configuration
self.config_pm(conf_tree, "jinja2")
# Ask suite engine to parse suite configuration
# and determine if it is up to date (unchanged)
suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf(
suite_name, opts.run_mode, opts.strict_mode, opts.debug_mode)
if opts.local_install_only_mode:
return
# Install suite files to each remote [user@]host
for name in ["", "log/", "share/", "share/cycle/", "work/"]:
uuid_file = os.path.abspath(name + uuid)
open(uuid_file, "w").close()
work_files.append(uuid_file)
# Install items to user@host
conf = ResourceLocator.default().get_conf()
auths = self.suite_engine_proc.get_tasks_auths(suite_name)
queue = [] # [[pipe, command, "ssh"|"rsync", auth], ...]
for auth in sorted(auths):
host = auth
if "@" in auth:
host = auth.split("@", 1)[1]
command = self.popen.get_cmd("ssh", auth, "bash", "--login", "-c")
rose_bin = "rose"
for name in [host, "*"]:
rose_home_node = conf.get(["rose-home-at", name],
no_ignore=True)
if rose_home_node is not None:
rose_bin = "%s/bin/rose" % (rose_home_node.value)
break
# Build remote "rose suite-run" command
rose_sr = "ROSE_VERSION=%s %s" % (my_rose_version, rose_bin)
rose_sr += " suite-run -v -v --name=%s" % suite_name
for key in ["new", "debug", "install-only"]:
attr = key.replace("-", "_") + "_mode"
if getattr(opts, attr, None) is not None:
rose_sr += " --" + key
if opts.log_keep:
rose_sr += " --log-keep=" + opts.log_keep
if opts.log_name:
rose_sr += " --log-name=" + opts.log_name
if not opts.log_archive_mode:
rose_sr += " --no-log-archive"
rose_sr += " --run=" + opts.run_mode
host_confs = [
"root-dir", "root-dir{share}", "root-dir{share/cycle}",
"root-dir{work}"
]
rose_sr += " --remote=uuid=" + uuid
locs_conf.set([auth])
for key in host_confs:
value = self._run_conf(key, host=host, conf_tree=conf_tree)
if value is not None:
val = self.popen.list_to_shell_str([str(value)])
rose_sr += "," + key + "=" + val
locs_conf.set([auth, key], value)
command += ["'" + rose_sr + "'"]
pipe = self.popen.run_bg(*command)
queue.append([pipe, command, "ssh", auth])
while queue:
sleep(self.SLEEP_PIPE)
pipe, command, command_name, auth = queue.pop(0)
if pipe.poll() is None:
queue.append([pipe, command, command_name, auth]) # put back
continue
ret_code = pipe.wait()
out, err = pipe.communicate()
if ret_code:
raise RosePopenError(command, ret_code, out, err)
if command_name == "rsync":
self.handle_event(out, level=Event.VV)
continue
else:
self.handle_event(out, level=Event.VV, prefix="[%s] " % auth)
for line in out.split("\n"):
if "/" + uuid == line.strip():
locs_conf.unset([auth])
break
else:
filters = {"excludes": [], "includes": []}
for name in ["", "log/", "share/", "share/cycle/", "work/"]:
filters["excludes"].append(name + uuid)
target = auth + ":" + suite_dir_rel
cmd = self._get_cmd_rsync(target, **filters)
queue.append([self.popen.run_bg(*cmd), cmd, "rsync", auth])
# Install ends
ConfigDumper()(locs_conf, os.path.join("log", "rose-suite-run.locs"))
if opts.install_only_mode:
return
elif opts.run_mode == "reload" and suite_conf_unchanged:
conf_name = self.suite_engine_proc.SUITE_CONF
self.handle_event(SkipReloadEvent(suite_name, conf_name))
return
# Start the suite
self.fs_util.chdir("log")
ret = 0
# FIXME: should sync files to suite host?
if opts.run_mode != "reload":
if opts.host:
hosts = [opts.host]
else:
names = shlex.split(
conf.get_value(["rose-suite-run", "hosts"], ""))
if names:
hosts += self.host_selector.expand(names)[0]
if (hosts and len(hosts) == 1
and self.host_selector.is_local_host(hosts[0])):
host = "localhost"
elif hosts:
host = self.host_selector(hosts)[0][0]
else:
host = "localhost"
self.handle_event(SuiteHostSelectEvent(suite_name, run_mode, host))
# FIXME: values in environ were expanded in the localhost
self.suite_engine_proc.run(suite_name, host, environ, opts.run_mode,
args)
open("rose-suite-run.host", "w").write(host + "\n")
# Disconnect log file handle, so monitoring tool command will no longer
# be associated with the log file.
self.event_handler.contexts[uuid].handle.close()
self.event_handler.contexts.pop(uuid)
# Launch the monitoring tool
# Note: maybe use os.ttyname(sys.stdout.fileno())?
if os.getenv("DISPLAY") and host and opts.gcontrol_mode:
self.suite_engine_proc.gcontrol(suite_name, host)
return ret
def run(self, repos, txn):
"""Apply the rule engine on transaction "txn" to repository "repos"."""
changes = set() # set([(status, path), ...])
for line in self._svnlook("changed", "-t", txn, repos).splitlines():
status, path = line.split(None, 1)
changes.add((status, path))
bad_changes = []
author = None
super_users = None
access_info_map = {} # {path-id: (owner, access-list), ...}
txn_access_info_map = {}
conf = ResourceLocator.default().get_conf()
ignores_str = conf.get_value(["rosa-svn", "ignores"], self.IGNORES)
ignores = shlex.split(ignores_str)
for status, path in sorted(changes):
if any([fnmatch(path, ignore) for ignore in ignores]):
continue
names = path.split("/", self.LEN_ID + 1)
if not names[-1]:
names.pop()
# Directories above the suites must match the ID patterns
is_bad = False
for name, pattern in zip(names, self.RE_ID_NAMES):
if not re.compile(r"\A" + pattern + r"\Z").match(name):
is_bad = True
break
if is_bad:
bad_changes.append(BadChange(status, path))
continue
# Can only add directories at levels above the suites
if len(names) < self.LEN_ID:
if status[0] != self.ST_ADD:
bad_changes.append(BadChange(status, path))
continue
else:
is_meta_suite = "".join(names[0:self.LEN_ID]) == "ROSIE"
# No need to check non-trunk changes
if len(names) > self.LEN_ID and names[self.LEN_ID] != "trunk":
continue
# New suite should have an info file
if status[0] == self.ST_ADD and len(names) == self.LEN_ID:
if (self.ST_ADD, path + "trunk/") not in changes:
bad_changes.append(
BadChange(status, path, BadChange.NO_TRUNK))
continue
path_trunk_info_file = path + self.TRUNK_INFO_FILE
if ((self.ST_ADD, path_trunk_info_file) not in changes and
(self.ST_UPDATE, path_trunk_info_file) not in changes):
bad_changes.append(
BadChange(status, path, BadChange.NO_INFO))
continue
# The rest are trunk changes in a suite
path_head = "/".join(names[0:self.LEN_ID]) + "/"
path_tail = path[len(path_head):]
# For meta suite, make sure keys in keys file can be parsed
if is_meta_suite and path_tail == self.TRUNK_KNOWN_KEYS_FILE:
out = self._svnlook("cat", "-t", txn, repos, path)
try:
shlex.split(out)
except ValueError:
bad_changes.append(
BadChange(status, path, BadChange.VALUE))
continue
# Suite trunk information file must have an owner
# User IDs of owner and access list must be real
if (status not in self.ST_DELETE and
path_tail == self.TRUNK_INFO_FILE):
txn_owner, txn_access_list = self._get_access_info(
repos, path_head, txn)
if not txn_owner:
bad_changes.append(
BadChange(status, path, BadChange.NO_OWNER))
continue
if self._verify_users(
status, path, txn_owner, txn_access_list, bad_changes):
continue
# New suite trunk: ignore the rest
if (self.ST_ADD, path_head + "trunk/") in changes:
continue
# Can only remove trunk information file with suite
if status == self.ST_DELETE and path_tail == self.TRUNK_INFO_FILE:
if (self.ST_DELETE, path_head) not in changes:
bad_changes.append(
BadChange(status, path, BadChange.NO_INFO))
continue
# Can only remove trunk with suite
if status == self.ST_DELETE and path_tail == "trunk/":
if (self.ST_DELETE, path_head) not in changes:
bad_changes.append(
BadChange(status, path, BadChange.NO_TRUNK))
continue
# See whether author has permission to make changes
if author is None:
author = self._svnlook("author", "-t", txn, repos).strip()
if super_users is None:
super_users = []
for s_key in ["rosa-svn", "rosa-svn-pre-commit"]:
value = conf.get_value([s_key, "super-users"])
if value is not None:
super_users = shlex.split(value)
break
if path_head not in access_info_map:
access_info = self._get_access_info(repos, path_head)
access_info_map[path_head] = access_info
owner, access_list = access_info_map[path_head]
admin_users = super_users + [owner]
# Only admin users can remove the suite
if author not in admin_users and not path_tail:
bad_changes.append(BadChange(status, path))
continue
# Admin users and those in access list can modify everything in
# trunk apart from specific entries in the trunk info file
if "*" in access_list or author in admin_users + access_list:
if path_tail != self.TRUNK_INFO_FILE:
continue
else:
bad_changes.append(BadChange(status, path))
continue
# The owner must not be deleted
if path_head not in txn_access_info_map:
txn_access_info = self._get_access_info(repos, path_head, txn)
txn_access_info_map[path_head] = txn_access_info
txn_owner, txn_access_list = txn_access_info_map[path_head]
if not txn_owner:
bad_changes.append(BadChange(status, path, BadChange.NO_OWNER))
continue
# Only the admin users can change owner and access list
if owner == txn_owner and access_list == txn_access_list:
continue
if author not in admin_users:
if owner != txn_owner:
bad_changes.append(BadChange(status, path, BadChange.PERM,
"owner=" + txn_owner))
else: # access list
bad_change = BadChange(
status, path, BadChange.PERM,
"access-list=" + " ".join(txn_access_list))
bad_changes.append(bad_change)
continue
if bad_changes:
raise BadChanges(bad_changes)
def taskjobs(self,
user,
suite,
page=1,
cycles=None,
tasks=None,
task_status=None,
job_status=None,
order=None,
per_page=None,
no_fuzzy_time="0",
form=None):
"""List task jobs.
user -- A string containing a valid user ID
suite -- A string containing a valid suite ID
page -- The page number to display
cycles -- Display only task jobs matching these cycles. A value in the
list can be a cycle, the string "before|after CYCLE", or a
glob to match cycles.
tasks -- Display only jobs for task names matching a list of names.
The list should be specified as a string which will be
shlex.split by this method. Values can be a valid task name or
a glob like pattern for matching valid task names.
task_status -- Select by task statuses.
job_status -- Select by job status. See RoseBushDAO.JOB_STATUS_COMBOS
for detail.
order -- Order search in a predetermined way. A valid value is one of
"time_desc", "time_asc",
"cycle_desc_name_desc", "cycle_desc_name_asc",
"cycle_asc_name_desc", "cycle_asc_name_asc",
"name_asc_cycle_asc", "name_desc_cycle_asc",
"name_asc_cycle_desc", "name_desc_cycle_desc",
"time_submit_desc", "time_submit_asc",
"time_run_desc", "time_run_asc",
"time_run_exit_desc", "time_run_exit_asc",
"duration_queue_desc", "duration_queue_asc",
"duration_run_desc", "duration_run_asc",
"duration_queue_run_desc", "duration_queue_run_asc"
per_page -- Number of entries to display per page (defualt=32)
no_fuzzy_time -- Don't display fuzzy time if this is True.
form -- Specify return format. If None, display HTML page. If "json",
return a JSON data structure.
"""
conf = ResourceLocator.default().get_conf()
per_page_default = int(
conf.get_value(["rose-bush", "jobs-per-page"], self.JOBS_PER_PAGE))
per_page_max = int(
conf.get_value(["rose-bush", "jobs-per-page-max"],
self.JOBS_PER_PAGE_MAX))
if not isinstance(per_page, int):
if per_page:
per_page = int(per_page)
else:
per_page = per_page_default
is_option_on = (cycles or tasks or task_status or job_status
or order is not None and order != "time_desc"
or per_page != per_page_default)
if page and per_page:
page = int(page)
else:
page = 1
task_statuses = ([[item, ""] for item in self.bush_dao.TASK_STATUSES])
if task_status:
if not isinstance(task_status, list):
task_status = [task_status]
for item in task_statuses:
if not task_status or item[0] in task_status:
item[1] = "1"
all_task_statuses = all([status[1] == "1" for status in task_statuses])
if all_task_statuses:
task_status = []
data = {
"cycles": cycles,
"host": self.host_name,
"is_option_on": is_option_on,
"logo": self.logo,
"method": "taskjobs",
"no_fuzzy_time": no_fuzzy_time,
"all_task_statuses": all_task_statuses,
"task_statuses": task_statuses,
"job_status": job_status,
"order": order,
"page": page,
"per_page": per_page,
"per_page_default": per_page_default,
"per_page_max": per_page_max,
"rose_version": self.rose_version,
"script": cherrypy.request.script_name,
"states": {},
"suite": suite,
"tasks": tasks,
"task_status_groups": self.bush_dao.TASK_STATUS_GROUPS,
"title": self.title,
"user": user,
}
if cycles:
cycles = shlex.split(str(cycles))
if tasks:
tasks = shlex.split(str(tasks))
data.update(self._get_suite_logs_info(user, suite))
data["states"].update(
self.bush_dao.get_suite_state_summary(user, suite))
data["states"]["last_activity_time"] = (self.get_last_activity_time(
user, suite))
entries, of_n_entries = self.bush_dao.get_suite_job_entries(
user, suite, cycles, tasks, task_status, job_status, order,
per_page, (page - 1) * per_page)
data["entries"] = entries
data["of_n_entries"] = of_n_entries
if per_page:
data["n_pages"] = of_n_entries / per_page
if of_n_entries % per_page != 0:
data["n_pages"] += 1
else:
data["n_pages"] = 1
data["time"] = strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
if form == "json":
return json.dumps(data)
try:
return self.template_env.get_template("taskjobs.html").render(
**data)
except jinja2.TemplateError:
traceback.print_exc()
def jobs(self, user, suite, page=1, cycles=None, tasks=None,
no_status=None, order=None, per_page=None, no_fuzzy_time="0",
form=None):
"""List jobs of a running or completed suite.
user -- A string containing a valid user ID
suite -- A string containing a valid suite ID
page -- The page number to display
cycles -- Display only task jobs matching these cycles. A value in the
list can be a cycle, the string "before|after CYCLE", or a
glob to match cycles.
tasks -- Display only jobs for task names matching a list of names.
The list should be specified as a string which will be
shlex.split by this method. Values can be a valid task name or
a glob like pattern for matching valid task names.
no_status -- Do not display jobs of tasks matching these statuses.
The values in the list should be "active", "success" or
"fail".
order -- Order search in a predetermined way. A valid value is one of
"time_desc", "time_asc",
"cycle_desc_name_desc", "cycle_desc_name_asc",
"cycle_asc_name_desc", "cycle_asc_name_asc",
"name_asc_cycle_asc", "name_desc_cycle_asc",
"name_asc_cycle_desc", "name_desc_cycle_desc",
"time_submit_desc", "time_submit_asc",
"time_run_desc", "time_run_asc",
"time_run_exit_desc", "time_run_exit_asc",
"duration_queue_desc", "duration_queue_asc",
"duration_run_desc", "duration_run_asc",
"duration_queue_run_desc", "duration_queue_run_asc"
per_page -- Number of entries to display per page (defualt=32)
no_fuzzy_time -- Don't display fuzzy time if this is True.
form -- Specify return format. If None, display HTML page. If "json",
return a JSON data structure.
"""
user_suite_dir = self._get_user_suite_dir(user, suite)
conf = ResourceLocator.default().get_conf()
per_page_default = int(conf.get_value(
["rose-bush", "jobs-per-page"], self.JOBS_PER_PAGE))
per_page_max = int(conf.get_value(
["rose-bush", "jobs-per-page-max"], self.JOBS_PER_PAGE_MAX))
is_option_on = (
cycles or
tasks or
no_status is not None or
order is not None and order != "time_desc" or
per_page is not None and per_page != per_page_default
)
if not isinstance(per_page, int):
if per_page:
per_page = int(per_page)
else:
per_page = per_page_default
if page and per_page:
page = int(page)
else:
page = 1
no_statuses = no_status
if no_status and not isinstance(no_status, list):
no_statuses = [no_status]
data = {
"cycles": cycles,
"host": self.host_name,
"is_option_on": is_option_on,
"logo": self.logo,
"method": "jobs",
"no_fuzzy_time": no_fuzzy_time,
"no_statuses": no_statuses,
"order": order,
"page": page,
"per_page": per_page,
"per_page_default": per_page_default,
"per_page_max": per_page_max,
"rose_version": self.rose_version,
"script": cherrypy.request.script_name,
"states": {},
"suite": suite,
"tasks": tasks,
"title": self.title,
"user": user,
}
# TODO: add paths to other suite files
if cycles:
cycles = shlex.split(str(cycles))
if tasks:
tasks = shlex.split(str(tasks))
data.update(self._get_suite_logs_info(user, suite))
data["states"].update(
self.suite_engine_proc.get_suite_state_summary(user, suite))
entries, of_n_entries = self.suite_engine_proc.get_suite_job_events(
user, suite, cycles, tasks, no_statuses, order, per_page,
(page - 1) * per_page)
data["entries"] = entries
data["of_n_entries"] = of_n_entries
if per_page:
data["n_pages"] = of_n_entries / per_page
if of_n_entries % per_page != 0:
data["n_pages"] += 1
else:
data["n_pages"] = 1
data["time"] = strftime("%Y-%m-%dT%H:%M:%S+0000", gmtime())
if form == "json":
return simplejson.dumps(data)
try:
template = self.template_env.get_template("jobs.html")
return template.render(**data)
except Exception as exc:
traceback.print_exc(exc)
templates_path = ['_templates']
# The suffix of source filenames.
source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = 'Rose Documentation'
copyright = (': British Crown Copyright 2012-8 Met Office. See Terms of Use. '
'This document is released under the Open Government Licence')
# The full version for the project you're documenting, acts as replacement for
# |version|.
release = ResourceLocator().get_version(ignore_environment=True)
# The short X.Y version, acts as replacement for |release|.
version = release
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = []
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'autumn'
# -- Options for HTML output ----------------------------------------------
html_theme = 'sphinx_rtd_theme'
# rtd_theme only handles 4 levels for the sidebar navigation.
def get_prepend_paths(event_handler=None,
path_root=None,
path_glob_args=[],
full_mode=False):
"""Return map of PATH-like env-var names to path lists to prepend to them.
event_handler -- An instance of rose.reporter.Reporter or an object with a
similar interface.
path_root -- If a glob is relative and this is defined, this is the root
directory of the relative path.
path_glob_args -- A list of strings in the form GLOB or NAME=GLOB. NAME is
"PATH" by default or should be PATH-like environment
variable name. GLOB should be a glob pattern for matching
file system paths to prepend to NAME.
full_mode -- If True, prepend relevant paths in site/user configuration and
the setting defined in "rose.task_env.PATH_GLOBS".
Return something like:
{"PATH": ["/opt/foo/bin", "/opt/bar/bin"],
# ... and so on
}
"""
prepend_paths_map = {}
# site/user configuration
if full_mode:
conf = ResourceLocator.default().get_conf()
my_conf = conf.get(["rose-task-run"], no_ignore=True)
if my_conf is not None:
for key, node in sorted(my_conf.value.items()):
if not key.startswith("path-prepend") or node.is_ignored():
continue
env_key = "PATH"
if key != "path-prepend":
env_key = key[len("path-prepend."):]
values = []
for v in node.value.split():
if os.path.exists(v):
values.append(v)
if values:
prepend_paths_map[env_key] = values
# Default or specified globs
path_globs_map = {}
if full_mode:
for name, path_globs in PATH_GLOBS.items():
path_globs_map[name] = path_globs
for path_glob_arg in path_glob_args:
if path_glob_arg is None:
continue
if "=" in path_glob_arg:
name, value = path_glob_arg.split("=", 1)
else:
name, value = "PATH", path_glob_arg
if name not in path_globs_map:
path_globs_map[name] = []
path_globs_map[name].append(value)
more_prepend_paths_map = {}
if not path_root:
path_root = os.getcwd()
for name, path_globs in path_globs_map.items():
if name not in more_prepend_paths_map:
more_prepend_paths_map[name] = []
for path_glob in path_globs:
if path_glob:
if path_glob.startswith("~"):
path_glob = os.path.expanduser(path_glob)
if not os.path.isabs(path_glob):
path_glob = os.path.join(path_root, path_glob)
for path in sorted(glob(path_glob)):
more_prepend_paths_map[name].append(path)
else:
more_prepend_paths_map[name] = [] # empty value resets
for name, more_prepend_paths in more_prepend_paths_map.items():
if name in prepend_paths_map:
prepend_paths_map[name].extend(more_prepend_paths)
elif more_prepend_paths:
prepend_paths_map[name] = more_prepend_paths
for key, prepend_paths in prepend_paths_map.items():
prepend_paths.reverse()
return prepend_paths_map
def _notify_trunk_changes(self, changeset_attribs, branch_attribs):
"""Email owner and/or access-list users on changes to trunk."""
# Notify only if users' email addresses can be determined
conf = ResourceLocator.default().get_conf()
user_tool_name = conf.get_value(["rosa-svn", "user-tool"])
if not user_tool_name:
return
notify_who_str = conf.get_value(
["rosa-svn", "notify-who-on-trunk-commit"], "")
if not notify_who_str.strip():
return
notify_who = shlex.split(notify_who_str)
# Build the message text
info_file_path = "%s/trunk/%s" % (
"/".join(branch_attribs["sid"]), self.INFO_FILE)
text = ""
for changed_line in branch_attribs["changed_lines"]:
text += changed_line
# For suite info file change, add diff as well
if (changed_line[4:].strip() == info_file_path and
branch_attribs["status_info_file"] == self.ST_MODIFIED):
old_strio = StringIO()
rose.config.dump(branch_attribs["old_info"], old_strio)
new_strio = StringIO()
rose.config.dump(branch_attribs["info"], new_strio)
for diff_line in unified_diff(
old_strio.getvalue().splitlines(True),
new_strio.getvalue().splitlines(True),
"@%d" % (int(changeset_attribs["revision"]) - 1),
"@%d" % (int(changeset_attribs["revision"]))):
text += " " * 4 + diff_line
# Determine who to notify
users = set()
for key in ["old_info", "info"]:
if branch_attribs[key] is not None:
info_conf = branch_attribs[key]
if "owner" in notify_who:
users.add(info_conf.get_value(["owner"]))
if "access-list" in notify_who:
users.update(
info_conf.get_value(["access-list"], "").split())
users.discard("*")
# Determine email addresses
user_tool = self.usertools_manager.get_handler(user_tool_name)
if "author" in notify_who:
users.add(changeset_attribs["author"])
else:
users.discard(changeset_attribs["author"])
emails = sorted(user_tool.get_emails(users))
if not emails:
return
# Send notification
msg = MIMEText(text)
msg.set_charset("utf-8")
msg["From"] = conf.get_value(
["rosa-svn", "notification-from"],
"notications@" + socket.getfqdn())
msg["To"] = ", ".join(emails)
msg["Subject"] = "%s-%s/trunk@%d" % (
changeset_attribs["prefix"],
branch_attribs["sid"],
int(changeset_attribs["revision"]))
smtp_host = conf.get_value(
["rosa-svn", "smtp-host"], default="localhost")
smtp = SMTP(smtp_host)
smtp.sendmail(msg["From"], emails, msg.as_string())
smtp.quit()
def expand(self, names=None, rank_method=None, thresholds=None):
"""Expand each name in names, and look up rank method for each name.
names, if specified, should be a list of host names or known groups in
the site / user configuration file. Otherwise, the default setting in
the site / user configuration file will be used.
rank_method, if specified, should be the name of a supported ranking
method. If not specified, use the default specified for a host group.
If the default differs in hosts, use "load:15".
"""
conf = ResourceLocator.default().get_conf()
if not names:
node = conf.get(["rose-host-select", "default"],
no_ignore=True)
if node:
names = [node.value]
else:
raise NoHostError()
host_names = []
rank_method_set = set()
thresholds_set = set()
while names:
name = names.pop()
key = "group{" + name + "}"
value = conf.get_value(["rose-host-select", key])
if value is None:
host_names.append(name)
else:
for val in value.split():
names.append(val)
if rank_method is None:
key = "method{" + name + "}"
method_str = conf.get_value(["rose-host-select", key])
if method_str is None:
rank_method_set.add(self.RANK_METHOD_DEFAULT)
else:
rank_method_set.add(method_str)
if thresholds is None:
key = "thresholds{" + name + "}"
threshold_str = conf.get_value(["rose-host-select", key])
if threshold_str is None:
thresholds_set.add(())
else:
thresholds_set.add(
tuple(sorted(shlex.split(threshold_str))))
# If default rank method differs in hosts, use load:15.
if rank_method is None:
if len(rank_method_set) == 1:
rank_method = rank_method_set.pop()
else:
rank_method = self.RANK_METHOD_DEFAULT
if thresholds is None:
if len(thresholds_set) == 1:
thresholds = thresholds_set.pop()
return host_names, rank_method, thresholds
def view(self, user, suite, path, path_in_tar=None, mode=None):
"""View a text log file."""
f_name = self._get_user_suite_dir(user, suite, path)
conf = ResourceLocator.default().get_conf()
view_size_max = int(conf.get_value(
["rose-bush", "view-size-max"], self.VIEW_SIZE_MAX))
if path_in_tar:
tar_f = tarfile.open(f_name, 'r:gz')
try:
tar_info = tar_f.getmember(path_in_tar)
except KeyError:
raise cherrypy.HTTPError(404)
f_size = tar_info.size
f = tar_f.extractfile(path_in_tar)
if f.read(2) == "#!":
mime = MIME_TEXT_PLAIN
else:
mime = mimetypes.guess_type(
urllib.pathname2url(path_in_tar))[0]
f.seek(0)
if (mode == "download" or
f_size > view_size_max or
mime and
(not mime.startswith("text/") or mime.endswith("html"))):
t = NamedTemporaryFile()
f_bsize = os.fstatvfs(t.fileno()).f_bsize
while True:
bytes = f.read(f_bsize)
if not bytes:
break
t.write(bytes)
cherrypy.response.headers["Content-Type"] = mime
try:
return cherrypy.lib.static.serve_file(t.name, mime)
finally:
t.close()
s = f.read()
f.close()
else:
f_size = os.stat(f_name).st_size
if open(f_name).read(2) == "#!":
mime = MIME_TEXT_PLAIN
else:
mime = mimetypes.guess_type(urllib.pathname2url(f_name))[0]
if (mode == "download" or
f_size > view_size_max or
mime and
(not mime.startswith("text/") or mime.endswith("html"))):
cherrypy.response.headers["Content-Type"] = mime
return cherrypy.lib.static.serve_file(f_name, mime)
s = open(f_name).read()
if mode == "text":
s = jinja2.escape(s)
try:
lines = [unicode(line) for line in s.splitlines()]
except UnicodeDecodeError:
return cherrypy.lib.static.serve_file(f_name, MIME_TEXT_PLAIN)
name = path
if path_in_tar:
name = path_in_tar
job_entry = None
if name.startswith("log/job/"):
names = self.suite_engine_proc.parse_job_log_rel_path(name)
if len(names) == 4:
cycle, task, submit_num, ext = names
entries = self.suite_engine_proc.get_suite_job_events(
user, suite, [cycle], [task], None, None, None, None)[0]
for entry in entries:
if entry["submit_num"] == int(submit_num):
job_entry = entry
break
if fnmatch(os.path.basename(path), "rose*.conf"):
file_content = "rose-conf"
else:
file_content = self.suite_engine_proc.is_conf(path)
template = self.template_env.get_template("view.html")
data = {}
data.update(self._get_suite_logs_info(user, suite))
return template.render(
rose_version=self.rose_version,
script=cherrypy.request.script_name,
method="view",
time=strftime("%Y-%m-%dT%H:%M:%S+0000", gmtime()),
logo=self.logo,
title=self.title,
host=self.host_name,
user=user,
suite=suite,
path=path,
path_in_tar=path_in_tar,
f_name=f_name,
mode=mode,
file_content=file_content,
lines=lines,
entry=job_entry,
**data)
def run(self, app_runner, conf_tree, opts, args, uuid, work_files):
"""Implement the "rose ana" command"""
# Initialise properties which will be needed later.
self._task = app_runner.suite_engine_proc.get_task_props()
self.task_name = self._task.task_name
self.opts = opts
self.args = args
self.config = conf_tree.node
self.app_runner = app_runner
# Attach to the main rose config (for retrieving settings from things
# like the user's ~/.metomi/rose.conf)
self.rose_conf = ResourceLocator.default().get_conf()
# Attach to a reporter instance for sending messages.
self._init_reporter(app_runner.event_handler)
# As part of the introduction of a re-written rose_ana, backwards
# compatibility is maintained here by detecting the lack of the
# newer syntax in the app config and falling back to the old version
# of the rose_ana app (renamed to rose_ana_v1)
# **Once the old behaviour is removed the below block can be too**.
new_style_app = False
for keys, node in self.config.walk(no_ignore=True):
task = keys[0]
if task.startswith("ana:"):
new_style_app = True
break
if not new_style_app:
# Use the previous app by instantiating and calling it explicitly
self.reporter("!!WARNING!! - Detected old style rose_ana app; "
"Using previous rose_ana version...")
from rose.apps.rose_ana_v1 import RoseAnaV1App
old_app = RoseAnaV1App(manager=self.manager)
return old_app.run(app_runner, conf_tree, opts, args, uuid,
work_files)
# Load any rose_ana specific configuration settings either from the
# site defaults or the user's personal config
self._get_global_ana_config()
# If the user's config indicates that it should be used - attach
# to the KGO database instance in case it is needed later.
use_kgo = self.ana_config.get("kgo-database", ".false.")
self.kgo_db = None
if use_kgo == TYPE_LOGICAL_VALUE_TRUE:
self.kgo_db = KGODatabase()
self.kgo_db.enter_task(self.task_name,
self.kgo_db.TASK_STATUS_RUNNING)
self.titlebar("Initialising KGO database")
self.kgo_db.buffer_to_db(self.reporter)
self.titlebar("Launching rose_ana")
# Load available methods for analysis and the tasks in the app.
self._load_analysis_modules()
self._load_analysis_methods()
self._load_tasks()
number_of_failures = 0
number_of_skips = 0
task_error = False
summary_status = []
for itask, task in enumerate(self.analysis_tasks):
# Report the name of the task and a banner line to aid readability.
self.titlebar("Running task #{0}".format(itask + 1))
self.reporter("Method: {0}".format(task.options["full_task_name"]))
# Since the run_analysis method is out of rose's control in many
# cases the safest thing to do is a blanket try/except; since we
# have no way of knowing what exceptions might be raised.
try:
task.run_analysis()
# In the case that the task didn't raise any exception,
# we can now check whether it passed or failed.
if task.passed:
msg = "Task #{0} passed".format(itask + 1)
summary_status.append(
("{0} ({1})".format(msg,
task.options["full_task_name"]),
self._prefix_pass))
self.reporter(msg, prefix=self._prefix_pass)
elif task.skipped:
number_of_skips += 1
msg = "Task #{0} skipped by method".format(itask + 1)
summary_status.append(
("{0} ({1})".format(msg,
task.options["full_task_name"]),
self._prefix_skip))
self.reporter(msg, prefix=self._prefix_skip)
else:
number_of_failures += 1
msg = "Task #{0} did not pass".format(itask + 1)
summary_status.append(
("{0} ({1})".format(msg,
task.options["full_task_name"]),
self._prefix_fail))
self.reporter(msg, prefix=self._prefix_fail)
except Exception as err:
# If an exception was raised, print a traceback and treat it
# as a failure.
task_error = True
number_of_failures += 1
msg = "Task #{0} encountered an error".format(itask + 1)
summary_status.append(
("{0} ({1})".format(msg, task.options["full_task_name"]),
self._prefix_fail))
self.reporter(msg + " (see stderr)", prefix=self._prefix_fail)
exception = traceback.format_exc()
self.reporter(msg,
prefix=self._prefix_fail,
kind=self.reporter.KIND_ERR)
self.reporter(exception,
prefix=self._prefix_fail,
kind=self.reporter.KIND_ERR)
# The KGO database (if needed by the task) also stores its status - to
# indicate whether there was some unexpected exception above.
if self.kgo_db is not None and not task_error:
self.kgo_db.enter_task(self.task_name,
self.kgo_db.TASK_STATUS_SUCCEEDED)
self.titlebar("Updating KGO database")
self.kgo_db.buffer_to_db(self.reporter)
# Summarise the results of the tasks
self.titlebar("Summary")
for line, prefix in summary_status:
self.reporter(line, prefix=prefix)
# And a final 1-line summary
total = len(summary_status)
plural = {1: ""}
prefix = self._prefix_pass
passed = total - number_of_failures - number_of_skips
msg = "{0} Task{1} Passed".format(passed, plural.get(passed, "s"))
if number_of_failures > 0:
msg += ", {0} Task{1} Failed".format(
number_of_failures, plural.get(number_of_failures, "s"))
prefix = self._prefix_fail
if number_of_skips > 0:
msg += ", {0} Task{1} Skipped".format(
number_of_skips, plural.get(number_of_skips, "s"))
msg += " (of {0} processed)".format(total)
self.titlebar("Final status")
self.reporter(msg, prefix=prefix)
self.titlebar("Completed rose_ana")
# Finally if there were legitimate test failures raise an exception
# so that the task is caught by cylc as failed. Also fail if it looks
# like every single task has been skipped
if number_of_failures > 0 or number_of_skips == total:
raise TestsFailedException(number_of_failures)
def _prompt(self, is_retry=False):
"""Prompt for the username and password, where necessary.
Prompt with zenity or raw_input/getpass.
"""
if (callable(self.prompt_func)
and not hasattr(self.password_store, "prompt_password")):
self.username, self.password = self.prompt_func(
self.username, self.password, is_retry)
return
icon_path = ResourceLocator.default().locate("images/rosie-icon.png")
if is_retry:
username = ""
prompt = self.PROMPT_USERNAME % {
"prefix": self.prefix,
"root": self.root
}
if self.popen.which("zenity") and os.getenv("DISPLAY"):
username = self.popen.run("zenity", "--entry", "--title=Rosie",
"--window-icon=" + icon_path,
"--text=" + prompt)[1].strip()
else:
username = raw_input(prompt)
if not username:
raise KeyboardInterrupt(self.STR_CANCELLED)
if username and username != self.username:
self.username = username
self._load_password()
if self.password:
return
if self.username and self.password is None or is_retry:
prompt = self.PROMPT_PASSWORD % {
"prefix": self.prefix,
"root": self.root,
"username": self.username
}
password = None
need_prompting = True
if hasattr(self.password_store, "prompt_password"):
try:
password = self.password_store.prompt_password(
prompt, self.scheme, self.host, self.username)
except RosieStoreRetrievalError as exc:
self.event_handler(exc)
else:
need_prompting = False
if not password and need_prompting:
if self.popen.which("zenity") and os.getenv("DISPLAY"):
password = self.popen.run("zenity", "--entry",
"--hide-text", "--title=Rosie",
"--window-icon=" + icon_path,
"--text=" + prompt)[1].strip()
else:
password = getpass(prompt)
if not password:
raise KeyboardInterrupt(self.STR_CANCELLED)
if password and password != self.password:
self.password = password
def generate_info_config(self, from_id=None, prefix=None):
"""Generate a rose.config.ConfigNode for a rose-suite.info.
This is suitable for passing into the create method of this
class.
If from_id is defined, copy items from it.
Return the rose.config.ConfigNode instance.
"""
from_project = None
from_title = None
if from_id is not None:
from_info_url = "%s/%s/rose-suite.info@%s" % (from_id.to_origin(),
from_id.branch,
from_id.revision)
out_data = self.popen("svn", "cat", from_info_url)[0]
from_config = rose.config.load(StringIO(out_data))
if from_config.get(["project"]) is not None:
from_project = from_config.get(["project"]).value
if from_config.get(["title"]) is not None:
from_title = from_config.get(["title"]).value
res_loc = ResourceLocator.default()
info_config = rose.config.ConfigNode()
# Determine prefix
if from_id is not None:
prefix = from_id.prefix
elif prefix is None:
prefix = SuiteId.get_prefix_default()
# Determine owner:
# 1. From user configuration [rosie-id]prefix-username
# 2. From username of a matching group in [groups] in
# ~/.subversion/servers
# 3. Current user ID
owner = res_loc.get_conf().get_value(
["rosie-id", "prefix-username." + prefix])
if not owner and self.subversion_servers_conf:
servers_conf = rose.config.load(self.subversion_servers_conf)
groups_node = servers_conf.get(["groups"])
if groups_node is not None:
group = None
prefix_loc = SuiteId.get_prefix_location(prefix)
prefix_host = urlparse(prefix_loc).hostname
for key, node in groups_node.value.items():
if fnmatch(prefix_host, node.value):
owner = servers_conf.get_value([key, "username"])
break
if not owner:
owner = pwd.getpwuid(os.getuid())[0]
info_config.set(["owner"], owner)
# Determine project and title
if from_project:
info_config.set(["project"], from_project)
else:
info_config.set(["project"], "")
if from_title:
info_config.set(["title"], "Copy of %s: %s" %
(from_id.to_string_with_version(), from_title))
else:
info_config.set(["title"], "")
# Determine access list
access_list_str = res_loc.get_conf().get_value(
["rosie-vc", "access-list-default"])
if access_list_str:
info_config.set(["access-list"], access_list_str)
return info_config
def start(is_main=False):
"""Create the server.
If is_main, invoke cherrypy.quickstart.
Otherwise, return a cherrypy.Application instance.
"""
# Environment variables (not normally defined in WSGI mode)
if os.getenv("ROSE_HOME") is None:
path = os.path.abspath(__file__)
while os.path.dirname(path) != path: # not root
if os.path.basename(path) == "lib":
os.environ["ROSE_HOME"] = os.path.dirname(path)
break
path = os.path.dirname(path)
for key, value in [("ROSE_NS", "rosa"), ("ROSE_UTIL", "ws")]:
if os.getenv(key) is None:
os.environ[key] = value
# CherryPy quick server configuration
rose_conf = ResourceLocator.default().get_conf()
if is_main and rose_conf.get_value(["rosie-ws", "log-dir"]) is not None:
log_dir_value = rose_conf.get_value(["rosie-ws", "log-dir"])
log_dir = env_var_process(os.path.expanduser(log_dir_value))
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
log_file = os.path.join(log_dir, "server.log")
log_error_file = os.path.join(log_dir, "server.err.log")
cherrypy.config["log.error_file"] = log_error_file
cherrypy.config["log.access_file"] = log_file
cherrypy.config["request.error_response"] = _handle_error
cherrypy.config["log.screen"] = False
# Configuration for dynamic pages
db_url_map = {}
for key, node in rose_conf.get(["rosie-db"]).value.items():
if key.startswith("db.") and key[3:]:
db_url_map[key[3:]] = node.value
res_loc = ResourceLocator.default()
html_lib = res_loc.get_util_home("lib", "html")
icon_path = res_loc.locate("images/rosie-icon-trim.png")
tmpl_loader = jinja2.FileSystemLoader(os.path.join(html_lib, "rosie-ws"))
root = Root(jinja2.Environment(loader=tmpl_loader), db_url_map)
# Configuration for static pages
config = {
"/etc": {
"tools.staticdir.dir": os.path.join(html_lib, "external"),
"tools.staticdir.on": True
},
"/favicon.ico": {
"tools.staticfile.on": True,
"tools.staticfile.filename": icon_path
}
}
if is_main:
port = int(rose_conf.get_value(["rosie-ws", "port"], 8080))
config.update({
"global": {
"server.socket_host": "0.0.0.0",
"server.socket_port": port
}
})
# Start server or return WSGI application
if is_main:
return cherrypy.quickstart(root, "/", config=config)
else:
return cherrypy.Application(root, script_name=None, config=config)