def __init__(self, *args, **kwargs): self.exposed = True self.props = {} rose_conf = ResourceLocator.default().get_conf() self.props["title"] = rose_conf.get_value(["rosie-disco", "title"], self.TITLE) self.props["host_name"] = rose_conf.get_value(["rosie-disco", "host"]) if self.props["host_name"] is None: self.props["host_name"] = HostSelector().get_local_host() if self.props["host_name"] and "." in self.props["host_name"]: self.props["host_name"] = (self.props["host_name"].split( ".", 1)[0]) self.props["rose_version"] = ResourceLocator.default().get_version() self.props["template_env"] = jinja2.Environment( loader=jinja2.FileSystemLoader(ResourceLocator.default( ).get_util_home("lib", "html", "template", "rosie-disco"))) db_url_map = {} for key, node in rose_conf.get(["rosie-db"]).value.items(): if key.startswith("db.") and key[3:]: db_url_map[key[3:]] = node.value self.db_url_map = db_url_map if not self.db_url_map: self.db_url_map = {} for key, db_url in self.db_url_map.items(): setattr(self, key, RosieDiscoService(self.props, key, db_url))
def __init__(self, *args, **kwargs): self.exposed = True self.props = {} rose_conf = ResourceLocator.default().get_conf() self.props["title"] = rose_conf.get_value( ["rosie-disco", "title"], self.TITLE) self.props["host_name"] = rose_conf.get_value(["rosie-disco", "host"]) if self.props["host_name"] is None: self.props["host_name"] = HostSelector().get_local_host() if self.props["host_name"] and "." in self.props["host_name"]: self.props["host_name"] = ( self.props["host_name"].split(".", 1)[0]) self.props["rose_version"] = ResourceLocator.default().get_version() self.props["template_env"] = jinja2.Environment( loader=jinja2.FileSystemLoader( ResourceLocator.default().get_util_home( "lib", "html", "template", "rosie-disco"))) db_url_map = {} for key, node in rose_conf.get(["rosie-db"]).value.items(): if key.startswith("db.") and key[3:]: db_url_map[key[3:]] = node.value self.db_url_map = db_url_map if not self.db_url_map: self.db_url_map = {} for key, db_url in self.db_url_map.items(): setattr(self, key, RosieDiscoService(self.props, key, db_url))
def start(is_main=False): """Create the server. If is_main, invoke cherrypy.quickstart. Otherwise, return a cherrypy.Application instance. """ # Environment variables (not normally defined in WSGI mode) if os.getenv("ROSE_HOME") is None: path = os.path.abspath(__file__) while os.path.dirname(path) != path: # not root if os.path.basename(path) == "lib": os.environ["ROSE_HOME"] = os.path.dirname(path) break path = os.path.dirname(path) for key, value in [("ROSE_NS", "rosa"), ("ROSE_UTIL", "ws")]: if os.getenv(key) is None: os.environ[key] = value # CherryPy quick server configuration rose_conf = ResourceLocator.default().get_conf() if is_main and rose_conf.get_value(["rosie-ws", "log-dir"]) is not None: log_dir_value = rose_conf.get_value(["rosie-ws", "log-dir"]) log_dir = env_var_process(os.path.expanduser(log_dir_value)) if not os.path.isdir(log_dir): os.makedirs(log_dir) log_file = os.path.join(log_dir, "server.log") log_error_file = os.path.join(log_dir, "server.err.log") cherrypy.config["log.error_file"] = log_error_file cherrypy.config["log.access_file"] = log_file cherrypy.config["request.error_response"] = _handle_error cherrypy.config["log.screen"] = False # Configuration for dynamic pages db_url_map = {} for key, node in rose_conf.get(["rosie-db"]).value.items(): if key.startswith("db.") and key[3:]: db_url_map[key[3:]] = node.value res_loc = ResourceLocator.default() html_lib = res_loc.get_util_home("lib", "html") icon_path = res_loc.locate("images/rosie-icon-trim.png") tmpl_loader = jinja2.FileSystemLoader(os.path.join(html_lib, "rosie-ws")) root = Root(jinja2.Environment(loader=tmpl_loader), db_url_map) # Configuration for static pages config = {"/etc": { "tools.staticdir.dir": os.path.join(html_lib, "external"), "tools.staticdir.on": True}, "/favicon.ico": { "tools.staticfile.on": True, "tools.staticfile.filename": icon_path}} if is_main: port = int(rose_conf.get_value(["rosie-ws", "port"], 8080)) config.update({"global": {"server.socket_host": "0.0.0.0", "server.socket_port": port}}) # Start server or return WSGI application if is_main: return cherrypy.quickstart(root, "/", config=config) else: return cherrypy.Application(root, script_name=None, config=config)
def main(): """rosa db-create.""" db_conf = ResourceLocator.default().get_conf().get(["rosie-db"]) if db_conf is not None: opts = RoseOptionParser().parse_args()[0] reporter = Reporter(opts.verbosity - opts.quietness) init = RosieDatabaseInitiator(event_handler=reporter) conf = ResourceLocator.default().get_conf() for key in db_conf.value: if key.startswith("db."): prefix = key.replace("db.", "", 1) db_url = conf.get_value(["rosie-db", "db." + prefix]) repos_path = conf.get_value(["rosie-db", "repos." + prefix]) init(db_url, repos_path)
def _configure(service_cls): """Configure cherrypy and return a dict for the specified cherrypy app.""" # Environment variables (not normally defined in WSGI mode) if not os.getenv("ROSE_HOME"): path = os.path.abspath(__file__) while os.path.dirname(path) != path: # not root if os.path.basename(path) == "lib": os.environ["ROSE_HOME"] = os.path.dirname(path) break path = os.path.dirname(path) for key, value in (("ROSE_NS", service_cls.NS), ("ROSE_UTIL", service_cls.UTIL)): if os.getenv(key) is None: os.environ[key] = value # Configuration for HTML library cherrypy.config["tools.encode.on"] = True cherrypy.config["tools.encode.encoding"] = "utf-8" config = {} static_lib = ResourceLocator.default().get_util_home( "lib", "html", "static") for name in os.listdir(static_lib): path = os.path.join(static_lib, name) if os.path.isdir(path): path_key = "tools.staticdir.dir" bool_key = "tools.staticdir.on" else: path_key = "tools.staticfile.filename" bool_key = "tools.staticfile.on" config["/" + name] = {path_key: path, bool_key: True} if name == service_cls.NS + "-favicon.png": config["/favicon.ico"] = config["/" + name] return config
def get_suite_log_url(self, user_name, suite_name): """Return the "rose bush" URL for a user's suite.""" prefix = "~" if user_name: prefix += user_name suite_d = os.path.join(prefix, self.get_suite_dir_rel(suite_name)) suite_d = os.path.expanduser(suite_d) if not os.path.isdir(suite_d): raise NoSuiteLogError(user_name, suite_name) rose_bush_url = None for f_name in glob(os.path.expanduser("~/.metomi/rose-bush*.status")): status = {} for line in open(f_name): k, v = line.strip().split("=", 1) status[k] = v if status.get("host"): rose_bush_url = "http://" + status["host"] if status.get("port"): rose_bush_url += ":" + status["port"] rose_bush_url += "/" break if not rose_bush_url: conf = ResourceLocator.default().get_conf() rose_bush_url = conf.get_value(["rose-suite-log", "rose-bush"]) if not rose_bush_url: return "file://" + suite_d if not rose_bush_url.endswith("/"): rose_bush_url += "/" if not user_name: user_name = pwd.getpwuid(os.getuid()).pw_name return rose_bush_url + "/".join(["list", user_name, suite_name])
def get_prefix_default(cls): """Return the default prefix.""" config = ResourceLocator.default().get_conf() value = config.get_value(["rosie-id", "prefix-default"]) if value is None: raise SuiteIdPrefixError() return value
def __init__(self, prefixes=None, prompt_func=None, popen=None, event_handler=None): if not event_handler: event_handler = Reporter() if not popen: popen = RosePopener(event_handler=event_handler) self.event_handler = event_handler self.popen = popen self.prompt_func = prompt_func self.prefixes = [] self.unreachable_prefixes = [] self.auth_managers = {} conf = ResourceLocator.default().get_conf() conf_rosie_id = conf.get(["rosie-id"], no_ignore=True) if conf_rosie_id is None: raise RosieWSClientConfError() for key, node in conf_rosie_id.value.items(): if node.is_ignored() or not key.startswith("prefix-ws."): continue prefix = key.replace("prefix-ws.", "") self.auth_managers[prefix] = RosieWSClientAuthManager( prefix, popen=self.popen, prompt_func=self.prompt_func) if not prefixes: prefixes_str = conf_rosie_id.get_value(["prefixes-ws-default"]) if prefixes_str: prefixes = shlex.split(prefixes_str) else: prefixes = sorted(self.auth_managers.keys()) self.set_prefixes(prefixes)
def get_prefix_default(cls): """Return the default prefix.""" config = ResourceLocator.default().get_conf() value = config.get_value(["rosie-id", "prefix-default"]) if not value or not value.strip(): raise SuiteIdPrefixError() return shlex.split(value)[0]
def __init__(self, prefix, popen=None, prompt_func=None): self.prefix = prefix root = self._get_conf_value("ws") if root is None: raise UndefinedRosiePrefixWS(self.prefix) if not root.endswith("/"): root += "/" self.root = root urlparse_res = urlparse(self.root) self.scheme = urlparse_res[0] self.host = urlparse_res[1] self.password_orig = None self.username_orig = None self.password = None self.username = None if popen is None: popen = RosePopener() self.popen = popen self.prompt_func = prompt_func res_loc = ResourceLocator.default() password_stores_str = res_loc.default().get_conf().get_value( keys=["rosie-id", "prefix-password-store." + self.prefix], default=self.PASSWORD_STORES_STR) for password_store_name in shlex.split(password_stores_str): password_store_cls = self.PASSWORD_STORE_CLASSES.get( password_store_name) if password_store_cls is not None and password_store_cls.usable(): self.password_store = password_store_cls() break else: self.password_store = None self.requests_kwargs = {} self._init_https_params()
def _search(cls, users, attr_idx): """Search LDAP directory for the indexed attr for users. Attr index can be UID_IDX, CN_IDX or MAIL_IDX. Return a list containing the results. """ conf = ResourceLocator.default().get_conf() uri = conf.get_value(["rosa-ldap", "uri"]) binddn = conf.get_value(["rosa-ldap", "binddn"]) passwd = "" passwd_file = conf.get_value(["rosa-ldap", "password-file"], cls.PASSWD_FILE) if passwd_file: passwd = open(os.path.expanduser(passwd_file)).read().strip() basedn = conf.get_value(["rosa-ldap", "basedn"], "") filter_str = "(|(uid=" + ")(uid=".join(users) + "))" filter_more_str = conf.get_value(["rosa-ldap", "filter-more"], "") if filter_more_str: filter_str = "(&" + filter_str + filter_more_str + ")" user_attr_str = conf.get_value(["rosa-ldap", "attrs"], cls.USER_ATTRS) attr = user_attr_str.split()[attr_idx] tls_ca_file = conf.get_value(["rosa-ldap", "tls-ca-file"]) if tls_ca_file: ldap.set_option(ldap.OPT_X_TLS_CACERTFILE, tls_ca_file) conn = ldap.initialize(uri) conn.bind_s(binddn, passwd) results = conn.search_s(basedn, ldap.SCOPE_SUBTREE, filter_str, [attr]) conn.unbind() return [result[1][attr][0] for result in results]
def get_suite_log_url(self, user_name, suite_name): """Return the "rose bush" URL for a user's suite.""" prefix = "~" if user_name: prefix += user_name suite_d = os.path.join(prefix, self.get_suite_dir_rel(suite_name)) suite_d = os.path.expanduser(suite_d) if not os.path.isdir(suite_d): raise NoSuiteLogError(user_name, suite_name) rose_bush_url = None for f_name in glob(os.path.expanduser("~/.metomi/rose-bush*.status")): status = {} for line in open(f_name): key, value = line.strip().split("=", 1) status[key] = value if status.get("host"): rose_bush_url = "http://" + status["host"] if status.get("port"): rose_bush_url += ":" + status["port"] rose_bush_url += "/" break if not rose_bush_url: conf = ResourceLocator.default().get_conf() rose_bush_url = conf.get_value(["rose-suite-log", "rose-bush"]) if not rose_bush_url: return "file://" + suite_d if not rose_bush_url.endswith("/"): rose_bush_url += "/" if not user_name: user_name = pwd.getpwuid(os.getuid()).pw_name return rose_bush_url + "/".join(["taskjobs", user_name, suite_name])
def _get_hosts(self, suite_name, host): if host: hosts = [host] else: conf = ResourceLocator.default().get_conf() hosts = None known_hosts = self.host_selector.expand( conf.get_value(["rose-suite-run", "hosts"], "").split() + conf.get_value(["rose-suite-run", "scan-hosts"], "").split() + ["localhost"])[0] known_hosts = list(set(known_hosts)) if known_hosts: hosts = self.suite_engine_proc.ping(suite_name, known_hosts) if not hosts: # Try the "rose-suite.host" file in the suite log directory log = self.suite_engine_proc.get_suite_dir(suite_name, "log") try: host_file = os.path.join(log, "rose-suite-run.host") hosts = [open(host_file).read().strip()] except IOError: pass if not hosts: hosts = ["localhost"] return hosts
def expand(self, names=None, rank_method=None, thresholds=None): """Expand each name in names, and look up rank method for each name. names, if specified, should be a list of host names or known groups in the site / user configuration file. Otherwise, the default setting in the site / user configuration file will be used. rank_method, if specified, should be the name of a supported ranking method. If not specified, use the default specified for a host group. If the default differs in hosts, use "load:15". """ conf = ResourceLocator.default().get_conf() if not names: node = conf.get(["rose-host-select", "default"], no_ignore=True) if node: names = [node.value] else: raise NoHostError() host_names = [] rank_method_set = set() thresholds_set = set() while names: name = names.pop() key = "group{" + name + "}" value = conf.get_value(["rose-host-select", key]) if value is None: host_names.append(name) else: for v in value.split(): names.append(v) if rank_method is None: key = "method{" + name + "}" m = conf.get_value(["rose-host-select", key]) if m is None: rank_method_set.add(self.RANK_METHOD_DEFAULT) else: rank_method_set.add(m) if thresholds is None: key = "thresholds{" + name + "}" t = conf.get_value(["rose-host-select", key]) if t is None: thresholds_set.add(()) else: thresholds_set.add(tuple(sorted(shlex.split(t)))) # If default rank method differs in hosts, use load:15. if rank_method is None: if len(rank_method_set) == 1: rank_method = rank_method_set.pop() else: rank_method = self.RANK_METHOD_DEFAULT if thresholds is None: if len(thresholds_set) == 1: thresholds = thresholds_set.pop() return host_names, rank_method, thresholds
def __init__(self, *args, **kwargs): self.exposed = True self.suite_engine_proc = SuiteEngineProcessor.get_processor() rose_conf = ResourceLocator.default().get_conf() self.logo = rose_conf.get_value(["rose-bush", "logo"]) self.title = rose_conf.get_value(["rose-bush", "title"], self.TITLE) self.host_name = rose_conf.get_value(["rose-bush", "host"]) if self.host_name is None: self.host_name = HostSelector().get_local_host() if self.host_name and "." in self.host_name: self.host_name = self.host_name.split(".", 1)[0] self.rose_version = ResourceLocator.default().get_version() template_env = jinja2.Environment(loader=jinja2.FileSystemLoader( ResourceLocator.default().get_util_home( "lib", "html", "template", "rose-bush"))) self.template_env = template_env
def run( self, suite_name, task_id, hook_event, hook_message=None, should_mail=False, mail_cc_list=None, should_shutdown=False, ): """ Invoke the hook for a suite. 1. For a task hook, if the task runs remotely, retrieve its log from the remote host. 2. If "should_mail", send an email notification to the current user, and those in the "mail_cc_list". 3. If "should_shutdown", shut down the suite. """ # Retrieve log and generate code view task_ids = [] if task_id: task_ids = [task_id] self.suite_engine_proc.job_logs_pull_remote(suite_name, task_ids) # Send email notification if required if should_mail: text = "" if task_id: text += "Task: %s\n" % task_id if hook_message: text += "Message: %s\n" % hook_message url = self.suite_engine_proc.get_suite_log_url(None, suite_name) text += "See: %s\n" % (url) user = pwd.getpwuid(os.getuid()).pw_name conf = ResourceLocator.default().get_conf() host = conf.get_value(["rose-suite-hook", "email-host"], default="localhost") msg = MIMEText(text) msg["From"] = user + "@" + host msg["To"] = msg["From"] if mail_cc_list: mail_cc_addresses = [] for mail_cc_address in mail_cc_list: if "@" not in mail_cc_address: mail_cc_address += "@" + host mail_cc_addresses.append(mail_cc_address) msg["Cc"] = ", ".join(mail_cc_addresses) mail_cc_list = mail_cc_addresses else: mail_cc_list = [] msg["Subject"] = "[%s] %s" % (hook_event, suite_name) smtp_host = conf.get_value(["rose-suite-hook", "smtp-host"], default="localhost") smtp = SMTP(smtp_host) smtp.sendmail(msg["From"], [msg["To"]] + mail_cc_list, msg.as_string()) smtp.quit() # Shut down if required if should_shutdown: self.suite_engine_proc.shutdown(suite_name, args=["--kill"])
def load_override_config(sections, my_globals=None): if my_globals is None: my_globals = globals() for section in sections: conf = ResourceLocator.default().get_conf().get([section]) if conf is None: continue for key, node in conf.value.items(): if node.is_ignored(): continue try: cast_value = ast.literal_eval(node.value) except Exception: cast_value = node.value name = key.replace("-", "_").upper() orig_value = my_globals[name] if (type(orig_value) is not type(cast_value) and orig_value is not None): sys.stderr.write(_OVERRIDE_WARNING_TYPE.format( section, key, cast_value, type(orig_value), type(cast_value)) ) continue if name.startswith("_"): sys.stderr.write(_OVERRIDE_WARNING_PRIVATE.format( section, key, name) ) continue my_globals[name] = cast_value
def get_cmd(self, key, *args): """Return default options and arguments of a known command as a list. If a setting [external] <key> is defined in the site/user configuration, use the setting. Otherwise, if RosePopener.ENVS_OF_CMDS[key] exists, it looks for each environment variable in the list in RosePopener.ENVS_OF_CMDS[key] in order. If the environment variable is defined and is not a null string, use the value of the environment variable. Otherwise, return RosePopener.CMDS[key] key: must be a key of RosePopener.CMDS args: if specified, will be added to the returned list """ if key not in self.cmds: root_node = ResourceLocator.default().get_conf() node = root_node.get(["external", key], no_ignore=True) if node is not None: self.cmds[key] = shlex.split(node.value) if key not in self.cmds: for name in self.ENVS_OF_CMDS.get(key, []): if os.getenv(name): # not None, not null str self.cmds[key] = shlex.split(os.getenv(name)) break if key not in self.cmds: self.cmds[key] = self.CMDS[key] return self.cmds[key] + list(args)
def run(self, app_runner, conf_tree, opts, args, uuid, work_files): """Implement the "rose ana" command""" # Get config file option for user-specified method paths method_paths = [ os.path.join(os.path.dirname(__file__), USRCOMPARISON_DIRNAME) ] conf = ResourceLocator.default().get_conf() my_conf = conf.get_value(["rose-ana", "method-path"]) if my_conf: for item in my_conf.split(): method_paths.append(item) # Initialise the analysis engine engine = Analyse(conf_tree.node, opts, args, method_paths, reporter=app_runner.event_handler, popen=app_runner.popen) # Run the analysis num_failed, tasks = engine.analyse() if num_failed != 0: raise TestsFailedException(num_failed)
def _configure(service_cls): """Configure cherrypy and return a dict for the specified cherrypy app.""" # Environment variables (not normally defined in WSGI mode) if not os.getenv("ROSE_HOME"): path = os.path.abspath(__file__) while os.path.dirname(path) != path: # not root if os.path.basename(path) == "lib": os.environ["ROSE_HOME"] = os.path.dirname(path) break path = os.path.dirname(path) for key, value in ( ("ROSE_NS", service_cls.NS), ("ROSE_UTIL", service_cls.UTIL)): if os.getenv(key) is None: os.environ[key] = value # Configuration for HTML library cherrypy.config["tools.encode.on"] = True cherrypy.config["tools.encode.encoding"] = "utf-8" config = {} static_lib = ResourceLocator.default().get_util_home( "lib", "html", "static") for name in os.listdir(static_lib): path = os.path.join(static_lib, name) if os.path.isdir(path): path_key = "tools.staticdir.dir" bool_key = "tools.staticdir.on" else: path_key = "tools.staticfile.filename" bool_key = "tools.staticfile.on" config["/" + name] = {path_key: path, bool_key: True} if name == service_cls.NS + "-favicon.png": config["/favicon.ico"] = config["/" + name] return config
def _get_hosts(self, suite_name, host): if host: hosts = [host] else: conf = ResourceLocator.default().get_conf() hosts = None known_hosts = self.host_selector.expand( conf.get_value(["rose-suite-run", "hosts"], "").split() + conf.get_value(["rose-suite-run", "scan-hosts"], "").split() + ["localhost"])[0] known_hosts = list(set(known_hosts)) if known_hosts: hosts = self.suite_engine_proc.ping( suite_name, known_hosts) if not hosts: # Try the "rose-suite.host" file in the suite log directory log = self.suite_engine_proc.get_suite_dir(suite_name, "log") try: host_file = os.path.join(log, "rose-suite-run.host") hosts = [open(host_file).read().strip()] except IOError: pass if not hosts: hosts = ["localhost"] return hosts
def __init__(self, prefixes=None, prompt_func=None, popen=None, event_handler=None): if not event_handler: event_handler = Reporter() if not popen: popen = RosePopener(event_handler=event_handler) self.event_handler = event_handler self.popen = popen self.prompt_func = prompt_func self.prefixes = [] self.auth_managers = {} conf = ResourceLocator.default().get_conf() conf_rosie_id = conf.get(["rosie-id"], no_ignore=True) if conf_rosie_id is not None: for key, node in conf_rosie_id.value.items(): if node.is_ignored() or not key.startswith("prefix-ws."): continue prefix = key.replace("prefix-ws.", "") self.auth_managers[prefix] = RosieWSClientAuthManager( prefix, popen=self.popen, prompt_func=self.prompt_func) if not prefixes: prefixes_str = conf_rosie_id.get_value(["prefixes-ws-default"]) if prefixes_str: prefixes = shlex.split(prefixes_str) else: prefixes = sorted(self.auth_managers.keys()) self.set_prefixes(prefixes)
def _run_conf(cls, key, default=None, host=None, conf_tree=None, r_opts=None): """Return the value of a setting given by a key for a given host. If r_opts is defined, we are alerady in a remote host, so there is no need to do a host match. Otherwise, the setting may be found in the run time configuration, or the default (i.e. site/user configuration). The value of each setting in the configuration would be in a line delimited list of PATTERN=VALUE pairs. """ if r_opts is not None: return r_opts.get(key, default) if host is None: host = "localhost" for conf, keys in [(conf_tree.node, []), (ResourceLocator.default().get_conf(), ["rose-suite-run"])]: if conf is None: continue node_value = conf.get_value(keys + [key]) if node_value is None: continue for line in node_value.strip().splitlines(): pattern, value = line.strip().split("=", 1) if pattern.startswith("jinja2:"): section, name = pattern.rsplit(":", 1) p_node = conf.get([section, name], no_ignore=True) # Values in "jinja2:*" section are quoted. pattern = ast.literal_eval(p_node.value) if fnmatchcase(host, pattern): return value.strip() return default
def _verify_users(self, status, path, txn_owner, txn_access_list, bad_changes): """Check txn_owner and txn_access_list. For any invalid users, append to bad_changes and return True. """ # The owner and names in access list must be real users conf = ResourceLocator.default().get_conf() user_tool_name = conf.get_value(["rosa-svn", "user-tool"]) if not user_tool_name: return False user_tool = self.usertools_manager.get_handler(user_tool_name) txn_users = set([txn_owner] + txn_access_list) txn_users.discard("*") bad_users = user_tool.verify_users(txn_users) for bad_user in bad_users: if txn_owner == bad_user: bad_change = BadChange( status, path, BadChange.USER, "owner=" + bad_user) bad_changes.append(bad_change) if bad_user in txn_access_list: bad_change = BadChange( status, path, BadChange.USER, "access-list=" + bad_user) bad_changes.append(bad_change) return bool(bad_users)
def expand(self, names=None, rank_method=None, thresholds=None): """Expand each name in names, and look up rank method for each name. names, if specified, should be a list of host names or known groups in the site / user configuration file. Otherwise, the default setting in the site / user configuration file will be used. rank_method, if specified, should be the name of a supported ranking method. If not specified, use the default specified for a host group. If the default differs in hosts, use "load:15". """ conf = ResourceLocator.default().get_conf() if not names: node = conf.get(["rose-host-select", "default"], no_ignore=True) if node: names = [node.value] else: raise NoHostError() host_names = [] rank_method_set = set() thresholds_set = set() while names: name = names.pop() key = "group{" + name + "}" value = conf.get_value(["rose-host-select", key]) if value is None: host_names.append(name) else: for val in value.split(): names.append(val) if rank_method is None: key = "method{" + name + "}" method_str = conf.get_value(["rose-host-select", key]) if method_str is None: rank_method_set.add(self.RANK_METHOD_DEFAULT) else: rank_method_set.add(method_str) if thresholds is None: key = "thresholds{" + name + "}" threshold_str = conf.get_value(["rose-host-select", key]) if threshold_str is None: thresholds_set.add(()) else: thresholds_set.add( tuple(sorted(shlex.split(threshold_str)))) # If default rank method differs in hosts, use load:15. if rank_method is None: if len(rank_method_set) == 1: rank_method = rank_method_set.pop() else: rank_method = self.RANK_METHOD_DEFAULT if thresholds is None: if len(thresholds_set) == 1: thresholds = thresholds_set.pop() return host_names, rank_method, thresholds
def _run_conf( cls, key, default=None, host=None, conf_tree=None, r_opts=None): """Return the value of a setting given by a key for a given host. If r_opts is defined, we are alerady in a remote host, so there is no need to do a host match. Otherwise, the setting may be found in the run time configuration, or the default (i.e. site/user configuration). The value of each setting in the configuration would be in a line delimited list of PATTERN=VALUE pairs. """ if r_opts is not None: return r_opts.get(key, default) if host is None: host = "localhost" for conf, keys in [ (conf_tree.node, []), (ResourceLocator.default().get_conf(), ["rose-suite-run"])]: if conf is None: continue node_value = conf.get_value(keys + [key]) if node_value is None: continue for line in node_value.strip().splitlines(): pattern, value = line.strip().split("=", 1) if pattern.startswith("jinja2:"): section, name = pattern.rsplit(":", 1) p_node = conf.get([section, name], no_ignore=True) # Values in "jinja2:*" section are quoted. pattern = ast.literal_eval(p_node.value) if fnmatchcase(host, pattern): return value.strip() return default
def cycles( self, user, suite, page=1, order=None, per_page=None, no_fuzzy_time="0", form=None): """List cycles of a running or completed suite.""" conf = ResourceLocator.default().get_conf() per_page_default = int(conf.get_value( ["rose-bush", "cycles-per-page"], self.CYCLES_PER_PAGE)) if not isinstance(per_page, int): if per_page: per_page = int(per_page) else: per_page = per_page_default if page and per_page: page = int(page) else: page = 1 data = { "logo": self.logo, "title": self.title, "host": self.host_name, "user": user, "suite": suite, "is_option_on": ( order is not None and order != "time_desc" or per_page is not None and per_page != per_page_default ), "order": order, "rose_version": self.rose_version, "script": cherrypy.request.script_name, "method": "cycles", "no_fuzzy_time": no_fuzzy_time, "states": {}, "per_page": per_page, "per_page_default": per_page_default, "page": page, "task_status_groups": self.bush_dao.TASK_STATUS_GROUPS, } data["entries"], data["of_n_entries"] = ( self.bush_dao.get_suite_cycles_summary( user, suite, order, per_page, (page - 1) * per_page)) if per_page: data["n_pages"] = data["of_n_entries"] / per_page if data["of_n_entries"] % per_page != 0: data["n_pages"] += 1 else: data["n_pages"] = 1 data.update(self._get_suite_logs_info(user, suite)) data["states"].update( self.bush_dao.get_suite_state_summary(user, suite)) data["states"]["last_activity_time"] = ( self.get_last_activity_time(user, suite)) data["time"] = strftime("%Y-%m-%dT%H:%M:%SZ", gmtime()) if form == "json": return simplejson.dumps(data) try: return self.template_env.get_template("cycles.html").render(**data) except jinja2.TemplateError: traceback.print_exc() return simplejson.dumps(data)
def cycles( self, user, suite, page=1, order=None, per_page=None, no_fuzzy_time="0", form=None): """List cycles of a running or completed suite.""" conf = ResourceLocator.default().get_conf() per_page_default = int(conf.get_value( ["rose-bush", "cycles-per-page"], self.CYCLES_PER_PAGE)) if not isinstance(per_page, int): if per_page: per_page = int(per_page) else: per_page = per_page_default if page and per_page: page = int(page) else: page = 1 data = { "logo": self.logo, "title": self.title, "host": self.host_name, "user": user, "suite": suite, "is_option_on": ( order is not None and order != "time_desc" or per_page is not None and per_page != per_page_default ), "order": order, "rose_version": self.rose_version, "script": cherrypy.request.script_name, "method": "cycles", "no_fuzzy_time": no_fuzzy_time, "states": {}, "per_page": per_page, "per_page_default": per_page_default, "page": page, "task_status_groups": self.bush_dao.TASK_STATUS_GROUPS, } data["entries"], data["of_n_entries"] = ( self.bush_dao.get_suite_cycles_summary( user, suite, order, per_page, (page - 1) * per_page)) if per_page: data["n_pages"] = data["of_n_entries"] / per_page if data["of_n_entries"] % per_page != 0: data["n_pages"] += 1 else: data["n_pages"] = 1 data.update(self._get_suite_logs_info(user, suite)) data["states"].update( self.bush_dao.get_suite_state_summary(user, suite)) data["states"]["last_activity_time"] = ( self.get_last_activity_time(user, suite)) data["time"] = strftime("%Y-%m-%dT%H:%M:%SZ", gmtime()) if form == "json": return json.dumps(data) try: return self.template_env.get_template("cycles.html").render(**data) except jinja2.TemplateError: traceback.print_exc() return json.dumps(data)
def __init__(self, *args, **kwargs): self.exposed = True self.bush_dao = RoseBushDAO() rose_conf = ResourceLocator.default().get_conf() self.logo = rose_conf.get_value(["rose-bush", "logo"]) self.title = rose_conf.get_value(["rose-bush", "title"], self.TITLE) self.host_name = rose_conf.get_value(["rose-bush", "host"]) if self.host_name is None: self.host_name = HostSelector().get_local_host() if self.host_name and "." in self.host_name: self.host_name = self.host_name.split(".", 1)[0] self.rose_version = ResourceLocator.default().get_version() template_env = jinja2.Environment( loader=jinja2.FileSystemLoader(ResourceLocator.default( ).get_util_home("lib", "html", "template", "rose-bush"))) template_env.filters['urlise'] = self.url2hyperlink self.template_env = template_env
def __init__(self, template_env): self.exposed = True self.suite_engine_proc = SuiteEngineProcessor.get_processor() self.template_env = template_env self.host_name = socket.gethostname() if self.host_name and "." in self.host_name: self.host_name = self.host_name.split(".", 1)[0] self.rose_version = ResourceLocator.default().get_version()
def __init__(self, *args, **kwargs): self.exposed = True self.bush_dao = RoseBushDAO() rose_conf = ResourceLocator.default().get_conf() self.logo = rose_conf.get_value(["rose-bush", "logo"]) self.title = rose_conf.get_value(["rose-bush", "title"], self.TITLE) self.host_name = rose_conf.get_value(["rose-bush", "host"]) if self.host_name is None: self.host_name = HostSelector().get_local_host() if self.host_name and "." in self.host_name: self.host_name = self.host_name.split(".", 1)[0] self.rose_version = ResourceLocator.default().get_version() template_env = jinja2.Environment(loader=jinja2.FileSystemLoader( ResourceLocator.default().get_util_home( "lib", "html", "template", "rose-bush"))) template_env.filters['urlise'] = self.url2hyperlink self.template_env = template_env
def _get_conf_value(self, name, default=None): """Return the value of a named conf setting for this prefix.""" conf = ResourceLocator.default().get_conf() value = conf.get_value( ["rosie-id", "prefix-%s.%s" % (name, self.prefix)], default=default) if value: value = env_var_process(value) return value
def restart(self, suite_name=None, host=None, gcontrol_mode=None, args=None): """Restart a "cylc" suite.""" # Check suite engine specific compatibility self.suite_engine_proc.check_global_conf_compat() if not suite_name: suite_name = get_suite_name(self.event_handler) suite_dir = self.suite_engine_proc.get_suite_dir(suite_name) if not os.path.exists(suite_dir): raise SuiteNotFoundError(suite_dir) # Ensure suite is not running self.suite_engine_proc.check_suite_not_running(suite_name) # Determine suite host to restart suite if host: hosts = [host] else: hosts = [] val = ResourceLocator.default().get_conf().get_value( ["rose-suite-run", "hosts"], "localhost") for known_host in val.split(): if known_host not in hosts: hosts.append(known_host) if hosts == ["localhost"]: host = hosts[0] else: host = self.host_selector(hosts)[0][0] self.handle_event(SuiteHostSelectEvent(suite_name, "restart", host)) # Suite host environment run_conf_file_name = self.suite_engine_proc.get_suite_dir( suite_name, "log", "rose-suite-run.conf") try: run_conf = ConfigLoader().load(run_conf_file_name) except (ConfigSyntaxError, IOError): environ = None else: run_conf_tree = ConfigTree() run_conf_tree.node = run_conf environ = self.config_pm(run_conf_tree, "env") # Restart the suite self.suite_engine_proc.run(suite_name, host, environ, "restart", args) # Launch the monitoring tool # Note: maybe use os.ttyname(sys.stdout.fileno())? if os.getenv("DISPLAY") and host and gcontrol_mode: self.suite_engine_proc.gcontrol(suite_name, host) return
def cycles( self, user, suite, page=1, order=None, per_page=None, form=None): """List cycles of a running or completed suite.""" user_suite_dir = self._get_user_suite_dir(user, suite) conf = ResourceLocator.default().get_conf() per_page_default = int(conf.get_value( ["rose-bush", "cycles-per-page"], self.CYCLES_PER_PAGE)) if not isinstance(per_page, int): if per_page: per_page = int(per_page) else: per_page = per_page_default if page and per_page: page = int(page) else: page = 1 data = { "logo": self.logo, "title": self.title, "host": self.host_name, "user": user, "suite": suite, "is_option_on": ( order is not None and order != "time_desc" or per_page is not None and per_page != per_page_default ), "order": order, "rose_version": self.rose_version, "script": cherrypy.request.script_name, "method": "cycles", "states": {}, "per_page": per_page, "per_page_default": per_page_default, "page": page, } data["entries"], data["of_n_entries"] = ( self.suite_engine_proc.get_suite_cycles_summary( user, suite, order, per_page, (page - 1) * per_page)) if per_page: data["n_pages"] = data["of_n_entries"] / per_page if data["of_n_entries"] % per_page != 0: data["n_pages"] += 1 else: data["n_pages"] = 1 data.update(self._get_suite_logs_info(user, suite)) data["states"].update( self.suite_engine_proc.get_suite_state_summary(user, suite)) data["time"] = strftime("%Y-%m-%dT%H:%M:%S+0000", gmtime()) if form == "json": return simplejson.dumps(data) try: template = self.template_env.get_template("cycles.html") return template.render(**data) except Exception as exc: traceback.print_exc(exc) return simplejson.dumps(data)
def get_prefix_web(cls, prefix=None): """Return a url for the prefix repository source url.""" if prefix is None: prefix = cls.get_prefix_default() key = "prefix-web." + prefix config = ResourceLocator.default().get_conf() value = config.get_value(["rosie-id", key]) if value is None: raise SuiteIdPrefixError(prefix) return value.rstrip("/")
def get_prefix_location(cls, prefix=None): """Return the repository location of a given prefix.""" if prefix is None: prefix = cls.get_prefix_default() key = "prefix-location." + prefix config = ResourceLocator.default().get_conf() value = config.get_value(["rosie-id", key]) if value is None: raise SuiteIdPrefixError(prefix) return value.rstrip("/")
def get_local_copy_root(cls): """Return the root directory for hosting the local suite copies.""" config = ResourceLocator.default().get_conf() value = config.get_value(["rosie-id", "local-copy-root"]) if value: local_copy_root = value else: local_copy_root = "$HOME/roses" local_copy_root = rose.env.env_var_process(local_copy_root) return local_copy_root
def _prompt(self, is_retry=False): """Prompt for the username and password, where necessary. Prompt with zenity or raw_input/getpass. """ if (callable(self.prompt_func) and not hasattr(self.password_store, "prompt_password")): self.username, self.password = self.prompt_func( self.username, self.password, is_retry) return icon_path = ResourceLocator.default().locate("images/rosie-icon.png") if is_retry: username = "" if self.username: username = "" prompt = self.PROMPT_USERNAME % { "prefix": self.prefix, "root": self.root} if self.popen.which("zenity") and os.getenv("DISPLAY"): username = self.popen.run( "zenity", "--entry", "--title=Rosie", "--window-icon=" + icon_path, "--text=" + prompt)[1].strip() else: username = raw_input(prompt) if not username: raise KeyboardInterrupt(self.STR_CANCELLED) if username and username != self.username: self.username = username self._load_password() if self.password: return if self.username and self.password is None or is_retry: prompt = self.PROMPT_PASSWORD % {"prefix": self.prefix, "root": self.root, "username": self.username} if hasattr(self.password_store, "prompt_password"): password = self.password_store.prompt_password( prompt, self.scheme, self.host, self.username) elif self.popen.which("zenity") and os.getenv("DISPLAY"): password = self.popen.run( "zenity", "--entry", "--hide-text", "--title=Rosie", "--window-icon=" + icon_path, "--text=" + prompt)[1].strip() else: password = getpass(prompt) if not password: raise KeyboardInterrupt(self.STR_CANCELLED) if password and password != self.password: self.password = password
def __init__(self, template_env, prefix, db_url): self.exposed = True self.template_env = template_env self.prefix = prefix source_option = "prefix-web." + self.prefix source_url_node = ResourceLocator.default().get_conf().get( ["rosie-id", source_option]) self.source_url = "" if source_url_node is not None: self.source_url = source_url_node.value self.dao = rosie.db.DAO(db_url)
def __init__(self, props, prefix, db_url): self.exposed = True self.props = props self.prefix = prefix source_option = "prefix-web." + self.prefix source_url_node = ResourceLocator.default().get_conf().get( ["rosie-id", source_option]) self.source_url = "" if source_url_node is not None: self.source_url = source_url_node.value self.dao = rosie.db.DAO(db_url)
def initialize(self, props, prefix, db_url, service_root): self.props = props self.prefix = prefix source_option = "prefix-web." + self.prefix source_url_node = ResourceLocator.default().get_conf().get( ["rosie-id", source_option]) self.source_url = "" if source_url_node is not None: self.source_url = source_url_node.value self.dao = rosie.db.DAO(db_url) self.service_root = service_root[:-1] # remove the '?' regex aspect
def __init__(self, *args, **kwargs): if hasattr(kwargs, "prog"): ns, util = kwargs["prog"].split(None, 1) resource_loc = ResourceLocator(ns=ns, util=util) else: resource_loc = ResourceLocator.default() kwargs["prog"] = resource_loc.get_util_name() if not hasattr(kwargs, "usage"): kwargs["usage"] = resource_loc.get_synopsis() OptionParser.__init__(self, *args, **kwargs) self.add_my_options("debug_mode", "quietness", "verbosity")
def load_override_config(): conf = ResourceLocator.default().get_conf().get(["rose-config-edit"]) if conf is None: return for key, node in conf.value.items(): if node.is_ignored(): continue try: cast_value = ast.literal_eval(node.value) except Exception: cast_value = node.value globals()[key.replace("-", "_").upper()] = cast_value
def __init__(self, *args, **kwargs): if hasattr(kwargs, "prog"): namespace, util = kwargs["prog"].split(None, 1) resource_loc = ResourceLocator(namespace=namespace, util=util) else: resource_loc = ResourceLocator.default() kwargs["prog"] = resource_loc.get_util_name() if not hasattr(kwargs, "usage"): kwargs["usage"] = resource_loc.get_synopsis() OptionParser.__init__(self, *args, **kwargs) self.add_my_options("debug_mode", "profile_mode", "quietness", "verbosity")
def cycles(self, user, suite, page=1, per_page=None, form=None): """List cycles of a running or completed suite.""" user_suite_dir = self._get_user_suite_dir(user, suite) if not os.path.isdir(user_suite_dir): raise cherrypy.HTTPError(400) if not isinstance(per_page, int): if per_page: per_page = int(per_page) else: conf = ResourceLocator.default().get_conf() per_page = int(conf.get_value( ["rose-bush", "cycles-per-page"], self.CYCLES_PER_PAGE)) if page and per_page: page = int(page) else: page = 1 data = { "host": self.host_name, "user": user, "suite": suite, "rose_version": self.rose_version, "script": cherrypy.request.script_name, "states": {}, "per_page": per_page, "page": page, } data["offset"] = (page - 1) * per_page entries, of_n_entries = ( self.suite_engine_proc.get_suite_cycles_summary( user, suite, per_page, data["offset"])) data["entries"] = entries data["of_n_entries"] = of_n_entries if per_page: data["n_pages"] = of_n_entries / per_page if of_n_entries % per_page != 0: data["n_pages"] += 1 else: data["n_pages"] = 1 data.update(self._get_suite_logs_info(user, suite)) data["states"].update( self.suite_engine_proc.get_suite_state_summary(user, suite)) data["time"] = strftime("%Y-%m-%dT%H:%M:%S+0000", gmtime()) if form == "json": return simplejson.dumps(data) try: template = self.template_env.get_template("cycles.html") return template.render(**data) except Exception as e: traceback.print_exc(e) return simplejson.dumps(data)
def get_prefix_locations(cls): """Return a dict containing the known prefixes and their repository locations. """ ret = {} config = ResourceLocator.default().get_conf() rosie_id_node = config.get(["rosie-id"], no_ignore=True) if rosie_id_node is None: return ret for key, node in rosie_id_node.value.items(): if node.state: continue if key.startswith("prefix-location."): ret[key[len("prefix-location."):]] = node.value return ret
def load_override_config(): """Load any overrides of the above settings.""" conf = ResourceLocator.default().get_conf() for s in ["rosie-browse", "rosie-go"]: node = conf.get([s], no_ignore=True) if node is None: continue for key, node in node.value.items(): if node.is_ignored(): continue try: cast_value = ast.literal_eval(node.value) except Exception: cast_value = node.value globals()[key.replace("-", "_").upper()] = cast_value