コード例 #1
0
ファイル: ws.py プロジェクト: csimag/rose
def start(is_main=False):
    """Create the server.

    If is_main, invoke cherrypy.quickstart.
    Otherwise, return a cherrypy.Application instance.

    """
    # Environment variables (not normally defined in WSGI mode)
    if os.getenv("ROSE_HOME") is None:
        path = os.path.abspath(__file__)
        while os.path.dirname(path) != path: # not root
            if os.path.basename(path) == "lib":
                os.environ["ROSE_HOME"] = os.path.dirname(path)
                break
            path = os.path.dirname(path)
    for key, value in [("ROSE_NS", "rosa"), ("ROSE_UTIL", "ws")]:
        if os.getenv(key) is None:
            os.environ[key] = value

    # CherryPy quick server configuration
    rose_conf = ResourceLocator.default().get_conf()
    if is_main and rose_conf.get_value(["rosie-ws", "log-dir"]) is not None:
        log_dir_value = rose_conf.get_value(["rosie-ws", "log-dir"])
        log_dir = env_var_process(os.path.expanduser(log_dir_value))
        if not os.path.isdir(log_dir):
            os.makedirs(log_dir)
        log_file = os.path.join(log_dir, "server.log")
        log_error_file = os.path.join(log_dir, "server.err.log")
        cherrypy.config["log.error_file"] = log_error_file
        cherrypy.config["log.access_file"] = log_file
        cherrypy.config["request.error_response"] = _handle_error
    cherrypy.config["log.screen"] = False
    # Configuration for dynamic pages
    db_url_map = {}
    for key, node in rose_conf.get(["rosie-db"]).value.items():
        if key.startswith("db.") and key[3:]:
            db_url_map[key[3:]] = node.value
    res_loc = ResourceLocator.default()
    html_lib = res_loc.get_util_home("lib", "html")
    icon_path = res_loc.locate("images/rosie-icon-trim.png")
    tmpl_loader = jinja2.FileSystemLoader(os.path.join(html_lib, "rosie-ws"))
    root = Root(jinja2.Environment(loader=tmpl_loader), db_url_map)

    # Configuration for static pages
    config = {"/etc": {
                    "tools.staticdir.dir": os.path.join(html_lib, "external"),
                    "tools.staticdir.on": True},
              "/favicon.ico": {
                    "tools.staticfile.on": True,
                    "tools.staticfile.filename": icon_path}}
    if is_main:
        port = int(rose_conf.get_value(["rosie-ws", "port"], 8080))
        config.update({"global": {"server.socket_host": "0.0.0.0",
                                  "server.socket_port": port}})

    # Start server or return WSGI application
    if is_main:
        return cherrypy.quickstart(root, "/", config=config)
    else:
        return cherrypy.Application(root, script_name=None, config=config)
コード例 #2
0
ファイル: ws.py プロジェクト: aosprey/rose
 def __init__(self, *args, **kwargs):
     self.exposed = True
     self.props = {}
     rose_conf = ResourceLocator.default().get_conf()
     self.props["title"] = rose_conf.get_value(
         ["rosie-disco", "title"], self.TITLE)
     self.props["host_name"] = rose_conf.get_value(["rosie-disco", "host"])
     if self.props["host_name"] is None:
         self.props["host_name"] = HostSelector().get_local_host()
         if self.props["host_name"] and "." in self.props["host_name"]:
             self.props["host_name"] = (
                 self.props["host_name"].split(".", 1)[0])
     self.props["rose_version"] = ResourceLocator.default().get_version()
     self.props["template_env"] = jinja2.Environment(
         loader=jinja2.FileSystemLoader(
             ResourceLocator.default().get_util_home(
                 "lib", "html", "template", "rosie-disco")))
     db_url_map = {}
     for key, node in rose_conf.get(["rosie-db"]).value.items():
         if key.startswith("db.") and key[3:]:
             db_url_map[key[3:]] = node.value
     self.db_url_map = db_url_map
     if not self.db_url_map:
         self.db_url_map = {}
     for key, db_url in self.db_url_map.items():
         setattr(self, key, RosieDiscoService(self.props, key, db_url))
コード例 #3
0
ファイル: opt_parse.py プロジェクト: jimbolton/rose
 def __init__(self, *args, **kwargs):
     if hasattr(kwargs, "prog"):
         ns, util = kwargs["prog"].split(None, 1)
         resource_loc = ResourceLocator(ns=ns, util=util)
     else:
         resource_loc = ResourceLocator.default()
     kwargs["prog"] = resource_loc.get_util_name()
     if not hasattr(kwargs, "usage"):
         kwargs["usage"] = resource_loc.get_synopsis()
     OptionParser.__init__(self, *args, **kwargs)
     self.add_my_options("debug_mode", "quietness", "verbosity")
コード例 #4
0
ファイル: db_create.py プロジェクト: ScottWales/rose
def main():
    """rosa db-create."""
    db_conf = ResourceLocator.default().get_conf().get(["rosie-db"])
    if db_conf is not None:
        opts = RoseOptionParser().parse_args()[0]
        reporter = Reporter(opts.verbosity - opts.quietness)
        init = RosieDatabaseInitiator(event_handler=reporter)
        conf = ResourceLocator.default().get_conf()
        for key in db_conf.value:
            if key.startswith("db."):
                prefix = key.replace("db.", "", 1)
                db_url = conf.get_value(["rosie-db", "db." + prefix])
                repos_path = conf.get_value(["rosie-db", "repos." + prefix])
                init(db_url, repos_path)
コード例 #5
0
ファイル: suite_hook.py プロジェクト: hjoliver/rose
    def run(
        self,
        suite_name,
        task_id,
        hook_event,
        hook_message=None,
        should_mail=False,
        mail_cc_list=None,
        should_shutdown=False,
    ):
        """
        Invoke the hook for a suite.

        1. For a task hook, if the task runs remotely, retrieve its log from
           the remote host.
        2. If "should_mail", send an email notification to the current user,
           and those in the "mail_cc_list".
        3. If "should_shutdown", shut down the suite.

        """
        # Retrieve log and generate code view
        task_ids = []
        if task_id:
            task_ids = [task_id]
            self.suite_engine_proc.job_logs_pull_remote(suite_name, task_ids)

        # Send email notification if required
        if should_mail:
            text = ""
            if task_id:
                text += "Task: %s\n" % task_id
            if hook_message:
                text += "Message: %s\n" % hook_message
            url = self.suite_engine_proc.get_suite_log_url(None, suite_name)
            text += "See: %s\n" % (url)
            user = pwd.getpwuid(os.getuid()).pw_name
            conf = ResourceLocator.default().get_conf()
            host = conf.get_value(["rose-suite-hook", "email-host"], default="localhost")
            msg = MIMEText(text)
            msg["From"] = user + "@" + host
            msg["To"] = msg["From"]
            if mail_cc_list:
                mail_cc_addresses = []
                for mail_cc_address in mail_cc_list:
                    if "@" not in mail_cc_address:
                        mail_cc_address += "@" + host
                    mail_cc_addresses.append(mail_cc_address)
                msg["Cc"] = ", ".join(mail_cc_addresses)
                mail_cc_list = mail_cc_addresses
            else:
                mail_cc_list = []
            msg["Subject"] = "[%s] %s" % (hook_event, suite_name)
            smtp_host = conf.get_value(["rose-suite-hook", "smtp-host"], default="localhost")
            smtp = SMTP(smtp_host)
            smtp.sendmail(msg["From"], [msg["To"]] + mail_cc_list, msg.as_string())
            smtp.quit()

        # Shut down if required
        if should_shutdown:
            self.suite_engine_proc.shutdown(suite_name, args=["--kill"])
コード例 #6
0
ファイル: ws_client_auth.py プロジェクト: ScottWales/rose
    def __init__(self, prefix, popen=None, prompt_func=None):
        self.prefix = prefix
        root = self._get_conf_value("ws")
        if root is None:
            raise UndefinedRosiePrefixWS(self.prefix)
        if not root.endswith("/"):
            root += "/"
        self.root = root
        urlparse_res = urlparse(self.root)
        self.scheme = urlparse_res[0]
        self.host = urlparse_res[1]
        self.password_orig = None
        self.username_orig = None
        self.password = None
        self.username = None
        if popen is None:
            popen = RosePopener()
        self.popen = popen
        self.prompt_func = prompt_func
        res_loc = ResourceLocator.default()
        password_stores_str = res_loc.default().get_conf().get_value(
            keys=["rosie-id", "prefix-password-store." + self.prefix],
            default=self.PASSWORD_STORES_STR)
        for password_store_name in shlex.split(password_stores_str):
            password_store_cls = self.PASSWORD_STORE_CLASSES.get(
                password_store_name)
            if password_store_cls is not None and password_store_cls.usable():
                self.password_store = password_store_cls()
                break
        else:
            self.password_store = None

        self.requests_kwargs = {}
        self._init_https_params()
コード例 #7
0
ファイル: host_select.py プロジェクト: craigmaclachlan/rose
    def expand(self, names=None, rank_method=None, thresholds=None):
        """Expand each name in names, and look up rank method for each name.

        names, if specified, should be a list of host names or known groups in
        the site / user configuration file. Otherwise, the default setting in
        the site / user configuration file will be used.

        rank_method, if specified, should be the name of a supported ranking
        method. If not specified, use the default specified for a host group.
        If the default differs in hosts, use "load:15".

        """
        conf = ResourceLocator.default().get_conf()
        if not names:
            node = conf.get(["rose-host-select", "default"],
                            no_ignore=True)
            if node:
                names = [node.value]
            else:
                raise NoHostError()

        host_names = []
        rank_method_set = set()
        thresholds_set = set()
        while names:
            name = names.pop()
            key = "group{" + name + "}"
            value = conf.get_value(["rose-host-select", key])
            if value is None:
                host_names.append(name)
            else:
                for v in value.split():
                    names.append(v)
                if rank_method is None:
                    key = "method{" + name + "}"
                    m = conf.get_value(["rose-host-select", key])
                    if m is None:
                        rank_method_set.add(self.RANK_METHOD_DEFAULT)
                    else:
                        rank_method_set.add(m)
                if thresholds is None:
                    key = "thresholds{" + name + "}"
                    t = conf.get_value(["rose-host-select", key])
                    if t is None:
                        thresholds_set.add(())
                    else:
                        thresholds_set.add(tuple(sorted(shlex.split(t))))

        # If default rank method differs in hosts, use load:15.
        if rank_method is None:
            if len(rank_method_set) == 1:
                rank_method = rank_method_set.pop()
            else:
                rank_method = self.RANK_METHOD_DEFAULT

        if thresholds is None:
            if len(thresholds_set) == 1:
                thresholds = thresholds_set.pop()

        return host_names, rank_method, thresholds
コード例 #8
0
ファイル: suite_control.py プロジェクト: jmancell/rose
    def _get_hosts(self, suite_name, host):
        if host:
            hosts = [host]
        else:
            conf = ResourceLocator.default().get_conf()
            hosts = None

            known_hosts = self.host_selector.expand(
              conf.get_value(["rose-suite-run", "hosts"], "").split() +
              conf.get_value(["rose-suite-run", "scan-hosts"], "").split() +
              ["localhost"])[0]
            known_hosts = list(set(known_hosts))

            if known_hosts:
                hosts = self.suite_engine_proc.ping(
                        suite_name,
                        known_hosts)

            if not hosts:
                # Try the "rose-suite.host" file in the suite log directory
                log = self.suite_engine_proc.get_suite_dir(suite_name, "log")
                try:
                    host_file = os.path.join(log, "rose-suite-run.host")
                    hosts = [open(host_file).read().strip()]
                except IOError:
                    pass
            if not hosts:
                hosts = ["localhost"]
        return hosts
コード例 #9
0
ファイル: bush.py プロジェクト: adammartins/rose
 def __init__(self, *args, **kwargs):
     self.exposed = True
     self.suite_engine_proc = SuiteEngineProcessor.get_processor()
     rose_conf = ResourceLocator.default().get_conf()
     self.logo = rose_conf.get_value(["rose-bush", "logo"])
     self.title = rose_conf.get_value(["rose-bush", "title"], self.TITLE)
     self.host_name = rose_conf.get_value(["rose-bush", "host"])
     if self.host_name is None:
         self.host_name = HostSelector().get_local_host()
         if self.host_name and "." in self.host_name:
             self.host_name = self.host_name.split(".", 1)[0]
     self.rose_version = ResourceLocator.default().get_version()
     template_env = jinja2.Environment(loader=jinja2.FileSystemLoader(
         ResourceLocator.default().get_util_home(
             "lib", "html", "template", "rose-bush")))
     self.template_env = template_env
コード例 #10
0
ファイル: suite_engine_proc.py プロジェクト: hjoliver/rose
 def get_suite_log_url(self, user_name, suite_name):
     """Return the "rose bush" URL for a user's suite."""
     prefix = "~"
     if user_name:
         prefix += user_name
     suite_d = os.path.join(prefix, self.get_suite_dir_rel(suite_name))
     suite_d = os.path.expanduser(suite_d)
     if not os.path.isdir(suite_d):
         raise NoSuiteLogError(user_name, suite_name)
     rose_bush_url = None
     for f_name in glob(os.path.expanduser("~/.metomi/rose-bush*.status")):
         status = {}
         for line in open(f_name):
             key, value = line.strip().split("=", 1)
             status[key] = value
         if status.get("host"):
             rose_bush_url = "http://" + status["host"]
             if status.get("port"):
                 rose_bush_url += ":" + status["port"]
         rose_bush_url += "/"
         break
     if not rose_bush_url:
         conf = ResourceLocator.default().get_conf()
         rose_bush_url = conf.get_value(["rose-suite-log", "rose-bush"])
     if not rose_bush_url:
         return "file://" + suite_d
     if not rose_bush_url.endswith("/"):
         rose_bush_url += "/"
     if not user_name:
         user_name = pwd.getpwuid(os.getuid()).pw_name
     return rose_bush_url + "/".join(["taskjobs", user_name, suite_name])
コード例 #11
0
ファイル: __init__.py プロジェクト: arjclark/rose
def load_override_config(sections, my_globals=None):
    if my_globals is None:
        my_globals = globals()
    for section in sections:
        conf = ResourceLocator.default().get_conf().get([section])
        if conf is None:
            continue
        for key, node in conf.value.items():
            if node.is_ignored():
                continue
            try:
                cast_value = ast.literal_eval(node.value)
            except Exception:
                cast_value = node.value
            name = key.replace("-", "_").upper()
            orig_value = my_globals[name]
            if (type(orig_value) is not type(cast_value) and
                    orig_value is not None):
                sys.stderr.write(_OVERRIDE_WARNING_TYPE.format(
                    section, key, cast_value,
                    type(orig_value), type(cast_value))
                )
                continue
            if name.startswith("_"):
                sys.stderr.write(_OVERRIDE_WARNING_PRIVATE.format(
                    section, key, name)
                )
                continue
            my_globals[name] = cast_value
コード例 #12
0
ファイル: popen.py プロジェクト: arjclark/rose
    def get_cmd(self, key, *args):
        """Return default options and arguments of a known command as a list.

        If a setting [external] <key> is defined in the site/user
        configuration, use the setting.

        Otherwise, if RosePopener.ENVS_OF_CMDS[key] exists, it looks for each
        environment variable in the list in RosePopener.ENVS_OF_CMDS[key] in
        order. If the environment variable is defined and is not a null string,
        use the value of the environment variable.

        Otherwise, return RosePopener.CMDS[key]

        key: must be a key of RosePopener.CMDS
        args: if specified, will be added to the returned list

        """
        if key not in self.cmds:
            root_node = ResourceLocator.default().get_conf()
            node = root_node.get(["external", key], no_ignore=True)
            if node is not None:
                self.cmds[key] = shlex.split(node.value)
        if key not in self.cmds:
            for name in self.ENVS_OF_CMDS.get(key, []):
                if os.getenv(name):  # not None, not null str
                    self.cmds[key] = shlex.split(os.getenv(name))
                    break
        if key not in self.cmds:
            self.cmds[key] = self.CMDS[key]
        return self.cmds[key] + list(args)
コード例 #13
0
ファイル: ws.py プロジェクト: dpmatthews/rose
def _configure(service_cls):
    """Configure cherrypy and return a dict for the specified cherrypy app."""
    # Environment variables (not normally defined in WSGI mode)
    if not os.getenv("ROSE_HOME"):
        path = os.path.abspath(__file__)
        while os.path.dirname(path) != path:  # not root
            if os.path.basename(path) == "lib":
                os.environ["ROSE_HOME"] = os.path.dirname(path)
                break
            path = os.path.dirname(path)
    for key, value in (
            ("ROSE_NS", service_cls.NS), ("ROSE_UTIL", service_cls.UTIL)):
        if os.getenv(key) is None:
            os.environ[key] = value

    # Configuration for HTML library
    cherrypy.config["tools.encode.on"] = True
    cherrypy.config["tools.encode.encoding"] = "utf-8"
    config = {}
    static_lib = ResourceLocator.default().get_util_home(
        "lib", "html", "static")
    for name in os.listdir(static_lib):
        path = os.path.join(static_lib, name)
        if os.path.isdir(path):
            path_key = "tools.staticdir.dir"
            bool_key = "tools.staticdir.on"
        else:
            path_key = "tools.staticfile.filename"
            bool_key = "tools.staticfile.on"
        config["/" + name] = {path_key: path, bool_key: True}
        if name == service_cls.NS + "-favicon.png":
            config["/favicon.ico"] = config["/" + name]
    return config
コード例 #14
0
ファイル: svn_pre_commit.py プロジェクト: harry-shepherd/rose
    def _verify_users(self, status, path, txn_owner, txn_access_list,
                      bad_changes):
        """Check txn_owner and txn_access_list.

        For any invalid users, append to bad_changes and return True.

        """
        # The owner and names in access list must be real users
        conf = ResourceLocator.default().get_conf()
        user_tool_name = conf.get_value(["rosa-svn", "user-tool"])
        if not user_tool_name:
            return False
        user_tool = self.usertools_manager.get_handler(user_tool_name)
        txn_users = set([txn_owner] + txn_access_list)
        txn_users.discard("*")
        bad_users = user_tool.verify_users(txn_users)
        for bad_user in bad_users:
            if txn_owner == bad_user:
                bad_change = BadChange(
                    status,
                    path,
                    BadChange.USER,
                    "owner=" + bad_user)
                bad_changes.append(bad_change)
            if bad_user in txn_access_list:
                bad_change = BadChange(
                    status,
                    path,
                    BadChange.USER,
                    "access-list=" + bad_user)
                bad_changes.append(bad_change)
        return bool(bad_users)
コード例 #15
0
ファイル: ldaptool.py プロジェクト: harry-shepherd/rose
    def _search(cls, users, attr_idx):
        """Search LDAP directory for the indexed attr for users.

        Attr index can be UID_IDX, CN_IDX or MAIL_IDX.

        Return a list containing the results.

        """
        conf = ResourceLocator.default().get_conf()
        uri = conf.get_value(["rosa-ldap", "uri"])
        binddn = conf.get_value(["rosa-ldap", "binddn"])
        passwd = ""
        passwd_file = conf.get_value(["rosa-ldap", "password-file"],
                                     cls.PASSWD_FILE)
        if passwd_file:
            passwd = open(os.path.expanduser(passwd_file)).read().strip()
        basedn = conf.get_value(["rosa-ldap", "basedn"], "")
        filter_str = "(|(uid=" + ")(uid=".join(users) + "))"
        filter_more_str = conf.get_value(["rosa-ldap", "filter-more"], "")
        if filter_more_str:
            filter_str = "(&" + filter_str + filter_more_str + ")"
        user_attr_str = conf.get_value(["rosa-ldap", "attrs"], cls.USER_ATTRS)
        attr = user_attr_str.split()[attr_idx]

        tls_ca_file = conf.get_value(["rosa-ldap", "tls-ca-file"])
        if tls_ca_file:
            ldap.set_option(ldap.OPT_X_TLS_CACERTFILE, tls_ca_file)
        conn = ldap.initialize(uri)
        conn.bind_s(binddn, passwd)
        results = conn.search_s(basedn, ldap.SCOPE_SUBTREE, filter_str, [attr])
        conn.unbind()
        return [result[1][attr][0] for result in results]
コード例 #16
0
ファイル: suite_run.py プロジェクト: csimag/rose
 def _run_conf(
         cls, key, default=None, host=None, conf_tree=None, r_opts=None):
     """Return the value of a setting given by a key for a given host. If
     r_opts is defined, we are alerady in a remote host, so there is no need
     to do a host match. Otherwise, the setting may be found in the run time
     configuration, or the default (i.e. site/user configuration). The value
     of each setting in the configuration would be in a line delimited list
     of PATTERN=VALUE pairs.
     """
     if r_opts is not None:
         return r_opts.get(key, default)
     if host is None:
         host = "localhost"
     for conf, keys in [
             (conf_tree.node, []),
             (ResourceLocator.default().get_conf(), ["rose-suite-run"])]:
         if conf is None:
             continue
         node_value = conf.get_value(keys + [key])
         if node_value is None:
             continue
         for line in node_value.strip().splitlines():
             pattern, value = line.strip().split("=", 1)
             if pattern.startswith("jinja2:"):
                 section, name = pattern.rsplit(":", 1)
                 p_node = conf.get([section, name], no_ignore=True)
                 # Values in "jinja2:*" section are quoted.
                 pattern = ast.literal_eval(p_node.value)
             if fnmatchcase(host, pattern):
                 return value.strip()
     return default
コード例 #17
0
ファイル: ws_client.py プロジェクト: lexual/rose
 def __init__(self, prefixes=None, prompt_func=None, popen=None,
              event_handler=None):
     if not event_handler:
         event_handler = Reporter()
     if not popen:
         popen = RosePopener(event_handler=event_handler)
     self.event_handler = event_handler
     self.popen = popen
     self.prompt_func = prompt_func
     self.prefixes = []
     self.unreachable_prefixes = []
     self.auth_managers = {}
     conf = ResourceLocator.default().get_conf()
     conf_rosie_id = conf.get(["rosie-id"], no_ignore=True)
     if conf_rosie_id is None:
         raise RosieWSClientConfError()
     for key, node in conf_rosie_id.value.items():
         if node.is_ignored() or not key.startswith("prefix-ws."):
             continue
         prefix = key.replace("prefix-ws.", "")
         self.auth_managers[prefix] = RosieWSClientAuthManager(
             prefix, popen=self.popen, prompt_func=self.prompt_func)
     if not prefixes:
         prefixes_str = conf_rosie_id.get_value(["prefixes-ws-default"])
         if prefixes_str:
             prefixes = shlex.split(prefixes_str)
         else:
             prefixes = sorted(self.auth_managers.keys())
     self.set_prefixes(prefixes)
コード例 #18
0
ファイル: suite_id.py プロジェクト: craigmaclachlan/rose
 def get_prefix_default(cls):
     """Return the default prefix."""
     config = ResourceLocator.default().get_conf()
     value = config.get_value(["rosie-id", "prefix-default"])
     if not value or not value.strip():
         raise SuiteIdPrefixError()
     return shlex.split(value)[0]
コード例 #19
0
ファイル: suite_id.py プロジェクト: csimag/rose
 def get_prefix_default(cls):
     """Return the default prefix."""
     config = ResourceLocator.default().get_conf()
     value = config.get_value(["rosie-id", "prefix-default"])
     if value is None:
         raise SuiteIdPrefixError()
     return value
コード例 #20
0
 def cycles(
         self, user, suite, page=1, order=None, per_page=None,
         no_fuzzy_time="0", form=None):
     """List cycles of a running or completed suite."""
     conf = ResourceLocator.default().get_conf()
     per_page_default = int(conf.get_value(
         ["rose-bush", "cycles-per-page"], self.CYCLES_PER_PAGE))
     if not isinstance(per_page, int):
         if per_page:
             per_page = int(per_page)
         else:
             per_page = per_page_default
     if page and per_page:
         page = int(page)
     else:
         page = 1
     data = {
         "logo": self.logo,
         "title": self.title,
         "host": self.host_name,
         "user": user,
         "suite": suite,
         "is_option_on": (
             order is not None and order != "time_desc" or
             per_page is not None and per_page != per_page_default
         ),
         "order": order,
         "rose_version": self.rose_version,
         "script": cherrypy.request.script_name,
         "method": "cycles",
         "no_fuzzy_time": no_fuzzy_time,
         "states": {},
         "per_page": per_page,
         "per_page_default": per_page_default,
         "page": page,
         "task_status_groups": self.bush_dao.TASK_STATUS_GROUPS,
     }
     data["entries"], data["of_n_entries"] = (
         self.bush_dao.get_suite_cycles_summary(
             user, suite, order, per_page, (page - 1) * per_page))
     if per_page:
         data["n_pages"] = data["of_n_entries"] / per_page
         if data["of_n_entries"] % per_page != 0:
             data["n_pages"] += 1
     else:
         data["n_pages"] = 1
     data.update(self._get_suite_logs_info(user, suite))
     data["states"].update(
         self.bush_dao.get_suite_state_summary(user, suite))
     data["states"]["last_activity_time"] = (
         self.get_last_activity_time(user, suite))
     data["time"] = strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
     if form == "json":
         return simplejson.dumps(data)
     try:
         return self.template_env.get_template("cycles.html").render(**data)
     except jinja2.TemplateError:
         traceback.print_exc()
     return simplejson.dumps(data)
コード例 #21
0
 def __init__(self, *args, **kwargs):
     self.exposed = True
     self.bush_dao = RoseBushDAO()
     rose_conf = ResourceLocator.default().get_conf()
     self.logo = rose_conf.get_value(["rose-bush", "logo"])
     self.title = rose_conf.get_value(["rose-bush", "title"], self.TITLE)
     self.host_name = rose_conf.get_value(["rose-bush", "host"])
     if self.host_name is None:
         self.host_name = HostSelector().get_local_host()
         if self.host_name and "." in self.host_name:
             self.host_name = self.host_name.split(".", 1)[0]
     self.rose_version = ResourceLocator.default().get_version()
     template_env = jinja2.Environment(loader=jinja2.FileSystemLoader(
         ResourceLocator.default().get_util_home(
             "lib", "html", "template", "rose-bush")))
     template_env.filters['urlise'] = self.url2hyperlink
     self.template_env = template_env
コード例 #22
0
ファイル: bush.py プロジェクト: m214089/rose
 def __init__(self, template_env):
     self.exposed = True
     self.suite_engine_proc = SuiteEngineProcessor.get_processor()
     self.template_env = template_env
     self.host_name = socket.gethostname()
     if self.host_name and "." in self.host_name:
         self.host_name = self.host_name.split(".", 1)[0]
     self.rose_version = ResourceLocator.default().get_version()
コード例 #23
0
ファイル: ws_client_auth.py プロジェクト: benfitzpatrick/rose
 def _get_conf_value(self, name, default=None):
     """Return the value of a named conf setting for this prefix."""
     conf = ResourceLocator.default().get_conf()
     value = conf.get_value(
         ["rosie-id", "prefix-%s.%s" % (name, self.prefix)],
         default=default)
     if value:
         value = env_var_process(value)
     return value
コード例 #24
0
ファイル: bush.py プロジェクト: adammartins/rose
 def cycles(
         self, user, suite, page=1, order=None, per_page=None, form=None):
     """List cycles of a running or completed suite."""
     user_suite_dir = self._get_user_suite_dir(user, suite)
     conf = ResourceLocator.default().get_conf()
     per_page_default = int(conf.get_value(
         ["rose-bush", "cycles-per-page"], self.CYCLES_PER_PAGE))
     if not isinstance(per_page, int):
         if per_page:
             per_page = int(per_page)
         else:
             per_page = per_page_default
     if page and per_page:
         page = int(page)
     else:
         page = 1
     data = {
         "logo": self.logo,
         "title": self.title,
         "host": self.host_name,
         "user": user,
         "suite": suite,
         "is_option_on": (
             order is not None and order != "time_desc" or
             per_page is not None and per_page != per_page_default
         ),
         "order": order,
         "rose_version": self.rose_version,
         "script": cherrypy.request.script_name,
         "method": "cycles",
         "states": {},
         "per_page": per_page,
         "per_page_default": per_page_default,
         "page": page,
     }
     data["entries"], data["of_n_entries"] = (
         self.suite_engine_proc.get_suite_cycles_summary(
             user, suite, order, per_page, (page - 1) * per_page))
     if per_page:
         data["n_pages"] = data["of_n_entries"] / per_page
         if data["of_n_entries"] % per_page != 0:
             data["n_pages"] += 1
     else:
         data["n_pages"] = 1
     data.update(self._get_suite_logs_info(user, suite))
     data["states"].update(
         self.suite_engine_proc.get_suite_state_summary(user, suite))
     data["time"] = strftime("%Y-%m-%dT%H:%M:%S+0000", gmtime())
     if form == "json":
         return simplejson.dumps(data)
     try:
         template = self.template_env.get_template("cycles.html")
         return template.render(**data)
     except Exception as exc:
         traceback.print_exc(exc)
     return simplejson.dumps(data)
コード例 #25
0
ファイル: suite_id.py プロジェクト: craigmaclachlan/rose
 def get_prefix_web(cls, prefix=None):
     """Return a url for the prefix repository source url."""
     if prefix is None:
         prefix = cls.get_prefix_default()
     key = "prefix-web." + prefix
     config = ResourceLocator.default().get_conf()
     value = config.get_value(["rosie-id", key])
     if value is None:
         raise SuiteIdPrefixError(prefix)
     return value.rstrip("/")
コード例 #26
0
ファイル: suite_id.py プロジェクト: csimag/rose
 def get_local_copy_root(cls):
     """Return the root directory for hosting the local suite copies."""
     config = ResourceLocator.default().get_conf()
     value = config.get_value(["rosie-id", "local-copy-root"])
     if value:
         local_copy_root = value
     else:
         local_copy_root = "$HOME/roses"
     local_copy_root = rose.env.env_var_process(local_copy_root)
     return local_copy_root
コード例 #27
0
ファイル: suite_id.py プロジェクト: craigmaclachlan/rose
 def get_prefix_location(cls, prefix=None):
     """Return the repository location of a given prefix."""
     if prefix is None:
         prefix = cls.get_prefix_default()
     key = "prefix-location." + prefix
     config = ResourceLocator.default().get_conf()
     value = config.get_value(["rosie-id", key])
     if value is None:
         raise SuiteIdPrefixError(prefix)
     return value.rstrip("/")
コード例 #28
0
ファイル: ws_client_auth.py プロジェクト: jonseddon/rose
    def _prompt(self, is_retry=False):
        """Prompt for the username and password, where necessary.

        Prompt with zenity or raw_input/getpass.

        """
        if (callable(self.prompt_func) and
                not hasattr(self.password_store, "prompt_password")):
            self.username, self.password = self.prompt_func(
                self.username, self.password, is_retry)
            return

        icon_path = ResourceLocator.default().locate("images/rosie-icon.png")
        if is_retry:
            username = ""
            if self.username:
                username = ""

            prompt = self.PROMPT_USERNAME % {
                "prefix": self.prefix, "root": self.root}
            if self.popen.which("zenity") and os.getenv("DISPLAY"):
                username = self.popen.run(
                    "zenity", "--entry",
                    "--title=Rosie",
                    "--window-icon=" + icon_path,
                    "--text=" + prompt)[1].strip()
            else:
                username = raw_input(prompt)
            if not username:
                raise KeyboardInterrupt(self.STR_CANCELLED)
            if username and username != self.username:
                self.username = username
                self._load_password()
                if self.password:
                    return

        if self.username and self.password is None or is_retry:
            prompt = self.PROMPT_PASSWORD % {"prefix": self.prefix,
                                             "root": self.root,
                                             "username": self.username}
            if hasattr(self.password_store, "prompt_password"):
                password = self.password_store.prompt_password(
                    prompt, self.scheme, self.host, self.username)
            elif self.popen.which("zenity") and os.getenv("DISPLAY"):
                password = self.popen.run(
                    "zenity", "--entry", "--hide-text",
                    "--title=Rosie",
                    "--window-icon=" + icon_path,
                    "--text=" + prompt)[1].strip()
            else:
                password = getpass(prompt)
            if not password:
                raise KeyboardInterrupt(self.STR_CANCELLED)
            if password and password != self.password:
                self.password = password
コード例 #29
0
ファイル: ws.py プロジェクト: matthewrmshin/rose
 def initialize(self, props, prefix, db_url, service_root):
     self.props = props
     self.prefix = prefix
     source_option = "prefix-web." + self.prefix
     source_url_node = ResourceLocator.default().get_conf().get(
         ["rosie-id", source_option])
     self.source_url = ""
     if source_url_node is not None:
         self.source_url = source_url_node.value
     self.dao = rosie.db.DAO(db_url)
     self.service_root = service_root[:-1]  # remove the '?' regex aspect
コード例 #30
0
ファイル: ws.py プロジェクト: csimag/rose
 def __init__(self, template_env, prefix, db_url):
     self.exposed = True
     self.template_env = template_env
     self.prefix = prefix
     source_option = "prefix-web." + self.prefix
     source_url_node = ResourceLocator.default().get_conf().get(
                                       ["rosie-id", source_option])
     self.source_url = ""
     if source_url_node is not None:
         self.source_url = source_url_node.value
     self.dao = rosie.db.DAO(db_url)
コード例 #31
0
    def suites(self,
               user,
               names=None,
               page=1,
               order=None,
               per_page=None,
               no_fuzzy_time="0",
               form=None):
        """List (installed) suites of a user.

        user -- A string containing a valid user ID
        form -- Specify return format. If None, display HTML page. If "json",
                return a JSON data structure.

        """
        user_suite_dir_root = self._get_user_suite_dir_root(user)
        conf = ResourceLocator.default().get_conf()
        per_page_default = int(
            conf.get_value(["rose-bush", "suites-per-page"],
                           self.SUITES_PER_PAGE))
        if not isinstance(per_page, int):
            if per_page:
                per_page = int(per_page)
            else:
                per_page = per_page_default
        if page and per_page:
            page = int(page)
        else:
            page = 1
        data = {
            "logo":
            self.logo,
            "title":
            self.title,
            "host":
            self.host_name,
            "rose_version":
            self.rose_version,
            "script":
            cherrypy.request.script_name,
            "method":
            "suites",
            "no_fuzzy_time":
            no_fuzzy_time,
            "user":
            user,
            "is_option_on":
            (names and shlex.split(str(names)) != ["*"]
             or order is not None and order != "time_desc"
             or per_page is not None and per_page != per_page_default),
            "names":
            names,
            "page":
            page,
            "order":
            order,
            "per_page":
            per_page,
            "per_page_default":
            per_page_default,
            "entries": [],
        }
        name_globs = ["*"]
        if names:
            name_globs = shlex.split(str(names))
        # Get entries
        sub_names = [
            ".service", "log", "share", "work", self.bush_dao.SUITE_CONF
        ]
        for dirpath, dnames, fnames in os.walk(user_suite_dir_root,
                                               followlinks=True):
            if dirpath != user_suite_dir_root and (any(
                    name in dnames or name in fnames for name in sub_names)):
                dnames[:] = []
            else:
                continue
            item = os.path.relpath(dirpath, user_suite_dir_root)
            if not any(fnmatch(item, glob_) for glob_ in name_globs):
                continue
            try:
                data["entries"].append({
                    "name":
                    item,
                    "info": {},
                    "last_activity_time":
                    (self.get_last_activity_time(user, item))
                })
            except OSError:
                continue

        if order == "name_asc":
            data["entries"].sort(key=lambda entry: entry["name"])
        elif order == "name_desc":
            data["entries"].sort(key=lambda entry: entry["name"], reverse=True)
        elif order == "time_asc":
            data["entries"].sort(self._sort_summary_entries, reverse=True)
        else:  # order == "time_desc"
            data["entries"].sort(self._sort_summary_entries)
        data["of_n_entries"] = len(data["entries"])
        if per_page:
            data["n_pages"] = data["of_n_entries"] / per_page
            if data["of_n_entries"] % per_page != 0:
                data["n_pages"] += 1
            offset = (page - 1) * per_page
            data["entries"] = data["entries"][offset:offset + per_page]
        else:
            data["n_pages"] = 1
        # Get suite info for each entry
        for entry in data["entries"]:
            user_suite_dir = os.path.join(user_suite_dir_root, entry["name"])
            rose_suite_info = os.path.join(user_suite_dir, "rose-suite.info")
            try:
                info_root = rose.config.load(rose_suite_info)
                for key, node in info_root.value.items():
                    if (node.is_ignored() or not isinstance(node.value, str)):
                        continue
                    entry["info"][key] = node.value
            except (IOError, rose.config.ConfigSyntaxError):
                pass
        data["time"] = strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
        if form == "json":
            return json.dumps(data)
        template = self.template_env.get_template("suites.html")
        return template.render(**data)
コード例 #32
0
    def get_file(self, user, suite, path, path_in_tar=None, mode=None):
        """Returns file information / content or a cherrypy response."""
        f_name = self._get_user_suite_dir(user, suite, path)
        conf = ResourceLocator.default().get_conf()
        view_size_max = int(
            conf.get_value(["rose-bush", "view-size-max"], self.VIEW_SIZE_MAX))
        if path_in_tar:
            tar_f = tarfile.open(f_name, "r:gz")
            try:
                tar_info = tar_f.getmember(path_in_tar)
            except KeyError:
                raise cherrypy.HTTPError(404)
            f_size = tar_info.size
            handle = tar_f.extractfile(path_in_tar)
            if handle.read(2) == "#!":
                mime = self.MIME_TEXT_PLAIN
            else:
                mime = mimetypes.guess_type(
                    urllib.pathname2url(path_in_tar))[0]
            handle.seek(0)
            if (mode == "download" or f_size > view_size_max or mime and
                (not mime.startswith("text/") or mime.endswith("html"))):
                temp_f = NamedTemporaryFile()
                f_bsize = os.fstatvfs(temp_f.fileno()).f_bsize
                while True:
                    bytes_ = handle.read(f_bsize)
                    if not bytes_:
                        break
                    temp_f.write(bytes_)
                cherrypy.response.headers["Content-Type"] = mime
                try:
                    return cherrypy.lib.static.serve_file(temp_f.name, mime)
                finally:
                    temp_f.close()
            text = handle.read()
        else:
            f_size = os.stat(f_name).st_size
            if open(f_name).read(2) == "#!":
                mime = self.MIME_TEXT_PLAIN
            else:
                mime = mimetypes.guess_type(urllib.pathname2url(f_name))[0]
            if not mime:
                mime = self.MIME_TEXT_PLAIN
            if (mode == "download" or f_size > view_size_max or mime and
                (not mime.startswith("text/") or mime.endswith("html"))):
                cherrypy.response.headers["Content-Type"] = mime
                return cherrypy.lib.static.serve_file(f_name, mime)
            text = open(f_name).read()
        try:
            if mode in [None, "text"]:
                text = jinja2.escape(text)
            lines = [unicode(line) for line in text.splitlines()]
        except UnicodeDecodeError:
            if path_in_tar:
                handle.seek(0)
                # file closed by cherrypy
                return cherrypy.lib.static.serve_fileobj(
                    handle, self.MIME_TEXT_PLAIN)
            else:
                return cherrypy.lib.static.serve_file(f_name,
                                                      self.MIME_TEXT_PLAIN)
        else:
            if path_in_tar:
                handle.close()
        name = path
        if path_in_tar:
            name = "log/" + path_in_tar
        job_entry = None
        if name.startswith("log/job"):
            names = self.bush_dao.parse_job_log_rel_path(name)
            if len(names) == 4:
                cycle, task, submit_num, _ = names
                entries = self.bush_dao.get_suite_job_entries(
                    user, suite, [cycle], [task], None, None, None, None,
                    None)[0]
                for entry in entries:
                    if entry["submit_num"] == int(submit_num):
                        job_entry = entry
                        break
        if fnmatch(os.path.basename(path), "rose*.conf"):
            file_content = "rose-conf"
        else:
            file_content = self.bush_dao.is_conf(path)

        return lines, job_entry, file_content, f_name
コード例 #33
0
ファイル: bush.py プロジェクト: kaday/rose
    def suites(self, user, names=None, page=1, order=None, per_page=None,
               form=None):
        """List (installed) suites of a user.

        user -- A string containing a valid user ID
        form -- Specify return format. If None, display HTML page. If "json",
                return a JSON data structure.

        """
        user_suite_dir_root = self._get_user_suite_dir_root(user)
        conf = ResourceLocator.default().get_conf()
        per_page_default = int(conf.get_value(
            ["rose-bush", "suites-per-page"], self.SUITES_PER_PAGE))
        if not isinstance(per_page, int):
            if per_page:
                per_page = int(per_page)
            else:
                per_page = per_page_default
        if page and per_page:
            page = int(page)
        else:
            page = 1
        data = {
            "logo": self.logo,
            "title": self.title,
            "host": self.host_name,
            "rose_version": self.rose_version,
            "script": cherrypy.request.script_name,
            "method": "suites",
            "user": user,
            "is_option_on": (
                names and shlex.split(str(names)) != ["*"] or
                order is not None and order != "time_desc" or
                per_page is not None and per_page != per_page_default
            ),
            "names": names,
            "page": page,
            "order": order,
            "per_page": per_page,
            "per_page_default": per_page_default,
            "entries": [],
        }
        name_globs = ["*"]
        if names:
            name_globs = shlex.split(str(names))
        # Get entries
        try:
            items = os.listdir(user_suite_dir_root)
        except OSError:
            items = []
        for item in items:
            if not any([fnmatch(item, glob_) for glob_ in name_globs]):
                continue
            user_suite_dir = os.path.join(user_suite_dir_root, item)
            suite_conf = os.path.join(
                user_suite_dir, self.suite_engine_proc.SUITE_CONF)
            job_logs_db = os.path.join(
                user_suite_dir, self.suite_engine_proc.JOB_LOGS_DB)
            if (not os.path.exists(job_logs_db) and
                    not os.path.exists(suite_conf)):
                continue
            suite_db = os.path.join(
                user_suite_dir, self.suite_engine_proc.SUITE_DB)
            try:
                last_activity_time = strftime(
                    "%Y-%m-%dT%H:%M:%S+0000",
                    gmtime(os.stat(suite_db).st_mtime))
            except OSError:
                last_activity_time = None
            data["entries"].append({
                "name": item,
                "info": {},
                "last_activity_time": last_activity_time})

        if order == "name_asc":
            data["entries"].sort(key=lambda entry: entry["name"])
        elif order == "name_desc":
            data["entries"].sort(key=lambda entry: entry["name"], reverse=True)
        elif order == "time_asc":
            data["entries"].sort(self._sort_summary_entries, reverse=True)
        else:  # order == "time_desc"
            data["entries"].sort(self._sort_summary_entries)
        data["of_n_entries"] = len(data["entries"])
        if per_page:
            data["n_pages"] = data["of_n_entries"] / per_page
            if data["of_n_entries"] % per_page != 0:
                data["n_pages"] += 1
            offset = (page - 1) * per_page
            data["entries"] = data["entries"][offset:offset + per_page]
        else:
            data["n_pages"] = 1
        # Get suite info for each entry
        for entry in data["entries"]:
            user_suite_dir = os.path.join(user_suite_dir_root, entry["name"])
            rose_suite_info = os.path.join(user_suite_dir, "rose-suite.info")
            try:
                info_root = rose.config.load(rose_suite_info)
                for key, node in info_root.value.items():
                    if (node.is_ignored() or
                            not isinstance(node.value, str)):
                        continue
                    entry["info"][key] = node.value
            except (IOError, rose.config.ConfigSyntaxError):
                pass
        data["time"] = strftime("%Y-%m-%dT%H:%M:%S+0000", gmtime())
        if form == "json":
            return simplejson.dumps(data)
        template = self.template_env.get_template("suites.html")
        return template.render(**data)
コード例 #34
0
ファイル: stem.py プロジェクト: craigmaclachlan/rose
 def _read_site_config_and_return_options(self):
     """Read the site rose.conf file."""
     return ResourceLocator.default().get_conf().get_value(
         ["rose-stem", "automatic-options"])
コード例 #35
0
    def run_impl(self, opts, args, uuid, work_files):
        # Log file, temporary
        if hasattr(self.event_handler, "contexts"):
            t_file = TemporaryFile()
            log_context = ReporterContext(None, self.event_handler.VV, t_file)
            self.event_handler.contexts[uuid] = log_context

        # Check suite engine specific compatibility
        self.suite_engine_proc.check_global_conf_compat()

        # Suite name from the current working directory
        if opts.conf_dir:
            self.fs_util.chdir(opts.conf_dir)
        opts.conf_dir = os.getcwd()

        # --remote=KEY=VALUE,...
        if opts.remote:
            # opts.name always set for remote.
            return self._run_remote(opts, opts.name)

        conf_tree = self.config_load(opts)
        self.fs_util.chdir(conf_tree.conf_dirs[0])

        suite_name = opts.name
        if not opts.name:
            suite_name = os.path.basename(os.getcwd())

        # Check suite.rc #! line for template scheme
        templ_scheme = "jinja2"
        if self.suite_engine_proc.SUITE_CONF in conf_tree.files:
            suiterc_path = os.path.join(
                conf_tree.files[self.suite_engine_proc.SUITE_CONF],
                self.suite_engine_proc.SUITE_CONF)
            with open(suiterc_path) as fh:
                line = fh.readline()
                if line.startswith("#!"):
                    templ_scheme = line[2:].strip().lower()
        suite_section = (templ_scheme + ':' +
                         self.suite_engine_proc.SUITE_CONF)

        extra_defines = []
        if opts.defines_suite:
            for define in opts.defines_suite:
                extra_defines.append("[" + suite_section + "]" + define)

        # Automatic Rose constants
        # ROSE_ORIG_HOST: originating host
        # ROSE_VERSION: Rose version (not retained in run_mode=="reload")
        # Suite engine version
        my_rose_version = ResourceLocator.default().get_version()
        suite_engine_key = self.suite_engine_proc.get_version_env_name()
        if opts.run_mode in ["reload", "restart"]:
            prev_config_path = self.suite_engine_proc.get_suite_dir(
                suite_name, "log", "rose-suite-run.conf")
            prev_config = ConfigLoader()(prev_config_path)
            suite_engine_version = prev_config.get_value(
                ["env", suite_engine_key])
        else:
            suite_engine_version = self.suite_engine_proc.get_version()
        resloc = ResourceLocator.default()
        auto_items = [(suite_engine_key, suite_engine_version),
                      ("ROSE_ORIG_HOST", self.host_selector.get_local_host()),
                      ("ROSE_SITE", resloc.get_conf().get_value(['site'], '')),
                      ("ROSE_VERSION", resloc.get_version())]
        for key, val in auto_items:
            requested_value = conf_tree.node.get_value(["env", key])
            if requested_value:
                if key == "ROSE_VERSION" and val != requested_value:
                    exc = VersionMismatchError(requested_value, val)
                    raise ConfigValueError(["env", key], requested_value, exc)
                val = requested_value
            else:
                conf_tree.node.set(["env", key],
                                   val,
                                   state=conf_tree.node.STATE_NORMAL)
            extra_defines.append('[%s]%s="%s"' % (suite_section, key, val))

        # Pass automatic Rose constants as suite defines
        self.conf_tree_loader.node_loader.load_defines(extra_defines,
                                                       conf_tree.node)

        # See if suite is running or not
        if opts.run_mode == "reload":
            # Check suite is running
            self.suite_engine_proc.get_suite_contact(suite_name)
        else:
            self.suite_engine_proc.check_suite_not_running(suite_name)

        # Install the suite to its run location
        suite_dir_rel = self._suite_dir_rel(suite_name)

        # Unfortunately a large try/finally block to ensure a temporary folder
        # created in validate only mode is cleaned up. Exceptions are not
        # caught here
        try:
            # Process Environment Variables
            environ = self.config_pm(conf_tree, "env")

            if opts.validate_suite_only_mode:
                temp_dir = mkdtemp()
                suite_dir = os.path.join(temp_dir, suite_dir_rel)
                os.makedirs(suite_dir, 0o0700)
            else:
                suite_dir = os.path.join(os.path.expanduser("~"),
                                         suite_dir_rel)

            suite_conf_dir = os.getcwd()
            locs_conf = ConfigNode()
            if opts.new_mode:
                if os.getcwd() == suite_dir:
                    raise NewModeError("PWD", os.getcwd())
                elif opts.run_mode in ["reload", "restart"]:
                    raise NewModeError("--run", opts.run_mode)
                self.suite_run_cleaner.clean(suite_name)
            if os.getcwd() != suite_dir:
                if opts.run_mode == "run":
                    self._run_init_dir(opts,
                                       suite_name,
                                       conf_tree,
                                       locs_conf=locs_conf)
                os.chdir(suite_dir)

            # Housekeep log files
            now_str = None
            if not opts.install_only_mode and not opts.local_install_only_mode:
                now_str = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
                self._run_init_dir_log(opts, now_str)
            self.fs_util.makedirs("log/suite")

            # Rose configuration and version logs
            self.fs_util.makedirs("log/rose-conf")
            run_mode = opts.run_mode
            if run_mode not in ["reload", "restart", "run"]:
                run_mode = "run"
            mode = run_mode
            if opts.validate_suite_only_mode:
                mode = "validate-suite-only"
            elif opts.install_only_mode:
                mode = "install-only"
            elif opts.local_install_only_mode:
                mode = "local-install-only"
            prefix = "rose-conf/%s-%s" % (strftime("%Y%m%dT%H%M%S"), mode)

            # Dump the actual configuration as rose-suite-run.conf
            ConfigDumper()(conf_tree.node, "log/" + prefix + ".conf")

            # Install version information file
            write_source_vc_info(suite_conf_dir, "log/" + prefix + ".version",
                                 self.popen)

            # If run through rose-stem, install version information files for
            # each source tree if they're a working copy
            if hasattr(opts, 'source') and hasattr(opts, 'project'):
                for i, url in enumerate(opts.source):
                    if os.path.isdir(url):
                        write_source_vc_info(
                            url, "log/" + opts.project[i] + "-" + str(i) +
                            ".version", self.popen)

            for ext in [".conf", ".version"]:
                self.fs_util.symlink(prefix + ext, "log/rose-suite-run" + ext)

            # Move temporary log to permanent log
            if hasattr(self.event_handler, "contexts"):
                log_file_path = os.path.abspath(
                    os.path.join("log", "rose-suite-run.log"))
                log_file = open(log_file_path, "ab")
                temp_log_file = self.event_handler.contexts[uuid].handle
                temp_log_file.seek(0)
                log_file.write(temp_log_file.read())
                self.event_handler.contexts[uuid].handle = log_file
                temp_log_file.close()

            # Process Files
            cwd = os.getcwd()
            for rel_path, conf_dir in conf_tree.files.items():
                if (conf_dir == cwd or any(
                        fnmatchcase(os.sep + rel_path, exclude)
                        for exclude in self.SYNC_EXCLUDES)
                        or conf_tree.node.get([templ_scheme + ":" + rel_path
                                               ]) is not None):
                    continue
                # No sub-directories, very slow otherwise
                if os.sep in rel_path:
                    rel_path = rel_path.split(os.sep, 1)[0]
                target_key = self.config_pm.get_handler(
                    "file").PREFIX + rel_path
                target_node = conf_tree.node.get([target_key])
                if target_node is None:
                    conf_tree.node.set([target_key])
                    target_node = conf_tree.node.get([target_key])
                elif target_node.is_ignored():
                    continue
                source_node = target_node.get("source")
                if source_node is None:
                    target_node.set(["source"],
                                    os.path.join(conf_dir, rel_path))
                elif source_node.is_ignored():
                    continue
            self.config_pm(conf_tree,
                           "file",
                           no_overwrite_mode=opts.no_overwrite_mode)

            # Process suite configuration template header
            # (e.g. Jinja2:suite.rc, EmPy:suite.rc)
            self.config_pm(conf_tree, templ_scheme, environ=environ)

            # Ask suite engine to parse suite configuration
            # and determine if it is up to date (unchanged)
            if opts.validate_suite_only_mode:
                suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf(
                    suite_dir, None, opts.strict_mode, debug_mode=True)
            else:
                suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf(
                    suite_name, opts.run_mode, opts.strict_mode,
                    opts.debug_mode)
        finally:
            # Ensure the temporary directory created is cleaned up regardless
            # of success or failure
            if opts.validate_suite_only_mode and os.path.exists(temp_dir):
                shutil.rmtree(temp_dir)

        # Only validating so finish now
        if opts.validate_suite_only_mode:
            return

        # Install share/work directories (local)
        for name in ["share", "share/cycle", "work"]:
            self._run_init_dir_work(opts,
                                    suite_name,
                                    name,
                                    conf_tree,
                                    locs_conf=locs_conf)

        if opts.local_install_only_mode:
            return

        # Install suite files to each remote [user@]host
        for name in ["", "log/", "share/", "share/cycle/", "work/"]:
            uuid_file = os.path.abspath(name + uuid)
            open(uuid_file, "w").close()
            work_files.append(uuid_file)

        # Install items to user@host
        auths = self.suite_engine_proc.get_tasks_auths(suite_name)
        proc_queue = []  # [[proc, command, "ssh"|"rsync", auth], ...]
        for auth in sorted(auths):
            host = auth
            if "@" in auth:
                host = auth.split("@", 1)[1]
            # Remote shell
            command = self.popen.get_cmd("ssh", "-n", auth)
            # Provide ROSE_VERSION and CYLC_VERSION in the environment
            shcommand = "env ROSE_VERSION=%s %s=%s" % (
                my_rose_version, suite_engine_key, suite_engine_version)
            # Use login shell?
            no_login_shell = self._run_conf("remote-no-login-shell",
                                            host=host,
                                            conf_tree=conf_tree)
            if not no_login_shell or no_login_shell.lower() != "true":
                shcommand += r""" bash -l -c '"$0" "$@"'"""
            # Path to "rose" command, if applicable
            rose_bin = self._run_conf("remote-rose-bin",
                                      host=host,
                                      conf_tree=conf_tree,
                                      default="rose")
            # Build remote "rose suite-run" command
            shcommand += " %s suite-run -vv -n %s" % (rose_bin, suite_name)
            for key in ["new", "debug", "install-only"]:
                attr = key.replace("-", "_") + "_mode"
                if getattr(opts, attr, None) is not None:
                    shcommand += " --%s" % key
            if opts.log_keep:
                shcommand += " --log-keep=%s" % opts.log_keep
            if opts.log_name:
                shcommand += " --log-name=%s" % opts.log_name
            if not opts.log_archive_mode:
                shcommand += " --no-log-archive"
            shcommand += " --run=%s" % opts.run_mode
            # Build --remote= option
            shcommand += " --remote=uuid=%s" % uuid
            if now_str is not None:
                shcommand += ",now-str=%s" % now_str
            host_confs = [
                "root-dir", "root-dir{share}", "root-dir{share/cycle}",
                "root-dir{work}"
            ]
            locs_conf.set([auth])
            for key in host_confs:
                value = self._run_conf(key, host=host, conf_tree=conf_tree)
                if value is not None:
                    val = self.popen.list_to_shell_str([str(value)])
                    shcommand += ",%s=%s" % (key, pipes.quote(val))
                    locs_conf.set([auth, key], value)
            command.append(shcommand)
            proc = self.popen.run_bg(*command)
            proc_queue.append([proc, command, "ssh", auth])

        while proc_queue:
            sleep(self.SLEEP_PIPE)
            proc, command, command_name, auth = proc_queue.pop(0)
            if proc.poll() is None:  # put it back in proc_queue
                proc_queue.append([proc, command, command_name, auth])
                continue
            ret_code = proc.wait()
            out, err = proc.communicate()
            if ret_code:
                raise RosePopenError(command, ret_code, out, err)
            if command_name == "rsync":
                self.handle_event(out, level=Event.VV)
                continue
            else:
                self.handle_event(out, level=Event.VV, prefix="[%s] " % auth)
            for line in out.split("\n"):
                if "/" + uuid == line.strip():
                    locs_conf.unset([auth])
                    break
            else:
                filters = {"excludes": [], "includes": []}
                for name in ["", "log/", "share/", "share/cycle/", "work/"]:
                    filters["excludes"].append(name + uuid)
                target = auth + ":" + suite_dir_rel
                cmd = self._get_cmd_rsync(target, **filters)
                proc_queue.append(
                    [self.popen.run_bg(*cmd), cmd, "rsync", auth])

        # Install ends
        ConfigDumper()(locs_conf, os.path.join("log", "rose-suite-run.locs"))
        if opts.install_only_mode:
            return
        elif opts.run_mode == "reload" and suite_conf_unchanged:
            conf_name = self.suite_engine_proc.SUITE_CONF
            self.handle_event(SkipReloadEvent(suite_name, conf_name))
            return

        # Start the suite
        self.fs_util.chdir("log")
        self.suite_engine_proc.run(suite_name, opts.host, opts.run_mode, args)

        # Disconnect log file handle, so monitoring tool command will no longer
        # be associated with the log file.
        self.event_handler.contexts[uuid].handle.close()
        self.event_handler.contexts.pop(uuid)

        # Launch the monitoring tool
        # Note: maybe use os.ttyname(sys.stdout.fileno())?
        if opts.gcontrol_mode and opts.run_mode != "reload":
            self.suite_engine_proc.gcontrol(suite_name)

        return 0
コード例 #36
0
    def restart(self,
                suite_name=None,
                host=None,
                gcontrol_mode=None,
                args=None):
        """Restart a "cylc" suite."""
        # Check suite engine specific compatibility
        self.suite_engine_proc.check_global_conf_compat()

        if not suite_name:
            suite_name = get_suite_name(self.event_handler)

        suite_dir = self.suite_engine_proc.get_suite_dir(suite_name)
        if not os.path.exists(suite_dir):
            raise SuiteNotFoundError(suite_dir)

        # Ensure suite is not running
        hosts = []
        if host:
            hosts.append(host)
        self.suite_engine_proc.check_suite_not_running(suite_name, hosts)

        # Determine suite host to restart suite
        if host:
            hosts = [host]
        else:
            hosts = []
            val = ResourceLocator.default().get_conf().get_value(
                ["rose-suite-run", "hosts"], "localhost")
            known_hosts = self.host_selector.expand(val.split())[0]
            for known_host in known_hosts:
                if known_host not in hosts:
                    hosts.append(known_host)

        if hosts == ["localhost"]:
            host = hosts[0]
        else:
            host = self.host_selector(hosts)[0][0]
        self.handle_event(SuiteHostSelectEvent(suite_name, "restart", host))

        # Suite host environment
        run_conf_file_name = self.suite_engine_proc.get_suite_dir(
            suite_name, "log", "rose-suite-run.conf")
        try:
            run_conf = ConfigLoader().load(run_conf_file_name)
        except (ConfigSyntaxError, IOError):
            environ = None
        else:
            run_conf_tree = ConfigTree()
            run_conf_tree.node = run_conf
            environ = self.config_pm(run_conf_tree, "env")

        # Restart the suite
        self.suite_engine_proc.run(suite_name, host, environ, "restart", args)

        # Write suite host name to host file
        host_file_name = self.suite_engine_proc.get_suite_dir(
            suite_name, "log", "rose-suite-run.host")
        open(host_file_name, "w").write(host + "\n")

        # Launch the monitoring tool
        # Note: maybe use os.ttyname(sys.stdout.fileno())?
        if os.getenv("DISPLAY") and host and gcontrol_mode:
            self.suite_engine_proc.gcontrol(suite_name, host)

        return
コード例 #37
0
 def cycles(self,
            user,
            suite,
            page=1,
            order=None,
            per_page=None,
            no_fuzzy_time="0",
            form=None):
     """List cycles of a running or completed suite."""
     conf = ResourceLocator.default().get_conf()
     per_page_default = int(
         conf.get_value(["rose-bush", "cycles-per-page"],
                        self.CYCLES_PER_PAGE))
     if not isinstance(per_page, int):
         if per_page:
             per_page = int(per_page)
         else:
             per_page = per_page_default
     if page and per_page:
         page = int(page)
     else:
         page = 1
     data = {
         "logo":
         self.logo,
         "title":
         self.title,
         "host":
         self.host_name,
         "user":
         user,
         "suite":
         suite,
         "is_option_on":
         (order is not None and order != "time_desc"
          or per_page is not None and per_page != per_page_default),
         "order":
         order,
         "rose_version":
         self.rose_version,
         "script":
         cherrypy.request.script_name,
         "method":
         "cycles",
         "no_fuzzy_time":
         no_fuzzy_time,
         "states": {},
         "per_page":
         per_page,
         "per_page_default":
         per_page_default,
         "page":
         page,
         "task_status_groups":
         self.bush_dao.TASK_STATUS_GROUPS,
     }
     data["entries"], data["of_n_entries"] = (
         self.bush_dao.get_suite_cycles_summary(user, suite, order,
                                                per_page,
                                                (page - 1) * per_page))
     if per_page:
         data["n_pages"] = data["of_n_entries"] / per_page
         if data["of_n_entries"] % per_page != 0:
             data["n_pages"] += 1
     else:
         data["n_pages"] = 1
     data.update(self._get_suite_logs_info(user, suite))
     data["states"].update(
         self.bush_dao.get_suite_state_summary(user, suite))
     data["states"]["last_activity_time"] = (self.get_last_activity_time(
         user, suite))
     data["time"] = strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
     if form == "json":
         return json.dumps(data)
     try:
         return self.template_env.get_template("cycles.html").render(**data)
     except jinja2.TemplateError:
         traceback.print_exc()
     return json.dumps(data)
コード例 #38
0
    def select(self,
               names=None,
               rank_method=None,
               thresholds=None,
               ssh_cmd_timeout=None):
        """Return a list. Element 0 is most desirable.
        Each element of the list is a tuple (host, score).

        names: a list of known host groups or host names.
        rank_method: the ranking method. Can be one of:
                     load:1, load:5, load:15 (=load =default), fs:FS and
                     random.  The "load" methods determines the load using the
                     average load as returned by the "uptime" command divided
                     by the number of CPUs. The "fs" method determines the load
                     using the usage in the file system specified by FS. The
                     "mem" method ranks by highest free memory. The "random"
                     method ranks everything by random.

        thresholds: a list of thresholds which each host must not exceed.
                    Should be in the format rank_method:value, where
                    rank_method is one of load:1, load:5, load:15 or fs:FS; and
                    value is number that must be be exceeded.

        ssh_cmd_timeout: timeout of SSH commands to hosts. A float in seconds.

        """

        host_names, rank_method, thresholds = self.expand(
            names, rank_method, thresholds)

        # Load scorers, ranking and thresholds
        rank_method_arg = None
        if rank_method:
            if ":" in rank_method:
                rank_method, rank_method_arg = rank_method.split(":", 1)
        else:
            rank_method = self.RANK_METHOD_DEFAULT
        rank_conf = ScorerConf(self.get_scorer(rank_method), rank_method_arg)
        self.handle_event(RankMethodEvent(rank_method, rank_method_arg))

        threshold_confs = []
        if thresholds:
            for threshold in thresholds:
                method = self.RANK_METHOD_DEFAULT
                method_arg = None
                value = threshold
                if ":" in threshold:
                    head, value = threshold.rsplit(":", 1)
                    method = head
                    if ":" in head:
                        method, method_arg = head.split(":", 1)
                try:
                    float(value)
                except ValueError:
                    raise ValueError(threshold)
                scorer = self.get_scorer(method)
                if method_arg is None:
                    method_arg = scorer.ARG
                threshold_conf = ScorerConf(self.get_scorer(method),
                                            method_arg, value)
                threshold_confs.append(threshold_conf)

        if ssh_cmd_timeout is None:
            conf = ResourceLocator.default().get_conf()
            ssh_cmd_timeout = float(
                conf.get_value(["rose-host-select", "timeout"],
                               self.SSH_CMD_TIMEOUT))

        host_name_list = list(host_names)
        host_names = []
        for host_name in host_name_list:
            if self.is_local_host(host_name):
                if self.get_local_host() not in host_names:
                    host_names.append(self.get_local_host())
            else:
                host_names.append(host_name)

        # Random selection with no thresholds. Return the 1st available host.
        if rank_conf.method == self.RANK_METHOD_RANDOM and not threshold_confs:
            shuffle(host_names)
            for host_name in host_names:
                if self.is_local_host(host_name):
                    return [("localhost", 1)]
                command = self.popen.get_cmd("ssh", host_name, "true")
                proc = self.popen.run_bg(*command, preexec_fn=os.setpgrp)
                time0 = time()
                while (proc.poll() is None
                       and time() - time0 <= ssh_cmd_timeout):
                    sleep(self.SSH_CMD_POLL_DELAY)
                if proc.poll() is None:
                    os.killpg(proc.pid, signal.SIGTERM)
                    proc.wait()
                    self.handle_event(TimedOutHostEvent(host_name))
                elif proc.wait():
                    self.handle_event(DeadHostEvent(host_name))
                else:
                    return [(host_name, 1)]
            else:
                raise NoHostSelectError()

        # ssh to each host to return its score(s).
        host_proc_dict = {}
        for host_name in sorted(host_names):
            command = []
            if not self.is_local_host(host_name):
                command_args = []
                command_args.append(host_name)
                command = self.popen.get_cmd("ssh", *command_args)
            command.append("bash")
            stdin = rank_conf.get_command()
            for threshold_conf in threshold_confs:
                stdin += threshold_conf.get_command()
            stdin += "exit\n"
            proc = self.popen.run_bg(*command,
                                     stdin=stdin,
                                     preexec_fn=os.setpgrp)
            proc.stdin.write(stdin)
            proc.stdin.flush()
            host_proc_dict[host_name] = proc

        # Retrieve score for each host name
        host_score_list = []
        time0 = time()
        while host_proc_dict:
            sleep(self.SSH_CMD_POLL_DELAY)
            for host_name, proc in host_proc_dict.items():
                if proc.poll() is None:
                    score = None
                elif proc.wait():
                    self.handle_event(DeadHostEvent(host_name))
                    host_proc_dict.pop(host_name)
                else:
                    out = proc.communicate()[0]
                    host_proc_dict.pop(host_name)
                    for threshold_conf in threshold_confs:
                        try:
                            is_bad = threshold_conf.check_threshold(out)
                            score = threshold_conf.command_out_parser(out)
                        except ValueError:
                            is_bad = True
                            score = None
                        if is_bad:
                            self.handle_event(
                                HostThresholdNotMetEvent(
                                    host_name, threshold_conf, score))
                            break
                    else:
                        try:
                            score = rank_conf.command_out_parser(out)
                            host_score_list.append((host_name, score))
                        except ValueError:
                            score = None
                        self.handle_event(
                            HostSelectScoreEvent(host_name, score))
            if time() - time0 > ssh_cmd_timeout:
                break

        # Report timed out hosts
        for host_name, proc in sorted(host_proc_dict.items()):
            self.handle_event(TimedOutHostEvent(host_name))
            os.killpg(proc.pid, signal.SIGTERM)
            proc.wait()

        if not host_score_list:
            raise NoHostSelectError()
        host_score_list.sort(lambda a, b: cmp(a[1], b[1]),
                             reverse=rank_conf.scorer.SIGN < 0)
        return host_score_list
コード例 #39
0
    def run(self, suite_name, task_id, hook_event, hook_message=None,
            should_mail=False, mail_cc_list=None, should_shutdown=False,
            should_retrieve_job_logs=False):
        """
        Invoke the hook for a suite.

        1. For a task hook, if the task runs remotely, retrieve its log from
           the remote host.
        2. If "should_mail", send an email notification to the current user,
           and those in the "mail_cc_list".
        3. If "should_shutdown", shut down the suite.

        """
        # Retrieve log and populate job logs database
        task_ids = []
        if task_id and should_retrieve_job_logs:
            task_ids = [task_id]
            self.suite_engine_proc.job_logs_pull_remote(suite_name, task_ids)

        # Send email notification if required
        email_exc = None
        if should_mail:
            text = ""
            if task_id:
                text += "Task: %s\n" % task_id
            if hook_message:
                text += "Message: %s\n" % hook_message
            url = self.suite_engine_proc.get_suite_log_url(None, suite_name)
            text += "See: %s\n" % (url)
            user = pwd.getpwuid(os.getuid()).pw_name
            conf = ResourceLocator.default().get_conf()
            host = conf.get_value(["rose-suite-hook", "email-host"],
                                  default="localhost")
            msg = MIMEText(text)
            msg["From"] = user + "@" + host
            msg["To"] = msg["From"]
            if mail_cc_list:
                mail_cc_addresses = []
                for mail_cc_address in mail_cc_list:
                    if "@" not in mail_cc_address:
                        mail_cc_address += "@" + host
                    mail_cc_addresses.append(mail_cc_address)
                msg["Cc"] = ", ".join(mail_cc_addresses)
                mail_cc_list = mail_cc_addresses
            else:
                mail_cc_list = []
            msg["Subject"] = "[%s] %s" % (hook_event, suite_name)
            smtp_host = conf.get_value(["rose-suite-hook", "smtp-host"],
                                       default="localhost")
            try:
                smtp = SMTP(smtp_host)
                smtp.sendmail(
                    msg["From"], [msg["To"]] + mail_cc_list, msg.as_string())
                smtp.quit()
            except (socket.error, SMTPException) as email_exc:
                pass

        # Shut down if required
        if should_shutdown:
            self.suite_engine_proc.shutdown(suite_name, args=["--kill"])

        if email_exc is not None:
            raise
コード例 #40
0
def main():
    """Implement the "rose config" command."""
    opt_parser = RoseOptionParser()
    opt_parser.add_my_options("default", "env_var_process_mode", "files",
                              "keys", "meta", "meta_key", "no_ignore",
                              "no_opts", "print_conf_mode")
    opts, args = opt_parser.parse_args()
    report = Reporter(opts.verbosity - opts.quietness)

    rose.macro.add_meta_paths()

    if opts.meta_key:
        opts.meta = True

    if opts.files and opts.meta_key:
        report(Exception("Cannot specify both a file and meta key."))
        sys.exit(1)

    config_loader = ConfigLoader()
    sources = []
    if opts.files:
        root_node = ConfigNode()
        for fname in opts.files:
            if fname == "-":
                sources.append(sys.stdin)
            else:
                if opts.meta:
                    try:
                        root_node = config_loader.load(fname)
                    except ConfigSyntaxError as exc:
                        report(exc)
                        sys.exit(1)
                    rel_path = os.sep.join(fname.split(os.sep)[:-1])
                    fpath = get_meta_path(root_node, rel_path)
                    if fpath is None:
                        report(MetadataNotFoundEvent(fname))
                    else:
                        sources.append(fpath)
                else:
                    sources.append(fname)
    elif opts.meta:
        root_node = ConfigNode()
        if opts.meta_key:
            root_node.set(["meta"], opts.meta_key)
        else:
            fname = os.path.join(os.getcwd(), rose.SUB_CONFIG_NAME)
            try:
                root_node = config_loader.load(fname)
            except ConfigSyntaxError as exc:
                report(exc)
                sys.exit(1)
        fpath = get_meta_path(root_node, meta_key=opts.meta_key)
        root_node.unset(["meta"])
        if fpath is None:
            report(Exception("Metadata not found"))
            sys.exit(1)
        else:
            sources.append(fpath)
    else:
        root_node = ResourceLocator.default().get_conf()

    for source in sources:
        try:
            if opts.meta or opts.no_opts:
                config_loader.load(source, root_node)
            else:
                config_loader.load_with_opts(source, root_node)
        except (ConfigSyntaxError, IOError) as exc:
            report(exc)
            sys.exit(1)
        if source is sys.stdin:
            source.close()

    if opts.quietness:
        sys.exit(root_node.get(args, opts.no_ignore) is None)

    if opts.keys_mode:
        try:
            keys = root_node.get(args, opts.no_ignore).value.keys()
        except AttributeError:
            sys.exit(1)
        keys.sort()
        for key in keys:
            print key
        sys.exit()

    conf_dump = ConfigDumper()
    if len(args) == 0:
        conf_dump(root_node, concat_mode=opts.print_conf_mode)
        sys.exit()

    node = root_node.get(args, opts.no_ignore)

    if node is not None and isinstance(node.value, dict):
        if opts.print_conf_mode:
            conf_dump(ConfigNode().set(args, node.value), concat_mode=True)
            sys.exit()

        keys = node.value.keys()
        keys.sort()
        for key in keys:
            node_of_key = node.get([key], opts.no_ignore)
            if node_of_key:
                value = node_of_key.value
                state = node_of_key.state
                string = "%s%s=%s" % (state, key, value)
                lines = string.splitlines()
                print lines[0]
                i_equal = len(state + key) + 1
                for line in lines[1:]:
                    print " " * i_equal + line
        sys.exit()

    if node is None:
        if opts.default is None:
            sys.exit(1)
        value = opts.default
    elif opts.env_var_process_mode:
        value = env_var_process(node.value)
    else:
        value = node.value
    if opts.print_conf_mode:
        conf_dump(ConfigNode().set(args, value), concat_mode=True)
    else:
        print value
    sys.exit()
コード例 #41
0
ファイル: vc.py プロジェクト: kaday/rose
    def generate_info_config(self, from_id=None, prefix=None, project=None):
        """Generate a rose.config.ConfigNode for a rose-suite.info.

        This is suitable for passing into the create method of this
        class.
        If from_id is defined, copy items from it.
        Return the rose.config.ConfigNode instance.

        """
        from_project = None
        from_title = None
        if from_id is not None:
            from_info_url = "%s/%s/rose-suite.info@%s" % (
                from_id.to_origin(), from_id.branch, from_id.revision)
            out_data = self.popen("svn", "cat", from_info_url)[0]
            from_config = rose.config.load(StringIO(out_data))

        res_loc = ResourceLocator.default()
        older_config = None
        info_config = rose.config.ConfigNode()

        # Determine project if given as a command-line option on create
        if from_id is None and project is not None:
            info_config.set(["project"], project)

        # Set the compulsory fields and use the project and metadata if
        #  available.
        meta_config = load_meta_config(info_config,
                                       config_type=rose.INFO_CONFIG_NAME)
        if from_id is None and project is not None:
            for node_keys, node in meta_config.walk(no_ignore=True):
                if isinstance(node.value, dict):
                    continue
                sect, key = node_keys
                value = node.value
                sect = sect.translate(None, "=")
                if key == "compulsory" and value == "true":
                    info_config.set([sect], "")
            info_config.set(["project"], project)
        else:
            if from_project is None:
                info_config.set(["project"], "")
            if from_title is None:
                info_config.set(["title"], "")

        # Determine prefix
        if prefix is None:
            if from_id is None:
                prefix = SuiteId.get_prefix_default()
            else:
                prefix = from_id.prefix

        # Determine owner:
        # 1. From user configuration [rosie-id]prefix-username
        # 2. From username of a matching group in [groups] in
        #    ~/.subversion/servers
        # 3. Current user ID
        owner = res_loc.get_conf().get_value(
            ["rosie-id", "prefix-username." + prefix])
        if not owner and self.subversion_servers_conf:
            servers_conf = rose.config.load(self.subversion_servers_conf)
            groups_node = servers_conf.get(["groups"])
            if groups_node is not None:
                prefix_loc = SuiteId.get_prefix_location(prefix)
                prefix_host = urlparse(prefix_loc).hostname
                for key, node in groups_node.value.items():
                    if fnmatch(prefix_host, node.value):
                        owner = servers_conf.get_value([key, "username"])
                        break
        if not owner:
            owner = pwd.getpwuid(os.getuid())[0]
        info_config.set(["owner"], owner)

        # Copy description
        try:
            from_id.to_string_with_version()
            info_config.set(["description"],
                            "Copy of %s" % (from_id.to_string_with_version()))
        except AttributeError:
            pass

        # Copy fields provided by the user
        try:
            from_config.walk(no_ignore=False)
            for node_keys, node in from_config.walk(no_ignore=False):
                if isinstance(node.value, dict):
                    continue
                sect, key = node_keys
                value = node.value
                if (key in ["description", "owner", "access-list"]
                        or (key == "project" and from_project is not None)):
                    pass
                else:
                    info_config.set([key], value)
        except UnboundLocalError:
            pass

        # Determine access list
        access_list_str = res_loc.get_conf().get_value(
            ["rosie-vc", "access-list-default"])
        if access_list_str:
            info_config.set(["access-list"], access_list_str)
        if from_id is None and project is not None:
            for node_keys, node in meta_config.walk(no_ignore=True):
                if isinstance(node.value, dict):
                    continue
                sect, key = node_keys
                value = node.value
                sect = sect.translate(None, "=")
                if key == "value-hints" or key == "values":
                    reminder = ("please remove all commented hints/lines " +
                                "in the main/top section before saving.")
                    info_config.set([sect],
                                    rose.variable.array_split(value)[0],
                                    comments=[value, reminder])
        if older_config is not None:
            for node_keys, node in older_config.walk(no_ignore=True):
                if isinstance(node.value, dict):
                    continue
                sect, key = node_keys
                value = node.value
                info_config.set([key], value)

        return info_config
コード例 #42
0
ファイル: suite_run.py プロジェクト: scwhitehouse/rose
    def run_impl(self, opts, args, uuid, work_files):
        # Log file, temporary
        if hasattr(self.event_handler, "contexts"):
            t_file = TemporaryFile()
            log_context = ReporterContext(None, self.event_handler.VV, t_file)
            self.event_handler.contexts[uuid] = log_context

        # Check suite engine specific compatibility
        self.suite_engine_proc.check_global_conf_compat()

        # Suite name from the current working directory
        if opts.conf_dir:
            self.fs_util.chdir(opts.conf_dir)
        opts.conf_dir = os.getcwd()

        if opts.defines_suite:
            suite_section = "jinja2:" + self.suite_engine_proc.SUITE_CONF
            if not opts.defines:
                opts.defines = []
            for define in opts.defines_suite:
                opts.defines.append("[" + suite_section + "]" + define)

        # --remote=KEY=VALUE,...
        if opts.remote:
            # opts.name always set for remote.
            return self._run_remote(opts, opts.name)

        conf_tree = self.config_load(opts)
        self.fs_util.chdir(conf_tree.conf_dirs[0])

        suite_name = opts.name
        if not opts.name:
            suite_name = os.path.basename(os.getcwd())

        # Automatic Rose constants
        # ROSE_ORIG_HOST: originating host
        # ROSE_VERSION: Rose version (not retained in run_mode=="reload")
        # Suite engine version
        jinja2_section = "jinja2:" + self.suite_engine_proc.SUITE_CONF
        my_rose_version = ResourceLocator.default().get_version()
        suite_engine_key = self.suite_engine_proc.get_version_env_name()
        if opts.run_mode in ["reload", "restart"]:
            prev_config_path = self.suite_engine_proc.get_suite_dir(
                suite_name, "log", "rose-suite-run.conf")
            prev_config = ConfigLoader()(prev_config_path)
            suite_engine_version = prev_config.get_value(
                ["env", suite_engine_key])
        else:
            suite_engine_version = self.suite_engine_proc.get_version()
        auto_items = {
            "ROSE_ORIG_HOST": self.host_selector.get_local_host(),
            "ROSE_VERSION": ResourceLocator.default().get_version(),
            suite_engine_key: suite_engine_version
        }
        for key, val in auto_items.items():
            requested_value = conf_tree.node.get_value(["env", key])
            if requested_value:
                if key == "ROSE_VERSION" and val != requested_value:
                    exc = VersionMismatchError(requested_value, val)
                    raise ConfigValueError(["env", key], requested_value, exc)
                val = requested_value
            else:
                conf_tree.node.set(["env", key],
                                   val,
                                   state=conf_tree.node.STATE_NORMAL)
            conf_tree.node.set([jinja2_section, key], '"' + val + '"')

        # See if suite is running or not
        hosts = []
        if opts.host:
            hosts.append(opts.host)
        if opts.run_mode == "reload":
            suite_run_hosts = self.suite_engine_proc.get_suite_run_hosts(
                None, suite_name, hosts)
            if not suite_run_hosts:
                raise SuiteNotRunningError(suite_name)
            hosts = suite_run_hosts
        else:
            self.suite_engine_proc.check_suite_not_running(suite_name, hosts)

        # Install the suite to its run location
        suite_dir_rel = self._suite_dir_rel(suite_name)
        suite_dir = os.path.join(os.path.expanduser("~"), suite_dir_rel)

        suite_conf_dir = os.getcwd()
        locs_conf = ConfigNode()
        if opts.new_mode:
            if os.getcwd() == suite_dir:
                raise NewModeError("PWD", os.getcwd())
            elif opts.run_mode in ["reload", "restart"]:
                raise NewModeError("--run", opts.run_mode)
            self.suite_run_cleaner.clean(suite_name)
        if os.getcwd() != suite_dir:
            if opts.run_mode == "run":
                self._run_init_dir(opts,
                                   suite_name,
                                   conf_tree,
                                   locs_conf=locs_conf)
            os.chdir(suite_dir)

        # Housekeep log files
        if not opts.install_only_mode and not opts.local_install_only_mode:
            self._run_init_dir_log(opts)
        self.fs_util.makedirs("log/suite")

        # Rose configuration and version logs
        self.fs_util.makedirs("log/rose-conf")
        run_mode = opts.run_mode
        if run_mode not in ["reload", "restart", "run"]:
            run_mode = "run"
        mode = run_mode
        if opts.install_only_mode:
            mode = "install-only"
        elif opts.local_install_only_mode:
            mode = "local-install-only"
        prefix = "rose-conf/%s-%s" % (strftime("%Y%m%dT%H%M%S"), mode)

        # Dump the actual configuration as rose-suite-run.conf
        ConfigDumper()(conf_tree.node, "log/" + prefix + ".conf")

        # Install version information file
        write_source_vc_info(suite_conf_dir, "log/" + prefix + ".version",
                             self.popen)

        # If run through rose-stem, install version information files for
        # each source tree if they're a working copy
        if hasattr(opts, 'source') and hasattr(opts, 'project'):
            for i, url in enumerate(opts.source):
                if os.path.isdir(url):
                    write_source_vc_info(
                        url,
                        "log/" + opts.project[i] + "-" + str(i) + ".version",
                        self.popen)

        for ext in [".conf", ".version"]:
            self.fs_util.symlink(prefix + ext, "log/rose-suite-run" + ext)

        # Move temporary log to permanent log
        if hasattr(self.event_handler, "contexts"):
            log_file_path = os.path.abspath(
                os.path.join("log", "rose-suite-run.log"))
            log_file = open(log_file_path, "ab")
            temp_log_file = self.event_handler.contexts[uuid].handle
            temp_log_file.seek(0)
            log_file.write(temp_log_file.read())
            self.event_handler.contexts[uuid].handle = log_file
            temp_log_file.close()

        # Install share/work directories (local)
        for name in ["share", "share/cycle", "work"]:
            self._run_init_dir_work(opts,
                                    suite_name,
                                    name,
                                    conf_tree,
                                    locs_conf=locs_conf)

        # Process Environment Variables
        environ = self.config_pm(conf_tree, "env")

        # Process Files
        cwd = os.getcwd()
        for rel_path, conf_dir in conf_tree.files.items():
            if (conf_dir == cwd or any([
                    fnmatchcase(os.sep + rel_path, exclude)
                    for exclude in self.SYNC_EXCLUDES
            ]) or conf_tree.node.get(["jinja2:" + rel_path]) is not None):
                continue
            # No sub-directories, very slow otherwise
            if os.sep in rel_path:
                rel_path = rel_path.split(os.sep, 1)[0]
            target_key = self.config_pm.get_handler("file").PREFIX + rel_path
            target_node = conf_tree.node.get([target_key])
            if target_node is None:
                conf_tree.node.set([target_key])
                target_node = conf_tree.node.get([target_key])
            elif target_node.is_ignored():
                continue
            source_node = target_node.get("source")
            if source_node is None:
                target_node.set(["source"], os.path.join(conf_dir, rel_path))
            elif source_node.is_ignored():
                continue
        self.config_pm(conf_tree,
                       "file",
                       no_overwrite_mode=opts.no_overwrite_mode)

        # Process Jinja2 configuration
        self.config_pm(conf_tree, "jinja2")

        # Ask suite engine to parse suite configuration
        # and determine if it is up to date (unchanged)
        suite_conf_unchanged = self.suite_engine_proc.cmp_suite_conf(
            suite_name, opts.run_mode, opts.strict_mode, opts.debug_mode)

        if opts.local_install_only_mode:
            return

        # Install suite files to each remote [user@]host
        for name in ["", "log/", "share/", "share/cycle/", "work/"]:
            uuid_file = os.path.abspath(name + uuid)
            open(uuid_file, "w").close()
            work_files.append(uuid_file)

        # Install items to user@host
        conf = ResourceLocator.default().get_conf()
        auths = self.suite_engine_proc.get_tasks_auths(suite_name)
        queue = []  # [[pipe, command, "ssh"|"rsync", auth], ...]
        for auth in sorted(auths):
            host = auth
            if "@" in auth:
                host = auth.split("@", 1)[1]
            command = self.popen.get_cmd("ssh", auth, "bash", "--login", "-c")
            rose_bin = "rose"
            for name in [host, "*"]:
                rose_home_node = conf.get(["rose-home-at", name],
                                          no_ignore=True)
                if rose_home_node is not None:
                    rose_bin = "%s/bin/rose" % (rose_home_node.value)
                    break
            # Build remote "rose suite-run" command
            rose_sr = "ROSE_VERSION=%s %s" % (my_rose_version, rose_bin)
            rose_sr += " suite-run -v -v --name=%s" % suite_name
            for key in ["new", "debug", "install-only"]:
                attr = key.replace("-", "_") + "_mode"
                if getattr(opts, attr, None) is not None:
                    rose_sr += " --" + key
            if opts.log_keep:
                rose_sr += " --log-keep=" + opts.log_keep
            if opts.log_name:
                rose_sr += " --log-name=" + opts.log_name
            if not opts.log_archive_mode:
                rose_sr += " --no-log-archive"
            rose_sr += " --run=" + opts.run_mode
            host_confs = [
                "root-dir", "root-dir{share}", "root-dir{share/cycle}",
                "root-dir{work}"
            ]
            rose_sr += " --remote=uuid=" + uuid
            locs_conf.set([auth])
            for key in host_confs:
                value = self._run_conf(key, host=host, conf_tree=conf_tree)
                if value is not None:
                    val = self.popen.list_to_shell_str([str(value)])
                    rose_sr += "," + key + "=" + val
                    locs_conf.set([auth, key], value)
            command += ["'" + rose_sr + "'"]
            pipe = self.popen.run_bg(*command)
            queue.append([pipe, command, "ssh", auth])

        while queue:
            sleep(self.SLEEP_PIPE)
            pipe, command, command_name, auth = queue.pop(0)
            if pipe.poll() is None:
                queue.append([pipe, command, command_name, auth])  # put back
                continue
            ret_code = pipe.wait()
            out, err = pipe.communicate()
            if ret_code:
                raise RosePopenError(command, ret_code, out, err)
            if command_name == "rsync":
                self.handle_event(out, level=Event.VV)
                continue
            else:
                self.handle_event(out, level=Event.VV, prefix="[%s] " % auth)
            for line in out.split("\n"):
                if "/" + uuid == line.strip():
                    locs_conf.unset([auth])
                    break
            else:
                filters = {"excludes": [], "includes": []}
                for name in ["", "log/", "share/", "share/cycle/", "work/"]:
                    filters["excludes"].append(name + uuid)
                target = auth + ":" + suite_dir_rel
                cmd = self._get_cmd_rsync(target, **filters)
                queue.append([self.popen.run_bg(*cmd), cmd, "rsync", auth])

        # Install ends
        ConfigDumper()(locs_conf, os.path.join("log", "rose-suite-run.locs"))
        if opts.install_only_mode:
            return
        elif opts.run_mode == "reload" and suite_conf_unchanged:
            conf_name = self.suite_engine_proc.SUITE_CONF
            self.handle_event(SkipReloadEvent(suite_name, conf_name))
            return

        # Start the suite
        self.fs_util.chdir("log")
        ret = 0
        # FIXME: should sync files to suite host?
        if opts.run_mode != "reload":
            if opts.host:
                hosts = [opts.host]
            else:
                names = shlex.split(
                    conf.get_value(["rose-suite-run", "hosts"], ""))
                if names:
                    hosts += self.host_selector.expand(names)[0]

        if (hosts and len(hosts) == 1
                and self.host_selector.is_local_host(hosts[0])):
            host = "localhost"
        elif hosts:
            host = self.host_selector(hosts)[0][0]
        else:
            host = "localhost"
        self.handle_event(SuiteHostSelectEvent(suite_name, run_mode, host))
        # FIXME: values in environ were expanded in the localhost
        self.suite_engine_proc.run(suite_name, host, environ, opts.run_mode,
                                   args)
        open("rose-suite-run.host", "w").write(host + "\n")

        # Disconnect log file handle, so monitoring tool command will no longer
        # be associated with the log file.
        self.event_handler.contexts[uuid].handle.close()
        self.event_handler.contexts.pop(uuid)

        # Launch the monitoring tool
        # Note: maybe use os.ttyname(sys.stdout.fileno())?
        if os.getenv("DISPLAY") and host and opts.gcontrol_mode:
            self.suite_engine_proc.gcontrol(suite_name, host)

        return ret
コード例 #43
0
    def run(self, repos, txn):
        """Apply the rule engine on transaction "txn" to repository "repos"."""

        changes = set()  # set([(status, path), ...])
        for line in self._svnlook("changed", "-t", txn, repos).splitlines():
            status, path = line.split(None, 1)
            changes.add((status, path))
        bad_changes = []
        author = None
        super_users = None
        access_info_map = {}  # {path-id: (owner, access-list), ...}
        txn_access_info_map = {}

        conf = ResourceLocator.default().get_conf()
        ignores_str = conf.get_value(["rosa-svn", "ignores"], self.IGNORES)
        ignores = shlex.split(ignores_str)

        for status, path in sorted(changes):
            if any([fnmatch(path, ignore) for ignore in ignores]):
                continue

            names = path.split("/", self.LEN_ID + 1)
            if not names[-1]:
                names.pop()

            # Directories above the suites must match the ID patterns
            is_bad = False
            for name, pattern in zip(names, self.RE_ID_NAMES):
                if not re.compile(r"\A" + pattern + r"\Z").match(name):
                    is_bad = True
                    break
            if is_bad:
                bad_changes.append(BadChange(status, path))
                continue

            # Can only add directories at levels above the suites
            if len(names) < self.LEN_ID:
                if status[0] != self.ST_ADD:
                    bad_changes.append(BadChange(status, path))
                continue
            else:
                is_meta_suite = "".join(names[0:self.LEN_ID]) == "ROSIE"

            # No need to check non-trunk changes
            if len(names) > self.LEN_ID and names[self.LEN_ID] != "trunk":
                continue

            # New suite should have an info file
            if status[0] == self.ST_ADD and len(names) == self.LEN_ID:
                if (self.ST_ADD, path + "trunk/") not in changes:
                    bad_changes.append(
                        BadChange(status, path, BadChange.NO_TRUNK))
                    continue
                path_trunk_info_file = path + self.TRUNK_INFO_FILE
                if ((self.ST_ADD, path_trunk_info_file) not in changes and
                        (self.ST_UPDATE, path_trunk_info_file) not in changes):
                    bad_changes.append(
                        BadChange(status, path, BadChange.NO_INFO))
                continue

            # The rest are trunk changes in a suite
            path_head = "/".join(names[0:self.LEN_ID]) + "/"
            path_tail = path[len(path_head):]

            # For meta suite, make sure keys in keys file can be parsed
            if is_meta_suite and path_tail == self.TRUNK_KNOWN_KEYS_FILE:
                out = self._svnlook("cat", "-t", txn, repos, path)
                try:
                    shlex.split(out)
                except ValueError:
                    bad_changes.append(
                        BadChange(status, path, BadChange.VALUE))
                    continue

            # Suite trunk information file must have an owner
            # User IDs of owner and access list must be real
            if (status not in self.ST_DELETE and
                    path_tail == self.TRUNK_INFO_FILE):
                txn_owner, txn_access_list = self._get_access_info(
                    repos, path_head, txn)
                if not txn_owner:
                    bad_changes.append(
                        BadChange(status, path, BadChange.NO_OWNER))
                    continue
                if self._verify_users(
                        status, path, txn_owner, txn_access_list, bad_changes):
                    continue

            # New suite trunk: ignore the rest
            if (self.ST_ADD, path_head + "trunk/") in changes:
                continue

            # Can only remove trunk information file with suite
            if status == self.ST_DELETE and path_tail == self.TRUNK_INFO_FILE:
                if (self.ST_DELETE, path_head) not in changes:
                    bad_changes.append(
                        BadChange(status, path, BadChange.NO_INFO))
                continue

            # Can only remove trunk with suite
            if status == self.ST_DELETE and path_tail == "trunk/":
                if (self.ST_DELETE, path_head) not in changes:
                    bad_changes.append(
                        BadChange(status, path, BadChange.NO_TRUNK))
                continue

            # See whether author has permission to make changes
            if author is None:
                author = self._svnlook("author", "-t", txn, repos).strip()
            if super_users is None:
                super_users = []
                for s_key in ["rosa-svn", "rosa-svn-pre-commit"]:
                    value = conf.get_value([s_key, "super-users"])
                    if value is not None:
                        super_users = shlex.split(value)
                        break
            if path_head not in access_info_map:
                access_info = self._get_access_info(repos, path_head)
                access_info_map[path_head] = access_info
            owner, access_list = access_info_map[path_head]
            admin_users = super_users + [owner]

            # Only admin users can remove the suite
            if author not in admin_users and not path_tail:
                bad_changes.append(BadChange(status, path))
                continue

            # Admin users and those in access list can modify everything in
            # trunk apart from specific entries in the trunk info file
            if "*" in access_list or author in admin_users + access_list:
                if path_tail != self.TRUNK_INFO_FILE:
                    continue
            else:
                bad_changes.append(BadChange(status, path))
                continue

            # The owner must not be deleted
            if path_head not in txn_access_info_map:
                txn_access_info = self._get_access_info(repos, path_head, txn)
                txn_access_info_map[path_head] = txn_access_info
            txn_owner, txn_access_list = txn_access_info_map[path_head]
            if not txn_owner:
                bad_changes.append(BadChange(status, path, BadChange.NO_OWNER))
                continue

            # Only the admin users can change owner and access list
            if owner == txn_owner and access_list == txn_access_list:
                continue
            if author not in admin_users:
                if owner != txn_owner:
                    bad_changes.append(BadChange(status, path, BadChange.PERM,
                                                 "owner=" + txn_owner))
                else:  # access list
                    bad_change = BadChange(
                        status, path, BadChange.PERM,
                        "access-list=" + " ".join(txn_access_list))
                    bad_changes.append(bad_change)
                continue

        if bad_changes:
            raise BadChanges(bad_changes)
コード例 #44
0
    def taskjobs(self,
                 user,
                 suite,
                 page=1,
                 cycles=None,
                 tasks=None,
                 task_status=None,
                 job_status=None,
                 order=None,
                 per_page=None,
                 no_fuzzy_time="0",
                 form=None):
        """List task jobs.

        user -- A string containing a valid user ID
        suite -- A string containing a valid suite ID
        page -- The page number to display
        cycles -- Display only task jobs matching these cycles. A value in the
                  list can be a cycle, the string "before|after CYCLE", or a
                  glob to match cycles.
        tasks -- Display only jobs for task names matching a list of names.
                 The list should be specified as a string which will be
                 shlex.split by this method. Values can be a valid task name or
                 a glob like pattern for matching valid task names.
        task_status -- Select by task statuses.
        job_status -- Select by job status. See RoseBushDAO.JOB_STATUS_COMBOS
                      for detail.
        order -- Order search in a predetermined way. A valid value is one of
            "time_desc", "time_asc",
            "cycle_desc_name_desc", "cycle_desc_name_asc",
            "cycle_asc_name_desc", "cycle_asc_name_asc",
            "name_asc_cycle_asc", "name_desc_cycle_asc",
            "name_asc_cycle_desc", "name_desc_cycle_desc",
            "time_submit_desc", "time_submit_asc",
            "time_run_desc", "time_run_asc",
            "time_run_exit_desc", "time_run_exit_asc",
            "duration_queue_desc", "duration_queue_asc",
            "duration_run_desc", "duration_run_asc",
            "duration_queue_run_desc", "duration_queue_run_asc"
        per_page -- Number of entries to display per page (defualt=32)
        no_fuzzy_time -- Don't display fuzzy time if this is True.
        form -- Specify return format. If None, display HTML page. If "json",
                return a JSON data structure.

        """
        conf = ResourceLocator.default().get_conf()
        per_page_default = int(
            conf.get_value(["rose-bush", "jobs-per-page"], self.JOBS_PER_PAGE))
        per_page_max = int(
            conf.get_value(["rose-bush", "jobs-per-page-max"],
                           self.JOBS_PER_PAGE_MAX))
        if not isinstance(per_page, int):
            if per_page:
                per_page = int(per_page)
            else:
                per_page = per_page_default
        is_option_on = (cycles or tasks or task_status or job_status
                        or order is not None and order != "time_desc"
                        or per_page != per_page_default)
        if page and per_page:
            page = int(page)
        else:
            page = 1
        task_statuses = ([[item, ""] for item in self.bush_dao.TASK_STATUSES])
        if task_status:
            if not isinstance(task_status, list):
                task_status = [task_status]
        for item in task_statuses:
            if not task_status or item[0] in task_status:
                item[1] = "1"
        all_task_statuses = all([status[1] == "1" for status in task_statuses])
        if all_task_statuses:
            task_status = []
        data = {
            "cycles": cycles,
            "host": self.host_name,
            "is_option_on": is_option_on,
            "logo": self.logo,
            "method": "taskjobs",
            "no_fuzzy_time": no_fuzzy_time,
            "all_task_statuses": all_task_statuses,
            "task_statuses": task_statuses,
            "job_status": job_status,
            "order": order,
            "page": page,
            "per_page": per_page,
            "per_page_default": per_page_default,
            "per_page_max": per_page_max,
            "rose_version": self.rose_version,
            "script": cherrypy.request.script_name,
            "states": {},
            "suite": suite,
            "tasks": tasks,
            "task_status_groups": self.bush_dao.TASK_STATUS_GROUPS,
            "title": self.title,
            "user": user,
        }
        if cycles:
            cycles = shlex.split(str(cycles))
        if tasks:
            tasks = shlex.split(str(tasks))
        data.update(self._get_suite_logs_info(user, suite))
        data["states"].update(
            self.bush_dao.get_suite_state_summary(user, suite))
        data["states"]["last_activity_time"] = (self.get_last_activity_time(
            user, suite))
        entries, of_n_entries = self.bush_dao.get_suite_job_entries(
            user, suite, cycles, tasks, task_status, job_status, order,
            per_page, (page - 1) * per_page)
        data["entries"] = entries
        data["of_n_entries"] = of_n_entries
        if per_page:
            data["n_pages"] = of_n_entries / per_page
            if of_n_entries % per_page != 0:
                data["n_pages"] += 1
        else:
            data["n_pages"] = 1
        data["time"] = strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
        if form == "json":
            return json.dumps(data)
        try:
            return self.template_env.get_template("taskjobs.html").render(
                **data)
        except jinja2.TemplateError:
            traceback.print_exc()
コード例 #45
0
ファイル: bush.py プロジェクト: kaday/rose
    def jobs(self, user, suite, page=1, cycles=None, tasks=None,
             no_status=None, order=None, per_page=None, no_fuzzy_time="0",
             form=None):
        """List jobs of a running or completed suite.

        user -- A string containing a valid user ID
        suite -- A string containing a valid suite ID
        page -- The page number to display
        cycles -- Display only task jobs matching these cycles. A value in the
                  list can be a cycle, the string "before|after CYCLE", or a
                  glob to match cycles.
        tasks -- Display only jobs for task names matching a list of names.
                 The list should be specified as a string which will be
                 shlex.split by this method. Values can be a valid task name or
                 a glob like pattern for matching valid task names.
        no_status -- Do not display jobs of tasks matching these statuses.
                     The values in the list should be "active", "success" or
                     "fail".
        order -- Order search in a predetermined way. A valid value is one of
            "time_desc", "time_asc",
            "cycle_desc_name_desc", "cycle_desc_name_asc",
            "cycle_asc_name_desc", "cycle_asc_name_asc",
            "name_asc_cycle_asc", "name_desc_cycle_asc",
            "name_asc_cycle_desc", "name_desc_cycle_desc",
            "time_submit_desc", "time_submit_asc",
            "time_run_desc", "time_run_asc",
            "time_run_exit_desc", "time_run_exit_asc",
            "duration_queue_desc", "duration_queue_asc",
            "duration_run_desc", "duration_run_asc",
            "duration_queue_run_desc", "duration_queue_run_asc"
        per_page -- Number of entries to display per page (defualt=32)
        no_fuzzy_time -- Don't display fuzzy time if this is True.
        form -- Specify return format. If None, display HTML page. If "json",
                return a JSON data structure.

        """
        user_suite_dir = self._get_user_suite_dir(user, suite)
        conf = ResourceLocator.default().get_conf()
        per_page_default = int(conf.get_value(
            ["rose-bush", "jobs-per-page"], self.JOBS_PER_PAGE))
        per_page_max = int(conf.get_value(
            ["rose-bush", "jobs-per-page-max"], self.JOBS_PER_PAGE_MAX))
        is_option_on = (
            cycles or
            tasks or
            no_status is not None or
            order is not None and order != "time_desc" or
            per_page is not None and per_page != per_page_default
        )
        if not isinstance(per_page, int):
            if per_page:
                per_page = int(per_page)
            else:
                per_page = per_page_default
        if page and per_page:
            page = int(page)
        else:
            page = 1
        no_statuses = no_status
        if no_status and not isinstance(no_status, list):
            no_statuses = [no_status]
        data = {
            "cycles": cycles,
            "host": self.host_name,
            "is_option_on": is_option_on,
            "logo": self.logo,
            "method": "jobs",
            "no_fuzzy_time": no_fuzzy_time,
            "no_statuses": no_statuses,
            "order": order,
            "page": page,
            "per_page": per_page,
            "per_page_default": per_page_default,
            "per_page_max": per_page_max,
            "rose_version": self.rose_version,
            "script": cherrypy.request.script_name,
            "states": {},
            "suite": suite,
            "tasks": tasks,
            "title": self.title,
            "user": user,
        }
        # TODO: add paths to other suite files
        if cycles:
            cycles = shlex.split(str(cycles))
        if tasks:
            tasks = shlex.split(str(tasks))
        data.update(self._get_suite_logs_info(user, suite))
        data["states"].update(
            self.suite_engine_proc.get_suite_state_summary(user, suite))
        entries, of_n_entries = self.suite_engine_proc.get_suite_job_events(
            user, suite, cycles, tasks, no_statuses, order, per_page,
            (page - 1) * per_page)
        data["entries"] = entries
        data["of_n_entries"] = of_n_entries
        if per_page:
            data["n_pages"] = of_n_entries / per_page
            if of_n_entries % per_page != 0:
                data["n_pages"] += 1
        else:
            data["n_pages"] = 1
        data["time"] = strftime("%Y-%m-%dT%H:%M:%S+0000", gmtime())
        if form == "json":
            return simplejson.dumps(data)
        try:
            template = self.template_env.get_template("jobs.html")
            return template.render(**data)
        except Exception as exc:
            traceback.print_exc(exc)
コード例 #46
0
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'Rose Documentation'
copyright = (': British Crown Copyright 2012-8 Met Office. See Terms of Use. '
             'This document is released under the Open Government Licence')

# The full version for the project you're documenting, acts as replacement for
# |version|.
release = ResourceLocator().get_version(ignore_environment=True)
# The short X.Y version, acts as replacement for |release|.
version = release

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = []

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'autumn'


# -- Options for HTML output ----------------------------------------------

html_theme = 'sphinx_rtd_theme'
# rtd_theme only handles 4 levels for the sidebar navigation.
コード例 #47
0
def get_prepend_paths(event_handler=None,
                      path_root=None,
                      path_glob_args=[],
                      full_mode=False):
    """Return map of PATH-like env-var names to path lists to prepend to them.

    event_handler -- An instance of rose.reporter.Reporter or an object with a
                     similar interface.
    path_root -- If a glob is relative and this is defined, this is the root
                 directory of the relative path.
    path_glob_args -- A list of strings in the form GLOB or NAME=GLOB. NAME is
                      "PATH" by default or should be PATH-like environment
                      variable name. GLOB should be a glob pattern for matching
                      file system paths to prepend to NAME.
    full_mode -- If True, prepend relevant paths in site/user configuration and
                 the setting defined in "rose.task_env.PATH_GLOBS".

    Return something like:
        {"PATH": ["/opt/foo/bin", "/opt/bar/bin"],
         # ... and so on
        }

    """

    prepend_paths_map = {}

    # site/user configuration
    if full_mode:
        conf = ResourceLocator.default().get_conf()
        my_conf = conf.get(["rose-task-run"], no_ignore=True)
        if my_conf is not None:
            for key, node in sorted(my_conf.value.items()):
                if not key.startswith("path-prepend") or node.is_ignored():
                    continue
                env_key = "PATH"
                if key != "path-prepend":
                    env_key = key[len("path-prepend."):]
                values = []
                for v in node.value.split():
                    if os.path.exists(v):
                        values.append(v)
                if values:
                    prepend_paths_map[env_key] = values

    # Default or specified globs
    path_globs_map = {}
    if full_mode:
        for name, path_globs in PATH_GLOBS.items():
            path_globs_map[name] = path_globs
    for path_glob_arg in path_glob_args:
        if path_glob_arg is None:
            continue
        if "=" in path_glob_arg:
            name, value = path_glob_arg.split("=", 1)
        else:
            name, value = "PATH", path_glob_arg
        if name not in path_globs_map:
            path_globs_map[name] = []
        path_globs_map[name].append(value)
    more_prepend_paths_map = {}
    if not path_root:
        path_root = os.getcwd()
    for name, path_globs in path_globs_map.items():
        if name not in more_prepend_paths_map:
            more_prepend_paths_map[name] = []
        for path_glob in path_globs:
            if path_glob:
                if path_glob.startswith("~"):
                    path_glob = os.path.expanduser(path_glob)
                if not os.path.isabs(path_glob):
                    path_glob = os.path.join(path_root, path_glob)
                for path in sorted(glob(path_glob)):
                    more_prepend_paths_map[name].append(path)
            else:
                more_prepend_paths_map[name] = []  # empty value resets
    for name, more_prepend_paths in more_prepend_paths_map.items():
        if name in prepend_paths_map:
            prepend_paths_map[name].extend(more_prepend_paths)
        elif more_prepend_paths:
            prepend_paths_map[name] = more_prepend_paths
    for key, prepend_paths in prepend_paths_map.items():
        prepend_paths.reverse()

    return prepend_paths_map
コード例 #48
0
ファイル: svn_post_commit.py プロジェクト: DamianAgius/rose
    def _notify_trunk_changes(self, changeset_attribs, branch_attribs):
        """Email owner and/or access-list users on changes to trunk."""

        # Notify only if users' email addresses can be determined
        conf = ResourceLocator.default().get_conf()
        user_tool_name = conf.get_value(["rosa-svn", "user-tool"])
        if not user_tool_name:
            return
        notify_who_str = conf.get_value(
            ["rosa-svn", "notify-who-on-trunk-commit"], "")
        if not notify_who_str.strip():
            return
        notify_who = shlex.split(notify_who_str)

        # Build the message text
        info_file_path = "%s/trunk/%s" % (
            "/".join(branch_attribs["sid"]), self.INFO_FILE)
        text = ""
        for changed_line in branch_attribs["changed_lines"]:
            text += changed_line
            # For suite info file change, add diff as well
            if (changed_line[4:].strip() == info_file_path and
                    branch_attribs["status_info_file"] == self.ST_MODIFIED):
                old_strio = StringIO()
                rose.config.dump(branch_attribs["old_info"], old_strio)
                new_strio = StringIO()
                rose.config.dump(branch_attribs["info"], new_strio)
                for diff_line in unified_diff(
                        old_strio.getvalue().splitlines(True),
                        new_strio.getvalue().splitlines(True),
                        "@%d" % (int(changeset_attribs["revision"]) - 1),
                        "@%d" % (int(changeset_attribs["revision"]))):
                    text += " " * 4 + diff_line

        # Determine who to notify
        users = set()
        for key in ["old_info", "info"]:
            if branch_attribs[key] is not None:
                info_conf = branch_attribs[key]
                if "owner" in notify_who:
                    users.add(info_conf.get_value(["owner"]))
                if "access-list" in notify_who:
                    users.update(
                        info_conf.get_value(["access-list"], "").split())
        users.discard("*")

        # Determine email addresses
        user_tool = self.usertools_manager.get_handler(user_tool_name)
        if "author" in notify_who:
            users.add(changeset_attribs["author"])
        else:
            users.discard(changeset_attribs["author"])
        emails = sorted(user_tool.get_emails(users))
        if not emails:
            return

        # Send notification
        msg = MIMEText(text)
        msg.set_charset("utf-8")
        msg["From"] = conf.get_value(
            ["rosa-svn", "notification-from"],
            "notications@" + socket.getfqdn())
        msg["To"] = ", ".join(emails)
        msg["Subject"] = "%s-%s/trunk@%d" % (
            changeset_attribs["prefix"],
            branch_attribs["sid"],
            int(changeset_attribs["revision"]))
        smtp_host = conf.get_value(
            ["rosa-svn", "smtp-host"], default="localhost")
        smtp = SMTP(smtp_host)
        smtp.sendmail(msg["From"], emails, msg.as_string())
        smtp.quit()
コード例 #49
0
ファイル: host_select.py プロジェクト: raghavendrads22/rose
    def expand(self, names=None, rank_method=None, thresholds=None):
        """Expand each name in names, and look up rank method for each name.

        names, if specified, should be a list of host names or known groups in
        the site / user configuration file. Otherwise, the default setting in
        the site / user configuration file will be used.

        rank_method, if specified, should be the name of a supported ranking
        method. If not specified, use the default specified for a host group.
        If the default differs in hosts, use "load:15".

        """
        conf = ResourceLocator.default().get_conf()
        if not names:
            node = conf.get(["rose-host-select", "default"],
                            no_ignore=True)
            if node:
                names = [node.value]
            else:
                raise NoHostError()

        host_names = []
        rank_method_set = set()
        thresholds_set = set()
        while names:
            name = names.pop()
            key = "group{" + name + "}"
            value = conf.get_value(["rose-host-select", key])
            if value is None:
                host_names.append(name)
            else:
                for val in value.split():
                    names.append(val)
                if rank_method is None:
                    key = "method{" + name + "}"
                    method_str = conf.get_value(["rose-host-select", key])
                    if method_str is None:
                        rank_method_set.add(self.RANK_METHOD_DEFAULT)
                    else:
                        rank_method_set.add(method_str)
                if thresholds is None:
                    key = "thresholds{" + name + "}"
                    threshold_str = conf.get_value(["rose-host-select", key])
                    if threshold_str is None:
                        thresholds_set.add(())
                    else:
                        thresholds_set.add(
                            tuple(sorted(shlex.split(threshold_str))))

        # If default rank method differs in hosts, use load:15.
        if rank_method is None:
            if len(rank_method_set) == 1:
                rank_method = rank_method_set.pop()
            else:
                rank_method = self.RANK_METHOD_DEFAULT

        if thresholds is None:
            if len(thresholds_set) == 1:
                thresholds = thresholds_set.pop()

        return host_names, rank_method, thresholds
コード例 #50
0
ファイル: bush.py プロジェクト: kaday/rose
 def view(self, user, suite, path, path_in_tar=None, mode=None):
     """View a text log file."""
     f_name = self._get_user_suite_dir(user, suite, path)
     conf = ResourceLocator.default().get_conf()
     view_size_max = int(conf.get_value(
         ["rose-bush", "view-size-max"], self.VIEW_SIZE_MAX))
     if path_in_tar:
         tar_f = tarfile.open(f_name, 'r:gz')
         try:
             tar_info = tar_f.getmember(path_in_tar)
         except KeyError:
             raise cherrypy.HTTPError(404)
         f_size = tar_info.size
         f = tar_f.extractfile(path_in_tar)
         if f.read(2) == "#!":
             mime = MIME_TEXT_PLAIN
         else:
             mime = mimetypes.guess_type(
                 urllib.pathname2url(path_in_tar))[0]
         f.seek(0)
         if (mode == "download" or
                 f_size > view_size_max or
                 mime and
                 (not mime.startswith("text/") or mime.endswith("html"))):
             t = NamedTemporaryFile()
             f_bsize = os.fstatvfs(t.fileno()).f_bsize
             while True:
                 bytes = f.read(f_bsize)
                 if not bytes:
                     break
                 t.write(bytes)
             cherrypy.response.headers["Content-Type"] = mime
             try:
                 return cherrypy.lib.static.serve_file(t.name, mime)
             finally:
                 t.close()
         s = f.read()
         f.close()
     else:
         f_size = os.stat(f_name).st_size
         if open(f_name).read(2) == "#!":
             mime = MIME_TEXT_PLAIN
         else:
             mime = mimetypes.guess_type(urllib.pathname2url(f_name))[0]
         if (mode == "download" or
                 f_size > view_size_max or
                 mime and
                 (not mime.startswith("text/") or mime.endswith("html"))):
             cherrypy.response.headers["Content-Type"] = mime
             return cherrypy.lib.static.serve_file(f_name, mime)
         s = open(f_name).read()
     if mode == "text":
         s = jinja2.escape(s)
     try:
         lines = [unicode(line) for line in s.splitlines()]
     except UnicodeDecodeError:
         return cherrypy.lib.static.serve_file(f_name, MIME_TEXT_PLAIN)
     name = path
     if path_in_tar:
         name = path_in_tar
     job_entry = None
     if name.startswith("log/job/"):
         names = self.suite_engine_proc.parse_job_log_rel_path(name)
         if len(names) == 4:
             cycle, task, submit_num, ext = names
             entries = self.suite_engine_proc.get_suite_job_events(
                 user, suite, [cycle], [task], None, None, None, None)[0]
             for entry in entries:
                 if entry["submit_num"] == int(submit_num):
                     job_entry = entry
                     break
     if fnmatch(os.path.basename(path), "rose*.conf"):
         file_content = "rose-conf"
     else:
         file_content = self.suite_engine_proc.is_conf(path)
     template = self.template_env.get_template("view.html")
     data = {}
     data.update(self._get_suite_logs_info(user, suite))
     return template.render(
         rose_version=self.rose_version,
         script=cherrypy.request.script_name,
         method="view",
         time=strftime("%Y-%m-%dT%H:%M:%S+0000", gmtime()),
         logo=self.logo,
         title=self.title,
         host=self.host_name,
         user=user,
         suite=suite,
         path=path,
         path_in_tar=path_in_tar,
         f_name=f_name,
         mode=mode,
         file_content=file_content,
         lines=lines,
         entry=job_entry,
         **data)
コード例 #51
0
    def run(self, app_runner, conf_tree, opts, args, uuid, work_files):
        """Implement the "rose ana" command"""
        # Initialise properties which will be needed later.
        self._task = app_runner.suite_engine_proc.get_task_props()
        self.task_name = self._task.task_name
        self.opts = opts
        self.args = args
        self.config = conf_tree.node
        self.app_runner = app_runner

        # Attach to the main rose config (for retrieving settings from things
        # like the user's ~/.metomi/rose.conf)
        self.rose_conf = ResourceLocator.default().get_conf()

        # Attach to a reporter instance for sending messages.
        self._init_reporter(app_runner.event_handler)

        # As part of the introduction of a re-written rose_ana, backwards
        # compatibility is maintained here by detecting the lack of the
        # newer syntax in the app config and falling back to the old version
        # of the rose_ana app (renamed to rose_ana_v1)
        # **Once the old behaviour is removed the below block can be too**.
        new_style_app = False
        for keys, node in self.config.walk(no_ignore=True):
            task = keys[0]
            if task.startswith("ana:"):
                new_style_app = True
                break
        if not new_style_app:
            # Use the previous app by instantiating and calling it explicitly
            self.reporter("!!WARNING!! - Detected old style rose_ana app; "
                          "Using previous rose_ana version...")
            from rose.apps.rose_ana_v1 import RoseAnaV1App
            old_app = RoseAnaV1App(manager=self.manager)
            return old_app.run(app_runner, conf_tree, opts, args, uuid,
                               work_files)

        # Load any rose_ana specific configuration settings either from the
        # site defaults or the user's personal config
        self._get_global_ana_config()

        # If the user's config indicates that it should be used - attach
        # to the KGO database instance in case it is needed later.
        use_kgo = self.ana_config.get("kgo-database", ".false.")
        self.kgo_db = None
        if use_kgo == TYPE_LOGICAL_VALUE_TRUE:
            self.kgo_db = KGODatabase()
            self.kgo_db.enter_task(self.task_name,
                                   self.kgo_db.TASK_STATUS_RUNNING)
            self.titlebar("Initialising KGO database")
            self.kgo_db.buffer_to_db(self.reporter)

        self.titlebar("Launching rose_ana")

        # Load available methods for analysis and the tasks in the app.
        self._load_analysis_modules()
        self._load_analysis_methods()
        self._load_tasks()

        number_of_failures = 0
        number_of_skips = 0
        task_error = False
        summary_status = []
        for itask, task in enumerate(self.analysis_tasks):
            # Report the name of the task and a banner line to aid readability.
            self.titlebar("Running task #{0}".format(itask + 1))
            self.reporter("Method: {0}".format(task.options["full_task_name"]))

            # Since the run_analysis method is out of rose's control in many
            # cases the safest thing to do is a blanket try/except; since we
            # have no way of knowing what exceptions might be raised.
            try:
                task.run_analysis()
                # In the case that the task didn't raise any exception,
                # we can now check whether it passed or failed.
                if task.passed:
                    msg = "Task #{0} passed".format(itask + 1)
                    summary_status.append(
                        ("{0} ({1})".format(msg,
                                            task.options["full_task_name"]),
                         self._prefix_pass))
                    self.reporter(msg, prefix=self._prefix_pass)
                elif task.skipped:
                    number_of_skips += 1
                    msg = "Task #{0} skipped by method".format(itask + 1)
                    summary_status.append(
                        ("{0} ({1})".format(msg,
                                            task.options["full_task_name"]),
                         self._prefix_skip))
                    self.reporter(msg, prefix=self._prefix_skip)
                else:
                    number_of_failures += 1
                    msg = "Task #{0} did not pass".format(itask + 1)
                    summary_status.append(
                        ("{0} ({1})".format(msg,
                                            task.options["full_task_name"]),
                         self._prefix_fail))
                    self.reporter(msg, prefix=self._prefix_fail)

            except Exception as err:
                # If an exception was raised, print a traceback and treat it
                # as a failure.
                task_error = True
                number_of_failures += 1
                msg = "Task #{0} encountered an error".format(itask + 1)
                summary_status.append(
                    ("{0} ({1})".format(msg, task.options["full_task_name"]),
                     self._prefix_fail))
                self.reporter(msg + " (see stderr)", prefix=self._prefix_fail)
                exception = traceback.format_exc()
                self.reporter(msg,
                              prefix=self._prefix_fail,
                              kind=self.reporter.KIND_ERR)
                self.reporter(exception,
                              prefix=self._prefix_fail,
                              kind=self.reporter.KIND_ERR)

        # The KGO database (if needed by the task) also stores its status - to
        # indicate whether there was some unexpected exception above.
        if self.kgo_db is not None and not task_error:
            self.kgo_db.enter_task(self.task_name,
                                   self.kgo_db.TASK_STATUS_SUCCEEDED)
            self.titlebar("Updating KGO database")
            self.kgo_db.buffer_to_db(self.reporter)

        # Summarise the results of the tasks
        self.titlebar("Summary")
        for line, prefix in summary_status:
            self.reporter(line, prefix=prefix)

        # And a final 1-line summary
        total = len(summary_status)
        plural = {1: ""}

        prefix = self._prefix_pass
        passed = total - number_of_failures - number_of_skips
        msg = "{0} Task{1} Passed".format(passed, plural.get(passed, "s"))

        if number_of_failures > 0:
            msg += ", {0} Task{1} Failed".format(
                number_of_failures, plural.get(number_of_failures, "s"))
            prefix = self._prefix_fail

        if number_of_skips > 0:
            msg += ", {0} Task{1} Skipped".format(
                number_of_skips, plural.get(number_of_skips, "s"))

        msg += " (of {0} processed)".format(total)
        self.titlebar("Final status")
        self.reporter(msg, prefix=prefix)

        self.titlebar("Completed rose_ana")

        # Finally if there were legitimate test failures raise an exception
        # so that the task is caught by cylc as failed.  Also fail if it looks
        # like every single task has been skipped
        if number_of_failures > 0 or number_of_skips == total:
            raise TestsFailedException(number_of_failures)
コード例 #52
0
ファイル: ws_client_auth.py プロジェクト: harry-shepherd/rose
    def _prompt(self, is_retry=False):
        """Prompt for the username and password, where necessary.

        Prompt with zenity or raw_input/getpass.

        """
        if (callable(self.prompt_func)
                and not hasattr(self.password_store, "prompt_password")):
            self.username, self.password = self.prompt_func(
                self.username, self.password, is_retry)
            return

        icon_path = ResourceLocator.default().locate("images/rosie-icon.png")
        if is_retry:
            username = ""

            prompt = self.PROMPT_USERNAME % {
                "prefix": self.prefix,
                "root": self.root
            }
            if self.popen.which("zenity") and os.getenv("DISPLAY"):
                username = self.popen.run("zenity", "--entry", "--title=Rosie",
                                          "--window-icon=" + icon_path,
                                          "--text=" + prompt)[1].strip()
            else:
                username = raw_input(prompt)
            if not username:
                raise KeyboardInterrupt(self.STR_CANCELLED)
            if username and username != self.username:
                self.username = username
                self._load_password()
                if self.password:
                    return

        if self.username and self.password is None or is_retry:
            prompt = self.PROMPT_PASSWORD % {
                "prefix": self.prefix,
                "root": self.root,
                "username": self.username
            }
            password = None
            need_prompting = True
            if hasattr(self.password_store, "prompt_password"):
                try:
                    password = self.password_store.prompt_password(
                        prompt, self.scheme, self.host, self.username)
                except RosieStoreRetrievalError as exc:
                    self.event_handler(exc)
                else:
                    need_prompting = False

            if not password and need_prompting:
                if self.popen.which("zenity") and os.getenv("DISPLAY"):
                    password = self.popen.run("zenity", "--entry",
                                              "--hide-text", "--title=Rosie",
                                              "--window-icon=" + icon_path,
                                              "--text=" + prompt)[1].strip()
                else:
                    password = getpass(prompt)
            if not password:
                raise KeyboardInterrupt(self.STR_CANCELLED)
            if password and password != self.password:
                self.password = password
コード例 #53
0
    def generate_info_config(self, from_id=None, prefix=None):
        """Generate a rose.config.ConfigNode for a rose-suite.info.

        This is suitable for passing into the create method of this
        class.
        If from_id is defined, copy items from it.
        Return the rose.config.ConfigNode instance.

        """
        from_project = None
        from_title = None
        if from_id is not None:
            from_info_url = "%s/%s/rose-suite.info@%s" % (from_id.to_origin(),
                                                          from_id.branch,
                                                          from_id.revision)
            out_data = self.popen("svn", "cat", from_info_url)[0]
            from_config = rose.config.load(StringIO(out_data))
            if from_config.get(["project"]) is not None:
                from_project = from_config.get(["project"]).value
            if from_config.get(["title"]) is not None:
                from_title = from_config.get(["title"]).value

        res_loc = ResourceLocator.default()
        info_config = rose.config.ConfigNode()

        # Determine prefix
        if from_id is not None:
            prefix = from_id.prefix
        elif prefix is None:
            prefix = SuiteId.get_prefix_default()

        # Determine owner:
        # 1. From user configuration [rosie-id]prefix-username
        # 2. From username of a matching group in [groups] in
        #    ~/.subversion/servers
        # 3. Current user ID
        owner = res_loc.get_conf().get_value(
                        ["rosie-id", "prefix-username." + prefix])
        if not owner and self.subversion_servers_conf:
            servers_conf = rose.config.load(self.subversion_servers_conf)
            groups_node = servers_conf.get(["groups"])
            if groups_node is not None:
                group = None
                prefix_loc = SuiteId.get_prefix_location(prefix)
                prefix_host = urlparse(prefix_loc).hostname
                for key, node in groups_node.value.items():
                    if fnmatch(prefix_host, node.value):
                        owner = servers_conf.get_value([key, "username"])
                        break
        if not owner:
            owner = pwd.getpwuid(os.getuid())[0]
        info_config.set(["owner"], owner)

        # Determine project and title
        if from_project:
            info_config.set(["project"], from_project)
        else:
            info_config.set(["project"], "")
        if from_title:
            info_config.set(["title"], "Copy of %s: %s" % 
                            (from_id.to_string_with_version(), from_title))
        else:
            info_config.set(["title"], "")

        # Determine access list
        access_list_str = res_loc.get_conf().get_value(
            ["rosie-vc", "access-list-default"])
        if access_list_str:
            info_config.set(["access-list"], access_list_str)
        return info_config
コード例 #54
0
ファイル: ws.py プロジェクト: craigmaclachlan/rose
def start(is_main=False):
    """Create the server.

    If is_main, invoke cherrypy.quickstart.
    Otherwise, return a cherrypy.Application instance.

    """
    # Environment variables (not normally defined in WSGI mode)
    if os.getenv("ROSE_HOME") is None:
        path = os.path.abspath(__file__)
        while os.path.dirname(path) != path:  # not root
            if os.path.basename(path) == "lib":
                os.environ["ROSE_HOME"] = os.path.dirname(path)
                break
            path = os.path.dirname(path)
    for key, value in [("ROSE_NS", "rosa"), ("ROSE_UTIL", "ws")]:
        if os.getenv(key) is None:
            os.environ[key] = value

    # CherryPy quick server configuration
    rose_conf = ResourceLocator.default().get_conf()
    if is_main and rose_conf.get_value(["rosie-ws", "log-dir"]) is not None:
        log_dir_value = rose_conf.get_value(["rosie-ws", "log-dir"])
        log_dir = env_var_process(os.path.expanduser(log_dir_value))
        if not os.path.isdir(log_dir):
            os.makedirs(log_dir)
        log_file = os.path.join(log_dir, "server.log")
        log_error_file = os.path.join(log_dir, "server.err.log")
        cherrypy.config["log.error_file"] = log_error_file
        cherrypy.config["log.access_file"] = log_file
        cherrypy.config["request.error_response"] = _handle_error
    cherrypy.config["log.screen"] = False
    # Configuration for dynamic pages
    db_url_map = {}
    for key, node in rose_conf.get(["rosie-db"]).value.items():
        if key.startswith("db.") and key[3:]:
            db_url_map[key[3:]] = node.value
    res_loc = ResourceLocator.default()
    html_lib = res_loc.get_util_home("lib", "html")
    icon_path = res_loc.locate("images/rosie-icon-trim.png")
    tmpl_loader = jinja2.FileSystemLoader(os.path.join(html_lib, "rosie-ws"))
    root = Root(jinja2.Environment(loader=tmpl_loader), db_url_map)

    # Configuration for static pages
    config = {
        "/etc": {
            "tools.staticdir.dir": os.path.join(html_lib, "external"),
            "tools.staticdir.on": True
        },
        "/favicon.ico": {
            "tools.staticfile.on": True,
            "tools.staticfile.filename": icon_path
        }
    }
    if is_main:
        port = int(rose_conf.get_value(["rosie-ws", "port"], 8080))
        config.update({
            "global": {
                "server.socket_host": "0.0.0.0",
                "server.socket_port": port
            }
        })

    # Start server or return WSGI application
    if is_main:
        return cherrypy.quickstart(root, "/", config=config)
    else:
        return cherrypy.Application(root, script_name=None, config=config)