class RosieSvnPostCommitHook(object): """A post-commit hook on a Rosie Subversion repository. Update the Rosie discovery database on changes to info file. Notify owner and users on access-list on trunk changes. """ DATE_FMT = "%Y-%m-%d %H:%M:%S %Z" ID_CHARS_LIST = ["abcdefghijklmnopqrstuvwxyz"] * 2 + ["0123456789"] * 3 LEN_ID = len(ID_CHARS_LIST) INFO_FILE = "rose-suite.info" KNOWN_KEYS_FILE_PATH = "R/O/S/I/E/trunk/rosie-keys" REC_COPY_INFO = re.compile(r"\A\s+\(from\s(\S+):r(\d+)\)\s*\Z") ST_ADDED = "A" ST_DELETED = "D" ST_MODIFIED = "M" ST_EMPTY = " " TRUNK = "trunk" def __init__(self, event_handler=None, popen=None): if event_handler is None: event_handler = Reporter() self.event_handler = event_handler if popen is None: popen = RosePopener(self.event_handler) self.popen = popen path = os.path.dirname(os.path.dirname(sys.modules["rosie"].__file__)) self.usertools_manager = SchemeHandlersManager( [path], "rosie.usertools", ["get_emails"]) def run(self, repos, revision, no_notification=False): """Update database with changes in a changeset.""" # Lookup prefix of repos # Do nothing if prefix is not registered conf = ResourceLocator.default().get_conf() rosie_db_node = conf.get(["rosie-db"], no_ignore=True) for key, node in rosie_db_node.value.items(): if node.is_ignored() or not key.startswith("repos."): continue if os.path.realpath(repos) == os.path.realpath(node.value): prefix = key[len("repos."):] break else: return # Locate Rosie DB of repos dao = RosieWriteDAO(conf.get_value(["rosie-db", "db." + prefix])) # Date-time of this commit os.environ["TZ"] = "UTC" date_time_str = self._svnlook("date", "-r", revision, repos) date, dtime, _ = date_time_str.split(None, 2) date = mktime(strptime(" ".join([date, dtime, "UTC"]), self.DATE_FMT)) # Detail of changes changeset_attribs = { "repos": repos, "revision": revision, "prefix": prefix, "author": self._svnlook("author", "-r", revision, repos).strip(), "date": date} branch_attribs_dict = self._get_suite_branch_changes(repos, revision) for key, branch_attribs in sorted(branch_attribs_dict.items()): # Update known keys in suite info database meta table if branch_attribs["has_changed_known_keys_file"]: self._update_known_keys(dao, changeset_attribs) # Update suite info database self._update_info_db(dao, changeset_attribs, branch_attribs) # Notification on trunk changes # Notification on owner and access-list changes if not no_notification and branch_attribs["branch"] == "trunk": self._notify_trunk_changes( changeset_attribs, branch_attribs) def _get_suite_branch_changes(self, repos, revision): """Retrieve changed statuses.""" branch_attribs_dict = {} changed_lines = self._svnlook( "changed", "--copy-info", "-r", revision, repos).splitlines(True) while changed_lines: changed_line = changed_lines.pop(0) # A normal status changed_line # Column 1: content status # Column 2: tree status # Column 3: "+" sign denotes a copy history # Column 5+: path path = changed_line[4:].strip() path_status = changed_line[0] if path.endswith("/") and path_status == "_": # Ignore property change on a directory continue # Path must be (under) a valid suite branch, including the special # ROSIE suite names = path.split("/", self.LEN_ID + 1) if (len(names) < self.LEN_ID + 1 or ( "".join(names[0:self.LEN_ID]) != "ROSIE" and any(name not in id_chars for name, id_chars in zip(names, self.ID_CHARS_LIST)))): continue sid = "".join(names[0:self.LEN_ID]) branch = names[self.LEN_ID] if branch: # Change to a path in a suite branch if (sid, branch) not in branch_attribs_dict: branch_attribs_dict[(sid, branch)] = ( self._new_suite_branch_change(sid, branch)) branch_attribs = branch_attribs_dict[(sid, branch)] try: tail = names[self.LEN_ID + 1] except IndexError: tail = None if tail == self.INFO_FILE: # Suite info file change if branch_attribs["info"] is None: branch_attribs["info"] = self._load_info( repos, revision, sid, branch) if path_status != self.ST_ADDED: branch_attribs["old_info"] = self._load_info( repos, int(revision) - 1, sid, branch) if (branch_attribs["old_info"] != branch_attribs["info"] and branch_attribs["status"] != self.ST_ADDED): branch_attribs["status_info_file"] = ( self.ST_MODIFIED) elif tail: # ROSIE meta known keys file change if path == self.KNOWN_KEYS_FILE_PATH: branch_attribs["has_changed_known_keys_file"] = True if branch_attribs["status"] == self.ST_EMPTY: branch_attribs["status"] = self.ST_MODIFIED elif path_status in [self.ST_ADDED, self.ST_DELETED]: # Branch add/delete branch_attribs["status"] = path_status # Load suite info and old info regardless if branch_attribs["info"] is None: branch_attribs["info"] = self._load_info( repos, revision, sid, branch) if (branch_attribs["old_info"] is None and branch_attribs["status"] == self.ST_DELETED): branch_attribs["old_info"] = self._load_info( repos, int(revision) - 1, sid, branch) # Append changed lines, so they can be used for notification branch_attribs["changed_lines"].append(changed_line) if changed_line[2] == "+": changed_line_2 = changed_lines.pop(0) branch_attribs["changed_lines"].append(changed_line_2) if path_status != self.ST_ADDED or tail: continue # A line containing the copy info for a branch # Column 5+ looks like: (from PATH:rREV) match = self.REC_COPY_INFO.match(changed_line_2) if match: from_path, from_rev = match.groups() branch_attribs_dict[(sid, branch)].update({ "from_path": from_path, "from_rev": from_rev}) elif path_status == self.ST_DELETED: # The suite has been deleted tree_out = self._svnlook( "tree", "-r", str(int(revision) - 1), "-N", repos, path) # Include all branches of the suite in the deletion info for tree_line in tree_out.splitlines()[1:]: del_branch = tree_line.strip().rstrip("/") branch_attribs_dict[(sid, del_branch)] = ( self._new_suite_branch_change(sid, del_branch, { "old_info": self._load_info( repos, int(revision) - 1, sid, del_branch), "status": self.ST_DELETED, "status_info_file": self.ST_EMPTY, "changed_lines": [ "D %s/%s/" % (path, del_branch)]})) return branch_attribs_dict def _load_info(self, repos, revision, sid, branch): """Load info file from branch_path in repos @revision.""" info_file_path = "%s/%s/%s" % ("/".join(sid), branch, self.INFO_FILE) t_handle = TemporaryFile() try: t_handle.write(self._svnlook( "cat", "-r", str(revision), repos, info_file_path)) except RosePopenError: return None t_handle.seek(0) config = rose.config.load(t_handle) t_handle.close() return config def _new_suite_branch_change(self, sid, branch, attribs=None): """Return a dict to represent a suite branch change.""" branch_attribs = { "sid": sid, "branch": branch, "from_path": None, "from_rev": None, "has_changed_known_keys_file": False, "old_info": None, "info": None, "status": self.ST_EMPTY, "status_info_file": self.ST_EMPTY, "changed_lines": []} if attribs: branch_attribs.update(attribs) return branch_attribs def _notify_trunk_changes(self, changeset_attribs, branch_attribs): """Email owner and/or access-list users on changes to trunk.""" # Notify only if users' email addresses can be determined conf = ResourceLocator.default().get_conf() user_tool_name = conf.get_value(["rosa-svn", "user-tool"]) if not user_tool_name: return notify_who_str = conf.get_value( ["rosa-svn", "notify-who-on-trunk-commit"], "") if not notify_who_str.strip(): return notify_who = shlex.split(notify_who_str) # Build the message text info_file_path = "%s/trunk/%s" % ( "/".join(branch_attribs["sid"]), self.INFO_FILE) text = "" for changed_line in branch_attribs["changed_lines"]: text += changed_line # For suite info file change, add diff as well if (changed_line[4:].strip() == info_file_path and branch_attribs["status_info_file"] == self.ST_MODIFIED): old_strio = StringIO() rose.config.dump(branch_attribs["old_info"], old_strio) new_strio = StringIO() rose.config.dump(branch_attribs["info"], new_strio) for diff_line in unified_diff( old_strio.getvalue().splitlines(True), new_strio.getvalue().splitlines(True), "@%d" % (int(changeset_attribs["revision"]) - 1), "@%d" % (int(changeset_attribs["revision"]))): text += " " * 4 + diff_line # Determine who to notify users = set() for key in ["old_info", "info"]: if branch_attribs[key] is not None: info_conf = branch_attribs[key] if "owner" in notify_who: users.add(info_conf.get_value(["owner"])) if "access-list" in notify_who: users.update( info_conf.get_value(["access-list"], "").split()) users.discard("*") # Determine email addresses user_tool = self.usertools_manager.get_handler(user_tool_name) if "author" in notify_who: users.add(changeset_attribs["author"]) else: users.discard(changeset_attribs["author"]) emails = sorted(user_tool.get_emails(users)) if not emails: return # Send notification msg = MIMEText(text) msg.set_charset("utf-8") msg["From"] = conf.get_value( ["rosa-svn", "notification-from"], "notications@" + socket.getfqdn()) msg["To"] = ", ".join(emails) msg["Subject"] = "%s-%s/trunk@%d" % ( changeset_attribs["prefix"], branch_attribs["sid"], int(changeset_attribs["revision"])) smtp_host = conf.get_value( ["rosa-svn", "smtp-host"], default="localhost") smtp = SMTP(smtp_host) smtp.sendmail(msg["From"], emails, msg.as_string()) smtp.quit() def _svnlook(self, *args): """Return the standard output from "svnlook".""" command = ["svnlook"] + list(args) return self.popen(*command)[0] def _update_info_db(self, dao, changeset_attribs, branch_attribs): """Update the suite info database for a suite branch.""" idx = changeset_attribs["prefix"] + "-" + branch_attribs["sid"] vc_attrs = { "idx": idx, "branch": branch_attribs["branch"], "revision": changeset_attribs["revision"]} for key in vc_attrs: vc_attrs[key] = vc_attrs[key].decode("utf-8") # Latest table try: dao.delete( LATEST_TABLE_NAME, idx=vc_attrs["idx"], branch=vc_attrs["branch"]) except al.exc.IntegrityError: # idx and branch were just added: there is no previous record. pass if branch_attribs["status"] != self.ST_DELETED: dao.insert(LATEST_TABLE_NAME, **vc_attrs) # N.B. deleted suite branch only has old info info_key = "info" if branch_attribs["status"] == self.ST_DELETED: info_key = "old_info" if branch_attribs[info_key] is None: return # Main table cols = dict(vc_attrs) cols.update({ "author": changeset_attribs["author"], "date": changeset_attribs["date"]}) for name in ["owner", "project", "title"]: cols[name] = branch_attribs[info_key].get_value([name]) if branch_attribs["from_path"] and vc_attrs["branch"] == u"trunk": from_names = branch_attribs["from_path"].split("/")[:self.LEN_ID] cols["from_idx"] = ( changeset_attribs["prefix"] + "-" + "".join(from_names)) cols["status"] = ( branch_attribs["status"] + branch_attribs["status_info_file"]) for key in cols: try: cols[key] = cols[key].decode("utf-8") except AttributeError: pass dao.insert(MAIN_TABLE_NAME, **cols) # Optional table for name in branch_attribs[info_key].value: if name in ["owner", "project", "title"]: continue value = branch_attribs[info_key].get_value([name]) if value is None: # setting may have ignore flag (!) continue cols = dict(vc_attrs) cols.update({ "name": name.decode("utf-8"), "value": value.decode("utf-8")}) dao.insert(OPTIONAL_TABLE_NAME, **cols) def _update_known_keys(self, dao, changeset_attribs): """Update the known_keys in the meta table.""" repos = changeset_attribs["repos"] revision = changeset_attribs["revision"] keys_str = self._svnlook( "cat", "-r", revision, repos, self.KNOWN_KEYS_FILE_PATH) keys_str = " ".join(shlex.split(keys_str)).decode("utf-8") if keys_str: try: dao.insert(META_TABLE_NAME, name=u"known_keys", value=keys_str) except al.exc.IntegrityError: dao.update( META_TABLE_NAME, (u"name",), name=u"known_keys", value=keys_str)
def _run(self, dao, app_runner, config): """Transform and archive suite files. This application is designed to work under "rose task-run" in a suite. """ path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__)) compress_manager = SchemeHandlersManager( [path], "rose.apps.rose_arch_compressions", ["compress_sources"], None, app_runner) # Set up the targets cycle = os.getenv("ROSE_TASK_CYCLE_TIME") targets = [] for t_key, t_node in sorted(config.value.items()): if t_node.is_ignored() or ":" not in t_key: continue s_key_head, s_key_tail = t_key.split(":", 1) if s_key_head != self.SECTION or not s_key_tail: continue target_prefix = self._get_conf( config, t_node, "target-prefix", default="") try: s_key_tail = env_var_process(s_key_tail) except UnboundEnvironmentVariableError as exc: raise ConfigValueError([t_key, ""], "", exc) target_name = target_prefix + s_key_tail target = RoseArchTarget(target_name) target.command_format = self._get_conf( config, t_node, "command-format", compulsory=True) try: target.command_format % {"sources": "", "target": ""} except KeyError as exc: target.status = target.ST_BAD app_runner.handle_event( RoseArchValueError( target.name, "command-format", target.command_format, type(exc).__name__, exc ) ) source_str = self._get_conf( config, t_node, "source", compulsory=True) source_prefix = self._get_conf( config, t_node, "source-prefix", default="") target.source_edit_format = self._get_conf( config, t_node, "source-edit-format", default="") try: target.source_edit_format % {"in": "", "out": ""} except KeyError as exc: target.status = target.ST_BAD app_runner.handle_event( RoseArchValueError( target.name, "source-edit-format", target.source_edit_format, type(exc).__name__, exc ) ) update_check_str = self._get_conf( config, t_node, "update-check", default="md5sum") try: checksum_func = get_checksum_func(update_check_str) except KeyError as exc: raise RoseArchValueError( target.name, "update-check", update_check_str, type(exc).__name__, exc ) for source_glob in shlex.split(source_str): paths = glob(source_prefix + source_glob) if not paths: exc = OSError(errno.ENOENT, os.strerror(errno.ENOENT), source_glob) app_runner.handle_event(ConfigValueError( [t_key, "source"], source_glob, exc)) target.status = target.ST_BAD continue for path in paths: # N.B. source_prefix may not be a directory name = path[len(source_prefix):] for path_, checksum, _ in get_checksum( path, checksum_func): if checksum is None: # is directory continue if path_: target.sources[checksum] = RoseArchSource( checksum, os.path.join(name, path_), os.path.join(path, path_)) else: # path is a file target.sources[checksum] = RoseArchSource( checksum, name, path) target.compress_scheme = self._get_conf(config, t_node, "compress") if target.compress_scheme: if (compress_manager.get_handler(target.compress_scheme) is None): app_runner.handle_event(ConfigValueError( [t_key, "compress"], target.compress_scheme, KeyError(target.compress_scheme))) target.status = target.ST_BAD else: target_base = target.name if "/" in target.name: target_base = target.name.rsplit("/", 1)[1] if "." in target_base: tail = target_base.split(".", 1)[1] if compress_manager.get_handler(tail): target.compress_scheme = tail rename_format = self._get_conf(config, t_node, "rename-format") if rename_format: rename_parser_str = self._get_conf(config, t_node, "rename-parser") if rename_parser_str: try: rename_parser = re.compile(rename_parser_str) except re.error as exc: raise RoseArchValueError( target.name, "rename-parser", rename_parser_str, type(exc).__name__, exc ) else: rename_parser = None for source in target.sources.values(): dict_ = {"cycle": cycle, "name": source.name} if rename_parser: match = rename_parser.match(source.name) if match: dict_.update(match.groupdict()) try: source.name = rename_format % dict_ except (KeyError, ValueError) as exc: raise RoseArchValueError( target.name, "rename-format", rename_format, type(exc).__name__, exc ) old_target = dao.select(target.name) if old_target is None or old_target != target: dao.delete(target) else: target.status = target.ST_OLD targets.append(target) # Delete from database items that are no longer relevant dao.delete_all(filter_targets=targets) # Update the targets for target in targets: if target.status == target.ST_OLD: app_runner.handle_event(RoseArchEvent(target)) continue target.command_rc = 1 dao.insert(target) if target.status == target.ST_BAD: app_runner.handle_event(RoseArchEvent(target)) continue work_dir = mkdtemp() t_init = time() t_tran, t_arch = t_init, t_init ret_code = None try: # Rename/edit sources target.status = target.ST_BAD rename_required = False for source in target.sources.values(): if source.name != source.orig_name: rename_required = True break if rename_required or target.source_edit_format: for source in target.sources.values(): source.path = os.path.join(work_dir, source.name) source_path_d = os.path.dirname(source.path) app_runner.fs_util.makedirs(source_path_d) if target.source_edit_format: fmt_args = {"in": source.orig_path, "out": source.path} command = target.source_edit_format % fmt_args app_runner.popen.run_ok(command, shell=True) else: app_runner.fs_util.symlink(source.orig_path, source.path) # Compress sources if target.compress_scheme: handler = compress_manager.get_handler( target.compress_scheme) handler.compress_sources(target, work_dir) t_tran = time() # Run archive command sources = [] if target.work_source_path: sources = [target.work_source_path] else: for source in target.sources.values(): sources.append(source.path) sources_str = app_runner.popen.list_to_shell_str(sources) target_str = app_runner.popen.list_to_shell_str([target.name]) command = target.command_format % {"sources": sources_str, "target": target_str} ret_code, out, err = app_runner.popen.run(command, shell=True) t_arch = time() if ret_code: app_runner.handle_event( RosePopenError([command], ret_code, out, err)) else: target.status = target.ST_NEW app_runner.handle_event(err, kind=Event.KIND_ERR) app_runner.handle_event(out) app_runner.handle_event(out) target.command_rc = ret_code dao.update_command_rc(target) finally: app_runner.fs_util.delete(work_dir) app_runner.handle_event( RoseArchEvent(target, [t_init, t_tran, t_arch], ret_code)) return [target.status for target in targets].count( RoseArchTarget.ST_BAD)
class RosieSvnPostCommitHook(object): """A post-commit hook on a Rosie Subversion repository. Update the Rosie discovery database on changes to info file. Notify owner and users on access-list on trunk changes. """ DATE_FMT = "%Y-%m-%d %H:%M:%S %Z" ID_CHARS_LIST = ["abcdefghijklmnopqrstuvwxyz"] * 2 + ["0123456789"] * 3 LEN_ID = len(ID_CHARS_LIST) INFO_FILE = "rose-suite.info" KNOWN_KEYS_FILE_PATH = "R/O/S/I/E/trunk/rosie-keys" REC_COPY_INFO = re.compile(r"\A\s+\(from\s(\S+):r(\d+)\)\s*\Z") ST_ADDED = "A" ST_DELETED = "D" ST_MODIFIED = "M" ST_EMPTY = " " TRUNK = "trunk" def __init__(self, event_handler=None, popen=None): if event_handler is None: event_handler = Reporter() self.event_handler = event_handler if popen is None: popen = RosePopener(self.event_handler) self.popen = popen path = os.path.dirname(os.path.dirname(sys.modules["rosie"].__file__)) self.usertools_manager = SchemeHandlersManager([path], "rosie.usertools", ["get_emails"]) def run(self, repos, revision, no_notification=False): """Update database with changes in a changeset.""" # Lookup prefix of repos # Do nothing if prefix is not registered conf = ResourceLocator.default().get_conf() rosie_db_node = conf.get(["rosie-db"], no_ignore=True) for key, node in rosie_db_node.value.items(): if node.is_ignored() or not key.startswith("repos."): continue if os.path.realpath(repos) == os.path.realpath(node.value): prefix = key[len("repos."):] break else: return # Locate Rosie DB of repos dao = RosieWriteDAO(conf.get_value(["rosie-db", "db." + prefix])) # Date-time of this commit os.environ["TZ"] = "UTC" date_time_str = self._svnlook("date", "-r", revision, repos) date, dtime, _ = date_time_str.split(None, 2) date = mktime(strptime(" ".join([date, dtime, "UTC"]), self.DATE_FMT)) # Detail of changes changeset_attribs = { "repos": repos, "revision": revision, "prefix": prefix, "author": self._svnlook("author", "-r", revision, repos).strip(), "date": date } branch_attribs_dict = self._get_suite_branch_changes(repos, revision) for key, branch_attribs in sorted(branch_attribs_dict.items()): # Update known keys in suite info database meta table if branch_attribs["has_changed_known_keys_file"]: self._update_known_keys(dao, changeset_attribs) # Update suite info database self._update_info_db(dao, changeset_attribs, branch_attribs) # Notification on trunk changes # Notification on owner and access-list changes if not no_notification and branch_attribs["branch"] == "trunk": self._notify_trunk_changes(changeset_attribs, branch_attribs) def _get_suite_branch_changes(self, repos, revision): """Retrieve changed statuses.""" branch_attribs_dict = {} changed_lines = self._svnlook("changed", "--copy-info", "-r", revision, repos).splitlines(True) while changed_lines: changed_line = changed_lines.pop(0) # A normal status changed_line # Column 1: content status # Column 2: tree status # Column 3: "+" sign denotes a copy history # Column 5+: path path = changed_line[4:].strip() path_status = changed_line[0] if path.endswith("/") and path_status == "_": # Ignore property change on a directory continue # Path must be (under) a valid suite branch, including the special # ROSIE suite names = path.split("/", self.LEN_ID + 1) if (len(names) < self.LEN_ID + 1 or ("".join(names[0:self.LEN_ID]) != "ROSIE" and any( name not in id_chars for name, id_chars in zip(names, self.ID_CHARS_LIST)))): continue sid = "".join(names[0:self.LEN_ID]) branch = names[self.LEN_ID] if branch: # Change to a path in a suite branch if (sid, branch) not in branch_attribs_dict: branch_attribs_dict[(sid, branch)] = ( self._new_suite_branch_change(sid, branch)) branch_attribs = branch_attribs_dict[(sid, branch)] try: tail = names[self.LEN_ID + 1] except IndexError: tail = None if tail == self.INFO_FILE: # Suite info file change if branch_attribs["info"] is None: branch_attribs["info"] = self._load_info( repos, revision, sid, branch) if path_status != self.ST_ADDED: branch_attribs["old_info"] = self._load_info( repos, int(revision) - 1, sid, branch) if (branch_attribs["old_info"] != branch_attribs["info"] and branch_attribs["status"] != self.ST_ADDED): branch_attribs["status_info_file"] = ( self.ST_MODIFIED) elif tail: # ROSIE meta known keys file change if path == self.KNOWN_KEYS_FILE_PATH: branch_attribs["has_changed_known_keys_file"] = True if branch_attribs["status"] == self.ST_EMPTY: branch_attribs["status"] = self.ST_MODIFIED elif path_status in [self.ST_ADDED, self.ST_DELETED]: # Branch add/delete branch_attribs["status"] = path_status # Load suite info and old info regardless if branch_attribs["info"] is None: branch_attribs["info"] = self._load_info( repos, revision, sid, branch) if (branch_attribs["old_info"] is None and branch_attribs["status"] == self.ST_DELETED): branch_attribs["old_info"] = self._load_info( repos, int(revision) - 1, sid, branch) # Append changed lines, so they can be used for notification branch_attribs["changed_lines"].append(changed_line) if changed_line[2] == "+": changed_line_2 = changed_lines.pop(0) branch_attribs["changed_lines"].append(changed_line_2) if path_status != self.ST_ADDED or tail: continue # A line containing the copy info for a branch # Column 5+ looks like: (from PATH:rREV) match = self.REC_COPY_INFO.match(changed_line_2) if match: from_path, from_rev = match.groups() branch_attribs_dict[(sid, branch)].update({ "from_path": from_path, "from_rev": from_rev }) elif path_status == self.ST_DELETED: # The suite has been deleted tree_out = self._svnlook("tree", "-r", str(int(revision) - 1), "-N", repos, path) # Include all branches of the suite in the deletion info for tree_line in tree_out.splitlines()[1:]: del_branch = tree_line.strip().rstrip("/") branch_attribs_dict[(sid, del_branch)] = ( self._new_suite_branch_change( sid, del_branch, { "old_info": self._load_info(repos, int(revision) - 1, sid, del_branch), "status": self.ST_DELETED, "status_info_file": self.ST_EMPTY, "changed_lines": ["D %s/%s/" % (path, del_branch)] })) return branch_attribs_dict def _load_info(self, repos, revision, sid, branch): """Load info file from branch_path in repos @revision.""" info_file_path = "%s/%s/%s" % ("/".join(sid), branch, self.INFO_FILE) t_handle = TemporaryFile() try: t_handle.write( self._svnlook("cat", "-r", str(revision), repos, info_file_path)) except RosePopenError: return None t_handle.seek(0) config = rose.config.load(t_handle) t_handle.close() return config def _new_suite_branch_change(self, sid, branch, attribs=None): """Return a dict to represent a suite branch change.""" branch_attribs = { "sid": sid, "branch": branch, "from_path": None, "from_rev": None, "has_changed_known_keys_file": False, "old_info": None, "info": None, "status": self.ST_EMPTY, "status_info_file": self.ST_EMPTY, "changed_lines": [] } if attribs: branch_attribs.update(attribs) return branch_attribs def _notify_trunk_changes(self, changeset_attribs, branch_attribs): """Email owner and/or access-list users on changes to trunk.""" # Notify only if users' email addresses can be determined conf = ResourceLocator.default().get_conf() user_tool_name = conf.get_value(["rosa-svn", "user-tool"]) if not user_tool_name: return notify_who_str = conf.get_value( ["rosa-svn", "notify-who-on-trunk-commit"], "") if not notify_who_str.strip(): return notify_who = shlex.split(notify_who_str) # Build the message text info_file_path = "%s/trunk/%s" % ("/".join( branch_attribs["sid"]), self.INFO_FILE) text = "" for changed_line in branch_attribs["changed_lines"]: text += changed_line # For suite info file change, add diff as well if (changed_line[4:].strip() == info_file_path and branch_attribs["status_info_file"] == self.ST_MODIFIED): old_strio = StringIO() rose.config.dump(branch_attribs["old_info"], old_strio) new_strio = StringIO() rose.config.dump(branch_attribs["info"], new_strio) for diff_line in unified_diff( old_strio.getvalue().splitlines(True), new_strio.getvalue().splitlines(True), "@%d" % (int(changeset_attribs["revision"]) - 1), "@%d" % (int(changeset_attribs["revision"]))): text += " " * 4 + diff_line # Determine who to notify users = set() for key in ["old_info", "info"]: if branch_attribs[key] is not None: info_conf = branch_attribs[key] if "owner" in notify_who: users.add(info_conf.get_value(["owner"])) if "access-list" in notify_who: users.update( info_conf.get_value(["access-list"], "").split()) users.discard("*") # Determine email addresses user_tool = self.usertools_manager.get_handler(user_tool_name) if "author" in notify_who: users.add(changeset_attribs["author"]) else: users.discard(changeset_attribs["author"]) emails = sorted(user_tool.get_emails(users)) if not emails: return # Send notification msg = MIMEText(text) msg.set_charset("utf-8") msg["From"] = conf.get_value(["rosa-svn", "notification-from"], "notications@" + socket.getfqdn()) msg["To"] = ", ".join(emails) msg["Subject"] = "%s-%s/trunk@%d" % ( changeset_attribs["prefix"], branch_attribs["sid"], int(changeset_attribs["revision"])) smtp_host = conf.get_value(["rosa-svn", "smtp-host"], default="localhost") smtp = SMTP(smtp_host) smtp.sendmail(msg["From"], emails, msg.as_string()) smtp.quit() def _svnlook(self, *args): """Return the standard output from "svnlook".""" command = ["svnlook"] + list(args) return self.popen(*command)[0] def _update_info_db(self, dao, changeset_attribs, branch_attribs): """Update the suite info database for a suite branch.""" idx = changeset_attribs["prefix"] + "-" + branch_attribs["sid"] vc_attrs = { "idx": idx, "branch": branch_attribs["branch"], "revision": changeset_attribs["revision"] } for key in vc_attrs: vc_attrs[key] = vc_attrs[key].decode("utf-8") # Latest table try: dao.delete(LATEST_TABLE_NAME, idx=vc_attrs["idx"], branch=vc_attrs["branch"]) except al.exc.IntegrityError: # idx and branch were just added: there is no previous record. pass if branch_attribs["status"] != self.ST_DELETED: dao.insert(LATEST_TABLE_NAME, **vc_attrs) # N.B. deleted suite branch only has old info info_key = "info" if branch_attribs["status"] == self.ST_DELETED: info_key = "old_info" if branch_attribs[info_key] is None: return # Main table cols = dict(vc_attrs) cols.update({ "author": changeset_attribs["author"], "date": changeset_attribs["date"] }) for name in ["owner", "project", "title"]: cols[name] = branch_attribs[info_key].get_value([name]) if branch_attribs["from_path"] and vc_attrs["branch"] == u"trunk": from_names = branch_attribs["from_path"].split("/")[:self.LEN_ID] cols["from_idx"] = (changeset_attribs["prefix"] + "-" + "".join(from_names)) cols["status"] = (branch_attribs["status"] + branch_attribs["status_info_file"]) for key in cols: try: cols[key] = cols[key].decode("utf-8") except AttributeError: pass dao.insert(MAIN_TABLE_NAME, **cols) # Optional table for name in branch_attribs[info_key].value: if name in ["owner", "project", "title"]: continue value = branch_attribs[info_key].get_value([name]) if value is None: # setting may have ignore flag (!) continue cols = dict(vc_attrs) cols.update({ "name": name.decode("utf-8"), "value": value.decode("utf-8") }) dao.insert(OPTIONAL_TABLE_NAME, **cols) def _update_known_keys(self, dao, changeset_attribs): """Update the known_keys in the meta table.""" repos = changeset_attribs["repos"] revision = changeset_attribs["revision"] keys_str = self._svnlook("cat", "-r", revision, repos, self.KNOWN_KEYS_FILE_PATH) keys_str = " ".join(shlex.split(keys_str)).decode("utf-8") if keys_str: try: dao.insert(META_TABLE_NAME, name=u"known_keys", value=keys_str) except al.exc.IntegrityError: dao.update(META_TABLE_NAME, (u"name", ), name=u"known_keys", value=keys_str)
class AppRunner(Runner): """Invoke a Rose application.""" OLD_DURATION_UNITS = {"h": 3600, "m": 60, "s": 1} NAME = "app" OPTIONS = ["app_mode", "command_key", "conf_dir", "defines", "install_only_mode", "new_mode", "no_overwrite_mode", "opt_conf_keys"] def __init__(self, *args, **kwargs): Runner.__init__(self, *args, **kwargs) path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__)) self.builtins_manager = SchemeHandlersManager( [path], "rose.apps", ["run"], None, *args, **kwargs) self.duration_parser = DurationParser() def run_impl(self, opts, args, uuid, work_files): """The actual logic for a run.""" # Preparation. conf_tree = self.config_load(opts) self._prep(conf_tree, opts) self._poll(conf_tree) # Run the application or the command. app_mode = conf_tree.node.get_value(["mode"]) if app_mode is None: app_mode = opts.app_mode if app_mode in [None, "command"]: return self._command(conf_tree, opts, args) else: builtin_app = self.builtins_manager.get_handler(app_mode) if builtin_app is None: raise UnknownBuiltinAppError(app_mode) return builtin_app.run(self, conf_tree, opts, args, uuid, work_files) def _poll(self, conf_tree): """Poll for prerequisites of applications.""" # Poll configuration poll_test = conf_tree.node.get_value(["poll", "test"]) poll_all_files_value = conf_tree.node.get_value(["poll", "all-files"]) poll_all_files = [] if poll_all_files_value: try: poll_all_files = shlex.split( env_var_process(poll_all_files_value)) except UnboundEnvironmentVariableError as exc: raise ConfigValueError(["poll", "all-files"], poll_all_files_value, exc) poll_any_files_value = conf_tree.node.get_value(["poll", "any-files"]) poll_any_files = [] if poll_any_files_value: try: poll_any_files = shlex.split( env_var_process(poll_any_files_value)) except UnboundEnvironmentVariableError as exc: raise ConfigValueError(["poll", "any-files"], poll_any_files_value, exc) poll_file_test = None if poll_all_files or poll_any_files: poll_file_test = conf_tree.node.get_value(["poll", "file-test"]) if poll_file_test and "{}" not in poll_file_test: raise ConfigValueError(["poll", "file-test"], poll_file_test, ConfigValueError.SYNTAX) poll_delays = [] if poll_test or poll_all_files or poll_any_files: # Parse something like this: delays=10,4*PT30S,PT2M30S,2*PT1H # R*DURATION: repeat the value R times conf_keys = ["poll", "delays"] poll_delays_value = conf_tree.node.get_value( conf_keys, default="").strip() if poll_delays_value: is_legacy0 = None for item in poll_delays_value.split(","): value = item.strip() repeat = 1 if "*" in value: repeat, value = value.split("*", 1) try: repeat = int(repeat) except ValueError as exc: raise ConfigValueError(conf_keys, poll_delays_value, ConfigValueError.SYNTAX) try: value = self.duration_parser.parse(value).get_seconds() is_legacy = False except ISO8601SyntaxError: # Legacy mode: nnnU # nnn is a float, U is the unit # No unit or s: seconds # m: minutes # h: hours unit = None if value[-1].lower() in self.OLD_DURATION_UNITS: unit = self.OLD_DURATION_UNITS[value[-1].lower()] value = value[:-1] try: value = float(value) except ValueError as exc: raise ConfigValueError(conf_keys, poll_delays_value, ConfigValueError.SYNTAX) if unit: value *= unit is_legacy = True if is_legacy0 is None: is_legacy0 = is_legacy elif is_legacy0 != is_legacy: raise ConfigValueError( conf_keys, poll_delays_value, ConfigValueError.DURATION_LEGACY_MIX) poll_delays += [value] * repeat else: poll_delays = [0] # poll once without a delay # Poll t_init = get_timepoint_for_now() while poll_delays and (poll_test or poll_any_files or poll_all_files): poll_delay = poll_delays.pop(0) if poll_delay: sleep(poll_delay) if poll_test: ret_code = self.popen.run( poll_test, shell=True, stdout=sys.stdout, stderr=sys.stderr)[0] self.handle_event(PollEvent(time(), poll_test, ret_code == 0)) if ret_code == 0: poll_test = None any_files = list(poll_any_files) for file_ in any_files: if self._poll_file(file_, poll_file_test): self.handle_event(PollEvent(time(), "any-files", True)) poll_any_files = [] break all_files = list(poll_all_files) for file_ in all_files: if self._poll_file(file_, poll_file_test): poll_all_files.remove(file_) if all_files and not poll_all_files: self.handle_event(PollEvent(time(), "all-files", True)) failed_items = [] if poll_test: failed_items.append("test") if poll_any_files: failed_items.append("any-files") if poll_all_files: failed_items.append("all-files:" + self.popen.list_to_shell_str(poll_all_files)) if failed_items: now = get_timepoint_for_now() raise PollTimeoutError(now, now - t_init, failed_items) def _poll_file(self, file_, poll_file_test): """Poll for existence of a file.""" is_done = False if poll_file_test: test = poll_file_test.replace( "{}", self.popen.list_to_shell_str([file_])) is_done = self.popen.run( test, shell=True, stdout=sys.stdout, stderr=sys.stderr)[0] == 0 else: is_done = bool(glob(file_)) self.handle_event(PollEvent(time(), "file:" + file_, is_done)) return is_done def _prep(self, conf_tree, opts): """Prepare to run the application.""" if opts.new_mode: conf_dir = opts.conf_dir if not conf_dir or os.path.abspath(conf_dir) == os.getcwd(): raise NewModeError(os.getcwd()) for path in os.listdir("."): self.fs_util.delete(path) # Dump the actual configuration as rose-app-run.conf ConfigDumper()(conf_tree.node, "rose-app-run.conf") # Environment variables: PATH paths = [] for conf_dir in conf_tree.conf_dirs: conf_bin_dir = os.path.join(conf_dir, "bin") if os.path.isdir(conf_bin_dir): paths.append(conf_bin_dir) if paths: value = os.pathsep.join(paths + [os.getenv("PATH")]) conf_tree.node.set(["env", "PATH"], value) else: conf_tree.node.set(["env", "PATH"], os.getenv("PATH")) # Free format files not defined in the configuration file file_section_prefix = self.config_pm.get_handler("file").PREFIX for rel_path, conf_dir in conf_tree.files.items(): if not rel_path.startswith("file" + os.sep): continue name = rel_path[len("file" + os.sep):] # No sub-directories, very slow otherwise if os.sep in name: name = name.split(os.sep, 1)[0] target_key = file_section_prefix + name target_node = conf_tree.node.get([target_key]) if target_node is None: conf_tree.node.set([target_key]) target_node = conf_tree.node.get([target_key]) elif target_node.is_ignored(): continue source_node = target_node.get("source") if source_node is None: target_node.set(["source"], os.path.join(conf_dir, "file", name)) elif source_node.is_ignored(): continue # Process Environment Variables self.config_pm(conf_tree, "env") # Process Files self.config_pm(conf_tree, "file", no_overwrite_mode=opts.no_overwrite_mode) def _command(self, conf_tree, opts, args): """Run the command.""" command = self.popen.list_to_shell_str(args) if not command: names = [opts.command_key, os.getenv("ROSE_TASK_NAME"), "default"] for name in names: if not name: continue command = conf_tree.node.get_value(["command", name]) if command is not None: break else: self.handle_event(CommandNotDefinedEvent()) return if os.access("STDIN", os.F_OK | os.R_OK): command += " <STDIN" self.handle_event("command: %s" % command) if opts.install_only_mode: return self.popen(command, shell=True, stdout=sys.stdout, stderr=sys.stderr)
class RosieSvnPostCommitHook(object): """A post-commit hook on a Rosie Subversion repository. Update the Rosie discovery database on changes. """ DATE_FMT = "%Y-%m-%d %H:%M:%S %Z" RE_ID_NAMES = [r"[a-z]", r"[a-z]", r"\d", r"\d", r"\d"] LEN_ID = len(RE_ID_NAMES) INFO_FILE = "rose-suite.info" KNOWN_KEYS_FILE = "rosie-keys" REC_COPY_INFO = re.compile("^\s+\(from\s([^\s]+)\)$") ST_ADDED = "A" ST_DELETED = "D" ST_MODIFIED = "M" ST_UPDATED = "U" TRUNK = "trunk" def __init__(self, event_handler=None, popen=None): if event_handler is None: event_handler = Reporter() self.event_handler = event_handler if popen is None: popen = RosePopener(self.event_handler) self.popen = popen path = os.path.dirname(os.path.dirname(sys.modules["rosie"].__file__)) self.usertools_manager = SchemeHandlersManager( [path], "rosie.usertools", ["get_emails"]) def _svnlook(self, *args): """Return the standard output from "svnlook".""" command = ["svnlook"] + list(args) return self.popen(*command)[0] def _check_path_is_sid(self, path): """Return whether the path contains a suffix-id.""" names = path.split("/") if len(names) < self.LEN_ID + 1: return False if "".join(names)[:self.LEN_ID] == "ROSIE": return True for name, pattern in zip(names, self.RE_ID_NAMES): if not re.compile(r"\A" + pattern + r"\Z").match(name): return False return True def run(self, repos, revision): """Update database with changes in a changeset.""" conf = ResourceLocator.default().get_conf() rosie_db_node = conf.get(["rosie-db"], no_ignore=True) for key, node in rosie_db_node.value.items(): if node.is_ignored() or not key.startswith("repos."): continue if os.path.realpath(repos) == os.path.realpath(node.value): prefix = key[len("repos."):] break else: return dao = RosieWriteDAO(conf.get_value(["rosie-db", "db." + prefix])) idx_branch_rev_info = {} author = self._svnlook("author", "-r", revision, repos).strip() os.environ["TZ"] = "UTC" date_time_str = self._svnlook("date", "-r", revision, repos) date, dtime, _ = date_time_str.split(None, 2) date = time.mktime(time.strptime(" ".join([date, dtime, "UTC"]), self.DATE_FMT)) # Retrieve copied information on the suite-idx-level. path_copies = {} path_num = -1 copy_changes = self._svnlook("changed", "--copy-info", "-r", revision, repos) for i, change in enumerate(copy_changes.splitlines()): copy_match = self.REC_COPY_INFO.match(change) if copy_match: path_copies[path_num] = copy_match.groups()[0] else: path_num += 1 changes = self._svnlook("changed", "-r", revision, repos) suite_statuses = {} path_statuses = [] for i, line in enumerate(changes.splitlines()): path_status, path = line.rsplit(None, 1) if not self._check_path_is_sid(path): # The path must contain a full suite id (e.g. a/a/0/0/1/). continue path_statuses.append((path_status, path, i)) # Loop through a stack of statuses, paths, and copy info pointers. configs = {} while path_statuses: path_status, path, path_num = path_statuses.pop(0) names = path.split("/") sid = "".join(names[0:self.LEN_ID]) idx = prefix + "-" + sid branch = names[self.LEN_ID] branch_path = "/".join(sid) + "/" + branch info_file_path = branch_path + "/" + self.INFO_FILE if not branch and path_status[0] == self.ST_DELETED: # The suite has been deleted at the a/a/0/0/1/ level. out = self._svnlook("tree", "-r", str(int(revision) - 1), "-N", repos, path) # Include all branches of the suite in the deletion info. for line in out.splitlines()[1:]: del_branch = line.strip().rstrip("/") path_statuses.append((path_status, path.rstrip("/") + "/" + del_branch, None)) if (sid == "ROSIE" and branch == self.TRUNK and path == branch_path + "/" + self.KNOWN_KEYS_FILE): # The known keys in the special R/O/S/I/E/ suite have changed. keys_str = self._svnlook("cat", "-r", revision, repos, path) keys_str = " ".join(shlex.split(keys_str)) if keys_str: try: dao.insert(META_TABLE_NAME, name="known_keys", value=keys_str) except al.exc.IntegrityError: dao.update(META_TABLE_NAME, ("name",), name="known_keys", value=keys_str) status_0 = " " from_idx = None if (path_num in path_copies and self._check_path_is_sid(path_copies[path_num])): copy_names = path_copies[path_num].split("/") copy_sid = "".join(copy_names[:self.LEN_ID]) if copy_sid != sid and branch: # This has been copied from a different suite. from_idx = prefix + "-" + copy_sid # Figure out our status information. if path.rstrip("/") == branch_path: if path_status[0] == self.ST_DELETED: status_0 = self.ST_DELETED if path_status[0] == self.ST_ADDED: status_0 = self.ST_ADDED if (len(path.rstrip("/")) > len(branch_path) and path != info_file_path and status_0.isspace()): status_0 = self.ST_MODIFIED suite_statuses.setdefault((idx, branch, revision), {0: status_0, 1: " "}) status_info = suite_statuses[(idx, branch, revision)] if not branch: continue if path.rstrip("/") not in [branch_path, info_file_path]: if branch_path not in [i[1] for i in path_statuses]: # Make sure the branch gets noticed. path_statuses.append((self.ST_UPDATED, branch_path, None)) continue # Start populating the idx+branch+revision info for this suite. suite_info = idx_branch_rev_info.setdefault( (idx, branch, revision), {}) suite_info["author"] = author suite_info["date"] = date suite_info.setdefault("from_idx", None) if from_idx is not None: suite_info["from_idx"] = from_idx suite_info.setdefault("owner", "") suite_info.setdefault("project", "") suite_info.setdefault("title", "") suite_info.setdefault("optional", {}) if (idx, branch, revision) not in configs: new_config = self._get_config_node( repos, info_file_path, revision) old_config = self._get_config_node( repos, info_file_path, str(int(revision) - 1)) configs[(idx, branch, revision)] = (new_config, old_config) if branch == self.TRUNK: self._notify_access_changes( "%s/%s@%s" % (idx, branch, revision), suite_info["author"], old_config, new_config) new_config, old_config = configs[(idx, branch, revision)] if new_config is None and old_config is None: # A technically-invalid commit (likely to be historical). idx_branch_rev_info.pop((idx, branch, revision)) continue if new_config is None and status_info[0] == self.ST_DELETED: new_config = old_config if old_config is None and status_info[0] == self.ST_ADDED: old_config = new_config if self._get_configs_differ(old_config, new_config): status_info[1] = self.ST_MODIFIED for key, node in new_config.value.items(): if node.is_ignored(): continue if key in ["owner", "project", "title"]: suite_info[key] = node.value else: suite_info["optional"][key] = node.value # Now loop over all idx+branch+revision suite groups. for suite_id, suite_info in idx_branch_rev_info.items(): idx, branch, revision = suite_id status = suite_statuses.get((idx, branch, revision), {0: " ", 1: " "}) suite_info["status"] = (status[0] + status[1]) optional = suite_info.pop("optional") for key, value in optional.items(): dao.insert(OPTIONAL_TABLE_NAME, idx=idx, branch=branch, revision=revision, name=key, value=value) dao.insert(MAIN_TABLE_NAME, idx=idx, branch=branch, revision=revision, **suite_info) try: dao.delete(LATEST_TABLE_NAME, idx=idx, branch=branch) except al.exc.IntegrityError: # idx and branch were just added: there is no previous record. pass if suite_info["status"][0] != self.ST_DELETED: dao.insert(LATEST_TABLE_NAME, idx=idx, branch=branch, revision=revision) __call__ = run def _get_config_node(self, repos, info_file_path, revision): """Load configuration file from info_file_path in repos @revision.""" t_handle = tempfile.TemporaryFile() try: t_handle.write( self._svnlook("cat", "-r", revision, repos, info_file_path)) except RosePopenError: return None t_handle.seek(0) config = rose.config.load(t_handle) t_handle.close() return config @classmethod def _get_configs_differ(cls, old_config, new_config): """Return True if old_config differs from new_config.""" for keys1, node1 in old_config.walk(no_ignore=True): node2 = new_config.get(keys1, no_ignore=True) if type(node1) != type(node2): return True if (not isinstance(node1.value, dict) and node1.value != node2.value): return True if node1.comments != node2.comments: return True for keys2, node2 in new_config.walk(no_ignore=True): node1 = old_config.get(keys2, no_ignore=True) if node1 is None: return True return False def _notify_access_changes(self, full_id, author, old_config, new_config): """Email owner and/or access-list users on changes.""" conf = ResourceLocator.default().get_conf() user_tool_name = conf.get_value(["rosa-svn", "user-tool"]) if not user_tool_name: return users = set() if old_config is None: new_access_str = new_config.get_value(["access-list"], "") changes = { ("owner", "+"): new_config.get_value(["owner"]), ("access-list", "+"): new_access_str, } users.update(new_access_str.split()) elif new_config is None: old_owner = old_config.get_value(["owner"]) old_access_str = old_config.get_value(["access-list"], "") changes = { ("owner", "-"): old_owner, ("access-list", "-"): old_access_str, } users.add(old_owner) users.update(old_access_str.split()) else: changes = {} old_owner = old_config.get_value(["owner"]) new_owner = new_config.get_value(["owner"]) if old_owner != new_owner: changes[("owner", "-")] = old_owner changes[("owner", "+")] = new_owner users.add(old_owner) users.add(new_owner) old_access_str = old_config.get_value(["access-list"], "") new_access_str = new_config.get_value(["access-list"], "") old_access_set = set(old_access_str.split()) new_access_set = set(new_access_str.split()) if old_access_set != new_access_set: changes[("access-list", "-")] = old_access_str changes[("access-list", "+")] = new_access_str users.update(old_access_set ^ new_access_set) users.discard("*") users.discard(author) if not users or users == set([author]): return user_tool = self.usertools_manager.get_handler(user_tool_name) users.add(author) emails = sorted(user_tool.get_emails(users)) from_email = conf.get_value(["rosa-svn", "notification-from"], "notications@" + socket.getfqdn()) text = "" for key, status in [ ("owner", "-"), ("owner", "+"), ("access-list", "-"), ("access-list", "+")]: if (key, status) in changes: text += "%s %s=%s\n" % (status, key, changes[(key, status)]) msg = MIMEText(text) msg.set_charset("utf-8") msg["From"] = from_email msg["To"] = ", ".join(emails) msg["Subject"] = "[%s] owner/access-list change" % full_id smtp_host = conf.get_value(["rosa-svn", "smtp-host"], default="localhost") smtp = SMTP(smtp_host) smtp.sendmail(msg["From"], emails, msg.as_string()) smtp.quit()
class AppRunner(Runner): """Invoke a Rose application.""" NAME = "app" OPTIONS = [ "app_mode", "command_key", "conf_dir", "defines", "install_only_mode", "new_mode", "no_overwrite_mode", "opt_conf_keys" ] def __init__(self, *args, **kwargs): Runner.__init__(self, *args, **kwargs) path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__)) self.builtins_manager = SchemeHandlersManager([path], "rose.apps", ["run"], None, *args, **kwargs) self.date_time_oper = RoseDateTimeOperator() def run_impl(self, opts, args, uuid, work_files): """The actual logic for a run.""" # Preparation. conf_tree = self.config_load(opts) self._prep(conf_tree, opts) self._poll(conf_tree) # Run the application or the command. app_mode = conf_tree.node.get_value(["mode"]) if app_mode is None: app_mode = opts.app_mode if app_mode is None: app_mode = os.getenv("ROSE_APP_MODE") if app_mode in [None, "command"]: return self._command(conf_tree, opts, args) else: builtin_app = self.builtins_manager.get_handler(app_mode) if builtin_app is None: raise UnknownBuiltinAppError(app_mode) return builtin_app.run(self, conf_tree, opts, args, uuid, work_files) def get_command(self, conf_tree, opts, args): """Get command to run.""" command = self.popen.list_to_shell_str(args) if not command: names = [ opts.command_key, os.getenv("ROSE_APP_COMMAND_KEY"), os.getenv("ROSE_TASK_NAME"), "default" ] for name in names: if not name: continue command = conf_tree.node.get_value(["command", name]) if command is not None: break return command def _prep(self, conf_tree, opts): """Prepare to run the application.""" if opts.new_mode: self._prep_new(opts) # Dump the actual configuration as rose-app-run.conf ConfigDumper()(conf_tree.node, "rose-app-run.conf") # Environment variables: PATH self._prep_path(conf_tree) # Free format files not defined in the configuration file file_section_prefix = self.config_pm.get_handler("file").PREFIX for rel_path, conf_dir in conf_tree.files.items(): if not rel_path.startswith("file" + os.sep): continue name = rel_path[len("file" + os.sep):] # No sub-directories, very slow otherwise if os.sep in name: name = name.split(os.sep, 1)[0] target_key = file_section_prefix + name target_node = conf_tree.node.get([target_key]) if target_node is None: conf_tree.node.set([target_key]) target_node = conf_tree.node.get([target_key]) elif target_node.is_ignored(): continue source_node = target_node.get(["source"]) if source_node is None: target_node.set(["source"], os.path.join(conf_dir, "file", name)) elif source_node.is_ignored(): continue # Process Environment Variables self.config_pm(conf_tree, "env") # Process Files self.config_pm(conf_tree, "file", no_overwrite_mode=opts.no_overwrite_mode) def _prep_new(self, opts): """Clear out run directory on a --new option if possible.""" conf_dir = opts.conf_dir if not conf_dir or os.path.abspath(conf_dir) == os.getcwd(): raise NewModeError(os.getcwd()) for path in os.listdir("."): self.fs_util.delete(path) @staticmethod def _prep_path(conf_tree): """Add bin directories to the PATH seen by the app command.""" paths = [] for conf_dir in conf_tree.conf_dirs: conf_bin_dir = os.path.join(conf_dir, "bin") if os.path.isdir(conf_bin_dir): paths.append(conf_bin_dir) if paths: value = os.pathsep.join(paths + [os.getenv("PATH")]) conf_tree.node.set(["env", "PATH"], value) else: conf_tree.node.set(["env", "PATH"], os.getenv("PATH")) def _poll(self, conf_tree): """Run any configured file polling.""" poller = Poller(self.popen, self.handle_event) poller.poll(conf_tree) def _command(self, conf_tree, opts, args): """Run the command.""" command = self.get_command(conf_tree, opts, args) if not command: self.handle_event(CommandNotDefinedEvent()) return if os.access("STDIN", os.F_OK | os.R_OK): command += " <STDIN" self.handle_event("command: %s" % command) if opts.install_only_mode: return self.popen(command, shell=True, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin)
class RosieSvnPreCommitHook(object): """A pre-commit hook on a Rosie Subversion repository. Ensure that commits conform to the rules of Rosie. """ IGNORES = "svnperms.conf" RE_ID_NAMES = [r"[Ra-z]", r"[Oa-z]", r"[S\d]", r"[I\d]", r"[E\d]"] LEN_ID = len(RE_ID_NAMES) ST_ADD = "A" ST_DELETE = "D" ST_UPDATE = "U" TRUNK_INFO_FILE = "trunk/rose-suite.info" TRUNK_KNOWN_KEYS_FILE = "trunk/rosie-keys" def __init__(self, event_handler=None, popen=None): if event_handler is None: event_handler = Reporter() self.event_handler = event_handler if popen is None: popen = RosePopener(self.event_handler) self.popen = popen path = os.path.dirname(os.path.dirname(sys.modules["rosie"].__file__)) self.usertools_manager = SchemeHandlersManager( [path], "rosie.usertools", ["verify_users"]) def _get_access_info(self, info_node): """Return (owner, access_list) from "info_node".""" owner = info_node.get_value(["owner"]) access_list = info_node.get_value(["access-list"], "").split() access_list.sort() return owner, access_list def _get_info(self, repos, path_head, txn=None): """Return a ConfigNode for the "rose-suite.info" of a suite. The suite is located under path_head. """ opt_txn = [] if txn is not None: opt_txn = ["-t", txn] t_handle = tempfile.TemporaryFile() path = path_head + self.TRUNK_INFO_FILE t_handle.write(self._svnlook("cat", repos, path, *opt_txn)) t_handle.seek(0) info_node = ConfigLoader()(t_handle) t_handle.close() return info_node def _svnlook(self, *args): """Return the standard output from "svnlook".""" command = ["svnlook"] + list(args) return self.popen(*command, stderr=sys.stderr)[0] def _verify_users(self, status, path, txn_owner, txn_access_list, bad_changes): """Check txn_owner and txn_access_list. For any invalid users, append to bad_changes and return True. """ # The owner and names in access list must be real users conf = ResourceLocator.default().get_conf() user_tool_name = conf.get_value(["rosa-svn", "user-tool"]) if not user_tool_name: return False user_tool = self.usertools_manager.get_handler(user_tool_name) txn_users = set([txn_owner] + txn_access_list) txn_users.discard("*") bad_users = user_tool.verify_users(txn_users) for bad_user in bad_users: if txn_owner == bad_user: bad_change = BadChange( status, path, BadChange.USER, "owner=" + bad_user) bad_changes.append(bad_change) if bad_user in txn_access_list: bad_change = BadChange( status, path, BadChange.USER, "access-list=" + bad_user) bad_changes.append(bad_change) return bool(bad_users) def run(self, repos, txn): """Apply the rule engine on transaction "txn" to repository "repos".""" changes = set() # set([(status, path), ...]) for line in self._svnlook("changed", "-t", txn, repos).splitlines(): status, path = line.split(None, 1) changes.add((status, path)) bad_changes = [] author = None super_users = None rev_info_map = {} # {path-id: (owner, access-list), ...} txn_info_map = {} conf = ResourceLocator.default().get_conf() ignores_str = conf.get_value(["rosa-svn", "ignores"], self.IGNORES) ignores = shlex.split(ignores_str) for status, path in sorted(changes): if any([fnmatch(path, ignore) for ignore in ignores]): continue names = path.split("/", self.LEN_ID + 1) if not names[-1]: names.pop() # Directories above the suites must match the ID patterns is_bad = False for name, pattern in zip(names, self.RE_ID_NAMES): if not re.compile(r"\A" + pattern + r"\Z").match(name): is_bad = True break if is_bad: bad_changes.append(BadChange(status, path)) continue # Can only add directories at levels above the suites if len(names) < self.LEN_ID: if status[0] != self.ST_ADD: bad_changes.append(BadChange(status, path)) continue else: is_meta_suite = "".join(names[0:self.LEN_ID]) == "ROSIE" # No need to check non-trunk changes if len(names) > self.LEN_ID and names[self.LEN_ID] != "trunk": continue # New suite should have an info file if status[0] == self.ST_ADD and len(names) == self.LEN_ID: if (self.ST_ADD, path + "trunk/") not in changes: bad_changes.append( BadChange(status, path, BadChange.NO_TRUNK)) continue path_trunk_info_file = path + self.TRUNK_INFO_FILE if ((self.ST_ADD, path_trunk_info_file) not in changes and (self.ST_UPDATE, path_trunk_info_file) not in changes): bad_changes.append( BadChange(status, path, BadChange.NO_INFO)) continue # The rest are trunk changes in a suite path_head = "/".join(names[0:self.LEN_ID]) + "/" path_tail = path[len(path_head):] # For meta suite, make sure keys in keys file can be parsed if is_meta_suite and path_tail == self.TRUNK_KNOWN_KEYS_FILE: out = self._svnlook("cat", "-t", txn, repos, path) try: shlex.split(out) except ValueError: bad_changes.append( BadChange(status, path, BadChange.VALUE)) continue # Suite trunk information file must have an owner # User IDs of owner and access list must be real if (status not in self.ST_DELETE and path_tail == self.TRUNK_INFO_FILE): if path_head not in txn_info_map: txn_info_map[path_head] = self._get_info( repos, path_head, txn) txn_owner, txn_access_list = self._get_access_info( txn_info_map[path_head]) if not txn_owner: bad_changes.append( BadChange(status, path, BadChange.NO_OWNER)) continue if self._verify_users( status, path, txn_owner, txn_access_list, bad_changes): continue reports = DefaultValidators().validate( txn_info_map[path_head], load_meta_config( txn_info_map[path_head], config_type=rose.INFO_CONFIG_NAME)) if reports: reports_str = get_reports_as_text({None: reports}, path) bad_changes.append( BadChange(status, path, BadChange.VALUE, reports_str)) continue # Can only remove trunk information file with suite if status == self.ST_DELETE and path_tail == self.TRUNK_INFO_FILE: if (self.ST_DELETE, path_head) not in changes: bad_changes.append( BadChange(status, path, BadChange.NO_INFO)) continue # Can only remove trunk with suite if status == self.ST_DELETE and path_tail == "trunk/": if (self.ST_DELETE, path_head) not in changes: bad_changes.append( BadChange(status, path, BadChange.NO_TRUNK)) continue # New suite trunk: ignore the rest if (self.ST_ADD, path_head + "trunk/") in changes: continue # See whether author has permission to make changes if author is None: author = self._svnlook("author", "-t", txn, repos).strip() if super_users is None: super_users = [] for s_key in ["rosa-svn", "rosa-svn-pre-commit"]: value = conf.get_value([s_key, "super-users"]) if value is not None: super_users = shlex.split(value) break if path_head not in rev_info_map: rev_info_map[path_head] = self._get_info( repos, path_head) owner, access_list = self._get_access_info(rev_info_map[path_head]) admin_users = super_users + [owner] # Only admin users can remove the suite if author not in admin_users and not path_tail: bad_changes.append(BadChange(status, path)) continue # Admin users and those in access list can modify everything in # trunk apart from specific entries in the trunk info file if "*" in access_list or author in admin_users + access_list: if path_tail != self.TRUNK_INFO_FILE: continue else: bad_changes.append(BadChange(status, path)) continue # The owner must not be deleted if path_head not in txn_info_map: txn_info_map[path_head] = self._get_info( repos, path_head, txn) txn_owner, txn_access_list = self._get_access_info( txn_info_map[path_head]) if not txn_owner: bad_changes.append(BadChange(status, path, BadChange.NO_OWNER)) continue # Only the admin users can change owner and access list if owner == txn_owner and access_list == txn_access_list: continue if author not in admin_users: if owner != txn_owner: bad_changes.append(BadChange(status, path, BadChange.PERM, "owner=" + txn_owner)) else: # access list bad_change = BadChange( status, path, BadChange.PERM, "access-list=" + " ".join(txn_access_list)) bad_changes.append(bad_change) continue if bad_changes: raise BadChanges(bad_changes) __call__ = run
class RosieSvnPreCommitHook(object): """A pre-commit hook on a Rosie Subversion repository. Ensure that commits conform to the rules of Rosie. """ IGNORES = "svnperms.conf" RE_ID_NAMES = [r"[Ra-z]", r"[Oa-z]", r"[S\d]", r"[I\d]", r"[E\d]"] LEN_ID = len(RE_ID_NAMES) ST_ADD = "A" ST_DELETE = "D" ST_UPDATE = "U" TRUNK_INFO_FILE = "trunk/rose-suite.info" TRUNK_KNOWN_KEYS_FILE = "trunk/rosie-keys" def __init__(self, event_handler=None, popen=None): if event_handler is None: event_handler = Reporter() self.event_handler = event_handler if popen is None: popen = RosePopener(self.event_handler) self.popen = popen path = os.path.dirname(os.path.dirname(sys.modules["rosie"].__file__)) self.usertools_manager = SchemeHandlersManager( [path], "rosie.usertools", ["verify_users"]) def _get_access_info(self, repos, path_head, txn=None): """Return the owner and the access list of a suite (path_head).""" opt_txn = [] if txn is not None: opt_txn = ["-t", txn] t_handle = tempfile.TemporaryFile() path = path_head + self.TRUNK_INFO_FILE t_handle.write(self._svnlook("cat", repos, path, *opt_txn)) t_handle.seek(0) node = ConfigLoader()(t_handle) t_handle.close() owner = node.get_value(["owner"]) access_list = node.get_value(["access-list"], "").split() access_list.sort() return owner, access_list def _svnlook(self, *args): """Return the standard output from "svnlook".""" command = ["svnlook"] + list(args) return self.popen(*command, stderr=sys.stderr)[0] def _verify_users(self, status, path, txn_owner, txn_access_list, bad_changes): """Check txn_owner and txn_access_list. For any invalid users, append to bad_changes and return True. """ # The owner and names in access list must be real users conf = ResourceLocator.default().get_conf() user_tool_name = conf.get_value(["rosa-svn", "user-tool"]) if not user_tool_name: return False user_tool = self.usertools_manager.get_handler(user_tool_name) txn_users = set([txn_owner] + txn_access_list) txn_users.discard("*") bad_users = user_tool.verify_users(txn_users) for bad_user in bad_users: if txn_owner == bad_user: bad_change = BadChange( status, path, BadChange.USER, "owner=" + bad_user) bad_changes.append(bad_change) if bad_user in txn_access_list: bad_change = BadChange( status, path, BadChange.USER, "access-list=" + bad_user) bad_changes.append(bad_change) return bool(bad_users) def run(self, repos, txn): """Apply the rule engine on transaction "txn" to repository "repos".""" changes = set() # set([(status, path), ...]) for line in self._svnlook("changed", "-t", txn, repos).splitlines(): status, path = line.split(None, 1) changes.add((status, path)) bad_changes = [] author = None super_users = None access_info_map = {} # {path-id: (owner, access-list), ...} txn_access_info_map = {} conf = ResourceLocator.default().get_conf() ignores_str = conf.get_value(["rosa-svn", "ignores"], self.IGNORES) ignores = shlex.split(ignores_str) for status, path in sorted(changes): if any([fnmatch(path, ignore) for ignore in ignores]): continue names = path.split("/", self.LEN_ID + 1) if not names[-1]: names.pop() # Directories above the suites must match the ID patterns is_bad = False for name, pattern in zip(names, self.RE_ID_NAMES): if not re.compile(r"\A" + pattern + r"\Z").match(name): is_bad = True break if is_bad: bad_changes.append(BadChange(status, path)) continue # Can only add directories at levels above the suites if len(names) < self.LEN_ID: if status[0] != self.ST_ADD: bad_changes.append(BadChange(status, path)) continue else: is_meta_suite = "".join(names[0:self.LEN_ID]) == "ROSIE" # No need to check non-trunk changes if len(names) > self.LEN_ID and names[self.LEN_ID] != "trunk": continue # New suite should have an info file if status[0] == self.ST_ADD and len(names) == self.LEN_ID: if (self.ST_ADD, path + "trunk/") not in changes: bad_changes.append( BadChange(status, path, BadChange.NO_TRUNK)) continue path_trunk_info_file = path + self.TRUNK_INFO_FILE if ((self.ST_ADD, path_trunk_info_file) not in changes and (self.ST_UPDATE, path_trunk_info_file) not in changes): bad_changes.append( BadChange(status, path, BadChange.NO_INFO)) continue # The rest are trunk changes in a suite path_head = "/".join(names[0:self.LEN_ID]) + "/" path_tail = path[len(path_head):] # For meta suite, make sure keys in keys file can be parsed if is_meta_suite and path_tail == self.TRUNK_KNOWN_KEYS_FILE: out = self._svnlook("cat", "-t", txn, repos, path) try: shlex.split(out) except ValueError: bad_changes.append( BadChange(status, path, BadChange.VALUE)) continue # Suite trunk information file must have an owner # User IDs of owner and access list must be real if (status not in self.ST_DELETE and path_tail == self.TRUNK_INFO_FILE): txn_owner, txn_access_list = self._get_access_info( repos, path_head, txn) if not txn_owner: bad_changes.append( BadChange(status, path, BadChange.NO_OWNER)) continue if self._verify_users( status, path, txn_owner, txn_access_list, bad_changes): continue # New suite trunk: ignore the rest if (self.ST_ADD, path_head + "trunk/") in changes: continue # Can only remove trunk information file with suite if status == self.ST_DELETE and path_tail == self.TRUNK_INFO_FILE: if (self.ST_DELETE, path_head) not in changes: bad_changes.append( BadChange(status, path, BadChange.NO_INFO)) continue # Can only remove trunk with suite if status == self.ST_DELETE and path_tail == "trunk/": if (self.ST_DELETE, path_head) not in changes: bad_changes.append( BadChange(status, path, BadChange.NO_TRUNK)) continue # See whether author has permission to make changes if author is None: author = self._svnlook("author", "-t", txn, repos).strip() if super_users is None: super_users = [] for s_key in ["rosa-svn", "rosa-svn-pre-commit"]: value = conf.get_value([s_key, "super-users"]) if value is not None: super_users = shlex.split(value) break if path_head not in access_info_map: access_info = self._get_access_info(repos, path_head) access_info_map[path_head] = access_info owner, access_list = access_info_map[path_head] admin_users = super_users + [owner] # Only admin users can remove the suite if author not in admin_users and not path_tail: bad_changes.append(BadChange(status, path)) continue # Admin users and those in access list can modify everything in # trunk apart from specific entries in the trunk info file if "*" in access_list or author in admin_users + access_list: if path_tail != self.TRUNK_INFO_FILE: continue else: bad_changes.append(BadChange(status, path)) continue # The owner must not be deleted if path_head not in txn_access_info_map: txn_access_info = self._get_access_info(repos, path_head, txn) txn_access_info_map[path_head] = txn_access_info txn_owner, txn_access_list = txn_access_info_map[path_head] if not txn_owner: bad_changes.append(BadChange(status, path, BadChange.NO_OWNER)) continue # Only the admin users can change owner and access list if owner == txn_owner and access_list == txn_access_list: continue if author not in admin_users: if owner != txn_owner: bad_changes.append(BadChange(status, path, BadChange.PERM, "owner=" + txn_owner)) else: # access list bad_change = BadChange( status, path, BadChange.PERM, "access-list=" + " ".join(txn_access_list)) bad_changes.append(bad_change) continue if bad_changes: raise BadChanges(bad_changes) __call__ = run
def _run(self, dao, app_runner, config): """Transform and archive suite files. This application is designed to work under "rose task-run" in a suite. """ path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__)) compress_manager = SchemeHandlersManager( [path], "rose.apps.rose_arch_compressions", ["compress_sources"], None, app_runner) # Set up the targets cycle = os.getenv("ROSE_TASK_CYCLE_TIME") targets = [] for t_key, t_node in sorted(config.value.items()): if t_node.is_ignored() or ":" not in t_key: continue s_key_head, s_key_tail = t_key.split(":", 1) if s_key_head != self.SECTION or not s_key_tail: continue target_prefix = self._get_conf(config, t_node, "target-prefix", default="") try: s_key_tail = env_var_process(s_key_tail) except UnboundEnvironmentVariableError as exc: raise ConfigValueError([t_key, ""], "", exc) target_name = target_prefix + s_key_tail target = RoseArchTarget(target_name) target.command_format = self._get_conf(config, t_node, "command-format", compulsory=True) try: target.command_format % {"sources": "", "target": ""} except KeyError as exc: target.status = target.ST_BAD app_runner.handle_event( RoseArchValueError(target.name, "command-format", target.command_format, type(exc).__name__, exc)) source_str = self._get_conf(config, t_node, "source", compulsory=True) source_prefix = self._get_conf(config, t_node, "source-prefix", default="") target.source_edit_format = self._get_conf(config, t_node, "source-edit-format", default="") try: target.source_edit_format % {"in": "", "out": ""} except KeyError as exc: target.status = target.ST_BAD app_runner.handle_event( RoseArchValueError(target.name, "source-edit-format", target.source_edit_format, type(exc).__name__, exc)) update_check_str = self._get_conf(config, t_node, "update-check", default="md5sum") try: checksum_func = get_checksum_func(update_check_str) except KeyError as exc: raise RoseArchValueError(target.name, "update-check", update_check_str, type(exc).__name__, exc) for source_glob in shlex.split(source_str): paths = glob(source_prefix + source_glob) if not paths: exc = OSError(errno.ENOENT, os.strerror(errno.ENOENT), source_glob) app_runner.handle_event( ConfigValueError([t_key, "source"], source_glob, exc)) target.status = target.ST_BAD continue for path in paths: # N.B. source_prefix may not be a directory name = path[len(source_prefix):] for path_, checksum, _ in get_checksum( path, checksum_func): if checksum is None: # is directory continue if path_: target.sources[checksum] = RoseArchSource( checksum, os.path.join(name, path_), os.path.join(path, path_)) else: # path is a file target.sources[checksum] = RoseArchSource( checksum, name, path) target.compress_scheme = self._get_conf(config, t_node, "compress") if target.compress_scheme: if (compress_manager.get_handler(target.compress_scheme) is None): app_runner.handle_event( ConfigValueError([t_key, "compress"], target.compress_scheme, KeyError(target.compress_scheme))) target.status = target.ST_BAD else: target_base = target.name if "/" in target.name: target_base = target.name.rsplit("/", 1)[1] if "." in target_base: tail = target_base.split(".", 1)[1] if compress_manager.get_handler(tail): target.compress_scheme = tail rename_format = self._get_conf(config, t_node, "rename-format") if rename_format: rename_parser_str = self._get_conf(config, t_node, "rename-parser") if rename_parser_str: try: rename_parser = re.compile(rename_parser_str) except re.error as exc: raise RoseArchValueError(target.name, "rename-parser", rename_parser_str, type(exc).__name__, exc) else: rename_parser = None for source in target.sources.values(): dict_ = {"cycle": cycle, "name": source.name} if rename_parser: match = rename_parser.match(source.name) if match: dict_.update(match.groupdict()) try: source.name = rename_format % dict_ except (KeyError, ValueError) as exc: raise RoseArchValueError(target.name, "rename-format", rename_format, type(exc).__name__, exc) old_target = dao.select(target.name) if old_target is None or old_target != target: dao.delete(target) else: target.status = target.ST_OLD targets.append(target) # Delete from database items that are no longer relevant dao.delete_all(filter_targets=targets) # Update the targets for target in targets: if target.status == target.ST_OLD: app_runner.handle_event(RoseArchEvent(target)) continue target.command_rc = 1 dao.insert(target) if target.status == target.ST_BAD: app_runner.handle_event(RoseArchEvent(target)) continue work_dir = mkdtemp() t_init = time() t_tran, t_arch = t_init, t_init ret_code = None try: # Rename/edit sources target.status = target.ST_BAD rename_required = False for source in target.sources.values(): if source.name != source.orig_name: rename_required = True break if rename_required or target.source_edit_format: for source in target.sources.values(): source.path = os.path.join(work_dir, source.name) source_path_d = os.path.dirname(source.path) app_runner.fs_util.makedirs(source_path_d) if target.source_edit_format: fmt_args = { "in": source.orig_path, "out": source.path } command = target.source_edit_format % fmt_args app_runner.popen.run_ok(command, shell=True) else: app_runner.fs_util.symlink(source.orig_path, source.path) # Compress sources if target.compress_scheme: handler = compress_manager.get_handler( target.compress_scheme) handler.compress_sources(target, work_dir) t_tran = time() # Run archive command sources = [] if target.work_source_path: sources = [target.work_source_path] else: for source in target.sources.values(): sources.append(source.path) sources_str = app_runner.popen.list_to_shell_str(sources) target_str = app_runner.popen.list_to_shell_str([target.name]) command = target.command_format % { "sources": sources_str, "target": target_str } ret_code, out, err = app_runner.popen.run(command, shell=True) t_arch = time() if ret_code: app_runner.handle_event( RosePopenError([command], ret_code, out, err)) else: target.status = target.ST_NEW app_runner.handle_event(err, kind=Event.KIND_ERR) app_runner.handle_event(out) target.command_rc = ret_code dao.update_command_rc(target) finally: app_runner.fs_util.delete(work_dir) app_runner.handle_event( RoseArchEvent(target, [t_init, t_tran, t_arch], ret_code)) return [target.status for target in targets].count(RoseArchTarget.ST_BAD)
class AppRunner(Runner): """Invoke a Rose application.""" NAME = "app" OPTIONS = ["app_mode", "command_key", "conf_dir", "defines", "install_only_mode", "new_mode", "no_overwrite_mode", "opt_conf_keys"] def __init__(self, *args, **kwargs): Runner.__init__(self, *args, **kwargs) path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__)) self.builtins_manager = SchemeHandlersManager( [path], "rose.apps", ["run"], None, *args, **kwargs) self.date_time_oper = RoseDateTimeOperator() def run_impl(self, opts, args, uuid, work_files): """The actual logic for a run.""" # Preparation. conf_tree = self.config_load(opts) self._prep(conf_tree, opts) self._poll(conf_tree) # Run the application or the command. app_mode = conf_tree.node.get_value(["mode"]) if app_mode is None: app_mode = opts.app_mode if app_mode is None: app_mode = os.getenv("ROSE_APP_MODE") if app_mode in [None, "command"]: return self._command(conf_tree, opts, args) else: builtin_app = self.builtins_manager.get_handler(app_mode) if builtin_app is None: raise UnknownBuiltinAppError(app_mode) return builtin_app.run(self, conf_tree, opts, args, uuid, work_files) def get_command(self, conf_tree, opts, args): """Get command to run.""" command = self.popen.list_to_shell_str(args) if not command: names = [opts.command_key, os.getenv("ROSE_APP_COMMAND_KEY"), os.getenv("ROSE_TASK_NAME"), "default"] for name in names: if not name: continue command = conf_tree.node.get_value(["command", name]) if command is not None: break return command def _prep(self, conf_tree, opts): """Prepare to run the application.""" if opts.new_mode: self._prep_new(opts) # Dump the actual configuration as rose-app-run.conf ConfigDumper()(conf_tree.node, "rose-app-run.conf") # Environment variables: PATH self._prep_path(conf_tree) # Free format files not defined in the configuration file file_section_prefix = self.config_pm.get_handler("file").PREFIX for rel_path, conf_dir in conf_tree.files.items(): if not rel_path.startswith("file" + os.sep): continue name = rel_path[len("file" + os.sep):] # No sub-directories, very slow otherwise if os.sep in name: name = name.split(os.sep, 1)[0] target_key = file_section_prefix + name target_node = conf_tree.node.get([target_key]) if target_node is None: conf_tree.node.set([target_key]) target_node = conf_tree.node.get([target_key]) elif target_node.is_ignored(): continue source_node = target_node.get(["source"]) if source_node is None: target_node.set(["source"], os.path.join(conf_dir, "file", name)) elif source_node.is_ignored(): continue # Process Environment Variables self.config_pm(conf_tree, "env") # Process Files self.config_pm(conf_tree, "file", no_overwrite_mode=opts.no_overwrite_mode) def _prep_new(self, opts): """Clear out run directory on a --new option if possible.""" conf_dir = opts.conf_dir if not conf_dir or os.path.abspath(conf_dir) == os.getcwd(): raise NewModeError(os.getcwd()) for path in os.listdir("."): self.fs_util.delete(path) @staticmethod def _prep_path(conf_tree): """Add bin directories to the PATH seen by the app command.""" paths = [] for conf_dir in conf_tree.conf_dirs: conf_bin_dir = os.path.join(conf_dir, "bin") if os.path.isdir(conf_bin_dir): paths.append(conf_bin_dir) if paths: value = os.pathsep.join(paths + [os.getenv("PATH")]) conf_tree.node.set(["env", "PATH"], value) else: conf_tree.node.set(["env", "PATH"], os.getenv("PATH")) def _poll(self, conf_tree): """Run any configured file polling.""" poller = Poller(self.popen, self.handle_event) poller.poll(conf_tree) def _command(self, conf_tree, opts, args): """Run the command.""" command = self.get_command(conf_tree, opts, args) if not command: self.handle_event(CommandNotDefinedEvent()) return if os.access("STDIN", os.F_OK | os.R_OK): command += " <STDIN" self.handle_event("command: %s" % command) if opts.install_only_mode: return self.popen( command, shell=True, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin)
class AppRunner(Runner): """Invoke a Rose application.""" OLD_DURATION_UNITS = {"h": 3600, "m": 60, "s": 1} NAME = "app" OPTIONS = [ "app_mode", "command_key", "conf_dir", "defines", "install_only_mode", "new_mode", "no_overwrite_mode", "opt_conf_keys" ] def __init__(self, *args, **kwargs): Runner.__init__(self, *args, **kwargs) path = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__)) self.builtins_manager = SchemeHandlersManager([path], "rose.apps", ["run"], None, *args, **kwargs) self.duration_parser = DurationParser() def run_impl(self, opts, args, uuid, work_files): """The actual logic for a run.""" # Preparation. conf_tree = self.config_load(opts) self._prep(conf_tree, opts) self._poll(conf_tree) # Run the application or the command. app_mode = conf_tree.node.get_value(["mode"]) if app_mode is None: app_mode = opts.app_mode if app_mode in [None, "command"]: return self._command(conf_tree, opts, args) else: builtin_app = self.builtins_manager.get_handler(app_mode) if builtin_app is None: raise UnknownBuiltinAppError(app_mode) return builtin_app.run(self, conf_tree, opts, args, uuid, work_files) def _poll(self, conf_tree): """Poll for prerequisites of applications.""" # Poll configuration poll_test = conf_tree.node.get_value(["poll", "test"]) poll_all_files_value = conf_tree.node.get_value(["poll", "all-files"]) poll_all_files = [] if poll_all_files_value: try: poll_all_files = shlex.split( env_var_process(poll_all_files_value)) except UnboundEnvironmentVariableError as exc: raise ConfigValueError(["poll", "all-files"], poll_all_files_value, exc) poll_any_files_value = conf_tree.node.get_value(["poll", "any-files"]) poll_any_files = [] if poll_any_files_value: try: poll_any_files = shlex.split( env_var_process(poll_any_files_value)) except UnboundEnvironmentVariableError as exc: raise ConfigValueError(["poll", "any-files"], poll_any_files_value, exc) poll_file_test = None if poll_all_files or poll_any_files: poll_file_test = conf_tree.node.get_value(["poll", "file-test"]) if poll_file_test and "{}" not in poll_file_test: raise ConfigValueError(["poll", "file-test"], poll_file_test, ConfigValueError.SYNTAX) poll_delays = [] if poll_test or poll_all_files or poll_any_files: # Parse something like this: delays=10,4*PT30S,PT2M30S,2*PT1H # R*DURATION: repeat the value R times conf_keys = ["poll", "delays"] poll_delays_value = conf_tree.node.get_value(conf_keys, default="").strip() if poll_delays_value: is_legacy0 = None for item in poll_delays_value.split(","): value = item.strip() repeat = 1 if "*" in value: repeat, value = value.split("*", 1) try: repeat = int(repeat) except ValueError as exc: raise ConfigValueError(conf_keys, poll_delays_value, ConfigValueError.SYNTAX) try: value = self.duration_parser.parse(value).get_seconds() is_legacy = False except ISO8601SyntaxError: # Legacy mode: nnnU # nnn is a float, U is the unit # No unit or s: seconds # m: minutes # h: hours unit = None if value[-1].lower() in self.OLD_DURATION_UNITS: unit = self.OLD_DURATION_UNITS[value[-1].lower()] value = value[:-1] try: value = float(value) except ValueError as exc: raise ConfigValueError(conf_keys, poll_delays_value, ConfigValueError.SYNTAX) if unit: value *= unit is_legacy = True if is_legacy0 is None: is_legacy0 = is_legacy elif is_legacy0 != is_legacy: raise ConfigValueError( conf_keys, poll_delays_value, ConfigValueError.DURATION_LEGACY_MIX) poll_delays += [value] * repeat else: poll_delays = [0] # poll once without a delay # Poll t_init = get_timepoint_for_now() while poll_delays and (poll_test or poll_any_files or poll_all_files): poll_delay = poll_delays.pop(0) if poll_delay: sleep(poll_delay) if poll_test: ret_code = self.popen.run(poll_test, shell=True, stdout=sys.stdout, stderr=sys.stderr)[0] self.handle_event(PollEvent(time(), poll_test, ret_code == 0)) if ret_code == 0: poll_test = None any_files = list(poll_any_files) for file_ in any_files: if self._poll_file(file_, poll_file_test): self.handle_event(PollEvent(time(), "any-files", True)) poll_any_files = [] break all_files = list(poll_all_files) for file_ in all_files: if self._poll_file(file_, poll_file_test): poll_all_files.remove(file_) if all_files and not poll_all_files: self.handle_event(PollEvent(time(), "all-files", True)) failed_items = [] if poll_test: failed_items.append("test") if poll_any_files: failed_items.append("any-files") if poll_all_files: failed_items.append("all-files:" + self.popen.list_to_shell_str(poll_all_files)) if failed_items: now = get_timepoint_for_now() raise PollTimeoutError(now, now - t_init, failed_items) def _poll_file(self, file_, poll_file_test): """Poll for existence of a file.""" is_done = False if poll_file_test: test = poll_file_test.replace( "{}", self.popen.list_to_shell_str([file_])) is_done = self.popen.run(test, shell=True, stdout=sys.stdout, stderr=sys.stderr)[0] == 0 else: is_done = bool(glob(file_)) self.handle_event(PollEvent(time(), "file:" + file_, is_done)) return is_done def _prep(self, conf_tree, opts): """Prepare to run the application.""" if opts.new_mode: conf_dir = opts.conf_dir if not conf_dir or os.path.abspath(conf_dir) == os.getcwd(): raise NewModeError(os.getcwd()) for path in os.listdir("."): self.fs_util.delete(path) # Dump the actual configuration as rose-app-run.conf ConfigDumper()(conf_tree.node, "rose-app-run.conf") # Environment variables: PATH paths = [] for conf_dir in conf_tree.conf_dirs: conf_bin_dir = os.path.join(conf_dir, "bin") if os.path.isdir(conf_bin_dir): paths.append(conf_bin_dir) if paths: value = os.pathsep.join(paths + [os.getenv("PATH")]) conf_tree.node.set(["env", "PATH"], value) else: conf_tree.node.set(["env", "PATH"], os.getenv("PATH")) # Free format files not defined in the configuration file file_section_prefix = self.config_pm.get_handler("file").PREFIX for rel_path, conf_dir in conf_tree.files.items(): if not rel_path.startswith("file" + os.sep): continue name = rel_path[len("file" + os.sep):] # No sub-directories, very slow otherwise if os.sep in name: name = name.split(os.sep, 1)[0] target_key = file_section_prefix + name target_node = conf_tree.node.get([target_key]) if target_node is None: conf_tree.node.set([target_key]) target_node = conf_tree.node.get([target_key]) elif target_node.is_ignored(): continue source_node = target_node.get("source") if source_node is None: target_node.set(["source"], os.path.join(conf_dir, "file", name)) elif source_node.is_ignored(): continue # Process Environment Variables self.config_pm(conf_tree, "env") # Process Files self.config_pm(conf_tree, "file", no_overwrite_mode=opts.no_overwrite_mode) def _command(self, conf_tree, opts, args): """Run the command.""" command = self.popen.list_to_shell_str(args) if not command: names = [opts.command_key, os.getenv("ROSE_TASK_NAME"), "default"] for name in names: if not name: continue command = conf_tree.node.get_value(["command", name]) if command is not None: break else: self.handle_event(CommandNotDefinedEvent()) return if os.access("STDIN", os.F_OK | os.R_OK): command += " <STDIN" self.handle_event("command: %s" % command) if opts.install_only_mode: return self.popen(command, shell=True, stdout=sys.stdout, stderr=sys.stderr)
class AppRunner(Runner): """Invoke a Rose application.""" NAME = "app" OPTIONS = ["app_mode", "command_key", "conf_dir", "defines", "install_only_mode", "new_mode", "no_overwrite_mode", "opt_conf_keys"] def __init__(self, *args, **kwargs): Runner.__init__(self, *args, **kwargs) p = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__)) self.builtins_manager = SchemeHandlersManager( [p], "rose.apps", ["run"], None, *args, **kwargs) def run_impl(self, opts, args, uuid, work_files): """The actual logic for a run.""" # Preparation. conf_tree = self.config_load(opts) self._prep(conf_tree, opts, args, uuid, work_files) self._poll(conf_tree, opts, args, uuid, work_files) # Run the application or the command. app_mode = conf_tree.node.get_value(["mode"]) if app_mode is None: app_mode = opts.app_mode if app_mode in [None, "command"]: return self._command(conf_tree, opts, args, uuid, work_files) else: builtin_app = self.builtins_manager.get_handler(app_mode) if builtin_app is None: raise UnknownBuiltinAppError(app_mode) return builtin_app.run(self, conf_tree, opts, args, uuid, work_files) def _poll(self, conf_tree, opts, args, uuid, work_files): """Poll for prerequisites of applications.""" # Poll configuration poll_test = conf_tree.node.get_value(["poll", "test"]) poll_all_files_value = conf_tree.node.get_value(["poll", "all-files"]) poll_all_files = [] if poll_all_files_value: try: poll_all_files = shlex.split( env_var_process(poll_all_files_value)) except UnboundEnvironmentVariableError as e: raise ConfigValueError(["poll", "all-files"], poll_all_files_value, e) poll_any_files_value = conf_tree.node.get_value(["poll", "any-files"]) poll_any_files = [] if poll_any_files_value: try: poll_any_files = shlex.split( env_var_process(poll_any_files_value)) except UnboundEnvironmentVariableError as e: raise ConfigValueError(["poll", "any-files"], poll_any_files_value, e) poll_file_test = None if poll_all_files or poll_any_files: poll_file_test = conf_tree.node.get_value(["poll", "file-test"]) if poll_file_test and "{}" not in poll_file_test: raise ConfigValueError(["poll", "file-test"], poll_file_test, ConfigValueError.SYNTAX) poll_delays = [] if poll_test or poll_all_files or poll_any_files: # Parse something like this: delays=10,4*30s,2.5m,2*1h # No unit or s: seconds # m: minutes # h: hours # N*: repeat the value N times poll_delays_value = conf_tree.node.get_value(["poll", "delays"], default="") poll_delays_value = poll_delays_value.strip() units = {"h": 3600, "m": 60, "s": 1} if poll_delays_value: for item in poll_delays_value.split(","): value = item.strip() repeat = 1 if "*" in value: repeat, value = value.split("*", 1) try: repeat = int(repeat) except ValueError as e: raise ConfigValueError(["poll", "delays"], poll_delays_value, ConfigValueError.SYNTAX) unit = None if value[-1].lower() in units.keys(): unit = units[value[-1]] value = value[:-1] try: value = float(value) except ValueError as e: raise ConfigValueError(["poll", "delays"], poll_delays_value, ConfigValueError.SYNTAX) if unit: value *= unit for i in range(repeat): poll_delays.append(value) else: poll_delays = [0] # poll once without a delay # Poll t_init = time() while poll_delays and (poll_test or poll_any_files or poll_all_files): poll_delay = poll_delays.pop(0) if poll_delay: sleep(poll_delay) if poll_test: rc, out, err = self.popen.run(poll_test, shell=True, stdout=sys.stdout, stderr=sys.stderr) self.handle_event(PollEvent(time(), poll_test, rc == 0)) if rc == 0: poll_test = None any_files = list(poll_any_files) for file in any_files: if self._poll_file(file, poll_file_test): self.handle_event(PollEvent(time(), "any-files", True)) poll_any_files = [] break all_files = list(poll_all_files) for file in all_files: if self._poll_file(file, poll_file_test): poll_all_files.remove(file) if all_files and not poll_all_files: self.handle_event(PollEvent(time(), "all-files", True)) failed_items = [] if poll_test: failed_items.append("test") if poll_any_files: failed_items.append("any-files") if poll_all_files: failed_items.append("all-files:" + self.popen.list_to_shell_str(poll_all_files)) if failed_items: now = time() raise PollTimeoutError(now, now - t_init, failed_items) def _poll_file(self, file, poll_file_test): ok = False if poll_file_test: test = poll_file_test.replace( "{}", self.popen.list_to_shell_str([file])) rc, out, err = self.popen.run(test, shell=True, stdout=sys.stdout, stderr=sys.stderr) ok = rc == 0 else: ok = os.path.exists(file) self.handle_event(PollEvent(time(), "file:" + file, ok)) return ok def _prep(self, conf_tree, opts, args, uuid, work_files): """Prepare to run the application.""" if opts.new_mode: conf_dir = opts.conf_dir if not conf_dir or os.path.abspath(conf_dir) == os.getcwd(): raise NewModeError(os.getcwd()) for p in os.listdir("."): self.fs_util.delete(p) # Dump the actual configuration as rose-app-run.conf ConfigDumper()(conf_tree.node, "rose-app-run.conf") # Environment variables: PATH paths = [] for conf_dir in conf_tree.conf_dirs: conf_bin_dir = os.path.join(conf_dir, "bin") if os.path.isdir(conf_bin_dir): paths.append(conf_bin_dir) if paths: value = os.pathsep.join(paths + [os.getenv("PATH")]) conf_tree.node.set(["env", "PATH"], value) else: conf_tree.node.set(["env", "PATH"], os.getenv("PATH")) # Free format files not defined in the configuration file file_section_prefix = self.config_pm.get_handler("file").PREFIX for rel_path, conf_dir in conf_tree.files.items(): if not rel_path.startswith("file" + os.sep): continue name = rel_path[len("file" + os.sep):] section = file_section_prefix + name if conf_tree.node.get([section], no_ignore=True) is None: conf_tree.node.set([section, "source"], os.path.join(conf_dir, rel_path)) # Process Environment Variables self.config_pm(conf_tree, "env") # Process Files self.config_pm(conf_tree, "file", no_overwrite_mode=opts.no_overwrite_mode) def _command(self, conf_tree, opts, args, uuid, work_files): """Run the command.""" command = self.popen.list_to_shell_str(args) if not command: names = [opts.command_key, os.getenv("ROSE_TASK_NAME"), "default"] for name in names: if not name: continue command = conf_tree.node.get_value(["command", name]) if command is not None: break else: self.handle_event(CommandNotDefinedEvent()) return if os.access("STDIN", os.F_OK | os.R_OK): command += " <STDIN" self.handle_event("command: %s" % command) if opts.install_only_mode: return # TODO: allow caller of app_run to specify stdout and stderr? self.popen(command, shell=True, stdout=sys.stdout, stderr=sys.stderr)